DroidDeck/tools/linuxfs/overlay/usr/local/bin/droiddeck-esync

#!/usr/bin/python3
# droiddeck-esync: picks and stages the esync pack for each Proton build.
# Credit: esync and ntsync were designed by Elizabeth Figura for Wine and Proton, and fsync by Elizabeth Figura
# and Paul Gofman; droiddeck-esync, droiddeck-ntsync and droiddeck-fsync build on that work.
import contextlib
import fcntl
import hashlib
import json
import os
import re
import resource
import shutil
import stat
import struct
import sys
import time
from pathlib import Path

NTDLL = "files/lib/wine/aarch64-unix/ntdll.so"
WINESERVER = "files/bin-arm64/wineserver"
PACK_FILES = (NTDLL, WINESERVER)
DEFAULT_COPY = ("files/bin-arm64/wine", "files/lib/wine/aarch64-unix/wine", "files/lib/wine/aarch64-unix/wine-preloader")
REAL_DIRS = (".", "files", "files/bin-arm64", "files/lib", "files/lib/wine", "files/lib/wine/aarch64-unix")
STOCK_LINKS = ("dist.lock", "files/steampipe_fixups_mtime")
SKIPPED = frozenset(("toolmanifest.vdf", "compatibilitytool.vdf", "droiddeck-proton-wrap", "bannerlator-proton-wrap"))
SKIPPED_SUFFIX = (".droiddeck-orig", ".bannerlator-orig")
RECORD = ".droiddeck-esync.json"
STORE = ".local/share/droiddeck-esync"
STORE_LOCK = "store.lock"
BUILTIN_MARKER = b"esync: up and running"
FSYNC_MARKER = b"fsync: up and running"
NTSYNC_MARKER = b"/dev/ntsync"
WINESERVER_ONLY = ("PROTON_NO_ESYNC", "PROTON_NO_FSYNC", "PROTON_NO_NTSYNC")
NOSYNC_APPS = frozenset(("2630", "1060210", "414740", "201510", "1233880"))
SOURCES = ("Proton Experimental (ARM64)", "Proton 11.0 (ARM64)")
SD_LIBRARY = "/mnt/droiddeck-sd"
OWN_TOOL_PREFIX = "droiddeck-proton-"
PACK_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}")
SHA256 = re.compile(r"[0-9a-f]{64}")
TEMP_NAME = re.compile(r"\..+\.tmp-[0-9]+")
MIN_NOFILE = 8192
LOG_LINES = 200
LIVE_FILE = "prefixes.json"
LIVE_ROWS = 200
LIVE_KEYS = ("WINESERVER", "BL_SYNC_PACK", "WINEESYNC", "PROTON_NO_ESYNC", "PROTON_NO_FSYNC", "BL_SYNC", "BL_FSYNC", "WINEFSYNC")
CACHE_ROWS = 512
TEMP_AGE = 3600
GC_GRACE = 600
CHUNK = 1 << 20


def say(message):
    with contextlib.suppress(Exception):
        print(message, file=sys.stderr, flush=True)


def truthy(value):
    return bool(value) and value != "0"


def store_root(env):
    home = env.get("HOME") or os.path.expanduser("~")
    root = Path(home) / STORE
    return root if root.is_dir() else None


def read_text(path):
    try:
        with open(path, encoding="utf-8", errors="surrogateescape") as source:
            return source.read()
    except FileNotFoundError:
        return ""


def write_text(path, text):
    folder, name = os.path.split(path)
    temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid()))
    try:
        with open(temp, "w", encoding="utf-8", errors="surrogateescape") as target:
            target.write(text)
        os.replace(temp, path)
    except BaseException:
        with contextlib.suppress(OSError):
            os.unlink(temp)
        raise


def acquire(path, block=True):
    for _ in range(64):
        fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o644)
        try:
            try:
                fcntl.flock(fd, fcntl.LOCK_EX if block else fcntl.LOCK_EX | fcntl.LOCK_NB)
            except BlockingIOError:
                os.close(fd)
                return None
            held = os.fstat(fd)
            try:
                current = os.stat(path)
            except FileNotFoundError:
                current = None
            if current is not None and (held.st_dev, held.st_ino) == (current.st_dev, current.st_ino):
                return fd
        except BaseException:
            os.close(fd)
            raise
        os.close(fd)
    raise OSError("%s keeps being replaced" % path)


@contextlib.contextmanager
def locked(path, block=True):
    fd = acquire(path, block)
    try:
        yield fd
    finally:
        if fd is not None:
            os.close(fd)


def update_file(root, name, change):
    path = os.path.join(root, name)
    with locked(os.path.join(root, STORE_LOCK)):
        old = read_text(path)
        new = change(old)
        if new != old:
            write_text(path, new)
    return new


def tsv(rows):
    return "".join("\t".join(re.sub(r"[\t\r\n]", " ", str(field)) for field in row) + "\n" for row in rows)


def parse_tsv(text, width):
    return [line.split("\t") for line in text.split("\n") if line and line.count("\t") == width - 1]


def exports_hash(path):
    with open(path, "rb") as source:
        data = source.read()
    if data[:4] != b"\x7fELF" or data[4:6] != b"\x02\x01":
        raise ValueError("%s is not a little-endian ELF64 file" % path)
    names = set()
    try:
        offset = struct.unpack_from("<Q", data, 40)[0]
        size, count = struct.unpack_from("<HH", data, 58)
        if size < 64:
            raise ValueError("%s has no usable section headers" % path)
        sections = []
        index = 0
        while index < count or (index == 0 and offset):
            sections.append(struct.unpack_from("<IIQQQQIIQQ", data, offset + index * size))
            if index == 0 and count == 0:
                count = sections[0][5]
            index += 1
        dynsym = next((section for section in sections if section[1] == 11), None)
        if dynsym is None:
            raise ValueError("%s has no dynamic symbol table" % path)
        strings = sections[dynsym[6]]
        start, end = strings[4], strings[4] + strings[5]
        for entry in range(dynsym[4], dynsym[4] + dynsym[5] - 23, 24):
            name, info, _, shndx, _, _ = struct.unpack_from("<IBBHQQ", data, entry)
            if name and shndx and info >> 4 in (1, 2):
                stop = data.index(b"\0", start + name, end)
                if stop > start + name:
                    names.add(data[start + name:stop])
    except (struct.error, IndexError) as error:
        raise ValueError("%s is a truncated ELF file (%s)" % (path, error))
    return hashlib.sha256(b"\n".join(sorted(names))).hexdigest()


def stat_key(info):
    return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns)


def digest(path):
    sha = hashlib.sha256()
    found = {BUILTIN_MARKER: False, FSYNC_MARKER: False, NTSYNC_MARKER: False}
    tail = b""
    keep = max(len(marker) for marker in found) - 1
    with open(path, "rb") as source:
        before = stat_key(os.fstat(source.fileno()))
        for chunk in iter(lambda: source.read(CHUNK), b""):
            sha.update(chunk)
            window = tail + chunk
            for marker in found:
                found[marker] = found[marker] or marker in window
            tail = window[-keep:]
        after = stat_key(os.fstat(source.fileno()))
    return sha.hexdigest(), found[BUILTIN_MARKER], found[FSYNC_MARKER], found[NTSYNC_MARKER], before if before == after else None


def parse_cache(text):
    rows = {}
    for fields in parse_tsv(text, 11):
        if not SHA256.fullmatch(fields[6]):
            continue
        try:
            key = tuple(int(value) for value in fields[1:6])
        except ValueError:
            continue
        rows.pop(fields[0], None)
        rows[fields[0]] = {"key": key, "sha": fields[6], "builtin": fields[7] == "1", "exports": fields[8], "fsync": fields[9] == "1",
                           "ntsync": fields[10] == "1"}
    return rows


def format_cache(items):
    return tsv([real] + list(row["key"]) + [row["sha"], "1" if row["builtin"] else "0", row["exports"], "1" if row["fsync"] else "0",
                                            "1" if row["ntsync"] else "0"]
               for real, row in items)


class HashCache:
    def __init__(self, root):
        self.root = os.fspath(root) if root is not None else None
        self.rows = parse_cache(read_text(os.path.join(self.root, "hash-cache.tsv"))) if self.root else {}
        self.changed = set()

    def lookup(self, path, exports=False):
        real = os.path.realpath(path)
        key = stat_key(os.stat(real))
        row = self.rows.get(real)
        if row is None or row["key"] != key:
            sha, builtin, fsync, ntsync, stable = digest(real)
            row = {"key": stable, "sha": sha, "builtin": builtin, "exports": "", "fsync": fsync, "ntsync": ntsync}
            if stable is not None:
                self.rows[real] = row
                self.changed.add(real)
        if exports and not row["exports"]:
            try:
                row["exports"] = exports_hash(real)
            except (OSError, ValueError):
                row["exports"] = "-"
            if row["key"] is not None:
                self.changed.add(real)
        return row

    def save(self, prune=False):
        if not self.root or not (self.changed or prune):
            return
        changed = {real: self.rows[real] for real in self.changed if real in self.rows}

        def merge(text):
            rows = parse_cache(text)
            for real, row in changed.items():
                rows.pop(real, None)
                rows[real] = row
            if prune:
                rows = {real: row for real, row in rows.items() if os.path.exists(real)}
            return format_cache(list(rows.items())[-CACHE_ROWS:])

        try:
            update_file(self.root, "hash-cache.tsv", merge)
        except OSError as error:
            say("droiddeck-esync: hash cache not saved (%s)" % error)
        self.changed.clear()


def version_line(tool_dir):
    try:
        with open(os.path.join(tool_dir, "version"), encoding="utf-8", errors="replace") as source:
            return source.readline().strip()
    except OSError:
        return ""


def version_token(line):
    fields = line.split()
    return fields[1] if len(fields) > 1 else ""


class Measured(dict):
    def __init__(self, values, exports):
        super().__init__(values)
        self.exports = exports

    def __missing__(self, key):
        if key != "exports":
            raise KeyError(key)
        value = self.exports()
        self[key] = value
        return value


def measure(tool_dir, cache=None):
    cache = cache if cache is not None else HashCache(None)
    tool = os.fspath(tool_dir)
    ntdll = cache.lookup(os.path.join(tool, NTDLL))
    server = cache.lookup(os.path.join(tool, WINESERVER))
    line = version_line(tool)
    return Measured({"tool": tool, "stock": os.path.realpath(tool), "ntdll": ntdll["sha"], "wineserver": server["sha"],
                     "builtin": server["builtin"], "fsync": server["fsync"], "ntsync": server["ntsync"], "version_line": line,
                     "token": version_token(line)},
                    lambda: cache.lookup(os.path.join(tool, NTDLL), True)["exports"])


def has_stock(tool):
    return all(os.path.isfile(os.path.join(tool, rel)) for rel in PACK_FILES)


def relative(folder, name):
    return name if folder == "." else folder + "/" + name


def valid_copy(rel):
    if not isinstance(rel, str) or not rel or rel.startswith("/") or "\0" in rel:
        return False
    if any(part in ("", ".", "..") for part in rel.split("/")):
        return False
    folder = os.path.dirname(rel) or "."
    return folder in REAL_DIRS and rel not in PACK_FILES and rel not in REAL_DIRS and rel not in STOCK_LINKS and rel != RECORD


def valid_hashes(value):
    return isinstance(value, dict) and all(isinstance(value.get(rel), str) and SHA256.fullmatch(value[rel]) for rel in PACK_FILES)


def is_int(value):
    return isinstance(value, int) and not isinstance(value, bool)


def valid_pack(data, name=None):
    if not isinstance(data, dict) or not is_int(data.get("format")) or data["format"] != 1:
        return False
    ident = data.get("id")
    copy = data.get("copy", list(DEFAULT_COPY))
    return bool(isinstance(ident, str) and PACK_ID.fullmatch(ident) and (name is None or ident == name)
                and isinstance(data.get("flavor"), str) and data["flavor"]
                and is_int(data.get("rev"))
                and isinstance(data.get("version"), str) and isinstance(data.get("version_line", ""), str)
                and valid_hashes(data.get("stock")) and valid_hashes(data.get("files"))
                and isinstance(data.get("exports"), str) and isinstance(data.get("source_match"), bool)
                and isinstance(copy, list) and all(valid_copy(rel) for rel in copy)
                and not data.get("revoked") and not data.get("ignored"))


def read_pack(pack_dir, name=None):
    with open(os.path.join(pack_dir, "pack.json"), encoding="utf-8") as source:
        data = json.load(source)
    if not valid_pack(data, name) or not all(os.path.isfile(os.path.join(pack_dir, rel)) for rel in PACK_FILES):
        raise ValueError("%s is not a usable droiddeck-esync pack" % pack_dir)
    return data


def load_packs(root):
    packs = []
    folder = os.path.join(root, "packs")
    try:
        names = sorted(os.listdir(folder))
    except OSError:
        return packs
    for name in names:
        pack_dir = os.path.join(folder, name)
        if name.startswith(".") or not os.path.isfile(os.path.join(pack_dir, ".complete")):
            continue
        if os.path.lexists(os.path.join(pack_dir, ".ignored")):
            continue
        try:
            packs.append(read_pack(pack_dir, name))
        except (OSError, ValueError):
            continue
    return packs


def line_epoch(line):
    fields = (line or "").split()
    try:
        return int(fields[0]) if fields else 0
    except ValueError:
        return 0


def match_kind(pack, measured):
    if pack.get("revoked") or pack.get("ignored") or measured["builtin"]:
        return None
    stock = pack["stock"]
    if stock[NTDLL] == measured["ntdll"] and stock[WINESERVER] == measured["wineserver"]:
        if pack.get("version") != measured["token"] and line_epoch(pack.get("version_line")) > line_epoch(measured["version_line"]) > 0:
            return None
        return "exact"
    if (pack.get("source_match") is True and measured["token"] and pack.get("version") == measured["token"]
            and pack.get("exports") == measured["exports"]):
        return "source"
    return None


def find_pack(packs, measured):
    best = None
    best_key = None
    for pack in packs:
        kind = match_kind(pack, measured)
        if kind is None:
            continue
        key = (kind == "exact", bool(measured["token"]) and pack.get("version") == measured["token"], pack["rev"], pack["id"])
        if best_key is None or key > best_key:
            best, best_key = pack, key
    return best


def absolute(path):
    path = os.fspath(path).rstrip("/") or "/"
    return path if os.path.isabs(path) else os.path.join(os.getcwd(), path)


def dist_name(pack_id, stock):
    return "%s~%s" % (pack_id, hashlib.sha256(os.fsencode(stock)).hexdigest()[:8])


def read_record(dist):
    try:
        with open(os.path.join(dist, RECORD), encoding="utf-8") as source:
            record = json.load(source)
    except (FileNotFoundError, NotADirectoryError, ValueError):
        return None
    return record if isinstance(record, dict) else None


def write_json(path, data):
    text = json.dumps(data, indent=1, sort_keys=True) + "\n"
    if read_text(path) != text:
        write_text(path, text)


def remove(path):
    if os.path.islink(path) or (os.path.lexists(path) and not os.path.isdir(path)):
        with contextlib.suppress(FileNotFoundError):
            os.unlink(path)
    elif os.path.isdir(path):
        shutil.rmtree(path, ignore_errors=True)


def age(path):
    try:
        return time.time() - os.lstat(path).st_mtime
    except OSError:
        return 0


def put_link(target, path):
    folder, name = os.path.split(path)
    temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid()))
    remove(temp)
    os.symlink(target, temp)
    try:
        os.replace(temp, path)
    except BaseException:
        remove(temp)
        raise


class PackDamaged(ValueError):
    pass


def install(source, path, expected=None):
    folder, name = os.path.split(path)
    temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid()))
    remove(temp)
    try:
        sha = hashlib.sha256()
        with open(source, "rb") as reader, open(temp, "xb") as writer:
            for chunk in iter(lambda: reader.read(CHUNK), b""):
                sha.update(chunk)
                writer.write(chunk)
        result = sha.hexdigest()
        if expected is not None and result != expected:
            raise PackDamaged("%s does not match its pack (sha256 %s)" % (source, result[:12]))
        os.chmod(temp, 0o755)
        os.replace(temp, path)
    except BaseException:
        remove(temp)
        raise
    return result


def intact(path, expected, cache):
    try:
        info = os.lstat(path)
    except OSError:
        return False
    return (stat.S_ISREG(info.st_mode) and (info.st_mode & 0o555) == 0o555
            and cache.lookup(path)["sha"] == expected)


def plan(tool, base, pack):
    real = set(PACK_FILES).union(pack.get("copy", DEFAULT_COPY))
    links = {}
    for folder in REAL_DIRS:
        try:
            names = os.listdir(os.path.join(tool, folder))
        except OSError:
            continue
        for name in names:
            rel = relative(folder, name)
            if rel in REAL_DIRS or rel in real or rel == RECORD or name in SKIPPED or name.endswith(SKIPPED_SUFFIX):
                continue
            text = None
            source = os.path.join(tool, rel)
            if os.path.islink(source):
                with contextlib.suppress(OSError):
                    text = os.readlink(source)
            links[rel] = text if text and not os.path.isabs(text) else os.path.join(base, rel)
    for rel in STOCK_LINKS:
        links[rel] = os.path.join(base, rel)
    return links


def sync(dist, tool, base, pack, pack_dir, record, cache):
    links = plan(tool, base, pack)
    copy = list(pack.get("copy", DEFAULT_COPY))
    real = set(PACK_FILES).union(copy)
    for folder in REAL_DIRS:
        path = os.path.join(dist, folder)
        if os.path.islink(path) or (os.path.lexists(path) and not os.path.isdir(path)):
            os.unlink(path)
        os.makedirs(path, exist_ok=True)
    for folder in REAL_DIRS:
        for name in os.listdir(os.path.join(dist, folder)):
            rel = relative(folder, name)
            if rel in links or rel in real or rel in REAL_DIRS or rel == RECORD:
                continue
            path = os.path.join(dist, rel)
            if os.path.islink(path) or (TEMP_NAME.fullmatch(name) and os.path.isfile(path)):
                os.unlink(path)
    for rel, target in sorted(links.items()):
        path = os.path.join(dist, rel)
        try:
            current = os.readlink(path)
        except OSError:
            current = None
        if current != target and not (current is None and os.path.isdir(path)):
            put_link(target, path)
    for rel in PACK_FILES:
        path = os.path.join(dist, rel)
        if not intact(path, pack["files"][rel], cache):
            install(os.path.join(pack_dir, rel), path, pack["files"][rel])
    previous = record.get("copies") if isinstance(record.get("copies"), dict) else {}
    copies = {}
    for rel in copy:
        source = os.path.join(tool, rel)
        path = os.path.join(dist, rel)
        entry = previous.get(rel)
        try:
            info = os.stat(source)
        except OSError:
            info = None
        if info is None or not stat.S_ISREG(info.st_mode):
            if entry is not None and os.path.lexists(path):
                copies[rel] = entry
            continue
        if (isinstance(entry, list) and len(entry) == 3 and entry[:2] == [info.st_size, info.st_mtime_ns]
                and isinstance(entry[2], str) and intact(path, entry[2], cache)):
            copies[rel] = entry
        else:
            copies[rel] = [info.st_size, info.st_mtime_ns, install(source, path)]
    write_json(os.path.join(dist, RECORD), {"pack": pack["id"], "stock": os.path.realpath(tool), "tool": base,
                                            "version": version_line(tool), "copies": copies})


def build_tree(tool, base, pack, pack_dir, out, cache):
    folder, name = os.path.split(out)
    temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid()))
    remove(temp)
    try:
        os.makedirs(temp)
        sync(temp, tool, base, pack, pack_dir, {}, cache)
    except BaseException:
        remove(temp)
        raise
    return temp


def place(temp, out):
    try:
        os.rename(temp, out)
    except OSError:
        remove(temp)
        if not os.path.isdir(out):
            raise
        return False
    return True


def build_shadow(tool_dir, pack_dir, out_dir, cache=None):
    tool = absolute(tool_dir)
    out = absolute(out_dir)
    pack = read_pack(pack_dir)
    os.makedirs(os.path.dirname(out), exist_ok=True)
    place(build_tree(tool, tool, pack, os.fspath(pack_dir), out, cache if cache is not None else HashCache(None)), out)
    return out


def ensure_dist(root, tool_dir, pack, cache=None):
    root = os.fspath(root)
    cache = cache if cache is not None else HashCache(root)
    tool = absolute(tool_dir)
    stock = os.path.realpath(tool)
    folder = os.path.join(root, "dist")
    os.makedirs(folder, exist_ok=True)
    name = dist_name(pack["id"], stock)
    dist = os.path.join(folder, name)
    pack_dir = os.path.join(root, "packs", pack["id"])
    with locked(dist + ".lock"):
        record = read_record(dist)
        if record is not None and record.get("pack") == pack["id"] and record.get("stock") == stock:
            previous = record.get("tool")
            base = previous if (isinstance(previous, str) and os.path.isabs(previous)
                                and os.path.realpath(previous) == stock) else tool
            sync(dist, tool, base, pack, pack_dir, record, cache)
        elif os.path.lexists(dist):
            fresh = build_tree(tool, tool, pack, pack_dir, dist, cache)
            old = os.path.join(folder, ".%s.old.tmp-%d" % (name, os.getpid()))
            remove(old)
            os.rename(dist, old)
            os.rename(fresh, dist)
            remove(old)
        elif not place(build_tree(tool, tool, pack, pack_dir, dist, cache), dist):
            sync(dist, tool, tool, pack, pack_dir, read_record(dist) or {}, cache)
        os.utime(dist)
    cache.save()
    return dist


def quarantine(root, pack, error):
    if not isinstance(error, PackDamaged):
        return
    with contextlib.suppress(OSError):
        os.unlink(os.path.join(root, "packs", pack["id"], ".complete"))
        say("droiddeck-esync: pack %s is damaged; the app fetches it again" % pack["id"])


def app_id(env):
    value = env.get("STEAM_COMPAT_APP_ID") or env.get("SteamAppId")
    if not value:
        value = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/").rsplit("/", 1)[-1]
    return value


def minimum_nofile(env):
    try:
        return int(env.get("BL_SYNC_MIN_NOFILE") or MIN_NOFILE)
    except ValueError:
        return MIN_NOFILE


def limit(value):
    return "unlimited" if value == resource.RLIM_INFINITY else str(value)


def log_launch(root, command, env, tool, pack_id):
    try:
        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
        line = tsv([[int(time.time()), app_id(env) or "-", command[1] if len(command) > 1 else "-", tool, pack_id or "-",
                     env.get("WINEESYNC") or "-", env.get("BL_SYNC") or "-", "%s/%s" % (limit(soft), limit(hard)),
                     env.get("BL_FSYNC") or "-"]])

        def append(text):
            lines = [entry for entry in text.split("\n") if entry]
            lines.append(line.rstrip("\n"))
            return "\n".join(lines[-LOG_LINES:]) + "\n"

        update_file(root, "launches.log", append)
    except Exception as error:
        say("droiddeck-esync: launch not logged (%s)" % error)


def wanted_row(measured):
    return [measured["ntdll"], measured["wineserver"], measured["token"] or "-", measured["exports"] or "-",
            measured["tool"], str(int(time.time()))]


def merge_wanted(text, fresh, drop):
    rows = []
    seen = set()
    for row in parse_tsv(text, 6):
        key = (row[0], row[1])
        if key in seen or key in drop:
            continue
        seen.add(key)
        if key in fresh:
            row[2:5] = fresh[key][2:5]
        rows.append(row)
    rows.extend(row for key, row in fresh.items() if key not in seen)
    return tsv(rows)


def want(root, measured):
    try:
        row = wanted_row(measured)
        update_file(root, "wanted.tsv", lambda text: merge_wanted(text, {(row[0], row[1]): row}, set()))
    except Exception as error:
        say("droiddeck-esync: wanted.tsv not updated (%s)" % error)


def select(command, env):
    result = (list(command), dict(env))
    try:
        command = list(command)
        env = dict(env)
        tool = os.path.dirname(command[0]) or "."
        if env.get("PROTON_USE_ARM64", "1") != "1" or not has_stock(tool) or os.path.lexists(os.path.join(tool, RECORD)):
            return result
        prefix = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/")
        last = prefix.rsplit("/", 1)[-1]
        if not prefix or last == "0" or last.startswith("0-"):
            return result
        if truthy(env.get("PROTON_NO_NTSYNC")):
            env["BL_SYNC"] = "0"
        result = (list(command), dict(env))
        root = store_root(env)
        if root is None:
            return result
        root = os.fspath(root)
        kept = keep_live(root, command, env, tool)
        if kept is not None:
            return kept
        chosen = choose(root, command, env, tool, result)
        remember(root, env, tool, chosen)
        return chosen
    except Exception as error:
        say("droiddeck-esync: skipped (%s); stock binaries" % error)
        record(result[0], result[1])
        return result


def record(command, env):
    try:
        root = store_root(env)
        if root is not None and command:
            remember(os.fspath(root), env, os.path.dirname(command[0]) or ".", (list(command), dict(env)))
    except Exception as error:
        say("droiddeck-esync: sync choice not recorded (%s)" % error)


def prefix_of(env):
    data = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/")
    return os.path.realpath(os.path.join(data, "pfx")) if data else None


def server_running(prefix):
    for entry in os.listdir("/proc"):
        if not entry.isdigit():
            continue
        try:
            with open("/proc/%s/comm" % entry, "rb") as source:
                if source.read().strip() != b"wineserver":
                    continue
            with open("/proc/%s/environ" % entry, "rb") as source:
                items = source.read().split(b"\0")
        except OSError:
            continue
        for item in items:
            if item.startswith(b"WINEPREFIX=") and os.path.realpath(os.fsdecode(item[11:])) == prefix:
                return True
    return False


def read_live(root):
    try:
        rows = json.loads(read_text(os.path.join(root, LIVE_FILE)) or "{}")
    except ValueError:
        return {}
    return rows if isinstance(rows, dict) else {}


def remember(root, env, tool, chosen):
    prefix = prefix_of(env)
    if prefix is None:
        return
    command, chosen_env = chosen
    row = {"tool": os.path.realpath(tool), "command": command[0], "env": {name: chosen_env.get(name) for name in LIVE_KEYS}}

    def merge(text):
        try:
            rows = json.loads(text or "{}")
        except ValueError:
            rows = {}
        if not isinstance(rows, dict):
            rows = {}
        rows.pop(prefix, None)
        rows[prefix] = row
        return json.dumps(dict(list(rows.items())[-LIVE_ROWS:]), sort_keys=False) + "\n"

    try:
        update_file(root, LIVE_FILE, merge)
    except OSError as error:
        say("droiddeck-esync: sync choice not recorded (%s)" % error)


def keep_live(root, command, env, tool):
    prefix = prefix_of(env)
    row = read_live(root).get(prefix) if prefix else None
    if not isinstance(row, dict) or not isinstance(row.get("env"), dict) or row.get("tool") != os.path.realpath(tool):
        return None
    kept_command = row.get("command")
    if not isinstance(kept_command, str) or os.path.basename(kept_command) != os.path.basename(command[0]):
        return None
    if not os.path.isfile(kept_command) or not server_running(prefix):
        return None
    kept_pack = row["env"].get("BL_SYNC_PACK")
    if isinstance(kept_pack, str) and kept_pack and command[1:2] == ["waitforexitandrun"]:
        try:
            cache = HashCache(root)
            try:
                pack = find_pack(load_packs(root), measure(tool, cache))
            finally:
                cache.save()
        except (OSError, ValueError):
            pack = {"id": kept_pack}
        if pack is None or pack["id"] != kept_pack:
            return None
    command, env = list(command), dict(env)
    command[0] = kept_command
    for name in LIVE_KEYS:
        value = row["env"].get(name)
        if isinstance(value, str):
            env[name] = value
        else:
            env.pop(name, None)
    say("droiddeck-esync: Wine is still running in this prefix; keeping its sync choice (%s)" % describe(env))
    log_launch(root, command, env, tool, env.get("BL_SYNC_PACK"))
    return command, env


def describe(env):
    if env.get("BL_SYNC") == "1":
        return "droiddeck-ntsync"
    if env.get("BL_FSYNC") == "1":
        return "droiddeck-fsync"
    if env.get("BL_SYNC_PACK") and truthy(env.get("WINEESYNC")):
        return "droiddeck-esync"
    if all(truthy(env.get(name)) for name in WINESERVER_ONLY):
        return "wineserver"
    return "Proton's own sync"


def fsync_ready(env, measured):
    listed = "PROTON_NO_FSYNC" not in env and app_id(env) in NOSYNC_APPS
    return measured["fsync"] and env.get("BL_SYNC") != "1" and not truthy(env.get("PROTON_NO_FSYNC")) and not listed


def with_fsync(root, command, env, tool, reason):
    command, env = list(command), dict(env)
    env["BL_FSYNC"] = "1"
    env["BL_SYNC"] = "0"
    env["WINEFSYNC"] = "1"
    say("droiddeck-fsync: %s; stock binaries with Proton's fsync" % reason)
    log_launch(root, command, env, tool, None)
    return command, env


def stand_in(root, command, env, tool, reason, measured=None):
    if measured is not None and fsync_ready(env, measured):
        return with_fsync(root, command, env, tool, reason)
    command, env = list(command), dict(env)
    if truthy(env.get("PROTON_NO_NTSYNC")):
        say("droiddeck-esync: %s; stock binaries" % reason)
    elif measured is not None and not measured["ntsync"]:
        env["BL_SYNC"] = "0"
        say("droiddeck-esync: %s; stock binaries (this Wine has no ntsync)" % reason)
    else:
        env["BL_SYNC"] = "1"
        say("droiddeck-esync: %s; stock binaries with droiddeck-ntsync" % reason)
    log_launch(root, command, env, tool, None)
    return command, env


def esync_unwanted(env):
    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    starved = hard != resource.RLIM_INFINITY and hard < minimum_nofile(env)
    listed = "PROTON_NO_ESYNC" not in env and app_id(env) in NOSYNC_APPS
    return truthy(env.get("PROTON_NO_ESYNC")) or env.get("WINEESYNC") == "0" or listed or starved


def wineserver_only(env):
    env = dict(env)
    for name in WINESERVER_ONLY:
        env.setdefault(name, "1")
    if truthy(env["PROTON_NO_ESYNC"]):
        env["WINEESYNC"] = "0"
    if truthy(env["PROTON_NO_FSYNC"]):
        env["WINEFSYNC"] = "0"
    if truthy(env["PROTON_NO_NTSYNC"]):
        env["BL_SYNC"] = "0"
    return env


def choose(root, command, env, tool, stock):
    name = os.path.basename(tool.rstrip("/")) or tool
    wants_fsync = env.get("BL_FSYNC_FIRST") == "1"
    if "BL_FSYNC" in env:
        env["BL_FSYNC"] = "0"
        stock = (stock[0], dict(stock[1], BL_FSYNC="0"))
    if env.get("BL_SYNC_FALLBACK") != "1" and not wants_fsync and env.get("BL_SYNC") != "1":
        stock = (stock[0], wineserver_only(stock[1]))
        say("droiddeck-esync: off; wineserver sync only")
        log_launch(root, stock[0], stock[1], tool, None)
        return stock
    cache = HashCache(root)
    try:
        measured = measure(tool, cache)
        if env.get("BL_SYNC") == "1" and not measured["ntsync"]:
            env["BL_SYNC"] = "0"
            stock = (stock[0], dict(stock[1], BL_SYNC="0"))
            wants_fsync = True
            say("droiddeck-ntsync: %s has no ntsync; trying droiddeck-fsync" % name)
        if wants_fsync and fsync_ready(env, measured):
            return with_fsync(root, command, env, tool, "chosen for %s" % name)
        if env.get("BL_SYNC_FALLBACK") != "1":
            say("droiddeck-esync: off")
            log_launch(root, command, env, tool, None)
            return stock
        if esync_unwanted(env) and fsync_ready(env, measured):
            return with_fsync(root, command, env, tool, "esync is off for this game")
        if measured["builtin"]:
            if esync_unwanted(env):
                env["PROTON_NO_ESYNC"] = "1"
                env["WINEESYNC"] = "0"
                return stand_in(root, command, env, tool, "esync is off for this game on %s" % name, measured)
            say("droiddeck-esync: %s has esync built in" % name)
            log_launch(root, command, env, tool, None)
            return stock
        pack = find_pack(load_packs(root), measured)
        if pack is None:
            want(root, measured)
            return stand_in(root, command, env, tool, "no pack for %s (%s, ntdll %s)"
                            % (name, measured["version_line"], measured["ntdll"][:12]), measured)
        try:
            dist = ensure_dist(root, tool, pack, cache)
        except Exception as error:
            quarantine(root, pack, error)
            return stand_in(root, command, env, tool, "pack %s unusable for %s (%s)" % (pack["id"], name, error), measured)
    finally:
        cache.save()
    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    if esync_unwanted(env):
        env["WINEESYNC"] = "0"
        env["PROTON_NO_ESYNC"] = "1"
        if not truthy(env.get("PROTON_NO_NTSYNC")) and measured["ntsync"]:
            env["BL_SYNC"] = "1"
    elif not env.get("WINEESYNC"):
        env["WINEESYNC"] = "1"
    if truthy(env["WINEESYNC"]) and "PROTON_NO_FSYNC" not in env:
        env["PROTON_NO_FSYNC"] = "1"
    env["WINESERVER"] = os.path.join(dist, WINESERVER)
    env["BL_SYNC_PACK"] = pack["id"]
    command[0] = os.path.join(dist, os.path.basename(command[0]))
    say("droiddeck-esync: pack %s for %s (%s, %s match); droiddeck-ntsync %s, droiddeck-esync %s, fds %s/%s"
        % (pack["id"], name, measured["version_line"], match_kind(pack, measured), "on" if env.get("BL_SYNC") == "1" else "off",
           "on" if truthy(env["WINEESYNC"]) else "off", limit(soft), limit(hard)))
    log_launch(root, command, env, tool, pack["id"])
    return command, env


def libraries(steam_root):
    found = [steam_root]
    try:
        with open(os.path.join(steam_root, "steamapps", "libraryfolders.vdf"), encoding="utf-8", errors="replace") as f:
            found += [path.replace("\\\\", "\\") for path in re.findall(r'"path"\s+"((?:[^"\\]|\\.)*)"', f.read())]
    except OSError:
        pass
    found.append(SD_LIBRARY)
    return list(dict.fromkeys(found))


def find_tools(steam_root):
    candidates = [os.path.join(library, "steamapps", "common", name) for library in libraries(steam_root) for name in SOURCES]
    folder = os.path.join(steam_root, "compatibilitytools.d")
    try:
        names = sorted(os.listdir(folder))
    except OSError:
        names = []
    candidates += [os.path.join(folder, name) for name in names if not name.startswith((OWN_TOOL_PREFIX, "."))]
    tools = []
    seen = set()
    for tool in candidates:
        if os.path.isdir(tool) and has_stock(tool):
            real = os.path.realpath(tool)
            if real not in seen:
                seen.add(real)
                tools.append(tool)
    return tools


def still_wanted(row, scanned, packs, cache):
    tool = row[4]
    try:
        if os.path.realpath(tool) in scanned or not has_stock(tool):
            return False
        measured = measure(tool, cache)
        return ((measured["ntdll"], measured["wineserver"]) == (row[0], row[1]) and not measured["builtin"]
                and find_pack(packs, measured) is None)
    except Exception:
        return True


def update_wanted(root, fresh, scanned, packs, cache):
    drop = set()
    for row in parse_tsv(read_text(os.path.join(root, "wanted.tsv")), 6):
        key = (row[0], row[1])
        if key not in fresh and not still_wanted(row, scanned, packs, cache):
            drop.add(key)
    return len(parse_tsv(update_file(root, "wanted.tsv", lambda text: merge_wanted(text, fresh, drop)), 6))


def reconcile_store(root, steam_root, env):
    cache = HashCache(root)
    packs = load_packs(root)
    enabled = env.get("BL_SYNC_FALLBACK") == "1"
    rows = []
    fresh = {}
    scanned = set()
    used = 0
    for tool in find_tools(os.fspath(steam_root)):
        try:
            measured = measure(tool, cache)
            pack = None if measured["builtin"] else find_pack(packs, measured)
            if measured["builtin"]:
                state = "builtin"
            elif pack is not None:
                state = "pack"
                used += 1
            else:
                state = "wanted"
                fresh.setdefault((measured["ntdll"], measured["wineserver"]), wanted_row(measured))
            scanned.add(measured["stock"])
        except Exception as error:
            say("droiddeck-esync: %s skipped (%s)" % (tool, error))
            continue
        rows.append([tool, measured["version_line"], measured["ntdll"], measured["wineserver"], state,
                     pack["id"] if pack is not None else "-"])
        if pack is not None and enabled:
            try:
                ensure_dist(root, tool, pack, cache)
            except Exception as error:
                say("droiddeck-esync: pack %s unusable for %s (%s)" % (pack["id"], tool, error))
                quarantine(root, pack, error)
    wanted = update_wanted(root, fresh, scanned, packs, cache)
    update_file(root, "tools.tsv", lambda text: tsv(sorted(rows)))
    cache.save(True)
    gc(root)
    return "droiddeck-esync: %d pack(s) in use, %d wanted" % (used, wanted)


def reconcile(steam_root, env):
    try:
        root = store_root(env)
        if root is None:
            return None
        return reconcile_store(os.fspath(root), steam_root, env)
    except Exception as error:
        say("droiddeck-esync: reconcile failed (%s)" % error)
        return None


def process_cmdlines():
    try:
        entries = os.listdir("/proc")
    except OSError:
        return None
    lines = []
    for entry in entries:
        if not entry.isdigit():
            continue
        try:
            with open(os.path.join("/proc", entry, "cmdline"), "rb") as source:
                lines.append(source.read())
        except OSError:
            continue
    return lines


def in_use(path, cmdlines):
    needles = {os.fsencode(path), os.fsencode(os.path.realpath(path))}
    return any(needle in line for line in cmdlines for needle in needles)


def stale(path, name, packs, cache):
    record = read_record(path)
    if record is None:
        return True
    pack = packs.get(record.get("pack"))
    stock = record.get("stock")
    if pack is None or not isinstance(stock, str) or name != dist_name(pack["id"], stock):
        return True
    if not has_stock(stock):
        return True
    try:
        measured = measure(stock, cache)
        if match_kind(pack, measured) is None:
            return True
        best = find_pack(list(packs.values()), measured)
        return best is None or best["id"] != pack["id"]
    except Exception:
        return False


def collect(root):
    folder = os.path.join(root, "dist")
    try:
        names = sorted(os.listdir(folder))
    except FileNotFoundError:
        return 0
    packs = {pack["id"]: pack for pack in load_packs(root)}
    cache = HashCache(root)
    candidates = []
    for name in names:
        path = os.path.join(folder, name)
        if name.startswith("."):
            if ".tmp-" in name and age(path) > TEMP_AGE:
                remove(path)
        elif (not name.endswith(".lock") and "~" in name and not os.path.islink(path) and os.path.isdir(path)
              and stale(path, name, packs, cache)):
            candidates.append(name)
    cmdlines = process_cmdlines() if candidates else []
    removed = 0
    for name in candidates:
        path = os.path.join(folder, name)
        lock = path + ".lock"
        with locked(lock, False) as fd:
            if (fd is None or cmdlines is None or age(path) < GC_GRACE or in_use(path, cmdlines)
                    or not stale(path, name, packs, cache)):
                continue
            doomed = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid()))
            remove(doomed)
            os.rename(path, doomed)
            os.unlink(lock)
        remove(doomed)
        removed += 1
    for name in sorted(os.listdir(folder)):
        if name.startswith(".") or not name.endswith(".lock"):
            continue
        lock = os.path.join(folder, name)
        dist = lock[:-len(".lock")]
        if os.path.lexists(dist):
            continue
        with locked(lock, False) as fd:
            if fd is not None and not os.path.lexists(dist):
                os.unlink(lock)
    cache.save()
    return removed


def gc(root):
    try:
        if root is None:
            return 0
        return collect(os.fspath(root))
    except Exception as error:
        say("droiddeck-esync: gc failed (%s)" % error)
        return 0


def status(root):
    root = os.fspath(root)
    packs = load_packs(root)
    try:
        dists = sorted(name for name in os.listdir(os.path.join(root, "dist"))
                       if "~" in name and not name.startswith(".") and not name.endswith(".lock"))
    except OSError:
        dists = []
    tools = parse_tsv(read_text(os.path.join(root, "tools.tsv")), 6)
    wanted = parse_tsv(read_text(os.path.join(root, "wanted.tsv")), 6)
    lines = ["droiddeck-esync store %s" % root, "packs: %d installed" % len(packs)]
    lines += ["  %s (%s, rev %d)" % (pack["id"], pack["flavor"], pack["rev"]) for pack in packs]
    lines.append("dists: %d" % len(dists))
    lines += ["  %s" % name for name in dists]
    lines.append("tools: %d" % len(tools))
    lines += ["  %s %s: %s (%s)" % (row[4], row[5], row[0], row[1]) for row in tools]
    lines.append("wanted: %d" % len(wanted))
    lines += ["  %s %s (%s, ntdll %s)" % (row[2], row[4], row[3][:12], row[0][:12]) for row in wanted]
    return "\n".join(lines)


def main(argv):
    command = argv[1:]
    if command == ["status"]:
        root = store_root(os.environ)
        print(status(root) if root is not None else "droiddeck-esync: no store at %s" % (Path(os.environ.get("HOME") or os.path.expanduser("~")) / STORE))
        return 0
    if len(command) == 2 and command[0] == "reconcile":
        if store_root(os.environ) is None:
            print("droiddeck-esync: no store")
            return 0
        summary = reconcile(command[1], os.environ)
        if summary is None:
            return 1
        print(summary)
        return 0
    if command == ["gc"]:
        root = store_root(os.environ)
        print("droiddeck-esync: %d dist(s) removed" % gc(root) if root is not None else "droiddeck-esync: no store")
        return 0
    if len(command) == 4 and command[0] == "build-shadow":
        print(build_shadow(command[1], command[2], command[3]))
        return 0
    print("usage: droiddeck-esync status|reconcile <steam_root>|gc|build-shadow <tool_dir> <pack_dir> <out_dir>", file=sys.stderr)
    return 2


if __name__ == "__main__":
    sys.exit(main(sys.argv))