#!/usr/bin/python3 # droiddeck-esync: picks and stages the esync pack for each Proton build. # Credit: esync and ntsync were designed by Elizabeth Figura for Wine and Proton, and fsync by Elizabeth Figura # and Paul Gofman; droiddeck-esync, droiddeck-ntsync and droiddeck-fsync build on that work. import contextlib import fcntl import hashlib import json import os import re import resource import shutil import stat import struct import sys import time from pathlib import Path NTDLL = "files/lib/wine/aarch64-unix/ntdll.so" WINESERVER = "files/bin-arm64/wineserver" PACK_FILES = (NTDLL, WINESERVER) DEFAULT_COPY = ("files/bin-arm64/wine", "files/lib/wine/aarch64-unix/wine", "files/lib/wine/aarch64-unix/wine-preloader") REAL_DIRS = (".", "files", "files/bin-arm64", "files/lib", "files/lib/wine", "files/lib/wine/aarch64-unix") STOCK_LINKS = ("dist.lock", "files/steampipe_fixups_mtime") SKIPPED = frozenset(("toolmanifest.vdf", "compatibilitytool.vdf", "droiddeck-proton-wrap", "bannerlator-proton-wrap")) SKIPPED_SUFFIX = (".droiddeck-orig", ".bannerlator-orig") RECORD = ".droiddeck-esync.json" STORE = ".local/share/droiddeck-esync" STORE_LOCK = "store.lock" BUILTIN_MARKER = b"esync: up and running" FSYNC_MARKER = b"fsync: up and running" NTSYNC_MARKER = b"/dev/ntsync" WINESERVER_ONLY = ("PROTON_NO_ESYNC", "PROTON_NO_FSYNC", "PROTON_NO_NTSYNC") NOSYNC_APPS = frozenset(("2630", "1060210", "414740", "201510", "1233880")) SOURCES = ("Proton Experimental (ARM64)", "Proton 11.0 (ARM64)") SD_LIBRARY = "/mnt/droiddeck-sd" OWN_TOOL_PREFIX = "droiddeck-proton-" PACK_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") SHA256 = re.compile(r"[0-9a-f]{64}") TEMP_NAME = re.compile(r"\..+\.tmp-[0-9]+") MIN_NOFILE = 8192 LOG_LINES = 200 LIVE_FILE = "prefixes.json" LIVE_ROWS = 200 LIVE_KEYS = ("WINESERVER", "BL_SYNC_PACK", "WINEESYNC", "PROTON_NO_ESYNC", "PROTON_NO_FSYNC", "BL_SYNC", "BL_FSYNC", "WINEFSYNC") CACHE_ROWS = 512 TEMP_AGE = 3600 GC_GRACE = 600 CHUNK = 1 << 20 def say(message): with contextlib.suppress(Exception): print(message, file=sys.stderr, flush=True) def truthy(value): return bool(value) and value != "0" def store_root(env): home = env.get("HOME") or os.path.expanduser("~") root = Path(home) / STORE return root if root.is_dir() else None def read_text(path): try: with open(path, encoding="utf-8", errors="surrogateescape") as source: return source.read() except FileNotFoundError: return "" def write_text(path, text): folder, name = os.path.split(path) temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid())) try: with open(temp, "w", encoding="utf-8", errors="surrogateescape") as target: target.write(text) os.replace(temp, path) except BaseException: with contextlib.suppress(OSError): os.unlink(temp) raise def acquire(path, block=True): for _ in range(64): fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o644) try: try: fcntl.flock(fd, fcntl.LOCK_EX if block else fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: os.close(fd) return None held = os.fstat(fd) try: current = os.stat(path) except FileNotFoundError: current = None if current is not None and (held.st_dev, held.st_ino) == (current.st_dev, current.st_ino): return fd except BaseException: os.close(fd) raise os.close(fd) raise OSError("%s keeps being replaced" % path) @contextlib.contextmanager def locked(path, block=True): fd = acquire(path, block) try: yield fd finally: if fd is not None: os.close(fd) def update_file(root, name, change): path = os.path.join(root, name) with locked(os.path.join(root, STORE_LOCK)): old = read_text(path) new = change(old) if new != old: write_text(path, new) return new def tsv(rows): return "".join("\t".join(re.sub(r"[\t\r\n]", " ", str(field)) for field in row) + "\n" for row in rows) def parse_tsv(text, width): return [line.split("\t") for line in text.split("\n") if line and line.count("\t") == width - 1] def exports_hash(path): with open(path, "rb") as source: data = source.read() if data[:4] != b"\x7fELF" or data[4:6] != b"\x02\x01": raise ValueError("%s is not a little-endian ELF64 file" % path) names = set() try: offset = struct.unpack_from("> 4 in (1, 2): stop = data.index(b"\0", start + name, end) if stop > start + name: names.add(data[start + name:stop]) except (struct.error, IndexError) as error: raise ValueError("%s is a truncated ELF file (%s)" % (path, error)) return hashlib.sha256(b"\n".join(sorted(names))).hexdigest() def stat_key(info): return (info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns) def digest(path): sha = hashlib.sha256() found = {BUILTIN_MARKER: False, FSYNC_MARKER: False, NTSYNC_MARKER: False} tail = b"" keep = max(len(marker) for marker in found) - 1 with open(path, "rb") as source: before = stat_key(os.fstat(source.fileno())) for chunk in iter(lambda: source.read(CHUNK), b""): sha.update(chunk) window = tail + chunk for marker in found: found[marker] = found[marker] or marker in window tail = window[-keep:] after = stat_key(os.fstat(source.fileno())) return sha.hexdigest(), found[BUILTIN_MARKER], found[FSYNC_MARKER], found[NTSYNC_MARKER], before if before == after else None def parse_cache(text): rows = {} for fields in parse_tsv(text, 11): if not SHA256.fullmatch(fields[6]): continue try: key = tuple(int(value) for value in fields[1:6]) except ValueError: continue rows.pop(fields[0], None) rows[fields[0]] = {"key": key, "sha": fields[6], "builtin": fields[7] == "1", "exports": fields[8], "fsync": fields[9] == "1", "ntsync": fields[10] == "1"} return rows def format_cache(items): return tsv([real] + list(row["key"]) + [row["sha"], "1" if row["builtin"] else "0", row["exports"], "1" if row["fsync"] else "0", "1" if row["ntsync"] else "0"] for real, row in items) class HashCache: def __init__(self, root): self.root = os.fspath(root) if root is not None else None self.rows = parse_cache(read_text(os.path.join(self.root, "hash-cache.tsv"))) if self.root else {} self.changed = set() def lookup(self, path, exports=False): real = os.path.realpath(path) key = stat_key(os.stat(real)) row = self.rows.get(real) if row is None or row["key"] != key: sha, builtin, fsync, ntsync, stable = digest(real) row = {"key": stable, "sha": sha, "builtin": builtin, "exports": "", "fsync": fsync, "ntsync": ntsync} if stable is not None: self.rows[real] = row self.changed.add(real) if exports and not row["exports"]: try: row["exports"] = exports_hash(real) except (OSError, ValueError): row["exports"] = "-" if row["key"] is not None: self.changed.add(real) return row def save(self, prune=False): if not self.root or not (self.changed or prune): return changed = {real: self.rows[real] for real in self.changed if real in self.rows} def merge(text): rows = parse_cache(text) for real, row in changed.items(): rows.pop(real, None) rows[real] = row if prune: rows = {real: row for real, row in rows.items() if os.path.exists(real)} return format_cache(list(rows.items())[-CACHE_ROWS:]) try: update_file(self.root, "hash-cache.tsv", merge) except OSError as error: say("droiddeck-esync: hash cache not saved (%s)" % error) self.changed.clear() def version_line(tool_dir): try: with open(os.path.join(tool_dir, "version"), encoding="utf-8", errors="replace") as source: return source.readline().strip() except OSError: return "" def version_token(line): fields = line.split() return fields[1] if len(fields) > 1 else "" class Measured(dict): def __init__(self, values, exports): super().__init__(values) self.exports = exports def __missing__(self, key): if key != "exports": raise KeyError(key) value = self.exports() self[key] = value return value def measure(tool_dir, cache=None): cache = cache if cache is not None else HashCache(None) tool = os.fspath(tool_dir) ntdll = cache.lookup(os.path.join(tool, NTDLL)) server = cache.lookup(os.path.join(tool, WINESERVER)) line = version_line(tool) return Measured({"tool": tool, "stock": os.path.realpath(tool), "ntdll": ntdll["sha"], "wineserver": server["sha"], "builtin": server["builtin"], "fsync": server["fsync"], "ntsync": server["ntsync"], "version_line": line, "token": version_token(line)}, lambda: cache.lookup(os.path.join(tool, NTDLL), True)["exports"]) def has_stock(tool): return all(os.path.isfile(os.path.join(tool, rel)) for rel in PACK_FILES) def relative(folder, name): return name if folder == "." else folder + "/" + name def valid_copy(rel): if not isinstance(rel, str) or not rel or rel.startswith("/") or "\0" in rel: return False if any(part in ("", ".", "..") for part in rel.split("/")): return False folder = os.path.dirname(rel) or "." return folder in REAL_DIRS and rel not in PACK_FILES and rel not in REAL_DIRS and rel not in STOCK_LINKS and rel != RECORD def valid_hashes(value): return isinstance(value, dict) and all(isinstance(value.get(rel), str) and SHA256.fullmatch(value[rel]) for rel in PACK_FILES) def is_int(value): return isinstance(value, int) and not isinstance(value, bool) def valid_pack(data, name=None): if not isinstance(data, dict) or not is_int(data.get("format")) or data["format"] != 1: return False ident = data.get("id") copy = data.get("copy", list(DEFAULT_COPY)) return bool(isinstance(ident, str) and PACK_ID.fullmatch(ident) and (name is None or ident == name) and isinstance(data.get("flavor"), str) and data["flavor"] and is_int(data.get("rev")) and isinstance(data.get("version"), str) and isinstance(data.get("version_line", ""), str) and valid_hashes(data.get("stock")) and valid_hashes(data.get("files")) and isinstance(data.get("exports"), str) and isinstance(data.get("source_match"), bool) and isinstance(copy, list) and all(valid_copy(rel) for rel in copy) and not data.get("revoked") and not data.get("ignored")) def read_pack(pack_dir, name=None): with open(os.path.join(pack_dir, "pack.json"), encoding="utf-8") as source: data = json.load(source) if not valid_pack(data, name) or not all(os.path.isfile(os.path.join(pack_dir, rel)) for rel in PACK_FILES): raise ValueError("%s is not a usable droiddeck-esync pack" % pack_dir) return data def load_packs(root): packs = [] folder = os.path.join(root, "packs") try: names = sorted(os.listdir(folder)) except OSError: return packs for name in names: pack_dir = os.path.join(folder, name) if name.startswith(".") or not os.path.isfile(os.path.join(pack_dir, ".complete")): continue if os.path.lexists(os.path.join(pack_dir, ".ignored")): continue try: packs.append(read_pack(pack_dir, name)) except (OSError, ValueError): continue return packs def line_epoch(line): fields = (line or "").split() try: return int(fields[0]) if fields else 0 except ValueError: return 0 def match_kind(pack, measured): if pack.get("revoked") or pack.get("ignored") or measured["builtin"]: return None stock = pack["stock"] if stock[NTDLL] == measured["ntdll"] and stock[WINESERVER] == measured["wineserver"]: if pack.get("version") != measured["token"] and line_epoch(pack.get("version_line")) > line_epoch(measured["version_line"]) > 0: return None return "exact" if (pack.get("source_match") is True and measured["token"] and pack.get("version") == measured["token"] and pack.get("exports") == measured["exports"]): return "source" return None def find_pack(packs, measured): best = None best_key = None for pack in packs: kind = match_kind(pack, measured) if kind is None: continue key = (kind == "exact", bool(measured["token"]) and pack.get("version") == measured["token"], pack["rev"], pack["id"]) if best_key is None or key > best_key: best, best_key = pack, key return best def absolute(path): path = os.fspath(path).rstrip("/") or "/" return path if os.path.isabs(path) else os.path.join(os.getcwd(), path) def dist_name(pack_id, stock): return "%s~%s" % (pack_id, hashlib.sha256(os.fsencode(stock)).hexdigest()[:8]) def read_record(dist): try: with open(os.path.join(dist, RECORD), encoding="utf-8") as source: record = json.load(source) except (FileNotFoundError, NotADirectoryError, ValueError): return None return record if isinstance(record, dict) else None def write_json(path, data): text = json.dumps(data, indent=1, sort_keys=True) + "\n" if read_text(path) != text: write_text(path, text) def remove(path): if os.path.islink(path) or (os.path.lexists(path) and not os.path.isdir(path)): with contextlib.suppress(FileNotFoundError): os.unlink(path) elif os.path.isdir(path): shutil.rmtree(path, ignore_errors=True) def age(path): try: return time.time() - os.lstat(path).st_mtime except OSError: return 0 def put_link(target, path): folder, name = os.path.split(path) temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid())) remove(temp) os.symlink(target, temp) try: os.replace(temp, path) except BaseException: remove(temp) raise class PackDamaged(ValueError): pass def install(source, path, expected=None): folder, name = os.path.split(path) temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid())) remove(temp) try: sha = hashlib.sha256() with open(source, "rb") as reader, open(temp, "xb") as writer: for chunk in iter(lambda: reader.read(CHUNK), b""): sha.update(chunk) writer.write(chunk) result = sha.hexdigest() if expected is not None and result != expected: raise PackDamaged("%s does not match its pack (sha256 %s)" % (source, result[:12])) os.chmod(temp, 0o755) os.replace(temp, path) except BaseException: remove(temp) raise return result def intact(path, expected, cache): try: info = os.lstat(path) except OSError: return False return (stat.S_ISREG(info.st_mode) and (info.st_mode & 0o555) == 0o555 and cache.lookup(path)["sha"] == expected) def plan(tool, base, pack): real = set(PACK_FILES).union(pack.get("copy", DEFAULT_COPY)) links = {} for folder in REAL_DIRS: try: names = os.listdir(os.path.join(tool, folder)) except OSError: continue for name in names: rel = relative(folder, name) if rel in REAL_DIRS or rel in real or rel == RECORD or name in SKIPPED or name.endswith(SKIPPED_SUFFIX): continue text = None source = os.path.join(tool, rel) if os.path.islink(source): with contextlib.suppress(OSError): text = os.readlink(source) links[rel] = text if text and not os.path.isabs(text) else os.path.join(base, rel) for rel in STOCK_LINKS: links[rel] = os.path.join(base, rel) return links def sync(dist, tool, base, pack, pack_dir, record, cache): links = plan(tool, base, pack) copy = list(pack.get("copy", DEFAULT_COPY)) real = set(PACK_FILES).union(copy) for folder in REAL_DIRS: path = os.path.join(dist, folder) if os.path.islink(path) or (os.path.lexists(path) and not os.path.isdir(path)): os.unlink(path) os.makedirs(path, exist_ok=True) for folder in REAL_DIRS: for name in os.listdir(os.path.join(dist, folder)): rel = relative(folder, name) if rel in links or rel in real or rel in REAL_DIRS or rel == RECORD: continue path = os.path.join(dist, rel) if os.path.islink(path) or (TEMP_NAME.fullmatch(name) and os.path.isfile(path)): os.unlink(path) for rel, target in sorted(links.items()): path = os.path.join(dist, rel) try: current = os.readlink(path) except OSError: current = None if current != target and not (current is None and os.path.isdir(path)): put_link(target, path) for rel in PACK_FILES: path = os.path.join(dist, rel) if not intact(path, pack["files"][rel], cache): install(os.path.join(pack_dir, rel), path, pack["files"][rel]) previous = record.get("copies") if isinstance(record.get("copies"), dict) else {} copies = {} for rel in copy: source = os.path.join(tool, rel) path = os.path.join(dist, rel) entry = previous.get(rel) try: info = os.stat(source) except OSError: info = None if info is None or not stat.S_ISREG(info.st_mode): if entry is not None and os.path.lexists(path): copies[rel] = entry continue if (isinstance(entry, list) and len(entry) == 3 and entry[:2] == [info.st_size, info.st_mtime_ns] and isinstance(entry[2], str) and intact(path, entry[2], cache)): copies[rel] = entry else: copies[rel] = [info.st_size, info.st_mtime_ns, install(source, path)] write_json(os.path.join(dist, RECORD), {"pack": pack["id"], "stock": os.path.realpath(tool), "tool": base, "version": version_line(tool), "copies": copies}) def build_tree(tool, base, pack, pack_dir, out, cache): folder, name = os.path.split(out) temp = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid())) remove(temp) try: os.makedirs(temp) sync(temp, tool, base, pack, pack_dir, {}, cache) except BaseException: remove(temp) raise return temp def place(temp, out): try: os.rename(temp, out) except OSError: remove(temp) if not os.path.isdir(out): raise return False return True def build_shadow(tool_dir, pack_dir, out_dir, cache=None): tool = absolute(tool_dir) out = absolute(out_dir) pack = read_pack(pack_dir) os.makedirs(os.path.dirname(out), exist_ok=True) place(build_tree(tool, tool, pack, os.fspath(pack_dir), out, cache if cache is not None else HashCache(None)), out) return out def ensure_dist(root, tool_dir, pack, cache=None): root = os.fspath(root) cache = cache if cache is not None else HashCache(root) tool = absolute(tool_dir) stock = os.path.realpath(tool) folder = os.path.join(root, "dist") os.makedirs(folder, exist_ok=True) name = dist_name(pack["id"], stock) dist = os.path.join(folder, name) pack_dir = os.path.join(root, "packs", pack["id"]) with locked(dist + ".lock"): record = read_record(dist) if record is not None and record.get("pack") == pack["id"] and record.get("stock") == stock: previous = record.get("tool") base = previous if (isinstance(previous, str) and os.path.isabs(previous) and os.path.realpath(previous) == stock) else tool sync(dist, tool, base, pack, pack_dir, record, cache) elif os.path.lexists(dist): fresh = build_tree(tool, tool, pack, pack_dir, dist, cache) old = os.path.join(folder, ".%s.old.tmp-%d" % (name, os.getpid())) remove(old) os.rename(dist, old) os.rename(fresh, dist) remove(old) elif not place(build_tree(tool, tool, pack, pack_dir, dist, cache), dist): sync(dist, tool, tool, pack, pack_dir, read_record(dist) or {}, cache) os.utime(dist) cache.save() return dist def quarantine(root, pack, error): if not isinstance(error, PackDamaged): return with contextlib.suppress(OSError): os.unlink(os.path.join(root, "packs", pack["id"], ".complete")) say("droiddeck-esync: pack %s is damaged; the app fetches it again" % pack["id"]) def app_id(env): value = env.get("STEAM_COMPAT_APP_ID") or env.get("SteamAppId") if not value: value = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/").rsplit("/", 1)[-1] return value def minimum_nofile(env): try: return int(env.get("BL_SYNC_MIN_NOFILE") or MIN_NOFILE) except ValueError: return MIN_NOFILE def limit(value): return "unlimited" if value == resource.RLIM_INFINITY else str(value) def log_launch(root, command, env, tool, pack_id): try: soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) line = tsv([[int(time.time()), app_id(env) or "-", command[1] if len(command) > 1 else "-", tool, pack_id or "-", env.get("WINEESYNC") or "-", env.get("BL_SYNC") or "-", "%s/%s" % (limit(soft), limit(hard)), env.get("BL_FSYNC") or "-"]]) def append(text): lines = [entry for entry in text.split("\n") if entry] lines.append(line.rstrip("\n")) return "\n".join(lines[-LOG_LINES:]) + "\n" update_file(root, "launches.log", append) except Exception as error: say("droiddeck-esync: launch not logged (%s)" % error) def wanted_row(measured): return [measured["ntdll"], measured["wineserver"], measured["token"] or "-", measured["exports"] or "-", measured["tool"], str(int(time.time()))] def merge_wanted(text, fresh, drop): rows = [] seen = set() for row in parse_tsv(text, 6): key = (row[0], row[1]) if key in seen or key in drop: continue seen.add(key) if key in fresh: row[2:5] = fresh[key][2:5] rows.append(row) rows.extend(row for key, row in fresh.items() if key not in seen) return tsv(rows) def want(root, measured): try: row = wanted_row(measured) update_file(root, "wanted.tsv", lambda text: merge_wanted(text, {(row[0], row[1]): row}, set())) except Exception as error: say("droiddeck-esync: wanted.tsv not updated (%s)" % error) def select(command, env): result = (list(command), dict(env)) try: command = list(command) env = dict(env) tool = os.path.dirname(command[0]) or "." if env.get("PROTON_USE_ARM64", "1") != "1" or not has_stock(tool) or os.path.lexists(os.path.join(tool, RECORD)): return result prefix = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/") last = prefix.rsplit("/", 1)[-1] if not prefix or last == "0" or last.startswith("0-"): return result if truthy(env.get("PROTON_NO_NTSYNC")): env["BL_SYNC"] = "0" result = (list(command), dict(env)) root = store_root(env) if root is None: return result root = os.fspath(root) kept = keep_live(root, command, env, tool) if kept is not None: return kept chosen = choose(root, command, env, tool, result) remember(root, env, tool, chosen) return chosen except Exception as error: say("droiddeck-esync: skipped (%s); stock binaries" % error) record(result[0], result[1]) return result def record(command, env): try: root = store_root(env) if root is not None and command: remember(os.fspath(root), env, os.path.dirname(command[0]) or ".", (list(command), dict(env))) except Exception as error: say("droiddeck-esync: sync choice not recorded (%s)" % error) def prefix_of(env): data = env.get("STEAM_COMPAT_DATA_PATH", "").rstrip("/") return os.path.realpath(os.path.join(data, "pfx")) if data else None def server_running(prefix): for entry in os.listdir("/proc"): if not entry.isdigit(): continue try: with open("/proc/%s/comm" % entry, "rb") as source: if source.read().strip() != b"wineserver": continue with open("/proc/%s/environ" % entry, "rb") as source: items = source.read().split(b"\0") except OSError: continue for item in items: if item.startswith(b"WINEPREFIX=") and os.path.realpath(os.fsdecode(item[11:])) == prefix: return True return False def read_live(root): try: rows = json.loads(read_text(os.path.join(root, LIVE_FILE)) or "{}") except ValueError: return {} return rows if isinstance(rows, dict) else {} def remember(root, env, tool, chosen): prefix = prefix_of(env) if prefix is None: return command, chosen_env = chosen row = {"tool": os.path.realpath(tool), "command": command[0], "env": {name: chosen_env.get(name) for name in LIVE_KEYS}} def merge(text): try: rows = json.loads(text or "{}") except ValueError: rows = {} if not isinstance(rows, dict): rows = {} rows.pop(prefix, None) rows[prefix] = row return json.dumps(dict(list(rows.items())[-LIVE_ROWS:]), sort_keys=False) + "\n" try: update_file(root, LIVE_FILE, merge) except OSError as error: say("droiddeck-esync: sync choice not recorded (%s)" % error) def keep_live(root, command, env, tool): prefix = prefix_of(env) row = read_live(root).get(prefix) if prefix else None if not isinstance(row, dict) or not isinstance(row.get("env"), dict) or row.get("tool") != os.path.realpath(tool): return None kept_command = row.get("command") if not isinstance(kept_command, str) or os.path.basename(kept_command) != os.path.basename(command[0]): return None if not os.path.isfile(kept_command) or not server_running(prefix): return None kept_pack = row["env"].get("BL_SYNC_PACK") if isinstance(kept_pack, str) and kept_pack and command[1:2] == ["waitforexitandrun"]: try: cache = HashCache(root) try: pack = find_pack(load_packs(root), measure(tool, cache)) finally: cache.save() except (OSError, ValueError): pack = {"id": kept_pack} if pack is None or pack["id"] != kept_pack: return None command, env = list(command), dict(env) command[0] = kept_command for name in LIVE_KEYS: value = row["env"].get(name) if isinstance(value, str): env[name] = value else: env.pop(name, None) say("droiddeck-esync: Wine is still running in this prefix; keeping its sync choice (%s)" % describe(env)) log_launch(root, command, env, tool, env.get("BL_SYNC_PACK")) return command, env def describe(env): if env.get("BL_SYNC") == "1": return "droiddeck-ntsync" if env.get("BL_FSYNC") == "1": return "droiddeck-fsync" if env.get("BL_SYNC_PACK") and truthy(env.get("WINEESYNC")): return "droiddeck-esync" if all(truthy(env.get(name)) for name in WINESERVER_ONLY): return "wineserver" return "Proton's own sync" def fsync_ready(env, measured): listed = "PROTON_NO_FSYNC" not in env and app_id(env) in NOSYNC_APPS return measured["fsync"] and env.get("BL_SYNC") != "1" and not truthy(env.get("PROTON_NO_FSYNC")) and not listed def with_fsync(root, command, env, tool, reason): command, env = list(command), dict(env) env["BL_FSYNC"] = "1" env["BL_SYNC"] = "0" env["WINEFSYNC"] = "1" say("droiddeck-fsync: %s; stock binaries with Proton's fsync" % reason) log_launch(root, command, env, tool, None) return command, env def stand_in(root, command, env, tool, reason, measured=None): if measured is not None and fsync_ready(env, measured): return with_fsync(root, command, env, tool, reason) command, env = list(command), dict(env) if truthy(env.get("PROTON_NO_NTSYNC")): say("droiddeck-esync: %s; stock binaries" % reason) elif measured is not None and not measured["ntsync"]: env["BL_SYNC"] = "0" say("droiddeck-esync: %s; stock binaries (this Wine has no ntsync)" % reason) else: env["BL_SYNC"] = "1" say("droiddeck-esync: %s; stock binaries with droiddeck-ntsync" % reason) log_launch(root, command, env, tool, None) return command, env def esync_unwanted(env): soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) starved = hard != resource.RLIM_INFINITY and hard < minimum_nofile(env) listed = "PROTON_NO_ESYNC" not in env and app_id(env) in NOSYNC_APPS return truthy(env.get("PROTON_NO_ESYNC")) or env.get("WINEESYNC") == "0" or listed or starved def wineserver_only(env): env = dict(env) for name in WINESERVER_ONLY: env.setdefault(name, "1") if truthy(env["PROTON_NO_ESYNC"]): env["WINEESYNC"] = "0" if truthy(env["PROTON_NO_FSYNC"]): env["WINEFSYNC"] = "0" if truthy(env["PROTON_NO_NTSYNC"]): env["BL_SYNC"] = "0" return env def choose(root, command, env, tool, stock): name = os.path.basename(tool.rstrip("/")) or tool wants_fsync = env.get("BL_FSYNC_FIRST") == "1" if "BL_FSYNC" in env: env["BL_FSYNC"] = "0" stock = (stock[0], dict(stock[1], BL_FSYNC="0")) if env.get("BL_SYNC_FALLBACK") != "1" and not wants_fsync and env.get("BL_SYNC") != "1": stock = (stock[0], wineserver_only(stock[1])) say("droiddeck-esync: off; wineserver sync only") log_launch(root, stock[0], stock[1], tool, None) return stock cache = HashCache(root) try: measured = measure(tool, cache) if env.get("BL_SYNC") == "1" and not measured["ntsync"]: env["BL_SYNC"] = "0" stock = (stock[0], dict(stock[1], BL_SYNC="0")) wants_fsync = True say("droiddeck-ntsync: %s has no ntsync; trying droiddeck-fsync" % name) if wants_fsync and fsync_ready(env, measured): return with_fsync(root, command, env, tool, "chosen for %s" % name) if env.get("BL_SYNC_FALLBACK") != "1": say("droiddeck-esync: off") log_launch(root, command, env, tool, None) return stock if esync_unwanted(env) and fsync_ready(env, measured): return with_fsync(root, command, env, tool, "esync is off for this game") if measured["builtin"]: if esync_unwanted(env): env["PROTON_NO_ESYNC"] = "1" env["WINEESYNC"] = "0" return stand_in(root, command, env, tool, "esync is off for this game on %s" % name, measured) say("droiddeck-esync: %s has esync built in" % name) log_launch(root, command, env, tool, None) return stock pack = find_pack(load_packs(root), measured) if pack is None: want(root, measured) return stand_in(root, command, env, tool, "no pack for %s (%s, ntdll %s)" % (name, measured["version_line"], measured["ntdll"][:12]), measured) try: dist = ensure_dist(root, tool, pack, cache) except Exception as error: quarantine(root, pack, error) return stand_in(root, command, env, tool, "pack %s unusable for %s (%s)" % (pack["id"], name, error), measured) finally: cache.save() soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) if esync_unwanted(env): env["WINEESYNC"] = "0" env["PROTON_NO_ESYNC"] = "1" if not truthy(env.get("PROTON_NO_NTSYNC")) and measured["ntsync"]: env["BL_SYNC"] = "1" elif not env.get("WINEESYNC"): env["WINEESYNC"] = "1" if truthy(env["WINEESYNC"]) and "PROTON_NO_FSYNC" not in env: env["PROTON_NO_FSYNC"] = "1" env["WINESERVER"] = os.path.join(dist, WINESERVER) env["BL_SYNC_PACK"] = pack["id"] command[0] = os.path.join(dist, os.path.basename(command[0])) say("droiddeck-esync: pack %s for %s (%s, %s match); droiddeck-ntsync %s, droiddeck-esync %s, fds %s/%s" % (pack["id"], name, measured["version_line"], match_kind(pack, measured), "on" if env.get("BL_SYNC") == "1" else "off", "on" if truthy(env["WINEESYNC"]) else "off", limit(soft), limit(hard))) log_launch(root, command, env, tool, pack["id"]) return command, env def libraries(steam_root): found = [steam_root] try: with open(os.path.join(steam_root, "steamapps", "libraryfolders.vdf"), encoding="utf-8", errors="replace") as f: found += [path.replace("\\\\", "\\") for path in re.findall(r'"path"\s+"((?:[^"\\]|\\.)*)"', f.read())] except OSError: pass found.append(SD_LIBRARY) return list(dict.fromkeys(found)) def find_tools(steam_root): candidates = [os.path.join(library, "steamapps", "common", name) for library in libraries(steam_root) for name in SOURCES] folder = os.path.join(steam_root, "compatibilitytools.d") try: names = sorted(os.listdir(folder)) except OSError: names = [] candidates += [os.path.join(folder, name) for name in names if not name.startswith((OWN_TOOL_PREFIX, "."))] tools = [] seen = set() for tool in candidates: if os.path.isdir(tool) and has_stock(tool): real = os.path.realpath(tool) if real not in seen: seen.add(real) tools.append(tool) return tools def still_wanted(row, scanned, packs, cache): tool = row[4] try: if os.path.realpath(tool) in scanned or not has_stock(tool): return False measured = measure(tool, cache) return ((measured["ntdll"], measured["wineserver"]) == (row[0], row[1]) and not measured["builtin"] and find_pack(packs, measured) is None) except Exception: return True def update_wanted(root, fresh, scanned, packs, cache): drop = set() for row in parse_tsv(read_text(os.path.join(root, "wanted.tsv")), 6): key = (row[0], row[1]) if key not in fresh and not still_wanted(row, scanned, packs, cache): drop.add(key) return len(parse_tsv(update_file(root, "wanted.tsv", lambda text: merge_wanted(text, fresh, drop)), 6)) def reconcile_store(root, steam_root, env): cache = HashCache(root) packs = load_packs(root) enabled = env.get("BL_SYNC_FALLBACK") == "1" rows = [] fresh = {} scanned = set() used = 0 for tool in find_tools(os.fspath(steam_root)): try: measured = measure(tool, cache) pack = None if measured["builtin"] else find_pack(packs, measured) if measured["builtin"]: state = "builtin" elif pack is not None: state = "pack" used += 1 else: state = "wanted" fresh.setdefault((measured["ntdll"], measured["wineserver"]), wanted_row(measured)) scanned.add(measured["stock"]) except Exception as error: say("droiddeck-esync: %s skipped (%s)" % (tool, error)) continue rows.append([tool, measured["version_line"], measured["ntdll"], measured["wineserver"], state, pack["id"] if pack is not None else "-"]) if pack is not None and enabled: try: ensure_dist(root, tool, pack, cache) except Exception as error: say("droiddeck-esync: pack %s unusable for %s (%s)" % (pack["id"], tool, error)) quarantine(root, pack, error) wanted = update_wanted(root, fresh, scanned, packs, cache) update_file(root, "tools.tsv", lambda text: tsv(sorted(rows))) cache.save(True) gc(root) return "droiddeck-esync: %d pack(s) in use, %d wanted" % (used, wanted) def reconcile(steam_root, env): try: root = store_root(env) if root is None: return None return reconcile_store(os.fspath(root), steam_root, env) except Exception as error: say("droiddeck-esync: reconcile failed (%s)" % error) return None def process_cmdlines(): try: entries = os.listdir("/proc") except OSError: return None lines = [] for entry in entries: if not entry.isdigit(): continue try: with open(os.path.join("/proc", entry, "cmdline"), "rb") as source: lines.append(source.read()) except OSError: continue return lines def in_use(path, cmdlines): needles = {os.fsencode(path), os.fsencode(os.path.realpath(path))} return any(needle in line for line in cmdlines for needle in needles) def stale(path, name, packs, cache): record = read_record(path) if record is None: return True pack = packs.get(record.get("pack")) stock = record.get("stock") if pack is None or not isinstance(stock, str) or name != dist_name(pack["id"], stock): return True if not has_stock(stock): return True try: measured = measure(stock, cache) if match_kind(pack, measured) is None: return True best = find_pack(list(packs.values()), measured) return best is None or best["id"] != pack["id"] except Exception: return False def collect(root): folder = os.path.join(root, "dist") try: names = sorted(os.listdir(folder)) except FileNotFoundError: return 0 packs = {pack["id"]: pack for pack in load_packs(root)} cache = HashCache(root) candidates = [] for name in names: path = os.path.join(folder, name) if name.startswith("."): if ".tmp-" in name and age(path) > TEMP_AGE: remove(path) elif (not name.endswith(".lock") and "~" in name and not os.path.islink(path) and os.path.isdir(path) and stale(path, name, packs, cache)): candidates.append(name) cmdlines = process_cmdlines() if candidates else [] removed = 0 for name in candidates: path = os.path.join(folder, name) lock = path + ".lock" with locked(lock, False) as fd: if (fd is None or cmdlines is None or age(path) < GC_GRACE or in_use(path, cmdlines) or not stale(path, name, packs, cache)): continue doomed = os.path.join(folder, ".%s.tmp-%d" % (name, os.getpid())) remove(doomed) os.rename(path, doomed) os.unlink(lock) remove(doomed) removed += 1 for name in sorted(os.listdir(folder)): if name.startswith(".") or not name.endswith(".lock"): continue lock = os.path.join(folder, name) dist = lock[:-len(".lock")] if os.path.lexists(dist): continue with locked(lock, False) as fd: if fd is not None and not os.path.lexists(dist): os.unlink(lock) cache.save() return removed def gc(root): try: if root is None: return 0 return collect(os.fspath(root)) except Exception as error: say("droiddeck-esync: gc failed (%s)" % error) return 0 def status(root): root = os.fspath(root) packs = load_packs(root) try: dists = sorted(name for name in os.listdir(os.path.join(root, "dist")) if "~" in name and not name.startswith(".") and not name.endswith(".lock")) except OSError: dists = [] tools = parse_tsv(read_text(os.path.join(root, "tools.tsv")), 6) wanted = parse_tsv(read_text(os.path.join(root, "wanted.tsv")), 6) lines = ["droiddeck-esync store %s" % root, "packs: %d installed" % len(packs)] lines += [" %s (%s, rev %d)" % (pack["id"], pack["flavor"], pack["rev"]) for pack in packs] lines.append("dists: %d" % len(dists)) lines += [" %s" % name for name in dists] lines.append("tools: %d" % len(tools)) lines += [" %s %s: %s (%s)" % (row[4], row[5], row[0], row[1]) for row in tools] lines.append("wanted: %d" % len(wanted)) lines += [" %s %s (%s, ntdll %s)" % (row[2], row[4], row[3][:12], row[0][:12]) for row in wanted] return "\n".join(lines) def main(argv): command = argv[1:] if command == ["status"]: root = store_root(os.environ) print(status(root) if root is not None else "droiddeck-esync: no store at %s" % (Path(os.environ.get("HOME") or os.path.expanduser("~")) / STORE)) return 0 if len(command) == 2 and command[0] == "reconcile": if store_root(os.environ) is None: print("droiddeck-esync: no store") return 0 summary = reconcile(command[1], os.environ) if summary is None: return 1 print(summary) return 0 if command == ["gc"]: root = store_root(os.environ) print("droiddeck-esync: %d dist(s) removed" % gc(root) if root is not None else "droiddeck-esync: no store") return 0 if len(command) == 4 and command[0] == "build-shadow": print(build_shadow(command[1], command[2], command[3])) return 0 print("usage: droiddeck-esync status|reconcile |gc|build-shadow ", file=sys.stderr) return 2 if __name__ == "__main__": sys.exit(main(sys.argv))