Files
DroidDeck/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-games
T
MaxsTechReview 6ee8866599 Make Games folders Steam libraries and skip the zero-filled reservation (#388)
With Second library on Automatic, every writable Games folder is now a
Steam library: it gets the same private prefix and tool binds as the card,
the app hands the session the list (BL_STEAM_LIBRARIES), and
droiddeck-steam-library registers, relabels and forgets entries from it.
Steam offers them in the install dialog as "<folder> (<volume>)", and
Proton mapping, esync, the redist seeder, desktop entries and the compat
beat read every registered library instead of only the card. Games folder
libraries stop at the fast path's bind budget.

Steam's preallocation on Android's FUSE storage either failed outright
(NTFS answers EPERM, so installs were cancelled as "not enough disk
space") or zero-filled the whole download first (exFAT, minutes for a
small game). After fallocate is refused, the Steam client's ftruncate
extension of its own staging files is reported as done; the writes that
follow fill the file. A 76 MB install to a USB SSD now preallocates
instantly instead of taking about 3.5 minutes.

A library that is not there is left out of the list and forgotten until
it returns, imports whose folder is missing are set aside rather than
offered as broken installs, and the front end refreshes when storage is
mounted or removed.
2026-10-07 09:31:33 -04:00

425 lines
18 KiB
Python
Executable File

#!/usr/bin/env python3
"""Resolve added games against the signed-in Steam client's licenses."""
import ctypes as C
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
import unicodedata
import urllib.parse
import urllib.request
STATE = ".droiddeck-games.json"
WINNATIVE = ".winnative-steam/steamapps"
LIBRARY = "/mnt/droiddeck-sd"
LEGACY_LIBRARY = "/mnt/bannerlator-sd"
PARKED = ".droiddeck-parked"
BASE_ID = 76561197960265728
def read_json(path, default):
try:
return json.loads(Path(path).read_text())
except (OSError, ValueError):
return default
def save_json(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(".tmp")
temp.write_text(json.dumps(value))
os.replace(temp, path)
def title_key(name):
return "".join(c for c in unicodedata.normalize("NFKC", name.replace("™", "").replace("®", "")).casefold() if c.isalnum())
def edition_key(name):
name = re.sub(r"(?i)\s+(?:windows edition|intergrade)\s*$", "", name)
return title_key(name)
def legacy_path(path):
path = str(path)
return LEGACY_LIBRARY + path[len(LIBRARY):] if path == LIBRARY or path.startswith(LIBRARY + "/") else path
def source_stamp(game):
try:
exe = Path(game["exe"])
stat = exe.stat()
hints = []
for parent in sorted({Path(game["folder"]), exe.parent}):
hint = parent / "steam_appid.txt"
hints.append(hint.read_text().strip() if hint.is_file() else "")
return [game["name"], str(exe), stat.st_size, stat.st_mtime_ns, hints]
except (OSError, ValueError):
return None
def manifest_ids(steamapps, name):
matches = []
for manifest in Path(steamapps).glob("appmanifest_*.acf"):
try:
install = re.search(r'"installdir"\s*"([^"\\]*)"', manifest.read_text(), re.I)
if install and install[1] == name:
matches.append(int(manifest.stem.removeprefix("appmanifest_")))
except (OSError, ValueError):
pass
return matches
def identify(game):
folder = Path(game["folder"])
steamapps = folder.parent.parent
for library in ([steamapps] if folder.parent.name == "common" and steamapps.name == "steamapps" else []) + [folder.parent / WINNATIVE]:
matches = manifest_ids(library, folder.name)
if matches:
return matches[0] if len(matches) == 1 else None
ids = set()
for parent in {folder, Path(game["exe"]).parent}:
try:
appid = int((parent / "steam_appid.txt").read_text().strip())
if 0 < appid < 0x80000000:
ids.add(appid)
except (OSError, ValueError):
pass
if ids:
return next(iter(ids)) if len(ids) == 1 else None
url = "https://store.steampowered.com/api/storesearch/?l=english&cc=US&term=" + urllib.parse.quote(game["name"])
try:
with urllib.request.urlopen(url, timeout=5) as response:
items = json.loads(response.read(1024 * 1024)).get("items", [])
for key in (title_key, edition_key):
ids = {int(item["id"]) for item in items
if key(item.get("name", "")) == key(game["name"]) and key(game["name"])}
if ids:
return next(iter(ids)) if len(ids) == 1 else None
return None
except (OSError, ValueError, KeyError):
return None
def method(obj, slot, result, *args):
if not obj:
raise RuntimeError("Steam interface unavailable")
table = C.cast(obj, C.POINTER(C.POINTER(C.c_void_p))).contents
return C.CFUNCTYPE(result, C.c_void_p, *args)(table[slot])
def probe(steam_root, ids):
# Slots belong to Valve's SteamClient020, SteamUser021 and STEAMAPPS_INTERFACE_VERSION008.
library = C.CDLL(str(Path(steam_root) / "steamrtarm64/steamclient.so"))
library.CreateInterface.argtypes = [C.c_char_p, C.c_void_p]
library.CreateInterface.restype = C.c_void_p
client = library.CreateInterface(b"SteamClient020", None)
pipe = method(client, 0, C.c_int)(client)
if not pipe:
raise RuntimeError("Steam pipe unavailable")
user = 0
try:
user = method(client, 2, C.c_int, C.c_int)(client, pipe)
if not user:
raise RuntimeError("No signed-in Steam user")
get_interface = lambda slot, version: method(client, slot, C.c_void_p, C.c_int, C.c_int, C.c_char_p)(client, user, pipe, version)
account = get_interface(5, b"SteamUser021")
if not method(account, 1, C.c_bool)(account):
raise RuntimeError("Steam is offline")
steamid = method(account, 2, C.c_uint64)(account)
apps = get_interface(15, b"STEAMAPPS_INTERFACE_VERSION008")
subscribed = method(apps, 6, C.c_bool, C.c_uint32)
return {"account": str(steamid - BASE_ID), "checked": time.time(),
"owned": {str(appid): bool(subscribed(apps, appid)) for appid in ids}}
finally:
if user:
method(client, 4, None, C.c_int, C.c_int)(client, pipe, user)
method(client, 1, C.c_bool, C.c_int)(client, pipe)
def game_running():
"""Whether a Proton game is up: a wineserver is there for as long as any prefix is in use."""
try:
pids = [p for p in os.listdir("/proc") if p.isdigit()]
except OSError:
return False
for pid in pids:
try:
with open("/proc/%s/comm" % pid) as f:
if f.read().strip() == "wineserver":
return True
except OSError:
pass
return False
def downloads_idle(steam_root):
try:
return not any(Path(steam_root, "steamapps/downloading").iterdir())
except OSError:
return True
def request_restart(steam_root):
"""The client reads manifests and shortcuts only at start, so an import needs one restart;
the session's loop runs the importer between the exit and the next start."""
Path(os.environ.get("HOME", "/root"), ".bl-restart-once").touch()
subprocess.run([str(Path(steam_root) / "steamrtarm64/steam"), "-shutdown"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
def unimported(steam_root, snapshot, candidates):
routes = read_json(Path(steam_root) / "userdata" / snapshot["account"] / "config/.droiddeck-routes.json", {})
routed = set(routes.values()) if isinstance(routes, dict) else set()
return sorted({appid for appid in candidates.values() if snapshot["owned"].get(str(appid)) is True} - routed)
def watch(steam_root, listing):
games = read_json(listing, [])
if os.environ.get("BL_STEAM_OFFLINE") == "1" or not games:
return
candidates, sources = {}, {}
next_lookup = 0
last_status = None
# The probe is a fresh python that loads steamclient.so, seconds of CPU under proot. Ownership
# only moves with a purchase or a new match, so it runs when the matches or their sources
# change, after a failed probe, and otherwise every five minutes - not on every 30 s beat.
# A failed probe is retried on a doubling delay up to those five minutes rather than on every
# beat (a client that is not logged on fails it for as long as it stays that way), and none
# starts while a game is running: the answer can wait, the game's cores cannot.
probed, next_probe, failures = None, 0, 0
pending, requested = [], []
while True:
now = time.monotonic()
for game in games:
folder = game["folder"]
stamp = source_stamp(game)
if now >= next_lookup or stamp != sources.get(folder):
sources[folder] = stamp
appid = identify(game)
if appid:
candidates[folder] = appid
else:
candidates.pop(folder, None)
if now >= next_lookup:
next_lookup = now + 300
status = "waiting for a matching Steam title"
key = (dict(candidates), dict(sources))
if candidates and (key != probed or now >= next_probe) and game_running():
status = last_status
elif candidates and (key != probed or now >= next_probe):
probed, next_probe = key, now + 300
try:
result = subprocess.run([sys.executable, __file__, "probe", steam_root,
*map(str, sorted(set(candidates.values())))],
capture_output=True, text=True, timeout=15)
if result.returncode != 0:
status = "ownership probe failed: " + (result.stderr.strip()[-300:] or str(result.returncode))
failures += 1
next_probe = now + min(300, 30 * 2 ** failures)
else:
failures = 0
line = next(line for line in result.stdout.splitlines() if line.startswith("DROIDDECK_OWNERSHIP="))
snapshot = json.loads(line.removeprefix("DROIDDECK_OWNERSHIP="))
if not snapshot["account"].isdigit() or int(snapshot["account"]) <= 0:
raise ValueError("invalid Steam account")
snapshot["candidates"] = candidates
snapshot["sources"] = sources
save_json(Path(steam_root) / "userdata" / snapshot["account"] / "config" / STATE, snapshot)
pending = unimported(steam_root, snapshot, candidates)
status = "matched %d/%d games; %d owned" % (len(candidates), len(games), sum(snapshot["owned"].values()))
except (OSError, ValueError, KeyError, StopIteration, subprocess.TimeoutExpired) as error:
status = "ownership check unavailable: " + str(error)
failures += 1
next_probe = now + min(300, 30 * 2 ** failures)
elif candidates:
status = last_status
if pending and pending != requested and not game_running() and downloads_idle(steam_root):
requested = pending
print("droiddeck-steam-games: restarting the client once to import %s" % pending, flush=True)
try:
request_restart(steam_root)
except (OSError, subprocess.TimeoutExpired) as error:
print("droiddeck-steam-games: restart unavailable: %s" % error, flush=True)
if status != last_status:
print("droiddeck-steam-games: " + status, flush=True)
last_status = status
time.sleep(30)
def installed_build(folder, appid, installdir):
"""The build another Steam client recorded for these files, so Steam updates from it
instead of verifying every file."""
try:
text = (folder.parent / WINNATIVE / f"appmanifest_{appid}.acf").read_text()
install = re.search(r'("installdir"\s*")([^"\\]*)(")', text, re.I)
flags = re.search(r'"StateFlags"\s*"(\d+)"', text)
if (install and install[2] == folder.name and flags and flags[1] == "4"
and re.search(rf'"appid"\s*"{appid}"', text) and '"InstalledDepots"' in text):
return text[:install.start(2)] + installdir + text[install.end(2):]
except OSError:
pass
if not (folder / ".download_complete").is_file():
return None
try:
ids = json.loads((folder / ".DepotDownloader/depot.config").read_text())["installedManifestIDs"]
depots = sorted((int(depot), int(gid)) for depot, gid in ids.items())
except (OSError, ValueError, KeyError, TypeError, AttributeError):
return None
if not depots or not all(0 < depot < 0x80000000 and 0 < gid < 0x7FFFFFFFFFFFFFFF for depot, gid in depots):
return None
return "".join(f' "{depot}"\n {{\n "manifest" "{gid}"\n }}\n' for depot, gid in depots)
def update_not_started(text, steamapps, appid, installdir):
fields = dict(re.findall(r'^\s*"(\w+)"\s*"([^"]*)"', text, re.M))
return (fields.get("installdir") == installdir and fields.get("buildid", "0") == "0"
and fields.get("BytesDownloaded", "0") == "0" and fields.get("BytesStaged", "0") == "0"
and re.search(r'"InstalledDepots"\s*\{\s*\}', text) is not None
and not (steamapps / "downloading" / str(appid)).exists())
def libraries(steam_root):
try:
text = (Path(steam_root) / "steamapps/libraryfolders.vdf").read_text(errors="replace")
except OSError:
text = ""
paths = [path.replace("\\\\", "\\") for path in re.findall(r'"path"\s+"((?:[^"\\]|\\.)*)"', text)]
return list(dict.fromkeys([Path(steam_root), Path(LIBRARY)] + [Path(path) for path in paths]))
def ours(manifest, appid):
try:
return re.search(rf'"installdir"\s*"DroidDeck-{appid}"', manifest.read_text(errors="replace")) is not None
except OSError:
return False
def unpark(steamapps):
for manifest in sorted((steamapps / PARKED).glob("appmanifest_*.acf")):
appid = manifest.stem.removeprefix("appmanifest_")
live = steamapps / manifest.name
if os.path.lexists(live):
manifest.unlink(missing_ok=True)
elif (steamapps / "common" / f"DroidDeck-{appid}").is_dir():
os.replace(manifest, live)
def park(steamapps):
for target in sorted((steamapps / "common").glob("DroidDeck-*")):
appid = target.name.removeprefix("DroidDeck-")
manifest = steamapps / f"appmanifest_{appid}.acf"
if appid.isdigit() and target.is_symlink() and not target.exists() and manifest.is_file() and ours(manifest, appid):
(steamapps / PARKED).mkdir(exist_ok=True)
os.replace(manifest, steamapps / PARKED / manifest.name)
print(f"droiddeck-steam-games: {appid} is not here this session; its install is set aside until it is")
def seed_install(steam_root, game, appid):
steamapps = Path(steam_root) / "steamapps"
common = steamapps / "common"
name = f"DroidDeck-{appid}"
target = common / name
source = Path(game["folder"])
title = game["name"].replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ")
minimal = ('"AppState"\n{\n' + f' "appid" "{appid}"\n "Universe" "1"\n "name" "{title}"\n'
+ f' "StateFlags" "4"\n "installdir" "{name}"\n' + '}\n')
legacy = minimal.replace(' "Universe" "1"\n', '')
build = installed_build(source, appid, name)
if build is None:
content = minimal
elif build.lstrip().startswith('"AppState"'):
content = build
else:
content = minimal[:-2] + ' "InstalledDepots"\n {\n' + build + ' }\n}\n'
manifest = steamapps / f"appmanifest_{appid}.acf"
if target.is_symlink() and os.readlink(target) != str(source) and os.readlink(target) == legacy_path(source):
relink = target.with_name(name + ".droiddeck")
relink.unlink(missing_ok=True)
relink.symlink_to(source, target_is_directory=True)
os.replace(relink, target)
for root in libraries(steam_root):
existing = root / "steamapps" / manifest.name
if existing.is_file():
if (existing == manifest and source.is_dir() and target.is_symlink()
and os.readlink(target) == str(source)
and existing.read_text() != content
and (existing.read_text() in (minimal, legacy, legacy.replace('"StateFlags" "4"', '"StateFlags" "2"'))
or (build is not None and update_not_started(existing.read_text(), steamapps, appid, name)))):
temp = manifest.with_suffix(".droiddeck")
temp.write_text(content)
os.replace(temp, manifest)
return True
if not source.is_dir():
return False
common.mkdir(parents=True, exist_ok=True)
if os.path.lexists(target):
if not target.is_symlink() or os.readlink(target) != str(source):
return False
else:
target.symlink_to(source, target_is_directory=True)
try:
with manifest.open("x") as output:
output.write(content)
except FileExistsError:
pass
return True
def route(steam_root, acct, games):
snapshot = read_json(Path(acct) / "config" / STATE, {})
if not isinstance(snapshot, dict):
snapshot = {}
owned = snapshot.get("owned", {}) if snapshot.get("account") == Path(acct).name else {}
candidates = snapshot.get("candidates", {})
sources = snapshot.get("sources", {})
if not all(isinstance(value, dict) for value in (owned, candidates, sources)):
owned, candidates, sources = {}, {}, {}
shortcuts, routes = [], {}
steamapps = Path(steam_root) / "steamapps"
try:
unpark(steamapps)
except OSError as error:
print(f"droiddeck-steam-games: set-aside installs not restored ({error})")
for game in games:
folder = game.get("folder", "")
appid = candidates.get(folder, candidates.get(legacy_path(folder)))
stamp = source_stamp(game)
recorded = sources.get(folder, sources.get(legacy_path(folder)))
confirmed = (isinstance(appid, int) and 0 < appid < 0x80000000
and owned.get(str(appid)) is True and stamp is not None
and recorded in (stamp, [stamp[0], legacy_path(stamp[1])] + stamp[2:]))
if confirmed:
try:
if seed_install(steam_root, game, appid):
routes[str(game["appid"])] = appid
continue
except OSError as error:
print(f"droiddeck-steam-games: import unavailable ({error})")
shortcuts.append(game)
try:
park(steamapps)
except OSError as error:
print(f"droiddeck-steam-games: missing installs not set aside ({error})")
save_json(Path(acct) / "config/.droiddeck-routes.json", routes)
return shortcuts, routes
if __name__ == "__main__":
if len(sys.argv) < 4:
sys.exit(2)
try:
if sys.argv[1] == "probe":
print("DROIDDECK_OWNERSHIP=" + json.dumps(probe(sys.argv[2], [int(value) for value in sys.argv[3:]])), flush=True)
elif sys.argv[1] == "watch":
watch(sys.argv[2], sys.argv[3])
else:
sys.exit(2)
except (OSError, RuntimeError, ValueError) as error:
print(str(error), file=sys.stderr)
sys.exit(1)