Files
DroidDeck/tools/linuxfs/overlay/usr/local/bin/droiddeck-fex
T
Kurt Himebauch eabcfc7b74 chore: consistent code branding (#2800)
* chore: consistent code branding

* fix: carry legacy overrides, queues and live sync rows across the rename

* fix: keep queued legacy overrides on a failed rename and separate merged queue entries
2026-11-17 23:31:26 -05:00

954 lines
37 KiB
Python
Executable File

#!/usr/bin/python3
import contextlib
import fcntl
import filecmp
import gzip
import json
import os
import re
import runpy
import shutil
import stat
import struct
import subprocess
import sys
import time
HOME = os.environ.get("HOME") or "/root"
STEAM_ROOT = os.path.join(HOME, ".local", "share", "Steam")
LIBRARIES = ("/mnt/droiddeck", "/mnt/droiddeck-sd")
STATE = os.path.join(HOME, ".local", "share", "droiddeck-fex")
CONFIG_DIR = os.path.join(STATE, "config")
DATA_DIR = os.path.join(STATE, "data")
CACHE_DIR = os.path.join(HOME, ".cache", "droiddeck-fex")
STATUS_FILE = os.path.join(STATE, "status.json")
WANTED = os.path.join(HOME, ".config", "droiddeck", "fex-wanted")
GAME_ENVIRONMENT = os.path.join(HOME, ".config", "droiddeck", "game-environment.json")
BIN_DIR = os.path.dirname(os.path.abspath(__file__))
PRELOADS = "/usr/local/lib/droiddeck-fex"
URUNTIME = "/usr/local/lib/droiddeck/uruntime"
HOST_PRELOAD = "/etc/ld.so.preload"
FEX_APPID = "3127680"
OWN_FEX = "/usr/local"
OWN_FEX_TAG = os.path.join(OWN_FEX, "share", "fex-emu", ".droiddeck-tag")
SNIPER_APPID = "1628350"
SLR4_APPID = "4183110"
INSTALL_DIRS = {FEX_APPID: "FEX-Emu", SNIPER_APPID: "SteamLinuxRuntime_sniper", SLR4_APPID: "SteamLinuxRuntime_4"}
RUNTIMES = {SNIPER_APPID: ("sniper_platform_", "Steam Linux Runtime 3.0 (sniper"),
SLR4_APPID: ("steamrt4_platform_", "Steam Linux Runtime 4.0 (steamrt4")}
MARKER = ".droiddeck-fex"
LAYOUT = 1
ROOTFS_LAYOUT = 10
BLOB_DIRS = re.compile(r"^(?:\.ref|[0-9a-f]{2})$")
ROOTFS_LIBRARY_DIRS = ("lib", "lib64")
ARCH_DIRS = ("x86_64-linux-gnu", "i386-linux-gnu")
INTERPRETERS = ("ld-linux-x86-64.so.2", "ld-linux.so.2")
PROVIDED = frozenset((
"ld-linux-x86-64.so.2", "ld-linux.so.2", "libc.so.6", "libm.so.6", "libmvec.so.1", "libpthread.so.0", "libdl.so.2",
"librt.so.1", "libutil.so.1", "libresolv.so.2", "libanl.so.1", "libBrokenLocale.so.1", "libnsl.so.1",
"libnss_compat.so.2", "libnss_dns.so.2", "libnss_files.so.2", "libnss_hesiod.so.2", "libthread_db.so.1",
"libmemusage.so", "libpcprofile.so", "libc_malloc_debug.so.0", "gconv", "libstdc++.so.6", "libgcc_s.so.1",
"libssl.so.3", "libcrypto.so.3",
"libfreetype.so.6", "libpng16.so.16", "libbrotlidec.so.1", "libbrotlicommon.so.1", "libbz2.so.1.0", "libz.so.1",
"libwayland-client.so.0", "libwayland-egl.so.1", "libwayland-server.so.0", "libffi.so.8",
"libgbm.so.1", "libdrm.so.2", "libexpat.so.1",
))
PROVIDED_NEEDS = {
"libwayland-client.so.0": ("libffi.so.8",),
"libwayland-server.so.0": ("libffi.so.8",),
"libgbm.so.1": ("libdrm.so.2", "libexpat.so.1"),
}
GLIBC_BUILD = re.compile(r"^(?:ld|lib[A-Za-z_]+)-2\.[0-9]+\.so$|^libSegFault\.so$")
GUEST_LIBDIRS = (("x86_64", "lib/x86_64-linux-gnu"), ("i386", "lib/i386-linux-gnu"))
PRELOAD_NAMES = ("libddsession.so", "libfakeinput.so")
FAULT_REPORT = "libfaultreport.so"
VULKAN_PIN = "libvulkan-thunk.so"
VULKAN_PIN_DIR = "lib/x86_64-linux-gnu/droiddeck/vulkan"
VULKAN_THUNK = "libvulkan-guest.so"
ASOUND_CONF = "pcm.!default {\n type pulse\n}\nctl.!default {\n type pulse\n}\n"
RUNTIME_LIBRARIES = (("libc.so.6", b"GLIBC_"), ("libstdc++.so.6", b"GLIBCXX_"))
VERSION_NAME = re.compile(rb"GLIBC(?:XX)?_[0-9][0-9.]*[0-9]")
BUNDLE_DEPTH = 3
THUNKS = {"GL": 1, "Vulkan": 1}
X86 = {3: "i386", 62: "x86_64"}
ARM = {40: "arm", 183: "arm64"}
SCAN_DEPTH = 4
SCAN_LIMIT = 4000
MODES = ("auto", "on", "off")
STEAM_VERBS = ("run", "waitforexitandrun")
ESCAPE = re.compile(rb"\\([0-7]{3})")
DEVICE_FONTS = "/usr/local/share/fonts/android"
FONT_RULES = "/etc/fonts/conf.d/66-droiddeck-device-fonts.conf"
def say(message):
print("droiddeck-fex: %s" % message, file=sys.stderr, flush=True)
def read_text(path):
try:
with open(path, encoding="utf-8", errors="replace") as f:
return f.read()
except OSError:
return None
def write_if_changed(path, data, mode=0o644):
if isinstance(data, str):
data = data.encode("utf-8")
try:
with open(path, "rb") as f:
if f.read() == data:
return False
except OSError:
pass
os.makedirs(os.path.dirname(path), exist_ok=True)
tmp = "%s.%d.tmp" % (path, os.getpid())
with open(tmp, "wb") as f:
f.write(data)
os.chmod(tmp, mode)
os.replace(tmp, path)
return True
def copy_if_changed(source, target):
if os.path.isfile(target) and filecmp.cmp(source, target, shallow=False):
return False
os.makedirs(os.path.dirname(target), exist_ok=True)
tmp = "%s.%d.tmp" % (target, os.getpid())
shutil.copyfile(source, tmp)
os.chmod(tmp, 0o755)
os.replace(tmp, target)
return True
@contextlib.contextmanager
def locked():
os.makedirs(STATE, exist_ok=True)
with open(os.path.join(STATE, "lock"), "w") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
def libraries():
roots = [STEAM_ROOT]
text = read_text(os.path.join(STEAM_ROOT, "steamapps", "libraryfolders.vdf")) or ""
roots += [path.replace("\\\\", "\\") for path in re.findall(r'"path"\s+"((?:[^"\\]|\\.)*)"', text)]
roots += LIBRARIES
seen, found = set(), []
for root in roots:
real = os.path.realpath(root)
if real not in seen and os.path.isdir(os.path.join(root, "steamapps")):
seen.add(real)
found.append(root)
return found
def app_dir(appid):
for library in libraries():
steamapps = os.path.join(library, "steamapps")
match = re.search(r'"installdir"\s+"([^"]+)"', read_text(os.path.join(steamapps, "appmanifest_%s.acf" % appid)) or "")
for name in filter(None, (match.group(1) if match else None, INSTALL_DIRS.get(appid))):
path = os.path.join(steamapps, "common", name)
if os.path.isdir(path):
return path
return None
def first_dir(prefix, candidates):
for candidate in candidates:
path = os.path.join(prefix, candidate)
if os.path.isdir(path):
return path
return None
def own_fex_loads():
for name in ("FEX", "FEXServer"):
binary = os.path.join(OWN_FEX, "bin", name)
if not os.access(binary, os.X_OK):
return False
try:
listed = subprocess.run(["/usr/lib/ld-linux-aarch64.so.1", "--list", binary],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=20)
except (OSError, subprocess.TimeoutExpired):
return False
if listed.returncode != 0 or b"not found" in listed.stdout:
return False
return True
def update_own_fex():
wanted = os.environ.get("DD_FEX_TAG")
if not wanted:
return
if (read_text(OWN_FEX_TAG) or "").strip() == wanted and own_fex_loads():
return
failed = os.path.join(STATE, "update-failed")
try:
if (read_text(failed) or "").strip() == wanted and time.time() - os.path.getmtime(failed) < 3600:
return
except OSError:
pass
say("installing DroidDeck's FEX build %s" % wanted)
try:
code = subprocess.call([os.path.join(BIN_DIR, "droiddeck-fex-update")], stdout=sys.stderr)
except OSError:
code = 1
if code == 0:
with contextlib.suppress(OSError):
os.remove(failed)
return
with contextlib.suppress(OSError):
with open(failed, "w", encoding="utf-8") as handle:
handle.write(wanted + "\n")
say("DroidDeck's FEX build %s did not install; using the FEX already here" % wanted)
def fex_install():
prefixes = []
if os.path.isfile(OWN_FEX_TAG) and own_fex_loads():
prefixes.append((OWN_FEX, False))
tool = app_dir(FEX_APPID)
if tool:
prefixes += [(os.path.join(tool, sub), True) for sub in ("usr", "", "files/usr", "files")]
prefixes.append(("/usr", False))
for prefix, portable in prefixes:
binary = next((os.path.join(prefix, "bin", name) for name in ("FEX", "FEXInterpreter")
if os.access(os.path.join(prefix, "bin", name), os.X_OK)), None)
if not binary:
continue
server = os.path.join(prefix, "bin", "FEXServer")
thunks_db = os.path.join(prefix, "share", "fex-emu", "ThunksDB.json")
return {
"bin": binary,
"server": server if os.access(server, os.X_OK) else shutil.which("FEXServer"),
"portable": portable,
"host_thunks": first_dir(prefix, ("lib/aarch64-linux-gnu/fex-emu/HostThunks", "lib/fex-emu/HostThunks", "lib64/fex-emu/HostThunks")),
"guest_thunks": first_dir(prefix, ("share/fex-emu/GuestThunks",)),
"thunks_db": thunks_db if os.path.isfile(thunks_db) else None,
}
return None
def platform_version(path):
return tuple(int(part) for part in re.findall(r"\d+", os.path.basename(path)))
def runtime_platform(app=SNIPER_APPID):
runtime = app_dir(app)
if not runtime:
return None
try:
entries = os.listdir(runtime)
except OSError:
return None
platforms = [os.path.join(runtime, entry) for entry in entries if entry.startswith(RUNTIMES[app][0])]
platforms = [path for path in platforms if os.path.isdir(os.path.join(path, "files"))
and os.path.isfile(os.path.join(path, "usr-mtree.txt.gz"))]
return max(platforms, key=platform_version, default=None)
def unescape(raw):
return ESCAPE.sub(lambda match: bytes([int(match.group(1), 8)]), raw)
def in_root(root, link, target):
if not target.startswith(b"/"):
return target
relative = target.lstrip(b"/")
if relative == b"usr" or relative.startswith(b"usr/"):
relative = relative[4:]
return os.path.relpath(os.path.join(root, relative), os.path.dirname(link))
def materialize(platform):
mtree = os.path.join(platform, "usr-mtree.txt.gz")
info = os.stat(mtree)
stamp = "%d %d %d\n" % (LAYOUT, info.st_size, info.st_mtime_ns)
root = os.fsencode(os.path.join(platform, "files"))
marker = os.path.join(os.fsdecode(root), MARKER)
if read_text(marker) == stamp:
return False
links = []
with gzip.open(mtree, "rb") as source:
for line in source:
parts = line.split()
if not parts or parts[0].startswith(b"#"):
continue
fields = dict(part.split(b"=", 1) for part in parts[1:] if b"=" in part)
relative = os.path.normpath(unescape(parts[0]))
if os.path.isabs(relative) or relative == b".." or relative.startswith(b"../"):
continue
dest = os.path.join(root, relative)
kind = fields.get(b"type")
if kind == b"dir":
os.makedirs(dest, exist_ok=True)
elif kind == b"file" and b"contents" in fields:
blob = os.path.join(root, os.path.normpath(unescape(fields[b"contents"])))
links.append((dest, os.path.relpath(blob, os.path.dirname(dest))))
elif kind == b"link" and b"link" in fields:
links.append((dest, in_root(root, dest, unescape(fields[b"link"]))))
for dest, target in links:
if os.path.islink(dest):
if os.readlink(dest) == target:
continue
os.unlink(dest)
elif os.path.lexists(dest):
continue
os.makedirs(os.path.dirname(dest), exist_ok=True)
os.symlink(target, dest)
usr = os.path.join(root, b"usr")
if not os.path.lexists(usr):
os.symlink(b".", usr)
write_if_changed(marker, stamp)
return True
def graphics_platform():
return runtime_platform(SLR4_APPID)
def provides_newer(base, provider):
for name, prefix in RUNTIME_LIBRARIES:
ours = {v for v in version_names(os.path.join(base, "files/lib/x86_64-linux-gnu", name)) if v.startswith(prefix)}
theirs = {v for v in version_names(os.path.join(provider, "files/lib/x86_64-linux-gnu", name)) if v.startswith(prefix)}
if not ours or not theirs or not ours < theirs:
return False
return True
def clone(source, target, parents=()):
real = os.path.realpath(source)
if real in parents or not os.path.lexists(real):
return
if os.path.isdir(real):
os.makedirs(target, exist_ok=True)
for entry in os.listdir(real):
clone(os.path.join(real, entry), os.path.join(target, entry), parents + (real,))
shutil.copystat(real, target)
return
shutil.copy2(real, target)
def remove_tree(path):
if os.path.islink(path) or not os.path.isdir(path):
with contextlib.suppress(FileNotFoundError):
os.unlink(path)
return
os.chmod(path, os.lstat(path).st_mode | stat.S_IRWXU)
for entry in os.listdir(path):
remove_tree(os.path.join(path, entry))
os.rmdir(path)
def stays_in_libraries(root, target, link):
resolved = os.path.normpath(os.path.join(os.path.relpath(target, root), link))
return resolved.split(os.sep)[0] in ROOTFS_LIBRARY_DIRS
def mirror(source, provider, name, target, root):
usable_provider = provider and all(os.path.exists(os.path.join(provider, needed))
for needed in (name,) + PROVIDED_NEEDS.get(name, ()))
if name in PROVIDED and usable_provider:
clone(os.path.join(provider, name), os.path.join(target, name))
return
path = os.path.join(source, name)
if not os.path.lexists(path):
return
link = os.readlink(path) if os.path.islink(path) else None
if link and GLIBC_BUILD.match(os.path.basename(link)):
if name not in INTERPRETERS:
return
link = os.path.join(os.path.dirname(link), name)
if provider and GLIBC_BUILD.match(name):
return
if link and not link.startswith("/") and stays_in_libraries(root, target, link):
os.symlink(link, os.path.join(target, name))
else:
clone(path, os.path.join(target, name))
def build_rootfs(base, provider):
base_files, provider_files = os.path.join(base, "files"), os.path.join(provider, "files")
rootfs = os.path.join(STATE, "rootfs")
stamp = "%d\n%s\n%s%s\n%s" % (ROOTFS_LAYOUT, os.path.realpath(base), read_text(os.path.join(base_files, MARKER)) or "",
os.path.realpath(provider), read_text(os.path.join(provider_files, MARKER)) or "")
for top in ("lib", "lib64"):
for arch in ARCH_DIRS:
for name in sorted(PROVIDED):
path = os.path.join(provider_files, top, arch, name)
try:
info = os.stat(path)
stamp += "\n%s %s %d %d %d %d" % (path, os.path.realpath(path), info.st_ino, info.st_size,
info.st_mtime_ns, info.st_ctime_ns)
except OSError:
stamp += "\n%s missing" % path
if read_text(os.path.join(rootfs, MARKER)) == stamp:
return False
for entry in os.listdir(STATE):
if entry.startswith("rootfs."):
remove_tree(os.path.join(STATE, entry))
tmp = "%s.%d.tmp" % (rootfs, os.getpid())
os.makedirs(tmp)
for entry in os.listdir(base_files):
if entry not in ("lib", "lib64", "usr", MARKER) and not BLOB_DIRS.match(entry):
clone(os.path.join(base_files, entry), os.path.join(tmp, entry))
os.symlink(".", os.path.join(tmp, "usr"))
for top in ("lib", "lib64"):
source = os.path.join(base_files, top)
os.makedirs(os.path.join(tmp, top))
for entry in os.listdir(source) if os.path.isdir(source) else ():
path = os.path.join(source, entry)
if entry in ARCH_DIRS and os.path.isdir(path) and not os.path.islink(path):
provided = os.path.join(provider_files, top, entry)
os.makedirs(os.path.join(tmp, top, entry))
for name in set(os.listdir(path)) | {name for name in PROVIDED if os.path.exists(os.path.join(provided, name))}:
mirror(path, provided, name, os.path.join(tmp, top, entry), tmp)
else:
mirror(source, None, entry, os.path.join(tmp, top), tmp)
write_if_changed(os.path.join(tmp, MARKER), stamp)
old = "%s.%d.old" % (rootfs, os.getpid())
if os.path.lexists(rootfs):
os.rename(rootfs, old)
os.rename(tmp, rootfs)
remove_tree(old)
return True
def host_preloads():
listed = read_text(HOST_PRELOAD) or ""
return [name for name in PRELOAD_NAMES if name in listed]
def install_preloads(root):
shipped = {}
for arch, libdir in GUEST_LIBDIRS:
for name in PRELOAD_NAMES + (FAULT_REPORT,):
source = os.path.join(PRELOADS, arch, name)
if os.path.isfile(source):
copy_if_changed(source, os.path.join(root, libdir, "droiddeck", name))
shipped.setdefault(name, set()).add(arch)
pin = os.path.join(PRELOADS, "x86_64", VULKAN_PIN)
if os.path.isfile(pin):
copy_if_changed(pin, os.path.join(root, VULKAN_PIN_DIR, "libvulkan.so.1"))
alias = os.path.join(root, VULKAN_PIN_DIR, "libvulkan.so")
if not os.path.islink(alias) or os.readlink(alias) != "libvulkan.so.1":
if os.path.lexists(alias):
os.unlink(alias)
os.symlink("libvulkan.so.1", alias)
wanted = [name for name in host_preloads() if name in shipped]
if len(shipped.get(FAULT_REPORT, ())) == len(GUEST_LIBDIRS):
wanted.append(FAULT_REPORT)
write_if_changed(os.path.join(root, "etc", "ld.so.preload"), "".join("/usr/$LIB/droiddeck/%s\n" % name for name in wanted))
write_if_changed(os.path.join(root, "etc", "asound.conf"), ASOUND_CONF)
def write_config(fex, root):
config = {"Config": {"RootFS": root}}
if fex["host_thunks"] and fex["guest_thunks"] and fex["thunks_db"]:
config["Config"]["ThunkHostLibs"] = fex["host_thunks"]
config["Config"]["ThunkGuestLibs"] = fex["guest_thunks"]
config["ThunksDB"] = dict(THUNKS)
write_if_changed(os.path.join(CONFIG_DIR, "ThunksConfig.json"), json.dumps({"ThunksDB": dict(THUNKS)}, indent=2) + "\n")
write_if_changed(os.path.join(CONFIG_DIR, "Config.json"), json.dumps(config, indent=2) + "\n")
def install_fonts(root):
target = os.path.join(root, FONT_RULES.lstrip("/"))
rules = read_text(FONT_RULES) or ""
if not os.path.isdir(DEVICE_FONTS) or "<fontconfig>" not in rules:
with contextlib.suppress(FileNotFoundError):
os.unlink(target)
return
write_if_changed(target, rules.replace("<fontconfig>", "<fontconfig>\n<dir>%s</dir>" % DEVICE_FONTS, 1))
def link_steam_sdk():
for sdk, candidates in (("sdk64", ("linux64", "ubuntu12_64")), ("sdk32", ("linux32", "ubuntu12_32"))):
link = os.path.join(HOME, ".steam", sdk)
want = "x86_64" if sdk == "sdk64" else "i386"
if elf_arch(os.path.join(link, "steamclient.so")) == want:
continue
for candidate in candidates:
directory = os.path.join(STEAM_ROOT, candidate)
if elf_arch(os.path.join(directory, "steamclient.so")) == want:
os.makedirs(os.path.dirname(link), exist_ok=True)
tmp = "%s.%d.tmp" % (link, os.getpid())
os.symlink(directory, tmp)
os.replace(tmp, link)
break
def link_guest_sdk(root):
for sdk, candidates in (("sdk64", ("linux64", "ubuntu12_64")), ("sdk32", ("linux32", "ubuntu12_32"))):
want = "x86_64" if sdk == "sdk64" else "i386"
directory = next((os.path.join(STEAM_ROOT, candidate) for candidate in candidates
if elf_arch(os.path.join(STEAM_ROOT, candidate, "steamclient.so")) == want), None)
if not directory:
continue
link = os.path.join(root, HOME.lstrip("/"), ".steam", sdk)
if os.path.islink(link) and os.readlink(link) == directory:
continue
os.makedirs(os.path.dirname(link), exist_ok=True)
if os.path.lexists(link):
remove_tree(link)
os.symlink(directory, link)
def steam_running():
for pid in os.listdir("/proc"):
if not pid.isdigit():
continue
try:
with open("/proc/%s/cmdline" % pid, "rb") as f:
first = f.read().split(b"\0", 1)[0]
except OSError:
continue
if first.endswith(b"steamrtarm64/steam"):
return True
return False
def request_install(app=SNIPER_APPID):
os.makedirs(os.path.dirname(WANTED), exist_ok=True)
write_if_changed(WANTED, "1\n")
if not steam_running():
return []
try:
compat = runpy.run_path(os.path.join(BIN_DIR, "steam-compatibility"))
if app != SNIPER_APPID and app_dir(app) is None:
return compat["request_app"](STEAM_ROOT, app)
return compat["request_fex"](STEAM_ROOT, set())
except Exception as error:
say("could not ask the client to install FEX (%s)" % error)
return []
def describe(fex, platform, graphics=None):
return {
"fex": fex["bin"] if fex else None,
"server": fex["server"] if fex else None,
"thunks": bool(fex and fex["host_thunks"] and fex["guest_thunks"] and fex["thunks_db"]),
"runtime": platform,
"steamclient64": os.path.realpath(os.path.join(HOME, ".steam", "sdk64", "steamclient.so"))
if elf_arch(os.path.join(HOME, ".steam", "sdk64", "steamclient.so")) == "x86_64" else None,
"ready": bool(fex and fex["server"] and platform),
"graphics": graphics,
}
def prepare(app=SNIPER_APPID):
with locked():
update_own_fex()
fex = fex_install()
platform = runtime_platform(app)
status = describe(fex, platform)
if not status["ready"]:
missing = []
tool = app_dir(FEX_APPID)
if not fex and tool:
say("FEX is installed at %s but holds no FEX or FEXInterpreter in usr/bin, bin or files; it has: %s"
% (tool, ", ".join(sorted(os.listdir(tool))[:20])))
if not fex:
missing.append("FEX (Steam app %s)" % FEX_APPID)
elif not fex["server"]:
missing.append("FEXServer beside %s" % fex["bin"])
if not platform:
missing.append("%s, Steam app %s)" % (RUNTIMES[app][1], app))
requested = request_install(app)
say("x86 Linux programs need %s; %s" % (" and ".join(missing),
"the Steam client was asked to install it" if requested else "open Steam once and it is installed for you"))
write_if_changed(STATUS_FILE, json.dumps(status, indent=2) + "\n")
return None
root = os.path.join(platform, "files")
if materialize(platform):
say("prepared %s as the x86 runtime" % os.path.basename(platform))
provider = None
if app == SNIPER_APPID:
provider = graphics_platform()
if provider and materialize(provider):
say("prepared %s for the x86 runtime's C and C++ libraries" % os.path.basename(provider))
if not provider:
say("Steam Linux Runtime 4.0 (Steam app %s) is not installed yet; x86 games may not find their graphics drivers"
% SLR4_APPID)
elif not provides_newer(platform, provider):
say("%s has no newer C and C++ libraries than %s" % (os.path.basename(provider), os.path.basename(platform)))
provider = None
else:
if build_rootfs(platform, provider):
say("built the x86 runtime from %s with the C and C++ libraries of %s"
% (os.path.basename(platform), os.path.basename(provider)))
root = os.path.join(STATE, "rootfs")
install_preloads(root)
install_fonts(root)
with contextlib.suppress(OSError):
link_guest_sdk(root)
write_config(fex, root)
fex["rootfs"] = root
try:
link_steam_sdk()
except OSError:
pass
status = describe(fex, platform, provider)
write_if_changed(STATUS_FILE, json.dumps(status, indent=2) + "\n")
with contextlib.suppress(OSError):
os.unlink(WANTED)
return fex
def elf_arch(path):
try:
with open(path, "rb") as f:
head = f.read(20)
except OSError:
return None
if len(head) < 20 or head[:4] != b"\x7fELF":
return None
machine = struct.unpack_from("<H" if head[5] == 1 else ">H", head, 18)[0]
return X86.get(machine) or ARM.get(machine) or "other"
def scan(folder):
counts = {"x86": 0, "arm": 0}
seen = 0
base = folder.rstrip("/").count("/")
for directory, dirs, files in os.walk(folder):
if directory.count("/") - base >= SCAN_DEPTH:
dirs[:] = []
for name in files:
seen += 1
if seen > SCAN_LIMIT:
return counts
arch = elf_arch(os.path.join(directory, name))
if arch in X86.values():
counts["x86"] += 1
elif arch in ARM.values():
counts["arm"] += 1
return counts
def needs_fex(path):
arch = elf_arch(path)
if arch is not None:
return arch in X86.values()
folder = path if os.path.isdir(path) else os.path.dirname(os.path.abspath(path))
counts = scan(folder)
return counts["x86"] > 0 and counts["arm"] == 0
def is_arm(path):
return elf_arch(path) in ARM.values()
def split_paths(value):
return [part for part in re.split(r"[:\s]+", value or "") if part]
def data_dir(env):
runtime = env.get("XDG_RUNTIME_DIR")
if runtime and os.path.isdir(runtime) and os.access(runtime, os.W_OK):
return os.path.join(runtime, "fex")
return DATA_DIR
def guest_environment(fex, env):
result = dict(env)
data = data_dir(env)
for directory in (CONFIG_DIR, data, CACHE_DIR):
os.makedirs(directory, exist_ok=True)
result["FEX_APP_CONFIG_LOCATION"] = CONFIG_DIR + "/"
result["FEX_APP_DATA_LOCATION"] = data + "/"
result["FEX_APP_CACHE_LOCATION"] = CACHE_DIR + "/"
if result.get("STEAM_COMPAT_DATA_PATH"):
short = data if data != DATA_DIR else "/tmp/fex"
os.makedirs(short, exist_ok=True)
result["STEAM_COMPAT_DATA_PATH"] = short
result.setdefault("FEX_SILENTLOG", "0")
result.setdefault("FEX_OUTPUTLOG", "stderr")
result.pop("GLIBC_TUNABLES", None)
result.setdefault("GC_DISABLE_INCREMENTAL", "1")
if fex["portable"]:
result["FEX_PORTABLE"] = "1"
else:
result.pop("FEX_PORTABLE", None)
for name in ("FEX_ROOTFS", "FEX_THUNKHOSTLIBS", "FEX_THUNKGUESTLIBS"):
if name in result and not os.path.exists(result[name]):
result.pop(name)
if fex.get("rootfs"):
result["FEX_ROOTFS"] = fex["rootfs"]
if fex.get("rootfs") and fex.get("host_thunks") and fex.get("guest_thunks") and fex.get("thunks_db"):
result["FEX_THUNKHOSTLIBS"] = fex["host_thunks"]
result["FEX_THUNKGUESTLIBS"] = fex["guest_thunks"]
result["FEX_THUNKCONFIG"] = os.path.join(CONFIG_DIR, "ThunksConfig.json")
preload = [part for part in split_paths(result.get("LD_PRELOAD"))
if "/" not in part or (os.path.exists(part) and not is_arm(part))]
library_path = [part for part in (result.get("LD_LIBRARY_PATH") or "").split(":")
if part and "arm64" not in part and "aarch64" not in part]
for name, parts in (("LD_PRELOAD", preload), ("LD_LIBRARY_PATH", library_path)):
if parts:
result[name] = ":".join(parts)
else:
result.pop(name, None)
return result
def version_names(path):
try:
with open(path, "rb") as f:
return set(VERSION_NAME.findall(f.read()))
except OSError:
return set()
def bundled_libraries(folder):
wanted = [name for name, _ in RUNTIME_LIBRARIES]
found = {}
seen = 0
base = folder.rstrip("/").count("/")
for directory, dirs, files in os.walk(folder):
seen += len(files)
if seen > SCAN_LIMIT:
break
if directory.count("/") - base >= BUNDLE_DEPTH:
dirs[:] = []
for name in wanted:
if name in files and name not in found:
found[name] = os.path.join(directory, name)
return found
def vulkan_thunk_loads(fex, folder):
needed = version_names(os.path.join(fex["guest_thunks"], VULKAN_THUNK))
if not needed:
return False
needed |= version_names(os.path.join(PRELOADS, "x86_64", VULKAN_PIN))
found = bundled_libraries(folder)
for name, prefix in RUNTIME_LIBRARIES:
library = found.get(name) or os.path.join(fex["rootfs"], "lib/x86_64-linux-gnu", name)
if not {version for version in needed if version.startswith(prefix)} <= version_names(library):
return False
return True
def required_runtime(program):
text = read_text(os.path.join(os.path.dirname(program), "toolmanifest.vdf")) or ""
match = re.search(r'"require_tool_appid"\s+"(\d+)"', text)
return match.group(1) if match and match.group(1) in RUNTIMES else SNIPER_APPID
def resolve(program, env):
if "/" in program:
return os.path.abspath(program)
return shutil.which(program, path=env.get("PATH")) or program
def launch(command, env, mode, decide=None):
program = resolve(command[0], env)
use = mode == "on" or (mode == "auto" and needs_fex(decide or program))
if not use:
return os.execvpe(command[0], command, env)
fex = prepare(required_runtime(program))
if fex is None:
return 126
guest = guest_environment(fex, env)
x86_64 = elf_arch(program) != "i386"
thunks = "on" if "FEX_THUNKCONFIG" in guest else "off"
folder = decide if decide and os.path.isdir(decide) else os.path.dirname(program)
if thunks == "on" and x86_64 and os.path.isfile(os.path.join(PRELOADS, "x86_64", VULKAN_PIN)) \
and vulkan_thunk_loads(fex, folder):
guest["LD_LIBRARY_PATH"] = ":".join(["/usr/" + VULKAN_PIN_DIR] + split_paths(guest.get("LD_LIBRARY_PATH")))
thunks = "on, its own Vulkan loader too"
say("running %s with %s (runtime %s, thunks %s)" % (program, fex["bin"], guest.get("FEX_ROOTFS", "none"), thunks))
return os.execve(fex["bin"], [fex["bin"], program] + command[1:], guest)
def game_environment(env, command):
try:
game_env = runpy.run_path(os.path.join(BIN_DIR, "droiddeck-game-env"))
env = game_env["game_gpu"](env)
app = game_env["game_id"](env) or env.get("SteamAppId") or env.get("SteamGameId")
if not app:
return env
fixes = {}
game = game_env["Game"](resolve(command[0], env), command[1:])
for check in game_env["FIXES"]:
found = check(game)
if found and found[0]:
fixes.update(found[0])
say(found[2])
try:
with open(GAME_ENVIRONMENT, encoding="utf-8") as source:
config = json.load(source)
except FileNotFoundError:
config = {"version": 1}
return game_env["apply_config"](env, config, str(app), fixes)
except FileNotFoundError:
return env
except Exception as error:
say("game settings were not applied (%s)" % error)
return env
def appimage_offset(path):
with open(path, "rb") as f:
head = f.read(64)
if head[:4] != b"\x7fELF":
raise ValueError("not an ELF file")
order = "<" if head[5] == 1 else ">"
if head[4] == 2:
shoff, = struct.unpack_from(order + "Q", head, 0x28)
shentsize, shnum = struct.unpack_from(order + "HH", head, 0x3A)
else:
shoff, = struct.unpack_from(order + "I", head, 0x20)
shentsize, shnum = struct.unpack_from(order + "HH", head, 0x2E)
return shoff + shentsize * shnum
def clear_extracted():
for stale in ("squashfs-root", "AppDir"):
if os.path.islink(stale):
os.unlink(stale)
else:
shutil.rmtree(stale, ignore_errors=True)
def extracted():
if os.path.isdir("AppDir") and not os.path.lexists("squashfs-root"):
os.symlink("AppDir", "squashfs-root")
return os.path.isdir("squashfs-root")
def want_fex():
with locked():
update_own_fex()
status = describe(fex_install(), runtime_platform(), graphics_platform())
if not status["ready"]:
request_install()
write_if_changed(STATUS_FILE, json.dumps(status, indent=2) + "\n")
XATTR_ONLY_NOISE = re.compile(r"^(proot (error|warning)|Failed to exec memfd|Further error messages of this type are suppressed)")
def only_xattrs_failed(errors):
seen = False
for line in errors.splitlines():
if "write_xattr" in line:
seen = True
elif re.search(r"fail|could not|cannot|error", line, re.I) and not XATTR_ONLY_NOISE.match(line):
return False
return seen
def extract(image):
image = os.path.abspath(image)
clear_extracted()
x86 = elf_arch(image) in X86.values()
if os.access(URUNTIME, os.X_OK):
run = subprocess.run([URUNTIME, "--appimage-extract"], stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
env=dict(os.environ, TARGET_APPIMAGE=image))
errors = run.stderr.decode("utf-8", "replace")
sys.stderr.write(errors)
status = run.returncode
if status == 2 and only_xattrs_failed(errors) and extracted() and os.path.lexists("squashfs-root/AppRun"):
say("uruntime could not keep the image's extended attributes; the files are unpacked")
status = 0
if status == 0 and extracted():
if x86:
want_fex()
return 0
say("uruntime could not extract it (status %d)" % status)
clear_extracted()
fex = prepare() if x86 else None
if fex is not None:
status = subprocess.call([fex["bin"], image, "--appimage-extract"], stdout=subprocess.DEVNULL,
env=guest_environment(fex, dict(os.environ)))
if status == 0 and extracted():
return 0
say("the image's own runtime could not extract it under FEX (status %d)" % status)
unsquashfs = shutil.which("unsquashfs")
if unsquashfs:
clear_extracted()
status = subprocess.call([unsquashfs, "-q", "-n", "-no-xattrs", "-o", str(appimage_offset(image)), "-d", "squashfs-root", image],
stdout=subprocess.DEVNULL)
if status == 0:
if x86:
want_fex()
return 0
say("unsquashfs could not extract it (status %d)" % status)
elif fex is None:
say("this AppImage could not be unpacked here: no AppImage runtime for ARM64, no FEX yet and no unsquashfs")
return 1
def parse_run(args):
mode, decide = os.environ.get("DROIDDECK_FEX", "auto"), None
while args and args[0].startswith("--"):
option = args.pop(0)
if option == "--":
break
if option in ("--mode", "--for") and args:
value = args.pop(0)
if option == "--mode":
mode = value
else:
decide = value
else:
raise ValueError("unknown option %s" % option)
if mode not in MODES:
raise ValueError("--mode takes %s" % ", ".join(MODES))
return mode, decide, args
def main(argv):
if not argv:
say("usage: droiddeck-fex steam VERB CMD... | run [--mode auto|on|off] [--for PATH] [--] CMD... | needs PATH | extract IMAGE | prepare | status")
return 2
action, args = argv[0], list(argv[1:])
if action == "steam":
if not args:
return 2
verb, command = args[0], args[1:]
if verb not in STEAM_VERBS:
return 0
if not command:
return 2
return launch(command, game_environment(dict(os.environ), command), os.environ.get("DROIDDECK_FEX", "auto"))
if action == "run":
try:
mode, decide, command = parse_run(args)
except ValueError as error:
say(str(error))
return 2
if not command:
return 2
return launch(command, dict(os.environ), mode, decide)
if action == "needs":
if len(args) != 1:
return 2
needed = needs_fex(args[0])
print("fex" if needed else "native")
return 0 if needed else 1
if action == "extract":
if len(args) != 1:
return 2
return extract(args[0])
if action == "prepare":
return 0 if prepare() else 1
if action == "status":
with locked():
status = describe(fex_install(), runtime_platform(), graphics_platform())
write_if_changed(STATUS_FILE, json.dumps(status, indent=2) + "\n")
print(json.dumps(status, indent=2))
return 0 if status["ready"] else 1
say("unknown action %s" % action)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))