fix(steam): Linux-only tools keep their native platform (#2313)

* fix(steam): reconcile tool dependencies by supported platform

* fix(steam): refresh tool metadata and clear stale GE mappings

* fix(steam): clear any Proton mapping from Linux-only tools

* fix(steam): watch folders the client creates after the helper starts

* Watch each missing directory level as it appears

* Make the watcher level test wait on inotify readiness

tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-compat

@@ -760,18 +760,50 @@ class Watcher:
     def __init__(self, files):
         self.files = files
         self.fd = -1
+        self.libc = None
+        wanted = {os.path.dirname(f) for f in files}
+        self.missing = {d for d in wanted if not os.path.isdir(d)}
+        self.watched = set()
         try:
             libc = ctypes.CDLL(None, use_errno=True)
             fd = libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC)
-            dirs = sorted({os.path.dirname(f) for f in files if os.path.isdir(os.path.dirname(f))})
+            dirs = sorted((wanted - self.missing) | {self.existing_parent(d) for d in self.missing} - {""})
             if fd >= 0 and dirs and all(libc.inotify_add_watch(fd, d.encode(), self.MASK) >= 0 for d in dirs):
+                self.watched = set(dirs)
                 self.fd = fd
+                self.libc = libc
             elif fd >= 0:
                 os.close(fd)
         except (OSError, AttributeError):
             self.fd = -1
         self.stamp = self.snapshot()
 
+    @staticmethod
+    def existing_parent(path):
+        while path and not os.path.isdir(path):
+            parent = os.path.dirname(path)
+            if parent == path:
+                return ""
+            path = parent
+        return path
+
+    def watch_new_dirs(self):
+        appeared = False
+        progressed = True
+        while progressed:
+            progressed = False
+            for d in sorted(self.missing):
+                parent = d if os.path.isdir(d) else self.existing_parent(d)
+                if not parent or parent in self.watched:
+                    continue
+                if self.libc.inotify_add_watch(self.fd, parent.encode(), self.MASK) >= 0:
+                    self.watched.add(parent)
+                    progressed = True
+                    appeared = appeared or parent == d
+        for d in [d for d in self.missing if d in self.watched]:
+            self.missing.discard(d)
+        return appeared
+
     def snapshot(self):
         stamp = []
         for f in self.files:
@@ -803,8 +835,9 @@ class Watcher:
                     at += 16 + size
         except BlockingIOError:
             pass
+        appeared = self.watch_new_dirs() if self.missing else False
         wanted = {os.path.basename(f) for f in self.files}
-        return any(n in wanted or n.startswith("appmanifest_") for n in names)
+        return appeared or any(n in wanted or n.startswith("appmanifest_") for n in names)
 
 
 class XMessage(ctypes.Structure):
@@ -1084,6 +1117,7 @@ class Helper:
             installed = COMPAT["installed_apps"](self.steam_root)
             self.installed = installed
             picks = COMPAT["valve_picks"](self.steam_root)
+            tools = COMPAT["tool_platforms"](self.steam_root)
             prior_auto = state.get("auto", {})
             if not isinstance(prior_auto, dict):
                 prior_auto = {}
@@ -1091,7 +1125,7 @@ class Helper:
             native_choices = set(prior_native if isinstance(prior_native, (list, tuple, set)) else ())
             picked = COMPAT["picked_apps"](state)
             runnable = {t["name"].lower() for t in live}
-            apps = sorted(set(installed) | set(picks) | set(prior_auto) | native_choices, key=COMPAT["app_order"])
+            apps = sorted(set(installed) | set(picks) | set(prior_auto) | native_choices | set(tools), key=COMPAT["app_order"])
             asked = [app for app, request in COMPAT["game_requests"]().items() if COMPAT["valid_game_request"](app, request)]
             apps = sorted(set(apps) | set(asked), key=COMPAT["app_order"])
             current = dict(page.compat("many(%s)" % json.dumps(apps)) or {})
@@ -1105,6 +1139,7 @@ class Helper:
                 if isinstance(state.get("auto"), dict):
                     state["auto"] = prior_auto
                 native_choices -= refused
+                tools = {app: platforms for app, platforms in tools.items() if app not in refused}
             current["0"] = top["selected"]
             native = COMPAT["native_targets"](self.steam_root, apps)
             raw_mapping = COMPAT["read_config_mapping"](
@@ -1137,7 +1172,7 @@ class Helper:
             changes, auto = COMPAT["plan_mapping"](
                 current, installed, listed, default,
                 COMPAT["followers"](state, current), native, picks, overrides,
-                prior_auto.keys(), native_choices)
+                prior_auto.keys(), native_choices, tools)
             changes = dict(moved, **changes)
             for app, tool in sorted(changes.items(), key=lambda item: COMPAT["app_order"](item[0])):
                 page.compat("set(%d, %s)" % (int(app), json.dumps(tool if tool is not None else "")))
@@ -1301,7 +1336,8 @@ def main():
                 failures += 1
             time.sleep(min(2 * max(failures, 1), 10))
     os.makedirs(COMPAT["COMPAT_DIR"], exist_ok=True)
-    watcher = Watcher([COMPAT["state_path"]("request.json"), COMPAT["state_path"]("games.json")] +
+    watcher = Watcher([COMPAT["state_path"]("request.json"), COMPAT["state_path"]("games.json"),
+                       os.path.join(steam_root, "appcache", "appinfo.vdf")] +
                       [os.path.join(lib, "steamapps", "libraryfolders.vdf") for lib in (steam_root, COMPAT["LIBRARY"] + "-sd")])
     steamos = os.environ.get("BL_STEAMDECK") == "1" and os.environ.get("BL_STEAM_UI") != "desktop"
     helper = Helper(steam_root, port, steamos)

tools/linuxfs/overlay/usr/local/bin/steam-compatibility

@@ -734,6 +734,7 @@ def libraries(steam_root):
 
 
 def installed_apps(steam_root):
+    entries = appinfo_entries(os.path.join(steam_root, "appcache", "appinfo.vdf"))
     apps = set()
     for library in libraries(steam_root):
         steamapps = os.path.join(library, "steamapps")
@@ -743,7 +744,7 @@ def installed_apps(steam_root):
             continue
         for name in names:
             m = re.fullmatch(r"appmanifest_(\d+)\.acf", name)
-            if m and m.group(1) not in NOT_GAMES:
+            if m and m.group(1) not in NOT_GAMES and entries.get(m.group(1), {}).get("type") != "tool":
                 apps.add(m.group(1))
     return sorted(apps, key=int)
 
@@ -927,6 +928,12 @@ def valve_picks(steam_root):
     return sorted(picks, key=int)
 
 
+def tool_platforms(steam_root):
+    entries = appinfo_entries(os.path.join(steam_root, "appcache", "appinfo.vdf"))
+    return {app: entry["oslist"] for app, entry in entries.items()
+            if entry["type"] == "tool" and entry["oslist"] & {"linux", "windows"}}
+
+
 def native_targets(steam_root, apps):
     wanted = {str(app) for app in apps if str(app) not in NOT_GAMES}
     if not wanted:
@@ -1064,10 +1071,11 @@ NATIVE_CHOICE = object()
 
 
 def plan_mapping(current, installed, runnable, default, auto, native=None, picks=(), overrides=None, managed=None,
-                 native_choices=()):
+                 native_choices=(), tools=None):
+    tools = tools or {}
     runnable = {n.lower() for n in runnable}
     installed = set(installed)
-    picks = set(picks)
+    picks = set(picks) | {app for app, platforms in tools.items() if "linux" not in platforms}
     native = native or {}
     overrides = overrides or {}
     managed = set(auto) if managed is None else set(managed)
@@ -1075,8 +1083,12 @@ def plan_mapping(current, installed, runnable, default, auto, native=None, picks
     changes, kept = {}, {}
     if current.get("0", "") != default:
         changes["0"] = default
-    for app in sorted((set(current) | installed | picks | set(auto) | set(overrides) | native_choices) - {"0"}, key=app_order):
+    for app in sorted((set(current) | installed | picks | set(auto) | set(overrides) | native_choices | set(tools)) - {"0"}, key=app_order):
         name = current.get(app, "")
+        if app in tools and "linux" in tools[app] and "windows" not in tools[app]:
+            if name and (name.lower() in runnable or "proton" in name.lower()):
+                changes[app] = None
+            continue
         if app in overrides:
             target = overrides[app]
             if target is DEFERRED_GAME_TOOL:
@@ -1560,7 +1572,7 @@ def register_default(config_path, apps=(), protect=(), default=None, catalog=Non
             current.setdefault(app, "")
         changes, auto = plan_mapping(
             current, apps, names, default, followers(state, current), native, picks,
-            overrides, managed, native_choices)
+            overrides, managed, native_choices, tool_platforms(steam_root))
         set_mappings(tokens, mapping, changes)
         text = serialize(tokens)
         if text != serialize(tokenize(original)):

tools/tests/test_steam_tool_platforms.py

@@ -0,0 +1,215 @@
+import json
+import os
+from pathlib import Path
+import select
+import tempfile
+import unittest
+from unittest import mock
+
+from tools.tests import test_droiddeck_fex as fixtures
+from tools.tests import test_steam_compat_live as live
+from tools.tests.test_steam_compat_mapping import config_text
+
+
+COMPAT = live.COMPAT
+TOOL = COMPAT["TOOL"]
+
+
+class ToolPlatformsTest(unittest.TestCase):
+    def setUp(self):
+        temporary = tempfile.TemporaryDirectory()
+        self.addCleanup(temporary.cleanup)
+        self.root = Path(temporary.name)
+        self.steam = self.root / "Steam"
+        self.state = self.root / "state"
+        self.config = self.steam / "config/config.vdf"
+        self.config.parent.mkdir(parents=True)
+        self.state.mkdir()
+        (self.steam / "appcache").mkdir()
+        (self.steam / "steamapps").mkdir()
+        self.entries = {
+            976730: ("windows", "64", "game"),
+            1826330: ("linux", "", "tool"),
+            1161040: ("linux", "", "tool"),
+            900001: ("windows", "64", "tool"),
+            900002: ("linux", "arm64", "tool"),
+            900003: ("osx", "64", "tool"),
+            900004: ("", "", "tool"),
+            900005: ("linux,windows", "64", "tool"),
+        }
+        self.write_appinfo()
+        self.catalog = [{"name": TOOL, "display": "Proton", "dir": "Proton", "valve": True}]
+        self.config.write_text(config_text('"0" { "name" "%s" }' % TOOL))
+        patcher = mock.patch.dict(COMPAT["register_default"].__globals__, COMPAT_DIR=str(self.state))
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+    def write_appinfo(self, magic=0x07564429):
+        (self.steam / "appcache/appinfo.vdf").write_bytes(fixtures.AppInfoTest.appinfo(self.entries, magic))
+
+    def platforms(self):
+        return COMPAT["tool_platforms"](str(self.steam))
+
+    def plan(self, current, auto=None, **kwargs):
+        return COMPAT["plan_mapping"](current, ["976730"], [TOOL], TOOL, auto or {},
+                                      tools=self.platforms(), **kwargs)
+
+    def test_metadata_selects_tools_across_cache_formats(self):
+        for magic in (0x07564427, 0x07564428, 0x07564429):
+            with self.subTest(magic=magic):
+                self.write_appinfo(magic)
+                self.assertEqual(set(self.platforms()), {"1826330", "1161040", "900001", "900002", "900005"})
+
+    def test_tools_are_not_installed_games(self):
+        for app in self.entries:
+            (self.steam / ("steamapps/appmanifest_%s.acf" % app)).write_text("")
+        self.assertEqual(COMPAT["installed_apps"](str(self.steam)), ["976730"])
+
+    def test_linux_tools_clear_proton_and_windows_tools_get_default_before_install(self):
+        current = {"0": TOOL, "976730": TOOL, "1826330": TOOL, "1161040": "proton_experimental"}
+        changes, auto = self.plan(current, {"1826330": TOOL, "976730": TOOL})
+        self.assertEqual(changes, {"1826330": None, "1161040": None, "900001": TOOL})
+        self.assertNotIn("1826330", auto)
+        self.assertEqual(auto["900001"], TOOL)
+        self.assertEqual(auto["976730"], TOOL)
+
+    def test_removed_ge_proton_is_cleared_with_or_without_saved_ownership(self):
+        for saved in ({}, {"1826330": "GE-Proton11-7"}):
+            with self.subTest(saved=saved):
+                changes, auto = self.plan({"0": TOOL, "1826330": "GE-Proton11-7"}, saved)
+                self.assertIsNone(changes["1826330"])
+                self.assertNotIn("1826330", auto)
+
+    def test_removed_ge_proton_game_selection_survives(self):
+        changes, auto = self.plan({"0": TOOL, "976730": "GE-Proton11-7"})
+        self.assertNotIn("976730", changes)
+        self.assertNotIn("976730", auto)
+
+    def test_saved_native_selections_survive(self):
+        for name in (COMPAT["FEX_TOOL"], "custom-native"):
+            with self.subTest(name=name):
+                changes, auto = self.plan({"0": TOOL, "1826330": name}, {"1826330": name})
+                self.assertNotIn("1826330", changes)
+                self.assertNotIn("1826330", auto)
+
+    def test_main_watches_cache_updates_and_replacements(self):
+        class StopWatching(BaseException):
+            pass
+
+        watchers = []
+
+        def capture(files):
+            watcher = live.LIVE["Watcher"](files)
+            watchers.append(watcher)
+            raise StopWatching()
+
+        namespace = live.LIVE["main"].__globals__
+        with mock.patch.dict(namespace, Watcher=capture), mock.patch.dict(os.environ, BL_CDP_PORT="1"), \
+                mock.patch.object(live.LIVE["sys"], "argv", ["compat", str(self.steam)]):
+            with self.assertRaises(StopWatching):
+                live.LIVE["main"]()
+        watcher = watchers[0]
+        if watcher.fd >= 0:
+            self.addCleanup(os.close, watcher.fd)
+        self.assertFalse(watcher.changed())
+        self.entries[900006] = ("linux", "", "tool")
+        self.write_appinfo()
+        self.assertTrue(watcher.changed())
+        self.assertIn("900006", self.platforms())
+        cache = self.steam / "appcache/appinfo.vdf"
+        replacement = cache.with_suffix(".tmp")
+        replacement.write_bytes(cache.read_bytes())
+        replacement.replace(cache)
+        self.assertTrue(watcher.changed())
+
+    def test_watcher_follows_an_appcache_created_after_startup(self):
+        root = self.root / "late"
+        (root / "state").mkdir(parents=True)
+        watcher = live.LIVE["Watcher"]([str(root / "state/request.json"), str(root / "appcache/appinfo.vdf")])
+        self.assertGreaterEqual(watcher.fd, 0)
+        self.addCleanup(os.close, watcher.fd)
+        self.assertFalse(watcher.changed())
+        (root / "appcache").mkdir()
+        self.assertTrue(watcher.changed())
+        (root / "appcache/appinfo.vdf").write_bytes(b"x")
+        self.assertTrue(watcher.changed())
+
+    def test_watcher_follows_each_missing_directory_level(self):
+        root = self.root / "levels"
+        root.mkdir()
+        target = root / "sd/steamapps/appmanifest_1.acf"
+        watcher = live.LIVE["Watcher"]([str(target)])
+        self.assertGreaterEqual(watcher.fd, 0)
+        self.addCleanup(os.close, watcher.fd)
+        (root / "sd").mkdir()
+        self.assertTrue(select.select([watcher.fd], [], [], 1)[0])
+        self.assertFalse(watcher.changed())
+        (root / "sd/steamapps").mkdir()
+        self.assertTrue(select.select([watcher.fd], [], [], 1)[0])
+        self.assertTrue(watcher.changed())
+        target.write_bytes(b"x")
+        self.assertTrue(watcher.changed())
+
+    def test_linux_tool_requests_cannot_restore_windows_platform(self):
+        changes, auto = self.plan({"0": TOOL, "1826330": TOOL}, overrides={"1826330": TOOL})
+        self.assertIsNone(changes["1826330"])
+        self.assertNotIn("1826330", auto)
+
+    def test_linux_execution_tools_and_custom_selections_survive(self):
+        current = {"0": TOOL, "1826330": COMPAT["FEX_TOOL"], "900002": "custom-native",
+                   "900001": "custom-windows", "900005": TOOL}
+        changes, auto = self.plan(current)
+        self.assertEqual(changes, {"976730": TOOL})
+        self.assertNotIn("1826330", auto)
+
+    def test_missing_or_malformed_cache_does_not_invent_platforms(self):
+        cache = self.steam / "appcache/appinfo.vdf"
+        cache.unlink()
+        self.assertEqual(self.platforms(), {})
+        cache.write_bytes(b"invalid")
+        self.assertEqual(self.platforms(), {})
+
+    def test_offline_repair_is_idempotent_and_preserves_game_and_neighbor_fields(self):
+        self.config.write_text(config_text(
+            '"0" { "name" "%s" } "976730" { "name" "custom-game" } '
+            '"1826330" { "name" "%s" }' % (TOOL, TOOL)))
+        for _ in range(2):
+            COMPAT["register_default"](str(self.config), ["976730"], catalog=self.catalog,
+                                       directory=str(self.state), default=TOOL)
+            self.assertEqual(COMPAT["read_config_mapping"](str(self.config)),
+                             {"0": TOOL, "976730": "custom-game", "900001": TOOL})
+            self.assertIn('"outside"', self.config.read_text())
+            state = json.loads((self.state / "state.json").read_text())
+            self.assertNotIn("1826330", state["auto"])
+
+    def reconcile(self, page):
+        helper = live.LIVE["Helper"](str(self.steam), 1)
+        helper.page = page
+        with mock.patch.dict(COMPAT, tool_catalog=lambda root: self.catalog):
+            helper.reconcile()
+        return json.loads((self.state / "state.json").read_text())
+
+    def test_live_repair_covers_uninstalled_dependencies_and_stays_idempotent(self):
+        page = live.FakePage({"0": TOOL, "1826330": TOOL, "976730": "custom-game"}, [TOOL])
+        state = self.reconcile(page)
+        self.assertEqual(page.sets, [("900001", TOOL), ("1826330", "")])
+        self.assertNotIn("1826330", state["auto"])
+        page.sets.clear()
+        self.reconcile(page)
+        self.assertEqual(page.sets, [])
+        self.assertEqual(page.mapping["976730"], "custom-game")
+
+    def test_live_removed_ge_proton_is_cleared_and_stays_cleared(self):
+        page = live.FakePage({"0": TOOL, "1826330": "GE-Proton11-7"}, [TOOL])
+        (self.state / "state.json").write_text(json.dumps({"auto": {"1826330": "GE-Proton11-7"}}))
+        state = self.reconcile(page)
+        self.assertIn(("1826330", ""), page.sets)
+        self.assertNotIn("1826330", state["auto"])
+        page.sets.clear()
+        self.reconcile(page)
+        self.assertEqual(page.sets, [])
+
+    def test_live_refused_tools_are_not_written(self):
+        page = live.FakePage({"0": TOOL, "1826330": TOOL}, [TOOL], refused={"1826330", "900001"})
+        self.reconcile(page)
+        self.assertEqual(page.sets, [])