Keep Windows components across runtime updates and reinstall lost ones (#1786)

app/src/main/java/com/droiddeck/launcher/FrontEndActivity.kt

@@ -2472,7 +2472,10 @@ class FrontEndActivity : ComponentActivity() {
             if (ready && (!componentSweepDone || endedSession != null)) {
                 componentSweepDone = true
                 endedSession?.let { componentSweepSession = it }
-                ui.postDelayed({ Thread({ com.droiddeck.launcher.frontend.AutoComponents.sweep(this, all) }, "components-sweep").start() }, 10_000)
+                ui.postDelayed({ Thread({
+                    com.droiddeck.launcher.frontend.AutoComponents.restoreMissing(this)
+                    com.droiddeck.launcher.frontend.AutoComponents.sweep(this, all)
+                }, "components-sweep").start() }, 10_000)
             }
             ui.post {
                 if (scanGeneration == libraryScanGeneration) {

app/src/main/java/com/droiddeck/launcher/frontend/AutoComponents.kt

@@ -185,18 +185,19 @@ object AutoComponents {
         val catalog = (catalogOf(context, false) ?: catalogOf(context, true))?.associateBy { it.name } ?: return
         val wanted = autoPicks(findings, catalog)
         val user = WinComponents.selection(context, key).user
+        val before = WinComponents.selection(context, key).auto
         val installed = installedOf(context)
         val kept = LinkedHashMap<String, WinComponents.AutoReason>()
         val missing = ArrayList<String>()
         for ((id, reason) in wanted) {
             if (user[id] == false) { kept[id] = reason; continue }
             if (id in installed) { kept[id] = reason; continue }
+            if (id in before) kept[id] = reason
             synchronized(waiting) { waiting.getOrPut(id) { ArrayList() }.let { if (it.none { g -> appKey(g) == key }) it.add(game) } }
             queueDownload(context, id, asked, inSession)
             missing.add(id)
         }
         if (inSession) Pending.set(context, key, missing)
-        val before = WinComponents.selection(context, key).auto
         if (before.keys != kept.keys) {
             WinComponents.setAuto(context, key, kept)
             Log.i(TAG, "${game.name}: on by itself ${kept.keys.joinToString().ifEmpty { "(none)" }}")
@@ -317,6 +318,20 @@ object AutoComponents {
 
     fun beforeLaunch(context: Context, game: Library.SteamGame) = queueGame(context, game, asked = false)
 
+    fun restoreMissing(context: Context): List<String> {
+        val selections = WinComponents.selections(context)
+        if (selections.isEmpty()) return emptyList()
+        val installed = installedOf(context)
+        val picked = selections.values.flatMap { it.effective }.filter { WinComponents.validId(it) && it !in installed }.distinct()
+        if (picked.isEmpty()) return emptyList()
+        val catalog = (catalogOf(context, false) ?: catalogOf(context, true))?.associateBy { it.name } ?: return emptyList()
+        val lost = picked.filter { it in catalog }
+        if (lost.isEmpty()) return emptyList()
+        Log.i(TAG, "wincomp: reinstalling ${lost.joinToString()}, picked for games but missing from the runtime")
+        lost.forEach { queueDownload(context, it, asked = true, inSession = true) }
+        return lost
+    }
+
     fun refreshLater(context: Context, game: Library.SteamGame) = queueGame(context, game, asked = true)
 
     fun sweep(context: Context, games: List<Library.SteamGame>) {

app/src/main/java/com/droiddeck/launcher/runtime/RuntimeFileTree.java

@@ -80,7 +80,7 @@ public final class RuntimeFileTree {
      * be carried.
      */
     static java.util.List<String> userPaths(File tree) throws IOException {
-        java.util.List<String> paths = new java.util.ArrayList<>(java.util.Arrays.asList("root", "opt/appimages/user", "opt/droiddeck-apps"));
+        java.util.List<String> paths = new java.util.ArrayList<>(java.util.Arrays.asList("root", "opt/appimages/user", "opt/droiddeck-apps", "opt/droiddeck/wincomponents"));
         for (String n : children(new File(tree, "usr/local/share/applications"))) {
             if (n.endsWith(".desktop") && (n.startsWith("droiddeck-app-") || n.startsWith("droiddeck-appimage-")))
                 paths.add("usr/local/share/applications/" + n);

app/src/test/java/com/droiddeck/launcher/runtime/RuntimeUserDataCarryTest.kt

@@ -25,6 +25,7 @@ class RuntimeUserDataCarryTest {
         put(old, "root/save.dat", "mine")
         put(old, "opt/appimages/user/app1/app.AppImage")
         put(old, "opt/droiddeck-apps/game/run.sh")
+        put(old, "opt/droiddeck/wincomponents/d3dx9/component.json")
         put(old, "usr/local/share/applications/droiddeck-app-game.desktop")
         put(old, "usr/local/share/applications/droiddeck-appimage-app1.desktop")
         put(old, "usr/local/share/applications/other.desktop")
@@ -43,6 +44,7 @@ class RuntimeUserDataCarryTest {
         assertEquals("mine", File(staging, "root/save.dat").readText())
         assertFalse("the shipped home is replaced, not merged", File(staging, "root/.bashrc").exists())
         for (rel in listOf("opt/appimages/user/app1/app.AppImage", "opt/droiddeck-apps/game/run.sh",
+                "opt/droiddeck/wincomponents/d3dx9/component.json",
                 "usr/local/share/applications/droiddeck-app-game.desktop", "usr/local/share/applications/droiddeck-appimage-app1.desktop",
                 "opt/appimages/dolphin.AppImage", "usr/share/applications/droiddeck-dolphin.desktop", ".droiddeck-pkg-dolphin")) {
             assertTrue(rel, File(staging, rel).exists())

tools/linuxfs/overlay/usr/local/bin/droiddeck-wincomponents

@@ -101,16 +101,18 @@ def registry_values(root):
     return good
 
 
-def expand(store, ids):
+def expand(store, ids, saved=None):
     """The picks with everything they bundle, each once, a bundle's parts before it."""
     out, seen = [], set()
+    saved = saved if isinstance(saved, dict) else {}
 
     def visit(cid, depth):
         if cid in seen or depth > 8:
             return
         seen.add(cid)
         found = component(store, cid)
-        for dep in (found[2] if found else []):
+        parts = found[2] if found else [d for d in saved.get(cid, []) if valid_id(d)]
+        for dep in parts:
             visit(dep, depth + 1)
         out.append(cid)
 
@@ -444,7 +446,7 @@ def list_line(name, current, item, separator, add, at_end=True):
     return reg_value_line(name, "expand_sz" if expand else "sz", separator.join(items))
 
 
-def apply_registry(pfx, values, state):
+def apply_registry(pfx, values, state, held=()):
     """Writes the picked components' registry values into the prefix and takes out the ones of
     components turned off: a value goes back to what it was before, an entry added to a list such
     as PATH just leaves the list. Returns (values written, notes)."""
@@ -478,6 +480,9 @@ def apply_registry(pfx, values, state):
     for ident, record in old.items():
         if ident in keep or not isinstance(record, dict) or not isinstance(record.get("key"), str):
             continue
+        if record.get("cid") in held:
+            keep[ident] = record
+            continue
         reg = files.get(record.get("hive"))
         if reg is None:
             keep[ident] = record
@@ -555,14 +560,30 @@ def apply(compat_path, app_id, selection=SELECTION, store=STORE, pending=PENDING
     copied = 0
     installed = []
     values = []
-    for cid in expand(store, ids):
+    held = set()
+    saved_overrides = state.get("overrides") if isinstance(state.get("overrides"), dict) else {}
+    saved_requires = state.get("requires") if isinstance(state.get("requires"), dict) else {}
+    kept_overrides, kept_requires = {}, {}
+    for cid in expand(store, ids, saved_requires):
         found = component(store, cid)
         if found is None:
-            notes.append("%s is not installed; skipped" % cid)
+            if cid in saved_requires or any(owner == cid for owner in state["files"].values()):
+                held.add(cid)
+                names = [n for n in saved_overrides.get(cid, []) if isinstance(n, str)]
+                overrides += names
+                kept_overrides[cid] = names
+                if isinstance(saved_requires.get(cid), list):
+                    kept_requires[cid] = saved_requires[cid]
+                notes.append("%s is not in the store; kept the files it placed until it is installed again" % cid)
+            else:
+                notes.append("%s is not installed; skipped" % cid)
             continue
         installed.append(cid)
-        names, files, _, registry = found
+        names, files, requires, registry = found
         overrides += names
+        kept_overrides[cid] = list(names)
+        if requires:
+            kept_requires[cid] = list(requires)
         values += [(cid, value) for value in registry]
         for rel, src in files:
             if place(src, pfx / rel, None if rel in state["files"] else compat / BACKUP / rel):
@@ -572,6 +593,9 @@ def apply(compat_path, app_id, selection=SELECTION, store=STORE, pending=PENDING
     for rel, cid in state["files"].items():
         if rel in keep:
             continue
+        if cid in held:
+            keep[rel] = cid
+            continue
         path = existing(pfx / rel)
         saved = compat / BACKUP / rel
         if saved.is_file():
@@ -587,10 +611,11 @@ def apply(compat_path, app_id, selection=SELECTION, store=STORE, pending=PENDING
         except FileNotFoundError:
             pass
     compat.mkdir(parents=True, exist_ok=True)
-    new_state = {"files": keep, "components": installed, "registry": state.get("registry", {})}
+    new_state = {"files": keep, "components": installed + sorted(held), "overrides": kept_overrides,
+                 "requires": kept_requires, "registry": state.get("registry", {})}
     written = 0
     if values or new_state["registry"]:
-        written, registry_notes = apply_registry(pfx, values, new_state)
+        written, registry_notes = apply_registry(pfx, values, new_state, held)
         notes += registry_notes
     save_state(compat, new_state)
     if copied or removed or restored or written:

tools/tests/test_wincomponents.py

@@ -123,6 +123,33 @@ class WinComponentsTest(unittest.TestCase):
         self.assertEqual(names, [])
         self.assertEqual(notes, ["physx is not installed; skipped"])
 
+    def test_a_component_gone_from_the_store_keeps_what_it_placed(self):
+        self.component("dmband", ["dmband"], {"syswow64/dmband.dll": b"band"})
+        meta = self.store / "directmusic"
+        meta.mkdir(parents=True)
+        (meta / "component.json").write_text(json.dumps({"id": "directmusic", "overrides": [], "requires": ["dmband"]}))
+        self.component("openal", ["openal32"], {"syswow64/OpenAL32.dll": b"x86"})
+        self.pick("directmusic", "openal")
+        self.apply()
+        for cid in ("dmband", "directmusic", "openal"):
+            for path in sorted((self.store / cid).rglob("*"), reverse=True):
+                path.rmdir() if path.is_dir() else path.unlink()
+            (self.store / cid).rmdir()
+        names, notes = self.apply()
+        self.assertEqual(MODULE["overrides_value"](names), "dmband=n,b;openal32=n,b")
+        self.assertEqual((self.pfx / "syswow64/dmband.dll").read_bytes(), b"band")
+        self.assertEqual((self.pfx / "syswow64/openal32.dll").read_bytes(), b"x86")
+        self.assertTrue(all("kept the files it placed" in note for note in notes), notes)
+        self.component("openal", ["openal32"], {"syswow64/OpenAL32.dll": b"x86 v2"})
+        os.utime(self.store / "openal/syswow64/OpenAL32.dll", (1, 1))
+        names, _ = self.apply()
+        self.assertEqual(MODULE["overrides_value"](names), "dmband=n,b;openal32=n,b")
+        self.assertEqual((self.pfx / "syswow64/openal32.dll").read_bytes(), b"x86 v2")
+        self.pick()
+        self.apply()
+        self.assertFalse((self.pfx / "syswow64/dmband.dll").exists())
+        self.assertFalse((self.pfx / "syswow64/openal32.dll").exists())
+
     def test_no_selection_touches_nothing(self):
         self.assertEqual(self.apply(), ([], []))
         self.assertFalse(self.compat.exists())