diff --git a/app/src/main/java/com/droiddeck/launcher/frontend/AddedGames.kt b/app/src/main/java/com/droiddeck/launcher/frontend/AddedGames.kt index 6d833d931..3a1510402 100644 --- a/app/src/main/java/com/droiddeck/launcher/frontend/AddedGames.kt +++ b/app/src/main/java/com/droiddeck/launcher/frontend/AddedGames.kt @@ -12,6 +12,8 @@ import java.util.zip.CRC32 /** Windows game folders shared with the Steam session. */ object AddedGames { private const val TAG = "AddedGames" + private const val LIBRARY = "/mnt/droiddeck-sd" + private const val LEGACY_LIBRARY = "/mnt/bannerlator-sd" class Game( val folder: File, val name: String, val exe: File, val guestExe: String, val guestDir: String, @@ -65,7 +67,7 @@ object AddedGames { if (path.startsWith("$roms/")) return "/root/ROMs/" + path.removePrefix("$roms/") } GameStorage.effective(context)?.let { lib -> - if (path.startsWith("${lib.path}/")) return "/mnt/bannerlator-sd/" + path.removePrefix("${lib.path}/") + if (path.startsWith("${lib.path}/")) return "$LIBRARY/" + path.removePrefix("${lib.path}/") } return null } @@ -108,7 +110,8 @@ object AddedGames { if (guestExe == null) { Log.w(TAG, "${folder.name}: the session cannot see ${exe.path}"); return } val guestDir = guestPath(context, exe.parentFile ?: folder) ?: return val name = folder.name - val crc = CRC32().apply { update(("\"$guestExe\"" + name).toByteArray()) }.value + // Keyed by the pre-rename path so shortcut ids, and the prefixes and saves under them, stay put. + val crc = CRC32().apply { update(("\"${guestExe.replaceFirst(Regex("^$LIBRARY/"), "$LEGACY_LIBRARY/")}\"" + name).toByteArray()) }.value val appId = crc or 0x80000000L val steamRoot = File(LinuxRuntime.rootDir(context), "root/.local/share/Steam") val steamId = steamRoute(steamRoot, appId) diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt index 515393370..9eb8a1d35 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt @@ -599,7 +599,7 @@ object SessionPrefs { // ── Game storage ──────────────────────────────────────────────────────────────────────── /** - * A second Steam library on this device: the folder bound at /mnt/bannerlator-sd and + * A second Steam library on this device: the folder bound at /mnt/droiddeck-sd and * registered with the client, which then asks where to install every game and shows both * on its Storage page. "" = automatic: the SD card when one is in the phone (the default, * so the choice is made inside the client like anywhere else); GAME_STORAGE_OFF = internal diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 5abd06c2c..4114c627e 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -697,9 +697,12 @@ class SessionService : Service() { if (library != null) { val problem = GameStorage.prepare(library.path) if (problem == null) { + File(LinuxRuntime.rootDir(this), "mnt/droiddeck-sd").mkdirs() + binds.add("${library.path}:/mnt/droiddeck-sd") + // Links, prefixes and Steam entries made before the rename still name the old path. File(LinuxRuntime.rootDir(this), "mnt/bannerlator-sd").mkdirs() binds.add("${library.path}:/mnt/bannerlator-sd") - Log.i(TAG, "game storage: ${library.path} -> /mnt/bannerlator-sd (\"${library.label}\")") + Log.i(TAG, "game storage: ${library.path} -> /mnt/droiddeck-sd (\"${library.label}\")") } else { Log.w(TAG, "game storage: $problem; internal only this session") } diff --git a/docs/development/steam-game-imports.md b/docs/development/steam-game-imports.md index 4639afa5b..4c4efc0e1 100644 --- a/docs/development/steam-game-imports.md +++ b/docs/development/steam-game-imports.md @@ -21,16 +21,21 @@ The resulting snapshot is stored under that account's `userdata//config` and refreshed every 30 seconds while logged in. Title lookups retry every five minutes and when the source changes, so an early network failure does not end the watcher. Changed status and probe failures are written to the session log. Startup and post-exit import -use the last successful snapshot for the selected account. A newly identified title -therefore starts as a shortcut and becomes a Steam entry after Steam exits and starts -again. An unavailable client or offline first launch falls back to shortcuts; existing +use the last successful snapshot for the selected account. The client reads manifests +and shortcuts only at start, so when a snapshot confirms owned titles that are not yet +imported, the watcher restarts the client once (not while a game runs or a download is in +progress) and the importer runs between the exit and the next start. An unavailable client or offline first launch falls back to shortcuts; existing snapshots remain usable offline. Steam still enforces the current license at launch. For a confirmed game without a manifest, the importer links the source folder into the internal Steam library and writes an installed manifest with `Universe=1` and -`StateFlags=4`. Steam ignores manifests without the universe field. This registers -existing files; it does not assert a verified build ID or depot manifest. Steam can -still require verification, updates or redistributables. Steam updates and uninstall +`StateFlags=4`. Steam ignores manifests without the universe field. A manifest without +installed depots makes Steam verify every file, so the importer carries the build another +client recorded: WinNative's `.winnative-steam/steamapps` manifest beside the game folder, +or a completed download's `.DepotDownloader/depot.config` depot manifest IDs. Steam then +only downloads what changed since that build. Bare manifests from earlier imports are +upgraded while Steam has not touched them. Without either record Steam still verifies. +WinNative's manifest also identifies the folder without a store lookup. Steam updates and uninstall operate on the linked game files, so imported folders must be writable. Existing Steam manifests and directories are preserved. Only the exact minimal manifests generated by the earlier importer are migrated, with the matching source symlink still present. If diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-desktop-games b/tools/linuxfs/overlay/usr/local/bin/bannerlator-desktop-games index 16fc736ec..7690fd219 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-desktop-games +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-desktop-games @@ -11,7 +11,7 @@ import sys HOME = os.environ.get("HOME", "/root") STEAM_ROOT = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HOME, ".local/share/Steam") -LIBRARIES = ((STEAM_ROOT, "Steam Games"), ("/mnt/bannerlator-sd", "SD Games")) +LIBRARIES = ((STEAM_ROOT, "Steam Games"), ("/mnt/droiddeck-sd", "SD Games")) # The client's own tools and runtimes live in steamapps beside the games; they are not titles. NOT_GAMES = {228980, 1493710, 3127680, 4183110, 4427310, 4185400} APPS_DIR = os.path.join(HOME, ".local/share/applications") diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-seed-redists b/tools/linuxfs/overlay/usr/local/bin/bannerlator-seed-redists index 61c8d35a6..ea1d413f5 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-seed-redists +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-seed-redists @@ -79,7 +79,7 @@ block=$(printf '%s\n' "$pairs" | awk -F'\t' -v now="$(date +%s)" ' for reg in "$steam_root"/steamapps/common/Proton*/files/share/default_pfx*/system.reg \ "$steam_root"/compatibilitytools.d/*/files/share/default_pfx*/system.reg \ "$steam_root"/steamapps/compatdata/*/pfx/system.reg \ - /mnt/bannerlator-sd/steamapps/compatdata/*/pfx/system.reg; do + /mnt/droiddeck-sd/steamapps/compatdata/*/pfx/system.reg; do [ -f "$reg" ] || continue seeded=$(dirname "$reg")/.bannerlator-redists [ "$(cat "$seeded" 2>/dev/null)" = "$stamp" ] && continue diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-session b/tools/linuxfs/overlay/usr/local/bin/bannerlator-session index f3b40af88..01fb68198 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-session +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-session @@ -234,7 +234,7 @@ case $mode in bannerlator-steam-install || exit $? step "Steam client ready" steam_root=$XDG_DATA_HOME/Steam - # Register the card library (bound at /mnt/bannerlator-sd) with the client - internal storage is + # Register the card library (bound at /mnt/droiddeck-sd) with the client - internal storage is # its main library already - and the ARM64 Proton that lets it launch the Windows ones. # A request left in ~/.bl-proton-extra installs a third-party Proton before the client starts, # so it is in the Compatibility dropdown the first time the user looks. One per line, each the @@ -446,7 +446,7 @@ case $mode in local d [ -e "$compat_stamp" ] && [ "$((beat % 12))" -ne 0 ] || return 0 for d in "$steam_root/steamapps" "$steam_root/steamapps/common" "$steam_root/config" \ - "$steam_root/compatibilitytools.d" /mnt/bannerlator-sd/steamapps; do + "$steam_root/compatibilitytools.d" /mnt/droiddeck-sd/steamapps; do [ "$d" -nt "$compat_stamp" ] && return 0 done return 1 diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-games b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-games index 1c2add014..46dcdb83e 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-games +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-games @@ -13,6 +13,9 @@ import urllib.parse import urllib.request STATE = ".droiddeck-games.json" +WINNATIVE = ".winnative-steam/steamapps" +LIBRARY = "/mnt/droiddeck-sd" +LEGACY_LIBRARY = "/mnt/bannerlator-sd" BASE_ID = 76561197960265728 @@ -40,6 +43,11 @@ def edition_key(name): return title_key(name) +def legacy_path(path): + path = str(path) + return LEGACY_LIBRARY + path[len(LIBRARY):] if path == LIBRARY or path.startswith(LIBRARY + "/") else path + + def source_stamp(game): try: exe = Path(game["exe"]) @@ -53,19 +61,23 @@ def source_stamp(game): return None +def manifest_ids(steamapps, name): + matches = [] + for manifest in Path(steamapps).glob("appmanifest_*.acf"): + try: + install = re.search(r'"installdir"\s*"([^"\\]*)"', manifest.read_text(), re.I) + if install and install[1] == name: + matches.append(int(manifest.stem.removeprefix("appmanifest_"))) + except (OSError, ValueError): + pass + return matches + + def identify(game): folder = Path(game["folder"]) steamapps = folder.parent.parent - if folder.parent.name == "common" and steamapps.name == "steamapps": - matches = [] - for manifest in steamapps.glob("appmanifest_*.acf"): - try: - text = manifest.read_text() - install = re.search(r'"installdir"\s*"([^"\\]*)"', text, re.I) - if install and install[1] == folder.name: - matches.append(int(manifest.stem.removeprefix("appmanifest_"))) - except (OSError, ValueError): - pass + for library in ([steamapps] if folder.parent.name == "common" and steamapps.name == "steamapps" else []) + [folder.parent / WINNATIVE]: + matches = manifest_ids(library, folder.name) if matches: return matches[0] if len(matches) == 1 else None ids = set() @@ -144,6 +156,27 @@ def game_running(): return False +def downloads_idle(steam_root): + try: + return not any(Path(steam_root, "steamapps/downloading").iterdir()) + except OSError: + return True + + +def request_restart(steam_root): + """The client reads manifests and shortcuts only at start, so an import needs one restart; + the session's loop runs the importer between the exit and the next start.""" + Path(os.environ.get("HOME", "/root"), ".bl-restart-once").touch() + subprocess.run([str(Path(steam_root) / "steamrtarm64/steam"), "-shutdown"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30) + + +def unimported(steam_root, snapshot, candidates): + routes = read_json(Path(steam_root) / "userdata" / snapshot["account"] / "config/.droiddeck-routes.json", {}) + routed = set(routes.values()) if isinstance(routes, dict) else set() + return sorted({appid for appid in candidates.values() if snapshot["owned"].get(str(appid)) is True} - routed) + + def watch(steam_root, listing): games = read_json(listing, []) if os.environ.get("BL_STEAM_OFFLINE") == "1" or not games: @@ -158,6 +191,7 @@ def watch(steam_root, listing): # beat (a client that is not logged on fails it for as long as it stays that way), and none # starts while a game is running: the answer can wait, the game's cores cannot. probed, next_probe, failures = None, 0, 0 + pending, requested = [], [] while True: now = time.monotonic() for game in games: @@ -195,6 +229,7 @@ def watch(steam_root, listing): snapshot["candidates"] = candidates snapshot["sources"] = sources save_json(Path(steam_root) / "userdata" / snapshot["account"] / "config" / STATE, snapshot) + pending = unimported(steam_root, snapshot, candidates) status = "matched %d/%d games; %d owned" % (len(candidates), len(games), sum(snapshot["owned"].values())) except (OSError, ValueError, KeyError, StopIteration, subprocess.TimeoutExpired) as error: status = "ownership check unavailable: " + str(error) @@ -202,12 +237,51 @@ def watch(steam_root, listing): next_probe = now + min(300, 30 * 2 ** failures) elif candidates: status = last_status + if pending and pending != requested and not game_running() and downloads_idle(steam_root): + requested = pending + print("bannerlator-steam-games: restarting the client once to import %s" % pending, flush=True) + try: + request_restart(steam_root) + except (OSError, subprocess.TimeoutExpired) as error: + print("bannerlator-steam-games: restart unavailable: %s" % error, flush=True) if status != last_status: print("bannerlator-steam-games: " + status, flush=True) last_status = status time.sleep(30) +def installed_build(folder, appid, installdir): + """The build another Steam client recorded for these files, so Steam updates from it + instead of verifying every file.""" + try: + text = (folder.parent / WINNATIVE / f"appmanifest_{appid}.acf").read_text() + install = re.search(r'("installdir"\s*")([^"\\]*)(")', text, re.I) + flags = re.search(r'"StateFlags"\s*"(\d+)"', text) + if (install and install[2] == folder.name and flags and flags[1] == "4" + and re.search(rf'"appid"\s*"{appid}"', text) and '"InstalledDepots"' in text): + return text[:install.start(2)] + installdir + text[install.end(2):] + except OSError: + pass + if not (folder / ".download_complete").is_file(): + return None + try: + ids = json.loads((folder / ".DepotDownloader/depot.config").read_text())["installedManifestIDs"] + depots = sorted((int(depot), int(gid)) for depot, gid in ids.items()) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + return None + if not depots or not all(0 < depot < 0x80000000 and 0 < gid < 0x7FFFFFFFFFFFFFFF for depot, gid in depots): + return None + return "".join(f' "{depot}"\n {{\n "manifest" "{gid}"\n }}\n' for depot, gid in depots) + + +def update_not_started(text, steamapps, appid, installdir): + fields = dict(re.findall(r'^\s*"(\w+)"\s*"([^"]*)"', text, re.M)) + return (fields.get("installdir") == installdir and fields.get("buildid", "0") == "0" + and fields.get("BytesDownloaded", "0") == "0" and fields.get("BytesStaged", "0") == "0" + and re.search(r'"InstalledDepots"\s*\{\s*\}', text) is not None + and not (steamapps / "downloading" / str(appid)).exists()) + + def seed_install(steam_root, game, appid): steamapps = Path(steam_root) / "steamapps" common = steamapps / "common" @@ -215,16 +289,30 @@ def seed_install(steam_root, game, appid): target = common / name source = Path(game["folder"]) title = game["name"].replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ") - content = ('"AppState"\n{\n' + f' "appid" "{appid}"\n "Universe" "1"\n "name" "{title}"\n' + minimal = ('"AppState"\n{\n' + f' "appid" "{appid}"\n "Universe" "1"\n "name" "{title}"\n' + f' "StateFlags" "4"\n "installdir" "{name}"\n' + '}\n') - legacy = content.replace(' "Universe" "1"\n', '') + legacy = minimal.replace(' "Universe" "1"\n', '') + build = installed_build(source, appid, name) + if build is None: + content = minimal + elif build.lstrip().startswith('"AppState"'): + content = build + else: + content = minimal[:-2] + ' "InstalledDepots"\n {\n' + build + ' }\n}\n' manifest = steamapps / f"appmanifest_{appid}.acf" - for root in [Path(steam_root), Path("/mnt/bannerlator-sd")]: + if target.is_symlink() and os.readlink(target) != str(source) and os.readlink(target) == legacy_path(source): + relink = target.with_name(name + ".droiddeck") + relink.unlink(missing_ok=True) + relink.symlink_to(source, target_is_directory=True) + os.replace(relink, target) + for root in [Path(steam_root), Path(LIBRARY)]: existing = root / "steamapps" / manifest.name if existing.is_file(): if (existing == manifest and source.is_dir() and target.is_symlink() and os.readlink(target) == str(source) - and existing.read_text() in (legacy, legacy.replace('"StateFlags" "4"', '"StateFlags" "2"'))): + and existing.read_text() != content + and (existing.read_text() in (minimal, legacy, legacy.replace('"StateFlags" "4"', '"StateFlags" "2"')) + or (build is not None and update_not_started(existing.read_text(), steamapps, appid, name)))): temp = manifest.with_suffix(".droiddeck") temp.write_text(content) os.replace(temp, manifest) @@ -256,11 +344,13 @@ def route(steam_root, acct, games): owned, candidates, sources = {}, {}, {} shortcuts, routes = [], {} for game in games: - appid = candidates.get(game.get("folder")) + folder = game.get("folder", "") + appid = candidates.get(folder, candidates.get(legacy_path(folder))) stamp = source_stamp(game) + recorded = sources.get(folder, sources.get(legacy_path(folder))) confirmed = (isinstance(appid, int) and 0 < appid < 0x80000000 and owned.get(str(appid)) is True and stamp is not None - and sources.get(game.get("folder")) == stamp) + and recorded in (stamp, [stamp[0], legacy_path(stamp[1])] + stamp[2:])) if confirmed: try: if seed_install(steam_root, game, appid): diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-library b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-library index e67878e6e..d065db6b5 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-library +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-library @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Registers the card the app installs to (bound at /mnt/bannerlator-sd) as a library folder of the +# Registers the card the app installs to (bound at /mnt/droiddeck-sd) as a library folder of the # native Steam client, so they show as installed there instead of being downloaded a second time. # # bannerlator-steam-library @@ -14,7 +14,7 @@ import sys # The app binds the storage the user chose (Steam cog -> Game storage) at this path and names it # in BL_LIBRARY_LABEL; nothing bound = internal only, and the entry is removed so the client does # not keep offering a place that is not there. -LIBRARIES = (("/mnt/bannerlator-sd", os.environ.get("BL_LIBRARY_LABEL", "SD Card")),) +LIBRARIES = (("/mnt/droiddeck-sd", os.environ.get("BL_LIBRARY_LABEL", "SD Card")),) # The client calls its own library "Local Drive (/)", which says nothing about where that is. # It is the phone's internal storage, so say so. MAIN_LABEL = "Internal Storage" @@ -66,6 +66,26 @@ def register(steam_root, library, label): RETIRED = ("/mnt/bannerlator",) +RENAMED = (("/mnt/bannerlator-sd", "/mnt/droiddeck-sd"),) + + +def rename(steam_root, old, new): + """Moves a library registered under an earlier mount point to the current one, keeping its + entry (content id, app sizes) rather than registering it anew.""" + path = os.path.join(steam_root, "steamapps", "libraryfolders.vdf") + if not os.path.isfile(path): + return + with open(path, encoding="utf-8", errors="replace") as f: + text = f.read() + old_path = re.compile(r'("path"\s+")%s(")' % re.escape(old)) + if not old_path.search(text): + return + if re.search(r'"path"\s+"%s"' % re.escape(new), text): + forget(steam_root, old) + return + with open(path, "w", encoding="utf-8") as f: + f.write(old_path.sub(lambda m: m.group(1) + new + m.group(2), text)) + print("bannerlator-steam-library: moved %s to %s" % (old, new)) def forget(steam_root, library): @@ -152,6 +172,8 @@ def main(): steam_root = sys.argv[1] for library in RETIRED: forget(steam_root, library) + for old, new in RENAMED: + rename(steam_root, old, new) relabel(steam_root, steam_root, MAIN_LABEL) for library, label in LIBRARIES: if os.path.isdir(os.path.join(library, "steamapps")): diff --git a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-shortcuts b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-shortcuts index c3fcd4ae3..0abb8b719 100755 --- a/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-shortcuts +++ b/tools/linuxfs/overlay/usr/local/bin/bannerlator-steam-shortcuts @@ -97,7 +97,7 @@ def write(obj): return bytes(out) -def shortcut(game): +def shortcut(game, launch_options=""): appid = int(game["appid"]) & 0xFFFFFFFF return { "appid": struct.unpack("