Windows components: picked automatically by each game's source (#554)

Each game's Windows components now come from the list its own source
trusts, and the ones that list names turn on by themselves. The user's own
on/off switches always win; each automatic row says why it is on.

- Steam games: the Steamworks Shared redistributables Steam installs with
  the game (appmanifest SharedDepots, else the client's appinfo.vdf).
  Steam keeps its install scripts marked as already run.
- GOG: the build manifest's dependencies. Epic: the manifest's
  prerequisite (Unreal's maps to VC++ and DirectX). Amazon: fuel.json
  PostInstall. Older GOG/Epic installs get their list fetched once with the
  saved sign-in; game files are never downloaded again.
- Custom games (the + button, Add all, migrated folders): one shared Steam
  match per game, also used for artwork. An appid from the game's files
  turns components on; a name-only match stays Recommended; no match falls
  back to the folder scan.
- Steam app info for apps the client never saw comes from an anonymous
  Steam PICS lookup (CM websocket, anonymous logon, product info), cached
  per appid; the user's Steam sign-in is never used.
- Missing components download in the background on one low-priority
  worker that pauses during sessions and waits on battery saver, heat or a
  metered network unless the user asked. A game Steam installs during a
  session is readied right away so it can be played without ending the
  session; the launch waits up to 10 s only for a component still on its
  way.
- Page: "On for this game" with reasons, a quiet "Checking what this game
  needs..." only for games never worked out, rows that ease in, pieces that
  need a download greyed out while offline. The intro names no other app.
- The folder scan skips Unity's *_Data/Managed (Mono, not .NET Framework).
This commit is contained in:
The412Banner authored and GitHub committed 2026-10-10 21:15:56 -04:00
1 parent d45933e559
commit a2e596a200
35 files changed
+3810 -146

No files matched your search

@@ -137,8 +137,7 @@ object AddedGameArt {
val out = ArrayList<Option>()
folderImages(game.folder).forEach { out.add(Option(FOLDER, it.path, it.path)) }
val cache = cacheDir(context, game)
val steamId = runCatching { GameIdentifier.identify(game.exe).appId?.toString() }.getOrNull()
?: File(cache, "steam-appid").takeIf { it.isFile }?.readText()?.trim()?.takeIf { it.isNotEmpty() && it != "none" }
val steamId = SteamMatch.resolve(context, game.folder.path, game.exe, game.name, legacy = File(cache, "steam-appid")).appId?.toString()
val remote = when (slot) { Slot.COVER -> "library_600x900.jpg"; Slot.BACKGROUND -> "library_hero.jpg"; Slot.LOGO -> "logo.png"; Slot.ICON -> null }
if (steamId != null && remote != null) {
val url = "$CDN$steamId/$remote".takeIf { exists(it) } ?: steamAssetUrl(steamId, remote)
@@ -222,10 +221,11 @@ object AddedGameArt {
val cache = cacheDir(context, game)
if (File(cache, "p.jpg").isFile) continue
cache.mkdirs()
val appId = runCatching { GameIdentifier.identify(game.exe).appId }.getOrNull()
val id = steamAppId(game.name, appId, File(cache, "steam-appid")) { name ->
runCatching { search(name) }.onFailure { Log.w(TAG, "${game.name}: store search failed (${it.message})") }.getOrNull()
val match = steamMatch(context, game) { name ->
runCatching { search(name) }.onFailure { Log.w(TAG, "${game.name}: store search failed (${it.message})") }.getOrNull()?.toIntOrNull()
}
val appId = match.appId?.takeIf { match.certainty == SteamMatch.Certainty.FILES }
val id = match.appId?.toString()
if (id != null && download(id, cache)) changed = true
if (File(cache, "p.jpg").isFile || sgdbKey.isEmpty()) continue
val gridId = remembered(File(cache, "sgdb-id")) {
@@ -237,16 +237,13 @@ object AddedGameArt {
}
/**
* The game's Steam appid for its art: the one its files name ([appId]) as it is, else the
* remembered or freshly searched one ([search] on [name]); null for none.
* The game's Steam app, the one record its Windows components use too ([SteamMatch]): what its
* files name, else a store search on its title ([search]) whose result has the same name.
*/
internal fun steamAppId(name: String, appId: Int?, lookup: File, search: (String) -> String?): String? {
if (appId != null) {
lookup.writeText(appId.toString())
Log.i(TAG, "$name: Steam app $appId (from its files)")
return appId.toString()
}
return remembered(lookup) { search(name)?.also { Log.i(TAG, "$name: Steam app $it") } }
internal fun steamMatch(context: Context, game: AddedGames.Game, search: (String) -> Int?): SteamMatch.Match {
val match = SteamMatch.resolve(context, game.folder.path, game.exe, game.name, legacy = File(cacheDir(context, game), "steam-appid"), search = search)
match.appId?.let { Log.i(TAG, "${game.name}: Steam app $it (${match.certainty.name.lowercase()})") }
return match
}
/** What [marker] remembers (an id, or "none" for a week), else [find]'s answer written to it. */
@@ -0,0 +1,419 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import android.net.ConnectivityManager
import android.os.Build
import android.os.PowerManager
import android.util.AtomicFile
import android.util.Log
import androidx.annotation.VisibleForTesting
import com.droiddeck.launcher.session.SessionState
import com.droiddeck.launcher.session.WinComponents
import com.droiddeck.launcher.stores.StoreGameSidecar
import kotlinx.coroutines.flow.MutableStateFlow
import org.json.JSONArray
import org.json.JSONObject
import java.io.File
import java.util.concurrent.PriorityBlockingQueue
/**
* Turns on, per game, the Windows components its source's list names ([WinCompSources]). Only the
* findings marked auto are turned on, only the ones that install here, and never one the user
* switched off: the user's switches are kept apart ([WinComponents.Selection.user]) and win.
*
* All the work is done by one background thread (low priority, so are the runtime helpers it
* starts), one job at a time from one queue: working out a game's list (kept, so a page opens on
* it) and downloading a component (each once, whichever games want it). The queue waits while a
* session runs; jobs nobody asked for (the start-up sweep, a game being launched) also wait on
* battery saver, a warm device or a metered network. Nothing here runs on the caller's thread.
*/
object AutoComponents {
private const val TAG = "AutoComponents"
/** The key a game's picks are kept under: the appid its prefix is named after. */
fun appKey(game: Library.SteamGame): String? =
if (game.library == Library.ADDED) Integer.toUnsignedString(game.appId) else game.appId.takeIf { it > 0 }?.toString()
/**
* What turns on by itself for [findings], given [catalog] and what is [installed]: the catalog
* name each auto finding installs as, with its reason. A component that needs a Windows
* installer, or is not in the catalog, stays out (the editor lists it as a suggestion).
*/
fun autoPicks(findings: List<WinCompSources.Finding>, catalog: Map<String, WinComponents.Component>): Map<String, WinComponents.AutoReason> {
val out = LinkedHashMap<String, WinComponents.AutoReason>()
for (f in findings) {
if (!f.auto || f.component == null) continue
val id = WinComponents.installable(f.component, catalog)
val c = catalog[id] ?: continue
if (WinComponents.support(c, catalog) != WinComponents.Support.READY) continue
if (id !in out) out[id] = WinComponents.AutoReason(f.origin.name, listOf(f.original) + f.args)
}
return out
}
// ---- seams for tests ----
@VisibleForTesting internal var catalogOf: (Context, Boolean) -> List<WinComponents.Component>? =
{ c, network -> if (network) WinComponents.fetch(c) else WinComponents.cached(c) }
@VisibleForTesting internal var installedOf: (Context) -> Set<String> = { WinComponents.installedIds(it).toSet() }
@VisibleForTesting internal var installer: (Context, WinComponents.Component, Map<String, WinComponents.Component>, (WinComponents.Progress) -> Unit) -> String? =
{ c, comp, all, progress -> WinComponents.install(c, comp, all, progress) }
@VisibleForTesting internal var sessionRunning: () -> Boolean = { SessionState.running }
/** Battery saver, a warm device or a metered network: no work nobody asked for. */
@VisibleForTesting internal var unfavourable: (Context) -> Boolean = { c -> constrained(c) }
/** A usable network: a download waits for one. */
@VisibleForTesting internal var online: (Context) -> Boolean = { c -> isOnline(c) }
/** Tests run the queue on the caller's thread ([drain]) instead of the worker. */
@VisibleForTesting @Volatile internal var runNow = false
/** Bumped whenever a game's list or automatic components change, so an open page reads them again. */
val changes = MutableStateFlow(0)
/** The components downloading for an automatic pick right now, with their progress. */
val downloading = MutableStateFlow<Map<String, WinComponents.Progress>>(emptyMap())
/** Games (by [appKey]) whose list is queued or being worked out: a page waits on it. */
val pendingGames = MutableStateFlow<Set<String>>(emptySet())
/** Components queued for an automatic pick, not yet downloaded. */
val pendingComponents = MutableStateFlow<Set<String>>(emptySet())
private fun mark(job: Job, on: Boolean) {
if (job.pics != null) return
val flow = if (job.component != null) pendingComponents else pendingGames
val id = job.component ?: job.game?.let { appKey(it) } ?: return
flow.value = if (on) flow.value + id else flow.value - id
}
/** One piece of work: a game's list ([game]) or one component's download ([component]). [asked]: the user is waiting on it. */
internal class Job(
val game: Library.SteamGame?, val component: String?, val asked: Boolean, val seq: Long, val pics: List<Int>? = null,
/** For a game Steam installed during the running session: may run in it, unless a game is running. */
val inSession: Boolean = false,
) {
val key get() = when {
pics != null -> "pics"
component != null -> "c:$component"
else -> "g:" + (game?.let { appKey(it) } ?: "")
}
/** Needs the network: a download, or Steam asked anonymously. */
val online get() = component != null || pics != null
}
private val seq = java.util.concurrent.atomic.AtomicLong()
/** What the user asked for first, then in the order queued. */
private val jobs = PriorityBlockingQueue<Job>(16, compareByDescending<Job> { it.asked }.thenBy { it.seq })
@Volatile private var draining = false
private val queuedKeys = java.util.Collections.synchronizedSet(HashSet<String>())
/** The games waiting on a component's download, to turn it on for when it lands. */
private val waiting = HashMap<String, MutableList<Library.SteamGame>>()
@Volatile private var worker: Thread? = null
private const val PAUSE_MS = 15_000L
/**
* [game]'s list worked out again on the worker, and its components queued: [asked] when the
* user is waiting (its page is open, or they installed or added it).
*/
fun queueGame(context: Context, game: Library.SteamGame, asked: Boolean) {
if (appKey(game) == null) return
enqueue(context, Job(game, null, asked, seq.incrementAndGet()))
}
private fun queueDownload(context: Context, id: String, asked: Boolean, inSession: Boolean = false) =
enqueue(context, Job(null, id, asked, seq.incrementAndGet(), inSession = inSession))
/** Adds [job] unless the same one is queued (a game or a component, once each). */
private fun enqueue(context: Context, job: Job) {
if (!queuedKeys.add(job.key)) {
// Asked for now: ahead of what nobody asked for, so the open page does not wait behind the sweep.
if (job.asked && jobs.removeIf { it.key == job.key && !it.asked }) jobs.add(job)
if (runNow && !draining) drain(context)
return
}
if (job.component != null) Log.i(TAG, "wincomp: queued ${job.component}")
mark(job, true)
jobs.add(job)
if (runNow) { if (!draining) drain(context) } else ensureWorker(context.applicationContext)
}
/** Whether [job] has to wait: always while a session runs; one nobody asked for also when it is unfavourable. */
internal fun mustWait(context: Context, job: Job): Boolean =
if (job.inSession && sessionRunning()) gameRunning() || (job.online && !online(context))
else sessionRunning() || (job.online && !online(context)) || (!job.asked && unfavourable(context))
/** A game is running in the session (Steam's reaper, or Proton waiting on its game). */
@VisibleForTesting internal var gameRunning: () -> Boolean = { gameProcessRunning() }
private fun gameProcessRunning(): Boolean = File("/proc").listFiles()?.any { p ->
p.name.all(Char::isDigit) && runCatching { String(File(p, "cmdline").readBytes()).replace('\u0000', ' ') }.getOrNull()
?.let { it.contains("SteamLaunch AppId=") || it.contains("proton waitforexitandrun") } == true
} == true
/**
* A game Steam installed during the running session: its list worked out and its components
* readied in the session (one job, low priority), so a launch right after finds them; it
* waits while a game runs. All other work stays paused until the session ends.
*/
fun queueInSession(context: Context, game: Library.SteamGame) {
if (appKey(game) == null) return
enqueue(context, Job(game, null, true, seq.incrementAndGet(), inSession = true))
}
/** Runs the queued jobs on this thread until the queue is empty or the next one has to wait. Tests. */
@VisibleForTesting
internal fun drain(context: Context) {
draining = true
try {
while (true) {
val job = next(context) ?: return
queuedKeys.remove(job.key)
run(context, job)
}
} finally {
draining = false
}
}
/** The first job in order that may run now, taken off the queue; null when none may. */
private fun next(context: Context): Job? = synchronized(jobs) {
val job = jobs.sortedWith(order).firstOrNull { !mustWait(context, it) } ?: return null
jobs.remove(job)
job
}
private val order = compareByDescending<Job> { it.asked }.thenBy { it.seq }
@VisibleForTesting internal fun queued(): List<String> = jobs.sortedWith(compareByDescending<Job> { it.asked }.thenBy { it.seq }).map { it.key }
@Synchronized
private fun ensureWorker(app: Context) {
if (worker?.isAlive == true) return
worker = Thread({
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_BACKGROUND)
while (true) {
// Something queued, then the first that may run (one for a game installed in the
// session goes ahead of paused work); else a look again in a while.
try { jobs.put(jobs.take()) } catch (e: InterruptedException) { return@Thread }
val job = next(app)
if (job == null) {
try { Thread.sleep(PAUSE_MS) } catch (e: InterruptedException) { return@Thread }
continue
}
queuedKeys.remove(job.key)
WinComponents.background { run(app, job) }
}
}, "auto-components").apply { isDaemon = true; start() }
}
private fun run(context: Context, job: Job) {
try {
when {
job.pics != null -> SteamAppInfo.lookup(context, job.pics)
job.component != null -> download(context, job.component, job.asked, job.inSession)
else -> job.game?.let { apply(context, it, job.asked, job.inSession) }
}
} catch (e: Exception) {
Log.w(TAG, "wincomp: ${job.key}: ${e.javaClass.simpleName}: ${e.message}")
} finally {
mark(job, false)
changes.value = changes.value + 1
}
}
/**
* [game]'s list (worked out again only when what it is read from changed), its automatic
* components that are downloaded turned on, and the rest queued for download.
*/
private fun apply(context: Context, game: Library.SteamGame, asked: Boolean, inSession: Boolean = false) {
val key = appKey(game) ?: return
val folder = game.gameFiles
if (game.library == Library.ADDED && game.source == Library.ADDED && folder != null && SteamMatch.get(context, folder.path) == null) {
AddedGames.single(context, folder.path)?.let { SteamMatch.resolve(context, folder.path, it.exe, it.name) }
}
// A GOG or Epic game installed before its sidecar kept the store's list: read it once from the store.
val store = com.droiddeck.launcher.stores.Store.byId(game.source)
if (folder != null && (store == com.droiddeck.launcher.stores.Store.GOG || store == com.droiddeck.launcher.stores.Store.EPIC)) StoreListBackfill.fill(context, folder)
// Its Steam app, when the client never saw it: Steam asked anonymously now if the user is
// waiting on this game (a pass over every game asks for all of them in one go, [sweep]).
if (asked && online(context)) steamAppOf(context, game)?.let { SteamAppInfo.lookup(context, listOf(it)) }
val findings = findings(context, game)
val catalog = (catalogOf(context, false) ?: catalogOf(context, true))?.associateBy { it.name } ?: return
val wanted = autoPicks(findings, catalog)
val user = WinComponents.selection(context, key).user
val installed = installedOf(context)
val kept = LinkedHashMap<String, WinComponents.AutoReason>()
val missing = ArrayList<String>()
for ((id, reason) in wanted) {
if (user[id] == false) { kept[id] = reason; continue }
if (id in installed) { kept[id] = reason; continue }
synchronized(waiting) { waiting.getOrPut(id) { ArrayList() }.let { if (it.none { g -> appKey(g) == key }) it.add(game) } }
queueDownload(context, id, asked, inSession)
missing.add(id)
}
// In a session the launch waits a little for these (droiddeck-wincomponents reads the list).
if (inSession) Pending.set(context, key, missing)
val before = WinComponents.selection(context, key).auto
if (before.keys != kept.keys) {
WinComponents.setAuto(context, key, kept)
Log.i(TAG, "${game.name}: on by itself ${kept.keys.joinToString().ifEmpty { "(none)" }}")
}
changes.value = changes.value + 1
}
/** One component downloaded and installed, then turned on for every game waiting on it. */
private fun download(context: Context, id: String, asked: Boolean, inSession: Boolean = false) {
val catalog = catalogOf(context, true)?.associateBy { it.name } ?: run { Pending.drop(context, id); return }
if (id !in installedOf(context)) {
val c = catalog[id] ?: run { Pending.drop(context, id); return }
val started = System.currentTimeMillis()
downloading.value = downloading.value + (id to WinComponents.Progress(id, "", WinComponents.Phase.DOWNLOAD, -1))
val problem = try {
installer(context, c, catalog) { p -> downloading.value = downloading.value + (id to p) }
} finally {
downloading.value = downloading.value - id
}
if (problem != null) { Log.w(TAG, "wincomp: $id did not install: $problem"); Pending.drop(context, id); return }
Log.i(TAG, "wincomp: $id installed in ${System.currentTimeMillis() - started}ms")
}
val games = synchronized(waiting) { waiting.remove(id).orEmpty() }
games.forEach { apply(context, it, asked, inSession) }
Pending.drop(context, id)
}
/**
* The components still being readied for games installed in the session, where the launch
* reads them (~/.config/droiddeck/wincomponents-pending.json in the runtime).
*/
internal object Pending {
private fun file(context: Context) = File(com.droiddeck.launcher.runtime.LinuxRuntime.rootDir(context), "root/.config/droiddeck/wincomponents-pending.json")
@Synchronized fun read(context: Context): JSONObject =
runCatching { JSONObject(file(context).readText()) }.getOrDefault(JSONObject()).let { it.optJSONObject("games") ?: JSONObject() }
@Synchronized fun set(context: Context, key: String, ids: List<String>) {
val games = read(context)
if (ids.isEmpty()) games.remove(key) else games.put(key, JSONArray(ids))
write(context, games)
}
@Synchronized fun drop(context: Context, id: String) {
val games = read(context)
for (k in games.keys().asSequence().toList()) {
val left = games.optJSONArray(k)?.let { a -> (0 until a.length()).map { a.optString(it) } }.orEmpty() - id
if (left.isEmpty()) games.remove(k) else games.put(k, JSONArray(left))
}
write(context, games)
}
private fun write(context: Context, games: JSONObject) {
val f = file(context)
runCatching {
f.parentFile?.mkdirs()
val tmp = File(f.path + ".tmp")
tmp.writeText(JSONObject().put("games", games).toString())
if (!tmp.renameTo(f)) { f.delete(); tmp.renameTo(f) }
}
}
}
// ---- the list, kept per game ----
private const val FINDINGS = "wincomp-findings.json"
/** What the list is read from, by modification time: a change means it is worked out again. */
internal fun signature(context: Context, game: Library.SteamGame): String {
val folder = game.gameFiles
val parts = listOfNotNull(
folder?.lastModified(),
folder?.let { SteamRedists.manifest(it, game.appId)?.lastModified() },
folder?.let { StoreGameSidecar.file(it).lastModified() },
folder?.let { File(it, "fuel.json").lastModified() },
SteamAppInfo.appinfoFile(context).lastModified(),
folder?.let { SteamMatch.get(context, it.path) }?.let { "${it.appId}/${it.certainty}" },
steamAppOf(context, game)?.let { SteamAppInfo.state(context, it) },
)
return parts.joinToString("|")
}
/** [game]'s list: the one kept when nothing it is read from changed, else worked out again and kept. Worker only. */
internal fun findings(context: Context, game: Library.SteamGame): List<WinCompSources.Finding> {
val key = appKey(game) ?: return emptyList()
val sig = signature(context, game)
val all = readFindings(context)
all.optJSONObject(key)?.takeIf { it.optString("sig") == sig }?.let { return decode(it.optJSONArray("findings")) }
val fresh = WinCompSources.forGame(context, game)
writeFindings(context, all.put(key, JSONObject().put("sig", sig).put("findings", encode(fresh))))
return fresh
}
/** The list kept for [appKey], without working anything out; null before the first time. */
fun cachedFindings(context: Context, appKey: String): List<WinCompSources.Finding>? =
readFindings(context).optJSONObject(appKey)?.let { decode(it.optJSONArray("findings")) }
@Synchronized
private fun readFindings(context: Context): JSONObject =
runCatching { JSONObject(AtomicFile(File(context.filesDir, FINDINGS)).readFully().toString(Charsets.UTF_8)) }.getOrDefault(JSONObject())
@Synchronized
private fun writeFindings(context: Context, json: JSONObject) {
val atomic = AtomicFile(File(context.filesDir, FINDINGS))
val out = atomic.startWrite()
try { out.write(json.toString().toByteArray()); atomic.finishWrite(out) } catch (e: Exception) { atomic.failWrite(out) }
}
private fun encode(list: List<WinCompSources.Finding>) = JSONArray().apply {
list.forEach { f ->
put(JSONObject().put("original", f.original).put("origin", f.origin.name).put("auto", f.auto).put("args", JSONArray(f.args))
.apply { f.component?.let { put("component", it) } })
}
}
private fun decode(arr: JSONArray?): List<WinCompSources.Finding> = if (arr == null) emptyList() else (0 until arr.length()).mapNotNull { i ->
val o = arr.optJSONObject(i) ?: return@mapNotNull null
val origin = runCatching { WinCompSources.Origin.valueOf(o.optString("origin")) }.getOrNull() ?: return@mapNotNull null
val args = o.optJSONArray("args")?.let { a -> (0 until a.length()).map { a.optString(it) } }.orEmpty()
WinCompSources.Finding(o.optString("component").takeIf { it.isNotEmpty() }, o.optString("original"), origin, o.optBoolean("auto"), args)
}
// ---- when ----
/** Before a launch from DroidDeck: nothing but a queued look (it waits for the session to end). */
fun beforeLaunch(context: Context, game: Library.SteamGame) = queueGame(context, game, asked = false)
/** After an install or an add: the user did it, so it does not wait on a metered network. */
fun refreshLater(context: Context, game: Library.SteamGame) = queueGame(context, game, asked = true)
/**
* Every game, whatever its source, at app start and after each session (Steam installs its
* games without a word to DroidDeck; older GOG and Epic installs get their list read once):
* only queued, one game at a time, behind anything asked for, under the same conditions. A
* game whose files did not change costs a look at their times.
*/
fun sweep(context: Context, games: List<Library.SteamGame>) {
// First every Steam app these games are that the client never saw, asked of Steam in one
// connection; then each game's list.
val apps = games.mapNotNull { steamAppOf(context, it) }.distinct()
if (apps.isNotEmpty() && SteamAppInfo.needLookup(context, apps).isNotEmpty()) enqueue(context, Job(null, null, false, seq.incrementAndGet(), pics = apps))
games.forEach { queueGame(context, it, asked = false) }
}
/** The Steam app whose list a game takes: a Steam game's own, or an added game's match. */
private fun steamAppOf(context: Context, game: Library.SteamGame): Int? =
if (game.library != Library.ADDED) game.appId.takeIf { it > 0 }
else game.gameFiles?.let { SteamMatch.get(context, it.path) }?.appId
/** The default network is there and validated (reaches the internet). */
fun isOnline(context: Context): Boolean {
val cm = context.getSystemService(ConnectivityManager::class.java) ?: return false
val caps = cm.getNetworkCapabilities(cm.activeNetwork ?: return false) ?: return false
return caps.hasCapability(android.net.NetworkCapabilities.NET_CAPABILITY_INTERNET) &&
caps.hasCapability(android.net.NetworkCapabilities.NET_CAPABILITY_VALIDATED)
}
/** Battery saver, thermal status moderate or worse, or a metered network. */
private fun constrained(context: Context): Boolean {
val power = context.getSystemService(PowerManager::class.java)
if (power?.isPowerSaveMode == true) return true
if (Build.VERSION.SDK_INT >= 29 && (power?.currentThermalStatus ?: 0) >= PowerManager.THERMAL_STATUS_MODERATE) return true
return context.getSystemService(ConnectivityManager::class.java)?.isActiveNetworkMetered == true
}
}
@@ -134,6 +134,9 @@ object DependencyDetector {
val lower = f.name.lowercase(Locale.US)
if (f.isDirectory) {
if (SKIP_DIR_RE.matches(lower)) continue
// Unity's <Game>_Data/Managed holds the game's own Mono assemblies, not a .NET
// Framework to install.
if (lower == "managed" && dir.name.endsWith("_Data", ignoreCase = true)) continue
val isRedistDir = insideRedist || REDIST_DIR_RE.matches(lower)
if (isRedistDir) consider(f, isDir = true, insideRedist = true)
// Aggressive coverage: descend redist containers fully, and otherwise sweep the game
@@ -157,6 +160,19 @@ object DependencyDetector {
}
}
/**
* The catalog components one installer stands for, by its path ("_CommonRedist/vcredist/2015/
* vc_redist.x64.exe", "DXSETUP.exe"): what a store's install list names. A VC++ installer
* with no year is the latest; empty for one that is none of ours.
*/
fun componentsFor(path: String): List<String> {
val p = path.replace('\\', '/').lowercase(Locale.US)
return classify(p, p.substringAfterLast('/'), isDir = false).map { if (it == VCREDIST_LOOSE) "vcredist2022" else it }
}
/** The name the catalog component [name] goes by, for a reason line. */
fun label(name: String): String = LABELS[name] ?: name
/** A loose VC++ installer resolves to the latest 14.x only if no specific vcredistYYYY was found. */
private fun resolveLooseVcredist(hits: LinkedHashMap<String, Recommendation>) {
val loose = hits.remove(VCREDIST_LOOSE) ?: return
@@ -0,0 +1,95 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import android.util.Log
import com.droiddeck.launcher.runtime.LinuxRuntime
import com.droiddeck.launcher.session.GameStorage
import com.droiddeck.launcher.session.SessionState
import java.io.File
/**
* A Steam game whose download finishes during a session, noticed by its appmanifest (a new one,
* or StateFlags turning to 4, fully installed) with a light look every five seconds while the
* session runs, and handed to [AutoComponents] as a single job that may run in the session, so its
* Windows components are in place when the user plays it right after the download.
*/
object SessionInstalls {
private const val TAG = "SessionInstalls"
private const val POLL_MS = 5_000L
/** One appmanifest as read: the app, its name and folder, whether it is fully installed. */
class Manifest(val appId: Int, val name: String, val installDir: String, val installed: Boolean, val steamapps: File)
private val APPID = Regex("\"appid\"\\s*\"(\\d+)\"", RegexOption.IGNORE_CASE)
private val NAME = Regex("\"name\"\\s*\"([^\"]*)\"", RegexOption.IGNORE_CASE)
private val DIR = Regex("\"installdir\"\\s*\"([^\"]*)\"", RegexOption.IGNORE_CASE)
private val STATE = Regex("\"StateFlags\"\\s*\"(\\d+)\"", RegexOption.IGNORE_CASE)
internal fun parse(file: File): Manifest? = runCatching {
val text = file.readText()
val id = APPID.find(text)?.groupValues?.get(1)?.toInt() ?: return null
Manifest(id, NAME.find(text)?.groupValues?.get(1).orEmpty(), DIR.find(text)?.groupValues?.get(1).orEmpty(),
STATE.find(text)?.groupValues?.get(1)?.toIntOrNull() == 4, file.parentFile!!)
}.getOrNull()
/** The libraries the session's client installs to, as host folders. */
internal fun steamappsDirs(context: Context): List<File> = listOfNotNull(
File(LinuxRuntime.rootDir(context), "root/.local/share/Steam/steamapps"),
GameStorage.effective(context)?.let { File(it.path, "steamapps") },
).filter { it.isDirectory }
/**
* Keeps what the libraries hold between looks: a manifest re-read only when its time changed.
* [poll] returns the apps that became fully installed since the last look (none on the first).
*/
internal class Watch(private val dirs: () -> List<File>) {
private val seen = HashMap<String, Pair<Long, Manifest?>>()
private var first = true
fun poll(): List<Manifest> {
val fresh = ArrayList<Manifest>()
for (dir in dirs()) for (f in dir.listFiles { x -> x.isFile && x.name.startsWith("appmanifest_") && x.name.endsWith(".acf") }.orEmpty()) {
val before = seen[f.path]
if (before != null && before.first == f.lastModified()) continue
val now = parse(f)
seen[f.path] = f.lastModified() to now
if (!first && now != null && now.installed && before?.second?.installed != true) fresh.add(now)
}
first = false
return fresh
}
}
/** A finished install as the Games tab would list it. */
internal fun gameOf(m: Manifest): Library.SteamGame =
Library.SteamGame(m.appId, m.name.ifEmpty { m.appId.toString() }, null, m.steamapps.parentFile?.path ?: "steam",
gameFiles = File(m.steamapps, "common/${m.installDir}"), source = Library.SOURCE_STEAM)
/**
* Looks every five seconds while the session runs, on a thread of its own at background
* priority; each finished install becomes one job for its Windows components.
*/
fun watch(context: Context) {
val app = context.applicationContext
Thread({
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_BACKGROUND)
val watch = Watch { steamappsDirs(app) }
try {
// The session is still starting when this is called: up to two minutes for it.
var waited = 0L
while (!SessionState.running && waited < 120_000L) { Thread.sleep(POLL_MS); waited += POLL_MS }
watch.poll()
while (SessionState.running) {
for (m in watch.poll()) {
Log.i(TAG, "wincomp: ${m.name} (${m.appId}) finished installing in the session")
AutoComponents.queueInSession(app, gameOf(m))
}
Thread.sleep(POLL_MS)
}
} catch (e: InterruptedException) {
} catch (e: Exception) {
Log.w(TAG, "install watch: ${e.message}")
}
}, "session-installs").apply { isDaemon = true }.start()
}
}
@@ -0,0 +1,202 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import android.util.AtomicFile
import android.util.Log
import com.droiddeck.launcher.runtime.LinuxRuntime
import org.json.JSONArray
import org.json.JSONObject
import java.io.DataInputStream
import java.io.File
import java.io.RandomAccessFile
import java.nio.ByteBuffer
import java.nio.ByteOrder
/**
* Steam's own data for an app, from the client in the runtime: its app-info cache
* (Steam/appcache/appinfo.vdf, the PICS data the client keeps for every app it has seen). What is
* read is which Steamworks Shared depots (the VC++, DirectX... redistributables of app 228980) the
* app's depots point at, and its name. Kept per appid in the app's files, so it works offline; an
* app the client has not seen is looked for again after a week.
*/
object SteamAppInfo {
private const val TAG = "SteamAppInfo"
private const val CACHE = "steam-appinfo.json"
private const val RETRY_AFTER_MS = 7L * 24 * 3600 * 1000
/** The Steamworks Shared app the redistributable depots belong to. */
const val SHARED_APP = "228980"
class Info(val name: String, val sharedDepots: List<String>)
fun appinfoFile(context: Context) = File(LinuxRuntime.rootDir(context), "root/.local/share/Steam/appcache/appinfo.vdf")
/** The shared depots for [appId]: from the cache, else the client's app-info file; null when unknown. */
@Synchronized
fun info(context: Context, appId: Int, appinfo: File = appinfoFile(context)): Info? {
val cacheFile = AtomicFile(File(context.filesDir, CACHE))
val cache = runCatching { JSONObject(cacheFile.readFully().toString(Charsets.UTF_8)) }.getOrDefault(JSONObject())
val now = System.currentTimeMillis()
val mtime = appinfo.lastModified()
val known = cache.optJSONObject(appId.toString())
known?.let { o ->
if (!o.optBoolean("miss")) return Info(o.optString("name"), o.optJSONArray("depots").strings())
// A miss is read again when the client has written its cache since (the user may have
// looked at the app in Steam), else a week later.
if (now - o.optLong("at") < RETRY_AFTER_MS && o.optLong("mtime") == mtime) return null
}
val read = runCatching { read(appinfo, appId) }.onFailure { Log.w(TAG, "appinfo: ${it.javaClass.simpleName}") }.getOrNull()
val entry = JSONObject().put("at", now).put("mtime", mtime)
// When Steam was last asked anonymously ([lookup]), kept across re-reads of the file.
known?.optLong("picsAt")?.takeIf { it > 0 }?.let { entry.put("picsAt", it) }
if (read == null) entry.put("miss", true) else entry.put("name", read.name).put("depots", JSONArray(read.sharedDepots))
cache.put(appId.toString(), entry)
val out = cacheFile.startWrite()
try { out.write(cache.toString().toByteArray()); cacheFile.finishWrite(out) } catch (e: Exception) { cacheFile.failWrite(out) }
return read
}
/** Steam asked anonymously ([SteamPics]); a test stands in for it. */
@androidx.annotation.VisibleForTesting
internal var pics: (List<Int>) -> Map<Int, Info?>? = { SteamPics.lookup(it) }
/**
* Of [appIds], those neither the client's cache (appinfo.vdf, read first: free) nor an earlier
* answer has, and that Steam was not asked about anonymously in the past week.
*/
fun needLookup(context: Context, appIds: List<Int>): List<Int> {
val wanted = appIds.distinct().filter { it > 0 && info(context, it) == null }
if (wanted.isEmpty()) return emptyList()
val cache = cache(context)
val now = System.currentTimeMillis()
return wanted.filter { now - (cache.optJSONObject(it.toString())?.optLong("picsAt") ?: 0L) >= RETRY_AFTER_MS }
}
/**
* [appIds] the client never saw, asked of Steam anonymously in one connection ([SteamPics], in
* batches) and every answer kept like the client's: an app's depots and name, or a miss tried
* again in a week. False when nothing needed asking or Steam could not be reached. Blocking.
*/
@Synchronized
fun lookup(context: Context, appIds: List<Int>): Boolean {
val need = needLookup(context, appIds)
if (need.isEmpty()) return false
val answers = pics(need) ?: return false
val cache = cache(context)
val now = System.currentTimeMillis()
val mtime = appinfoFile(context).lastModified()
for (id in need) {
val info = answers[id]
val entry = JSONObject().put("at", now).put("mtime", mtime).put("picsAt", now)
if (info == null) entry.put("miss", true) else entry.put("name", info.name).put("depots", JSONArray(info.sharedDepots)).put("source", "pics")
cache.put(id.toString(), entry)
}
val file = AtomicFile(File(context.filesDir, CACHE))
val out = file.startWrite()
try { out.write(cache.toString().toByteArray()); file.finishWrite(out) } catch (e: Exception) { file.failWrite(out) }
return true
}
/** What is kept for [appId]: "hit", "miss" or "none" (part of a game's list fingerprint). */
fun state(context: Context, appId: Int): String = cache(context).optJSONObject(appId.toString())?.let { if (it.optBoolean("miss")) "miss" else "hit" } ?: "none"
private fun cache(context: Context): JSONObject =
runCatching { JSONObject(AtomicFile(File(context.filesDir, CACHE)).readFully().toString(Charsets.UTF_8)) }.getOrDefault(JSONObject())
private fun JSONArray?.strings(): List<String> = if (this == null) emptyList() else (0 until length()).map { optString(it) }
/**
* [appId]'s entry in an appinfo.vdf (versions 27, 28 and 29, the last with its keys in a string
* table), or null when the file does not have it.
*/
internal fun read(file: File, appId: Int): Info? {
if (!file.isFile) return null
RandomAccessFile(file, "r").use { raf ->
val head = ByteBuffer.allocate(16).order(ByteOrder.LITTLE_ENDIAN)
raf.readFully(head.array(), 0, 8)
val magic = head.getInt(0)
val version = magic and 0xFF
if (magic ushr 8 != 0x075644 || version !in 0x27..0x29) return null
var strings: List<String>? = null
var pos = 8L
if (version >= 0x29) {
raf.readFully(head.array(), 8, 8)
val tableAt = head.getLong(8)
strings = readStrings(raf, tableAt)
pos = 16L
}
val fixed = if (version >= 0x28) 4 + 4 + 8 + 20 + 4 + 20 else 4 + 4 + 8 + 20 + 4
val entry = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN)
while (pos + 8 <= raf.length()) {
raf.seek(pos)
raf.readFully(entry.array(), 0, 8)
val id = entry.getInt(0)
val size = entry.getInt(4).toLong() and 0xFFFFFFFFL
if (id == 0) return null
if (id == appId) {
if (size > 64L * 1024 * 1024) return null
val body = ByteArray(size.toInt())
raf.readFully(body)
val kv = ByteBuffer.wrap(body, fixed, body.size - fixed).order(ByteOrder.LITTLE_ENDIAN)
val tree = readMap(kv, strings)
@Suppress("UNCHECKED_CAST")
val app = (tree["appinfo"] as? Map<String, Any>) ?: tree
return infoOf(app)
}
pos += 8 + size
}
return null
}
}
@Suppress("UNCHECKED_CAST")
internal fun infoOf(app: Map<String, Any>): Info {
val name = (app["common"] as? Map<String, Any>)?.get("name") as? String ?: ""
val depots = (app["depots"] as? Map<String, Any>).orEmpty()
val shared = depots.mapNotNull { (key, value) ->
val depot = value as? Map<String, Any> ?: return@mapNotNull null
key.takeIf { (depot["depotfromapp"] as? String) == SHARED_APP || (key in SteamRedists.DEPOTS && depot["sharedinstall"] != null) }
}
return Info(name, shared)
}
private fun readStrings(raf: RandomAccessFile, at: Long): List<String> {
raf.seek(at)
val count = Integer.reverseBytes(raf.readInt())
val rest = ByteArray((raf.length() - at - 4).toInt().coerceAtLeast(0))
raf.readFully(rest)
val out = ArrayList<String>(count)
var start = 0
for (i in rest.indices) {
if (out.size == count) break
if (rest[i] == 0.toByte()) { out.add(String(rest, start, i - start, Charsets.UTF_8)); start = i + 1 }
}
return out
}
private fun cstring(b: ByteBuffer): String {
val start = b.position()
while (b.get() != 0.toByte()) { }
val bytes = ByteArray(b.position() - start - 1)
b.position(start); b.get(bytes); b.get()
return String(bytes, Charsets.UTF_8)
}
/** One binary key-value map, to its end marker; keys by name, or by index into [strings]. */
private fun readMap(b: ByteBuffer, strings: List<String>?): Map<String, Any> {
val out = LinkedHashMap<String, Any>()
while (b.hasRemaining()) {
val type = b.get().toInt() and 0xFF
if (type == 0x08) return out
val key = if (strings != null) strings.getOrElse(b.int) { "" } else cstring(b)
when (type) {
0x00 -> out[key] = readMap(b, strings)
0x01 -> out[key] = cstring(b)
0x02 -> out[key] = b.int.toString()
0x03 -> out[key] = b.float.toString()
0x07, 0x0A -> out[key] = b.long.toString()
else -> return out
}
}
return out
}
}
@@ -0,0 +1,77 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import android.util.AtomicFile
import org.json.JSONObject
import java.io.File
/**
* Which Steam app a game added in DroidDeck is, kept once per game (by its folder) and used by
* everything that asks: its art ([AddedGameArt]) and its Windows components ([WinCompSources]),
* so the two never disagree.
*
* [Certainty.FILES]: the game's own files name the appid (steam_appid.txt, also under
* steam_settings/ or a parent folder; an emulator ini; Steam's appmanifest) - certain.
* [Certainty.NAME]: Steam's store search returned a game of the same name - a guess.
* [Certainty.NONE]: neither; a miss is tried again after a week.
*/
object SteamMatch {
enum class Certainty { FILES, NAME, NONE }
/** [source]: what gave the appid (GameIdentifier's source, or "store search"). */
class Match(val appId: Int?, val certainty: Certainty, val source: String, val checkedAt: Long)
private const val FILE = "steam-matches.json"
private const val RETRY_AFTER_MS = 7L * 24 * 3600 * 1000
private fun store(context: Context) = AtomicFile(File(context.filesDir, FILE))
@Synchronized
private fun all(context: Context): JSONObject =
runCatching { JSONObject(store(context).readFully().toString(Charsets.UTF_8)) }.getOrDefault(JSONObject())
@Synchronized
fun get(context: Context, key: String): Match? = all(context).optJSONObject(key)?.let { o ->
Match(
o.optInt("appId", 0).takeIf { it > 0 },
runCatching { Certainty.valueOf(o.optString("certainty")) }.getOrDefault(Certainty.NONE),
o.optString("source"), o.optLong("at"),
)
}
@Synchronized
fun put(context: Context, key: String, match: Match) {
val json = all(context).put(
key, JSONObject().put("certainty", match.certainty.name).put("source", match.source).put("at", match.checkedAt)
.apply { match.appId?.let { put("appId", it) } },
)
val atomic = store(context)
val out = atomic.startWrite()
try { out.write(json.toString().toByteArray()); atomic.finishWrite(out) } catch (e: Exception) { atomic.failWrite(out) }
}
/**
* The match for the game in [key] (its folder), [exe] its exe and [name] its title: its files
* first, always read again; else the one kept (a name match stays, a miss for a week); else
* [legacy], an appid an earlier build's art lookup kept; else [search] on the name, when given.
* Blocking ([search] asks the network).
*/
fun resolve(context: Context, key: String, exe: File, name: String, legacy: File? = null, search: ((String) -> Int?)? = null): Match {
val now = System.currentTimeMillis()
val identity = runCatching { GameIdentifier.identify(exe) }.getOrNull()
identity?.appId?.let { id ->
return Match(id, Certainty.FILES, identity.source.name.lowercase(), now).also { put(context, key, it) }
}
val known = get(context, key)
if (known != null && known.certainty == Certainty.NAME) return known
if (known != null && known.certainty == Certainty.NONE && now - known.checkedAt < RETRY_AFTER_MS) return known
legacy?.takeIf { it.isFile }?.let { runCatching { it.readText().trim().toInt() }.getOrNull() }?.takeIf { it > 0 }?.let { id ->
return Match(id, Certainty.NAME, "store search", now).also { put(context, key, it) }
}
if (search == null) return known?.takeIf { it.certainty != Certainty.FILES } ?: Match(null, Certainty.NONE, "", 0)
val id = runCatching { search(name) }.getOrNull()
val match = if (id != null && id > 0) Match(id, Certainty.NAME, "store search", now) else Match(null, Certainty.NONE, "", now)
put(context, key, match)
return match
}
}
@@ -0,0 +1,334 @@
package com.droiddeck.launcher.frontend
import android.util.Log
import androidx.annotation.VisibleForTesting
import org.json.JSONObject
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.DataInputStream
import java.io.IOException
import java.io.InputStream
import java.io.OutputStream
import java.net.HttpURLConnection
import java.net.URL
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.security.SecureRandom
import java.util.zip.GZIPInputStream
import javax.net.ssl.SSLSocket
import javax.net.ssl.SSLSocketFactory
/**
* Steam's own app info (PICS) for apps the client in the runtime has never seen, asked of Steam
* anonymously: a connection to one of Steam's connection managers (the list from Steam's public
* ISteamDirectory/GetCMListForConnect, the WebSocket transport Steam's web clients use), an
* anonymous logon - no account, nothing of the user's Steam sign-in - a request for the apps'
* access tokens and then their product info, the app KeyValues read for their Steamworks Shared
* depots and name, and the connection closed. What steamcmd's anonymous `app_info_print` sees.
*/
object SteamPics {
private const val TAG = "SteamPics"
private const val CM_LIST = "https://api.steampowered.com/ISteamDirectory/GetCMListForConnect/v1/?cellid=0&format=json"
/** At most this many apps per request. */
const val BATCH = 25
private const val PROTO = 0x80000000.toInt()
private const val EMSG_MULTI = 1
private const val EMSG_LOGON = 5514
private const val EMSG_LOGON_RESPONSE = 751
private const val EMSG_LOGOFF = 706
private const val EMSG_TOKENS_REQUEST = 8905
private const val EMSG_TOKENS_RESPONSE = 8906
private const val EMSG_INFO_REQUEST = 8903
private const val EMSG_INFO_RESPONSE = 8904
private const val PROTOCOL_VERSION = 65580
/** An anonymous user's SteamID: universe Public, account type AnonUser, instance 0. */
private const val ANON_STEAMID = (1L shl 56) or (10L shl 52)
/**
* [appIds]' info, one connection for all of them in batches of [BATCH]: each known app's
* info, null for an app Steam does not know. Null when Steam could not be reached. Blocking.
*/
fun lookup(appIds: List<Int>): Map<Int, SteamAppInfo.Info?>? {
if (appIds.isEmpty()) return emptyMap()
val host = runCatching { cmHost() }.getOrNull() ?: return null
return try {
Wss(host, 20_000).use { ws ->
ws.send(message(EMSG_LOGON, header(ANON_STEAMID, 0), Proto().varint(1, PROTOCOL_VERSION.toLong()).varint(7, 16).string(6, "english").bytes()))
var steamId = 0L
var session = 0
// The logon's answer (other messages before it are passed over).
loop@ while (true) for ((emsg, hdr, body) in messages(ws.receive() ?: return null)) if (emsg == EMSG_LOGON_RESPONSE) {
val h = fields(hdr); val b = fields(body)
if ((b[1]?.firstOrNull() as? Long ?: 0L) != 1L) { Log.w(TAG, "anonymous logon refused: ${b[1]}"); return null }
steamId = h[1]?.firstOrNull() as? Long ?: 0L
session = (h[2]?.firstOrNull() as? Long ?: 0L).toInt()
break@loop
}
val out = HashMap<Int, SteamAppInfo.Info?>()
for (batch in batches(appIds)) {
val tokens = HashMap<Int, Long>()
ws.send(message(EMSG_TOKENS_REQUEST, header(steamId, session), Proto().apply { batch.forEach { varint(2, it.toLong()) } }.bytes()))
tokens@ while (true) for ((emsg, _, body) in messages(ws.receive() ?: return null)) if (emsg == EMSG_TOKENS_RESPONSE) {
for (t in fields(body)[3].orEmpty()) {
val f = fields(t as ByteArray)
val id = (f[1]?.firstOrNull() as? Long)?.toInt() ?: continue
tokens[id] = f[2]?.firstOrNull() as? Long ?: 0L
}
break@tokens
}
ws.send(message(EMSG_INFO_REQUEST, header(steamId, session), Proto().apply {
batch.forEach { id -> bytes(2, Proto().varint(1, id.toLong()).apply { tokens[id]?.takeIf { it != 0L }?.let { varint(2, it) } }.bytes()) }
varint(3, 0)
}.bytes()))
var pending = true
while (pending) for ((emsg, _, body) in messages(ws.receive() ?: return null)) if (emsg == EMSG_INFO_RESPONSE) {
val r = parseInfoResponse(body)
out.putAll(r.apps)
r.unknown.forEach { out[it] = null }
pending = r.pending
}
batch.forEach { if (it !in out) out[it] = null }
}
runCatching { ws.send(message(EMSG_LOGOFF, header(steamId, session), ByteArray(0))) }
Log.i(TAG, "steam pics: ${out.size} apps, ${out.values.count { it?.sharedDepots?.isNotEmpty() == true }} with shared redists")
out
}
} catch (e: Exception) {
Log.w(TAG, "steam pics: ${e.javaClass.simpleName}: ${e.message}")
null
}
}
/** [appIds] in requests of at most [BATCH], each once. */
fun batches(appIds: List<Int>): List<List<Int>> = appIds.distinct().chunked(BATCH)
/** A connection manager for the WebSocket transport, least loaded first as Steam lists them. */
private fun cmHost(): String? {
val c = URL(CM_LIST).openConnection() as HttpURLConnection
c.connectTimeout = 10_000; c.readTimeout = 10_000
val body = try { c.inputStream.bufferedReader().readText() } finally { c.disconnect() }
val list = JSONObject(body).optJSONObject("response")?.optJSONArray("serverlist") ?: return null
return (0 until list.length()).mapNotNull { list.optJSONObject(it) }
.firstOrNull { it.optString("type") == "websockets" }?.optString("endpoint")?.substringBefore(':')
}
// ---- the product info answer ----
class InfoResponse(val apps: Map<Int, SteamAppInfo.Info>, val unknown: List<Int>, val pending: Boolean)
/** A CMsgClientPICSProductInfoResponse: each app's KeyValues read, the unknown ones, whether more follows. */
@VisibleForTesting
internal fun parseInfoResponse(body: ByteArray): InfoResponse {
val f = fields(body)
val apps = HashMap<Int, SteamAppInfo.Info>()
for (a in f[1].orEmpty()) {
val app = fields(a as ByteArray)
val id = (app[1]?.firstOrNull() as? Long)?.toInt() ?: continue
val buffer = app[5]?.firstOrNull() as? ByteArray ?: continue
val text = String(buffer, Charsets.UTF_8).trimEnd('\u0000')
infoFromKeyValues(text)?.let { apps[id] = it }
}
val unknown = f[2].orEmpty().mapNotNull { (it as? Long)?.toInt() } + f[2].orEmpty().filterIsInstance<ByteArray>().flatMap { packed(it) }
return InfoResponse(apps, unknown, (f[6]?.firstOrNull() as? Long ?: 0L) != 0L)
}
/** An app's KeyValues text ("appinfo" { "common" {...} "depots" {...} }) to its name and shared depots. */
@VisibleForTesting
internal fun infoFromKeyValues(text: String): SteamAppInfo.Info? {
// The buffer ends in a NUL, kept or not by whoever handed it over.
val tree = runCatching { KeyValues(text.trimEnd('\u0000', ' ', '\n', '\r', '\t')).read() }.getOrNull() ?: return null
@Suppress("UNCHECKED_CAST")
val app = tree["appinfo"] as? Map<String, Any> ?: return null
return SteamAppInfo.infoOf(app)
}
/** Valve's text KeyValues: quoted keys and values, nested braces. */
private class KeyValues(private val s: String) {
private var i = 0
fun read(): Map<String, Any> = map(top = true)
private fun map(top: Boolean): Map<String, Any> {
val out = LinkedHashMap<String, Any>()
while (true) {
skip()
if (i >= s.length) { if (top) return out else throw IOException("unclosed") }
if (s[i] == '}') { i++; return out }
val key = token()
skip()
if (i < s.length && s[i] == '{') { i++; out[key] = map(top = false) } else out[key] = token()
}
}
private fun skip() {
while (i < s.length) {
if (s[i].isWhitespace() || s[i] == '\u0000') i++
else if (s.startsWith("//", i)) { while (i < s.length && s[i] != '\n') i++ }
else break
}
}
private fun token(): String {
if (i < s.length && s[i] == '"') {
i++
val b = StringBuilder()
while (i < s.length && s[i] != '"') {
if (s[i] == '\\' && i + 1 < s.length) { i++; b.append(when (s[i]) { 'n' -> '\n'; 't' -> '\t'; else -> s[i] }) } else b.append(s[i])
i++
}
i++
return b.toString()
}
val start = i
while (i < s.length && !s[i].isWhitespace() && s[i] != '{' && s[i] != '}') i++
if (start == i) throw IOException("token expected at $i")
return s.substring(start, i)
}
}
// ---- messages ----
private fun header(steamId: Long, session: Int) = Proto().fixed64(1, steamId).varint(2, session.toLong()).bytes()
private fun message(emsg: Int, header: ByteArray, body: ByteArray): ByteArray =
ByteBuffer.allocate(8 + header.size + body.size).order(ByteOrder.LITTLE_ENDIAN)
.putInt(emsg or PROTO).putInt(header.size).put(header).put(body).array()
/** One message off the wire as (emsg, header, body); a Multi unpacked (gunzipped when it says so) into the ones inside. */
@VisibleForTesting
internal fun messages(data: ByteArray): List<Triple<Int, ByteArray, ByteArray>> {
if (data.size < 8) return emptyList()
val b = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)
val raw = b.int
if (raw and PROTO == 0) return emptyList()
val emsg = raw and PROTO.inv()
val hl = b.int
if (hl < 0 || 8 + hl > data.size) return emptyList()
val header = data.copyOfRange(8, 8 + hl)
val body = data.copyOfRange(8 + hl, data.size)
if (emsg != EMSG_MULTI) return listOf(Triple(emsg, header, body))
val f = fields(body)
var inner = f[2]?.firstOrNull() as? ByteArray ?: return emptyList()
if ((f[1]?.firstOrNull() as? Long ?: 0L) > 0) inner = GZIPInputStream(ByteArrayInputStream(inner)).use { it.readBytes() }
val out = ArrayList<Triple<Int, ByteArray, ByteArray>>()
val ib = ByteBuffer.wrap(inner).order(ByteOrder.LITTLE_ENDIAN)
while (ib.remaining() >= 4) {
val len = ib.int
if (len < 0 || len > ib.remaining()) break
val one = ByteArray(len).also { ib.get(it) }
out.addAll(messages(one))
}
return out
}
// ---- protobuf, the little needed ----
/** Writes protobuf fields. */
@VisibleForTesting
internal class Proto {
private val out = ByteArrayOutputStream()
private fun raw(v: Long) { var x = v; while (true) { val b = (x and 0x7F).toInt(); x = x ushr 7; if (x == 0L) { out.write(b); return }; out.write(b or 0x80) } }
fun varint(field: Int, v: Long) = apply { raw((field shl 3).toLong()); raw(v) }
fun fixed64(field: Int, v: Long) = apply { raw(((field shl 3) or 1).toLong()); for (k in 0 until 8) out.write(((v ushr (8 * k)) and 0xFF).toInt()) }
fun bytes(field: Int, b: ByteArray) = apply { raw(((field shl 3) or 2).toLong()); raw(b.size.toLong()); out.write(b) }
fun string(field: Int, s: String) = bytes(field, s.toByteArray())
fun bytes(): ByteArray = out.toByteArray()
}
/** Reads protobuf fields: number to values (Long for varint and fixed, ByteArray for length-delimited). */
@VisibleForTesting
internal fun fields(b: ByteArray): Map<Int, List<Any>> {
val out = HashMap<Int, MutableList<Any>>()
var i = 0
fun varint(): Long { var v = 0L; var s = 0; while (true) { val c = b[i++].toInt() and 0xFF; v = v or ((c and 0x7F).toLong() shl s); s += 7; if (c < 0x80) return v } }
while (i < b.size) {
val key = varint()
val field = (key ushr 3).toInt()
val value: Any = when ((key and 7).toInt()) {
0 -> varint()
1 -> { var v = 0L; for (k in 0 until 8) v = v or ((b[i + k].toLong() and 0xFF) shl (8 * k)); i += 8; v }
5 -> { var v = 0L; for (k in 0 until 4) v = v or ((b[i + k].toLong() and 0xFF) shl (8 * k)); i += 4; v }
2 -> { val n = varint().toInt(); val v = b.copyOfRange(i, i + n); i += n; v }
else -> throw IOException("wire type ${key and 7}")
}
out.getOrPut(field) { ArrayList() }.add(value)
}
return out
}
private fun packed(b: ByteArray): List<Int> {
val out = ArrayList<Int>(); var i = 0
while (i < b.size) { var v = 0L; var s = 0; while (true) { val c = b[i++].toInt() and 0xFF; v = v or ((c and 0x7F).toLong() shl s); s += 7; if (c < 0x80) break }; out.add(v.toInt()) }
return out
}
// ---- WebSocket over TLS, binary frames ----
private class Wss(host: String, timeoutMs: Int) : AutoCloseable {
// Made with the host name, so it goes out as SNI.
private val socket: SSLSocket = (SSLSocketFactory.getDefault().createSocket(host, 443) as SSLSocket).apply { soTimeout = timeoutMs }
private val out: OutputStream
private val input: DataInputStream
init {
// Host name checked against the certificate, as HTTPS does.
socket.sslParameters = socket.sslParameters.apply { endpointIdentificationAlgorithm = "HTTPS" }
socket.startHandshake()
out = socket.outputStream
input = DataInputStream(socket.inputStream)
val key = ByteArray(16).also { SecureRandom().nextBytes(it) }
out.write(("GET /cmsocket/ HTTP/1.1\r\nHost: $host\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" +
"Sec-WebSocket-Key: ${android.util.Base64.encodeToString(key, android.util.Base64.NO_WRAP)}\r\nSec-WebSocket-Version: 13\r\n\r\n").toByteArray())
out.flush()
val status = line(input) ?: throw IOException("no upgrade response")
if (!status.contains(" 101 ")) throw IOException("upgrade refused: $status")
while (true) { val l = line(input) ?: break; if (l.isEmpty()) break }
}
private fun line(s: InputStream): String? {
val buf = ByteArrayOutputStream()
while (true) {
val b = s.read()
if (b < 0) return if (buf.size() == 0) null else buf.toString("UTF-8")
if (b == '\n'.code) return buf.toString("UTF-8").trimEnd('\r')
buf.write(b)
}
}
fun send(payload: ByteArray) {
val mask = ByteArray(4).also { SecureRandom().nextBytes(it) }
val frame = ByteArrayOutputStream()
frame.write(0x82)
when {
payload.size < 126 -> frame.write(0x80 or payload.size)
payload.size < 65536 -> { frame.write(0x80 or 126); frame.write(payload.size ushr 8); frame.write(payload.size and 0xFF) }
else -> { frame.write(0x80 or 127); for (k in 7 downTo 0) frame.write(((payload.size.toLong() ushr (8 * k)) and 0xFF).toInt()) }
}
frame.write(mask)
for (k in payload.indices) frame.write((payload[k].toInt() xor mask[k % 4].toInt()) and 0xFF)
out.write(frame.toByteArray()); out.flush()
}
/** The next binary message (fragments joined); null on close. */
fun receive(): ByteArray? {
val message = ByteArrayOutputStream()
while (true) {
val b0 = input.read(); if (b0 < 0) return null
val b1 = input.read(); if (b1 < 0) return null
val fin = b0 and 0x80 != 0
val opcode = b0 and 0x0F
var len = (b1 and 0x7F).toLong()
if (len == 126L) len = input.readUnsignedShort().toLong() else if (len == 127L) len = input.readLong()
if (len > 32L * 1024 * 1024) throw IOException("frame too large")
val data = ByteArray(len.toInt()).also { input.readFully(it) }
when (opcode) {
0x8 -> return null
0x9 -> { out.write(byteArrayOf(0x8A.toByte(), 0x80.toByte(), 0, 0, 0, 0)); out.flush() }
0x2, 0x0 -> { message.write(data); if (fin) return message.toByteArray() }
else -> {}
}
}
}
override fun close() {
runCatching { out.write(byteArrayOf(0x88.toByte(), 0x80.toByte(), 0, 0, 0, 0)); out.flush() }
runCatching { socket.close() }
}
}
}
@@ -0,0 +1,98 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import android.util.AtomicFile
import android.util.Log
import androidx.annotation.VisibleForTesting
import com.droiddeck.launcher.stores.Store
import com.droiddeck.launcher.stores.StoreGameSidecar
import com.droiddeck.launcher.stores.epic.EpicApiClient
import com.droiddeck.launcher.stores.epic.EpicCredentialStore
import com.droiddeck.launcher.stores.epic.EpicDownloadManager
import com.droiddeck.launcher.stores.gog.GogAuth
import com.droiddeck.launcher.stores.gog.GogDownloadManager
import org.json.JSONObject
import java.io.File
/**
* For a GOG or Epic game installed before its sidecar kept the store's list (GOG's build
* `dependencies`, Epic's manifest prerequisite): the list read once from the store's build
* manifest with the saved sign-in - the manifest alone, never a game file - and kept in the sidecar
* the way an install keeps it. Runs on the components' worker ([AutoComponents]). Without a
* sign-in or a network it is skipped quietly and tried again a week later.
*/
object StoreListBackfill {
private const val TAG = "StoreListBackfill"
private const val FILE = "store-list-backfill.json"
private const val RETRY_AFTER_MS = 7L * 24 * 3600 * 1000
/** The store's list for the game in a sidecar: GOG's dependency ids, or Epic's prerequisite {name, path}; null when it could not be had. */
fun interface Fetcher {
fun fetch(context: Context, sidecar: StoreGameSidecar): Map<String, String>?
}
/** The store itself, with the saved sign-in. */
internal val storeFetcher = Fetcher { context, sidecar -> fromStore(context, sidecar) }
@VisibleForTesting internal var fetcher: Fetcher = storeFetcher
/** Whether [sidecar] was written before the list was kept. */
/**
* Whether [sidecar] has no list read whole: written before the list was kept, or by a build
* that could have written an empty one for a manifest it did not read right.
*/
fun needs(sidecar: StoreGameSidecar): Boolean =
(sidecar.store == Store.GOG || sidecar.store == Store.EPIC) && WinCompSources.LIST_READ !in sidecar.extra
/**
* The list for the game in [folder], fetched and kept when its sidecar lacks one and it was not
* tried in the past week. True when the sidecar changed. Blocking (network).
*/
@Synchronized
fun fill(context: Context, folder: File): Boolean {
val sidecar = StoreGameSidecar.read(folder)?.takeIf { it.isInstalled && needs(it) } ?: return false
val tries = runCatching { JSONObject(AtomicFile(File(context.filesDir, FILE)).readFully().toString(Charsets.UTF_8)) }.getOrDefault(JSONObject())
val now = System.currentTimeMillis()
if (now - tries.optLong(folder.path, 0L) < RETRY_AFTER_MS) return false
// Offline is no try: it waits for a network rather than a week.
if (!AutoComponents.online(context)) return false
val list = runCatching { fetcher.fetch(context, sidecar) }.getOrNull()
if (list == null) {
tries.put(folder.path, now)
val atomic = AtomicFile(File(context.filesDir, FILE))
val out = atomic.startWrite()
try { out.write(tries.toString().toByteArray()); atomic.finishWrite(out) } catch (e: Exception) { atomic.failWrite(out) }
Log.i(TAG, "${sidecar.title}: the ${sidecar.store.label} list could not be had; again in a week")
return false
}
// Read fresh and written whole, as an install writes it: only the list is added.
val current = StoreGameSidecar.read(folder) ?: return false
current.copy(extra = current.extra + list + (WinCompSources.LIST_READ to "1")).write(folder)
Log.i(TAG, "backfill: ${sidecar.title} " + when (sidecar.store) {
Store.GOG -> "gog deps=[${list[WinCompSources.GOG_DEPENDENCIES].orEmpty()}]"
else -> "epic prereq=" + (list[WinCompSources.EPIC_PREREQ_NAME].orEmpty().ifEmpty { list[WinCompSources.EPIC_PREREQ_PATH].orEmpty() }.ifEmpty { "none" })
})
return true
}
private fun fromStore(context: Context, sidecar: StoreGameSidecar): Map<String, String>? {
return when (sidecar.store) {
Store.GOG -> {
val token = GogAuth.validToken(context) ?: return null
GogDownloadManager.fetchDependencies(sidecar.id, sidecar.installVersion, token)
?.let { mapOf(WinCompSources.GOG_DEPENDENCIES to it.joinToString(",")) }
}
Store.EPIC -> {
val token = EpicCredentialStore.getValidAccessToken(context) ?: return null
val namespace = sidecar.extra["namespace"].orEmpty()
val catalogItemId = sidecar.extra["catalogItemId"].orEmpty()
if (namespace.isEmpty() || catalogItemId.isEmpty()) return null
val json = EpicApiClient.getManifestApiJson(token, namespace, catalogItemId, sidecar.extra["appName"] ?: sidecar.id) ?: return null
EpicDownloadManager.fetchPrereq(json)?.let { (name, path) ->
mapOf(WinCompSources.EPIC_PREREQ_NAME to name, WinCompSources.EPIC_PREREQ_PATH to path)
}
}
else -> null
}
}
}
@@ -0,0 +1,160 @@
package com.droiddeck.launcher.frontend
import android.content.Context
import com.droiddeck.launcher.stores.Store
import com.droiddeck.launcher.stores.StoreGameSidecar
import org.json.JSONObject
import java.io.File
import java.util.Locale
/**
* Which Windows components a game needs, from the list its source keeps - the source badge decides
* which list is trusted:
*
* - **Steam**: the Steamworks Shared redistributables Steam installs with the game (its
* appmanifest's SharedDepots, else the client's app info) - turned on by themselves.
* - **GOG**: the build manifest's `dependencies`, kept in the sidecar at install - on by themselves.
* - **Epic Games**: the manifest's prerequisite (usually UE4/UE5's prerequisite setup: VC++ and the
* DirectX June 2010 runtime), kept in the sidecar at install - on by themselves.
* - **Amazon Games**: fuel.json's PostInstall installers - on by themselves.
* - **Custom** whose files name a Steam appid ([SteamMatch], certain): Steam's list for that app -
* on by themselves. Matched by name only: the same list as Recommended. No match: the folder scan
* ([DependencyDetector]) as Recommended.
*
* Every store entry is mapped to catalog names here, in one place; one that maps to nothing is a
* suggestion under its own name. The folder scan is offered as Recommended for every game.
*/
object WinCompSources {
/** Whose list a finding came from; the reason line says it. */
enum class Origin { STEAM, GOG, EPIC, AMAZON, STEAM_NAME, FOLDER }
/**
* One thing a list names: [component] the catalog name it maps to (null: none, a suggestion),
* [original] what the list called it, [auto] whether it is turned on by itself, [args] what
* the reason line names (Steam: the label, the game, the appid).
*/
data class Finding(val component: String?, val original: String, val origin: Origin, val auto: Boolean, val args: List<String> = emptyList())
// ---- mapping tables: a store's names -> catalog components ----
private val VC_YEAR = mapOf(
"2005" to "vcredist2005", "2008" to "vcredist2008", "2010" to "vcredist2010", "2012" to "vcredist2012",
"2013" to "vcredist2013", "2015" to "vcredist2015", "2017" to "vcredist2015", "2019" to "vcredist2019", "2022" to "vcredist2022",
)
private val DOTNET = mapOf(
"35" to "dotnet35", "40" to "dotnet40", "45" to "dotnet45", "451" to "dotnet452", "452" to "dotnet452",
"46" to "dotnet46", "461" to "dotnet46", "462" to "dotnet46", "47" to "dotnet472", "471" to "dotnet472",
"472" to "dotnet472", "48" to "dotnet48", "481" to "dotnet48",
)
/**
* A GOG dependency id ("MSVC2017_x64", "DirectX", "DOTNET48", "PhysX", "OpenAL", "XNA4") to its
* components; empty for one that is none of ours (ISI scripts, GOG's own helpers).
*/
fun gog(id: String): List<String> {
val key = id.uppercase(Locale.US).replace(Regex("[^A-Z0-9]"), "")
Regex("""^(?:MSVC|VCREDIST|VC)(\d{4})""").find(key)?.let { m -> return listOfNotNull(VC_YEAR[m.groupValues[1]]) }
Regex("""^(?:DOTNET|NETFRAMEWORK|DOTNETFRAMEWORK)(\d{2,3})""").find(key)?.let { m -> return listOfNotNull(DOTNET[m.groupValues[1]]) }
return when {
key.startsWith("DIRECTX") || key.startsWith("DX9") || key == "DXSETUP" || key.startsWith("D3DX") -> listOf("d3dx9")
key.startsWith("PHYSX") -> listOf("physx")
key.startsWith("OPENAL") -> listOf("oalinst")
key == "XNA31" || key == "XNA3" -> listOf("xna31")
key.startsWith("XNA") -> listOf("xna40")
key.startsWith("XLIVE") || key.startsWith("GFWL") -> listOf("XLiveRedist")
key.startsWith("XACT") -> listOf("xact")
else -> emptyList()
}
}
/** Unreal's prerequisite setup (UE4PrereqSetup_x64.exe, UEPrereqSetup_x64.exe): VC++ 2015-2022 and DirectX June 2010. */
private val UE_PREREQ = Regex("""(?i)ue\d?prereqsetup""")
/** An Epic prerequisite ([path] the installer it runs, [name] what it is called) to its components. */
fun epic(path: String, name: String): List<String> = when {
UE_PREREQ.containsMatchIn(path) || name.contains("UE4 Prerequisites", true) || name.contains("UE5 Prerequisites", true) ||
name.contains("Unreal Engine Prerequisites", true) -> listOf("vcredist2022", "d3dx9")
else -> DependencyDetector.componentsFor(path).ifEmpty { if (path.isEmpty()) gog(name) else emptyList() }
}
/** An installer a store runs after an install (Amazon's PostInstall "Command") to its components. */
fun installer(path: String): List<String> = if (UE_PREREQ.containsMatchIn(path)) listOf("vcredist2022", "d3dx9") else DependencyDetector.componentsFor(path)
/** Steamworks Shared depot ids to (component, Steam's label). */
fun steamDepots(ids: List<String>): List<Pair<String, String>> = ids.mapNotNull { SteamRedists.DEPOTS[it] }.distinctBy { it.first }
// ---- what each store left at install ----
/** Sidecar keys the store installs fill. */
const val GOG_DEPENDENCIES = "dependencies"
const val EPIC_PREREQ_PATH = "prereqPath"
const val EPIC_PREREQ_NAME = "prereqName"
/** Set beside the list when it was read whole (an empty list then means the store lists none). */
const val LIST_READ = "listRead"
/** fuel.json's PostInstall commands. */
fun amazonPostInstall(folder: File): List<String> = runCatching {
val arr = JSONObject(File(folder, "fuel.json").readText()).optJSONArray("PostInstall") ?: return emptyList()
(0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("Command")?.takeIf { c -> c.isNotEmpty() } }
}.getOrDefault(emptyList())
// ---- per game ----
/** Steam's list for [appId]: the local appmanifest's SharedDepots, else the client's app info. */
fun steamList(context: Context, gameDir: File?, appId: Int, gameName: String, auto: Boolean): List<Finding> {
val local = SteamRedists.detect(gameDir, appId)
val pairs = if (local.isNotEmpty()) local.map { it.componentName to it.label }
else SteamAppInfo.info(context, appId)?.let { steamDepots(it.sharedDepots) }.orEmpty()
return pairs.map { (component, label) ->
Finding(component, label, if (auto) Origin.STEAM else Origin.STEAM_NAME, auto, listOf(label, gameName, appId.toString()))
}
}
/** Everything known for [game], blocking (files; the client's app info; no network). */
fun forGame(context: Context, game: Library.SteamGame): List<Finding> {
val folder = game.gameFiles
val out = ArrayList<Finding>()
val store = Store.byId(game.source)
val sidecar = folder?.let { StoreGameSidecar.read(it) }
when {
game.library != Library.ADDED -> if (game.appId > 0) out += steamList(context, folder, game.appId, game.name, auto = true)
store == Store.GOG -> sidecar?.extra?.get(GOG_DEPENDENCIES)?.split(',')?.map { it.trim() }?.filter { it.isNotEmpty() }?.forEach { id ->
val parts = gog(id)
if (parts.isEmpty()) out += Finding(null, id, Origin.GOG, false)
parts.forEach { out += Finding(it, id, Origin.GOG, true) }
}
store == Store.EPIC -> {
val path = sidecar?.extra?.get(EPIC_PREREQ_PATH).orEmpty()
val name = sidecar?.extra?.get(EPIC_PREREQ_NAME).orEmpty()
if (path.isNotEmpty() || name.isNotEmpty()) {
val parts = epic(path, name)
val original = name.ifEmpty { path.substringAfterLast('/') }
if (parts.isEmpty()) out += Finding(null, original, Origin.EPIC, false)
parts.forEach { out += Finding(it, original, Origin.EPIC, true) }
}
}
store == Store.AMAZON -> folder?.let { amazonPostInstall(it) }?.forEach { command ->
val parts = installer(command)
val original = command.replace('\\', '/').substringAfterLast('/')
if (parts.isEmpty()) out += Finding(null, original, Origin.AMAZON, false)
parts.forEach { out += Finding(it, original, Origin.AMAZON, true) }
}
else -> {}
}
// A game added in DroidDeck (Custom, or a store's) that is a Steam app: Steam's list for it,
// on by itself when its files say so, else only recommended.
if (game.library == Library.ADDED && folder != null) SteamMatch.get(context, folder.path)?.let { match ->
val id = match.appId ?: return@let
when (match.certainty) {
SteamMatch.Certainty.FILES -> out += steamList(context, folder, id, game.name, auto = true)
SteamMatch.Certainty.NAME -> out += steamList(context, folder, id, game.name, auto = false)
else -> {}
}
}
// The folder scan, for every game: what it bundles, as Recommended.
folder?.let { DependencyDetector.detect(it) }?.forEach { rec ->
out += Finding(rec.componentName, rec.reason, Origin.FOLDER, false, listOf(rec.reason, rec.kind.name))
}
return out
}
}