From c5d5ac904eb7425d4a5a3836b20ec8c2d059dbae Mon Sep 17 00:00:00 2001 From: xXJsonDeruloXx Date: Thu, 8 Oct 2026 15:47:27 -0400 Subject: [PATCH] feat: DroidDeck Proton (Auto) assembles a per-game Proton from recipes --- app/src/main/assets/recipes/default.json | 4 + .../launcher/session/ComponentsManager.kt | 59 +-- .../launcher/session/DdProtonRecipes.kt | 157 ++++++++ .../launcher/session/SessionFiles.kt | 1 + .../launcher/session/SessionService.kt | 5 + .../launcher/session/DdProtonRecipesTest.kt | 138 +++++++ docs/development/game-environment.md | 45 ++- .../overlay/usr/local/bin/droiddeck-game-env | 40 +++ .../overlay/usr/local/bin/droiddeck-recipe | 338 ++++++++++++++++++ .../overlay/usr/local/bin/steam-compatibility | 26 +- tools/tests/test_game_environment.py | 35 +- tools/tests/test_recipe.py | 332 +++++++++++++++++ 12 files changed, 1148 insertions(+), 32 deletions(-) create mode 100644 app/src/main/assets/recipes/default.json create mode 100644 app/src/main/java/com/droiddeck/launcher/session/DdProtonRecipes.kt create mode 100644 app/src/test/java/com/droiddeck/launcher/session/DdProtonRecipesTest.kt create mode 100644 tools/linuxfs/overlay/usr/local/bin/droiddeck-recipe create mode 100644 tools/tests/test_recipe.py diff --git a/app/src/main/assets/recipes/default.json b/app/src/main/assets/recipes/default.json new file mode 100644 index 0000000000..7dd9d2eecf --- /dev/null +++ b/app/src/main/assets/recipes/default.json @@ -0,0 +1,4 @@ +{ + "version": 1, + "games": {} +} diff --git a/app/src/main/java/com/droiddeck/launcher/session/ComponentsManager.kt b/app/src/main/java/com/droiddeck/launcher/session/ComponentsManager.kt index 0e7045a716..41b8bf8194 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/ComponentsManager.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/ComponentsManager.kt @@ -483,20 +483,7 @@ object ComponentsManager { } else safeName(file.removeSuffix(".wcp")) to File(packagesDir(context), file) if (!wcp.isFile) continue val unpacked = File(store, key) - if (!File(unpacked, ".complete").isFile) { - unpacked.deleteRecursively() - readWcp(wcp) { tar -> - while (true) { - val e = tar.nextTarEntry ?: break - val rel = normalize(e.name) - if (e.isDirectory || !rel.startsWith("files/") || rel.split('/').any { it == ".." }) continue - val out = File(unpacked, rel) - out.parentFile?.mkdirs() - FileOutputStream(out).use { tar.copyTo(it) } - } - } - File(unpacked, ".complete").writeText("1\n") - } + unpack(wcp, unpacked) keep += key val files = a.optJSONObject("files") ?: continue for (rel in files.keys()) { @@ -515,6 +502,34 @@ object ComponentsManager { tmp.renameTo(list) } + /** + * A package's files/ unpacked into [dir], with `.complete` written last; a directory that already + * has `.complete` is left as it is. The launch wrappers and droiddeck-recipe read only complete ones. + */ + internal fun unpack(wcp: File, dir: File) { + if (File(dir, ".complete").isFile) return + dir.deleteRecursively() + readWcp(wcp) { tar -> + while (true) { + val e = tar.nextTarEntry ?: break + val rel = normalize(e.name) + if (e.isDirectory || !rel.startsWith("files/") || rel.split('/').any { it == ".." }) continue + val out = File(dir, rel) + out.parentFile?.mkdirs() + FileOutputStream(out).use { tar.copyTo(it) } + } + } + File(dir, "files").mkdirs() + File(dir, ".complete").writeText("1\n") + } + + /** The stored package file for a catalog file name (present or not). */ + internal fun packageFile(context: Context, file: String): File = File(packagesDir(context), safeName(file)) + + /** A stored package's details, or null when it is missing or unreadable. */ + internal fun storedPackage(context: Context, file: String): Package? = + packageFile(context, file).takeIf { it.isFile }?.let { runCatching { packageInfo(context, it) }.getOrNull() } + // ------------------------------------------------------------------ actions /** Applies a package now, or queues it when a game is running on that Proton. Returns what happened. */ @@ -737,7 +752,7 @@ object ComponentsManager { forcedPackageWanted(active?.optString("file"), active?.optString("protonVersion") == p.version) } if (targets.isEmpty()) continue - val missing = missingPackage(context, rule, lookup, fetch) + val missing = missingPackage(context, rule.comp, rule.file, rule.release, lookup, fetch) if (missing != null) { lines.add(missing) continue @@ -747,18 +762,20 @@ object ComponentsManager { return lines.joinToString("; ").ifEmpty { null } } - /** Null when [rule]'s package is stored and readable. Otherwise the log line, after a fetch. */ - private fun missingPackage( + /** Null when [file]'s package is stored and readable. Otherwise the log line, after a fetch. */ + internal fun missingPackage( context: Context, - rule: ForcedPackage, + comp: String, + file: String, + release: String, lookup: (file: String, release: String) -> CatalogItem?, fetch: (CatalogItem) -> Unit, ): String? { - val wcp = File(packagesDir(context), safeName(rule.file)) + val wcp = packageFile(context, file) if (wcp.isFile && runCatching { packageInfo(context, wcp) }.isSuccess) return null wcp.delete() - val item = lookup(rule.file, rule.release) - ?: return "${rule.file} is not in the Nightlies listing; ${LABEL.getValue(rule.comp)} stays on the Proton's copy" + val item = lookup(file, release) + ?: return "$file is not in the Nightlies listing; ${LABEL.getValue(comp)} stays on the Proton's copy" fetch(item) return null } diff --git a/app/src/main/java/com/droiddeck/launcher/session/DdProtonRecipes.kt b/app/src/main/java/com/droiddeck/launcher/session/DdProtonRecipes.kt new file mode 100644 index 0000000000..4a923e661a --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/session/DdProtonRecipes.kt @@ -0,0 +1,157 @@ +package com.droiddeck.launcher.session + +import android.content.Context +import android.util.AtomicFile +import com.droiddeck.launcher.core.GameEnvironment +import com.droiddeck.launcher.runtime.LinuxRuntime +import org.json.JSONObject +import java.io.File + +/** + * The recipes behind "DroidDeck Proton (Auto)": per game, a DXVK, VKD3D-Proton and/or FEX package + * from the Nightlies "-Linux" releases and a few environment variables. + * + * The app keeps the recipes ([OVERRIDE] in its files, else the bundled [ASSET]), makes sure every + * package they name is stored (the same download Components uses), unpacks each into the runtime's + * recipe store, and publishes [GUEST_FILE] with those paths. droiddeck-recipe reads that file when + * the Auto tool launches a game and starts Proton from a tree assembled for it; nothing is swapped + * into any Proton, so every other tool stays as it shipped. + * + * A recipe may only set tuning variables: the same rule droiddeck-recipe applies, kept here so a + * bad file is refused before it is published. + */ +object DdProtonRecipes { + private const val TAG = "DdProtonRecipes" + const val ASSET = "recipes/default.json" + const val OVERRIDE = "recipes.json" + private const val GUEST_FILE = "root/.config/droiddeck/recipes.json" + private const val STORE = "root/.local/share/droiddeck-recipes/store" + + private val ENV_PREFIXES = listOf("DXVK_", "VKD3D_", "FEX_", "MESA_", "PROTON_") + private val ENV_EXACT = setOf("mesa_glthread", "WINEDLLOVERRIDES") + private val ENV_DENIED_PARTS = listOf("PATH", "_DIR", "_FILE", "LIBRARY", "ROOTFS", "THUNK", "PRELOAD") + private val DLL_OVERRIDE = Regex("[A-Za-z0-9_.*-]+(?:,[A-Za-z0-9_.*-]+)*=(?:n|b|n,b|b,n|d)?") + + data class PackageRef(val comp: String, val file: String, val release: String) + data class Recipe(val id: String, val packages: List, val env: Map, val note: String) + + fun envAllowed(name: String, value: String): Boolean { + if (!GameEnvironment.validName(name) || name.startsWith("LD_")) return false + if (ENV_DENIED_PARTS.any { it in name.uppercase() }) return false + if (name !in ENV_EXACT && ENV_PREFIXES.none { name.startsWith(it) }) return false + if (!GameEnvironment.validValue(value)) return false + if (name == "WINEDLLOVERRIDES") return value.split(';').filter { it.isNotEmpty() }.all { DLL_OVERRIDE.matches(it) } + return true + } + + /** The recipes in [text]; throws IllegalArgumentException naming the first thing wrong. */ + fun parse(text: String): List { + val json = JSONObject(text) + require(json.optInt("version") == 1) { "recipes: version must be 1" } + val games = json.optJSONObject("games") ?: return emptyList() + return games.keys().asSequence().sorted().map { id -> + require(id.isNotEmpty() && GameEnvironment.validScope(id)) { "recipes: $id is not a Steam app or shortcut id" } + val game = games.getJSONObject(id) + val packages = ComponentsManager.COMPONENTS.mapNotNull { comp -> + val ref = game.optJSONObject(comp) ?: return@mapNotNull null + val file = ref.optString("file") + val release = ref.optString("release") + require(file.endsWith(".wcp") && ComponentsManager.safeName(file) == file) { "recipes: $id: $comp file '$file'" } + require(release.isNotEmpty() && ComponentsManager.safeName(release) == release) { "recipes: $id: $comp release '$release'" } + PackageRef(comp, file, release) + } + val env = buildMap { + game.optJSONObject("env")?.let { vars -> + vars.keys().forEach { name -> + val value = vars.get(name) + require(value is String && envAllowed(name, value)) { "recipes: $id: $name is not a variable a recipe may set" } + put(name, value) + } + } + } + Recipe(id, packages, env, game.optString("note").take(200)) + }.toList() + } + + /** The recipe store directory a package unpacks into. */ + internal fun storeKey(file: String) = ComponentsManager.safeName(file.removeSuffix(".wcp")) + + /** + * The guest file: each game's components as unpacked store paths. A component whose package is + * not in [available] (file name -> guest path) is left out, so that game runs on the depot's copy. + */ + internal fun publishJson(recipes: List, available: Map): JSONObject { + val games = JSONObject() + for (recipe in recipes) { + val game = JSONObject() + for (ref in recipe.packages) available[ref.file]?.let { game.put(ref.comp, it) } + if (recipe.env.isNotEmpty()) game.put("env", JSONObject(recipe.env)) + if (recipe.note.isNotEmpty()) game.put("note", recipe.note) + games.put(recipe.id, game) + } + return JSONObject().put("version", 1).put("games", games) + } + + private fun source(context: Context): String? { + val override = File(context.filesDir, OVERRIDE) + if (override.isFile) return override.readText() + return runCatching { context.assets.open(ASSET).use { it.readBytes().decodeToString() } }.getOrNull() + } + + /** + * Fetches and unpacks what the recipes need, then publishes them; returns a line for the session + * log. A package that cannot be fetched is left out of the published file and named in the line. + */ + fun ensure(context: Context): String = ensure( + context, + { file, release -> ComponentsManager.catalogItem(context, file, release) }, + { ComponentsManager.download(context, it) {} }, + ) + + internal fun ensure( + context: Context, + lookup: (file: String, release: String) -> ComponentsManager.CatalogItem?, + fetch: (ComponentsManager.CatalogItem) -> Unit, + kind: (file: String) -> String? = { ComponentsManager.storedPackage(context, it)?.comp }, + unpack: (file: String, dir: File) -> Unit = { file, dir -> ComponentsManager.unpack(ComponentsManager.packageFile(context, file), dir) }, + ): String { + val text = source(context) + val recipes = if (text == null) emptyList() else parse(text) + val root = LinuxRuntime.rootDir(context) + val store = File(root, STORE).apply { mkdirs() } + val available = LinkedHashMap() + val problems = ArrayList() + for (ref in recipes.flatMap { it.packages }.distinctBy { it.file }) { + val fetched = runCatching { ComponentsManager.missingPackage(context, ref.comp, ref.file, ref.release, lookup, fetch) } + val missing = fetched.getOrElse { "${ref.file}: ${it.message}" } + if (missing != null) { problems += missing; continue } + when (kind(ref.file)) { + ref.comp -> Unit + null -> { problems += "${ref.file} could not be read"; continue } + else -> { problems += "${ref.file} is not a ${ref.comp} package"; continue } + } + val dir = File(store, storeKey(ref.file)) + runCatching { unpack(ref.file, dir) } + .onFailure { dir.deleteRecursively(); problems += "${ref.file}: ${it.message}" } + .onSuccess { available[ref.file] = "/" + dir.absolutePath.removePrefix(root.absolutePath.trimEnd('/')).trimStart('/') } + } + val keep = available.keys.map { storeKey(it) }.toSet() + store.listFiles()?.filter { it.isDirectory && it.name !in keep }?.forEach { it.deleteRecursively() } + write(File(root, GUEST_FILE), publishJson(recipes, available).toString()) + val line = "$TAG: ${recipes.size} recipe(s), ${available.size} package(s) ready" + return if (problems.isEmpty()) line else "$line; " + problems.joinToString("; ") + } + + private fun write(file: File, text: String) { + check(file.parentFile!!.isDirectory || file.parentFile!!.mkdirs()) + val atomic = AtomicFile(file) + val output = atomic.startWrite() + try { + output.write(text.toByteArray(Charsets.UTF_8)) + atomic.finishWrite(output) + } catch (error: Exception) { + atomic.failWrite(output) + throw error + } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt index a6478cd109..77d51788e3 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionFiles.kt @@ -36,6 +36,7 @@ object SessionFiles { "usr/local/bin/droiddeck-wincomponents" to "usr/local/bin/droiddeck-wincomponents", "usr/local/bin/droiddeck-msi-install" to "usr/local/bin/droiddeck-msi-install", "usr/local/bin/droiddeck-esync" to "usr/local/bin/droiddeck-esync", + "usr/local/bin/droiddeck-recipe" to "usr/local/bin/droiddeck-recipe", "usr/local/bin/droiddeck-steam-compat" to "usr/local/bin/droiddeck-steam-compat", "usr/local/bin/droiddeck-fex" to "usr/local/bin/droiddeck-fex", "libblsession.so" to "usr/local/lib/libblsession.so", diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 69d4ea65ee..2e3c3f65d0 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -343,6 +343,11 @@ class SessionService : Service() { runCatching { ComponentsManager.ensureForcedPackages(this) } .onSuccess { if (it != null) Log.i(TAG, it) } .onFailure { Log.w(TAG, "forced components", it) } + // DroidDeck Proton (Auto)'s recipes: their packages fetched and unpacked, the file + // droiddeck-recipe reads published. Only that tool reads it; a failure leaves it stock. + runCatching { DdProtonRecipes.ensure(this) } + .onSuccess { Log.i(TAG, it) } + .onFailure { Log.w(TAG, "DroidDeck Proton (Auto) recipes", it) } } addClientEnvironment(guest, steamHere) // Where the fast path's description of proot's view goes, once the binds are known. diff --git a/app/src/test/java/com/droiddeck/launcher/session/DdProtonRecipesTest.kt b/app/src/test/java/com/droiddeck/launcher/session/DdProtonRecipesTest.kt new file mode 100644 index 0000000000..999f22cc41 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/DdProtonRecipesTest.kt @@ -0,0 +1,138 @@ +package com.droiddeck.launcher.session + +import android.content.Context +import com.droiddeck.launcher.runtime.LinuxRuntime +import org.json.JSONObject +import org.junit.Assert.* +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.io.File + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [28], manifest = Config.NONE) +class DdProtonRecipesTest { + private val dxvk = "dxvk-2.4-linux.wcp" + private val vkd3d = "vkd3d-proton-2.14-linux.wcp" + private lateinit var context: Context + + @Before fun setUp() { + context = RuntimeEnvironment.getApplication() + File(context.filesDir, "components").deleteRecursively() + File(context.filesDir, DdProtonRecipes.OVERRIDE).delete() + LinuxRuntime.rootDir(context).deleteRecursively() + } + + @Test fun recipesParseIntoPackagesEnvAndNote() { + val recipes = DdProtonRecipes.parse( + """{"version":1,"games":{"42":{"dxvk":{"file":"$dxvk","release":"Dxvk-Linux"}, + "env":{"DXVK_HUD":"version","WINEDLLOVERRIDES":"xinput1_3=n,b;libsentry=d"},"note":"test"}, + "4294967295":{"fex":{"file":"fex-2607-linux.wcp","release":"FexCore-Linux"}}}}""", + ) + assertEquals(listOf("42", "4294967295"), recipes.map { it.id }) + assertEquals(listOf(DdProtonRecipes.PackageRef("dxvk", dxvk, "Dxvk-Linux")), recipes[0].packages) + assertEquals(mapOf("DXVK_HUD" to "version", "WINEDLLOVERRIDES" to "xinput1_3=n,b;libsentry=d"), recipes[0].env) + assertEquals("test", recipes[0].note) + assertEquals("fex", recipes[1].packages.single().comp) + } + + @Test fun anythingOutsideTheFormatIsRefused() { + val bad = listOf( + """{"version":2,"games":{}}""", + """{"version":1,"games":{"0":{}}}""", + """{"version":1,"games":{"steam":{}}}""", + """{"version":1,"games":{"42":{"dxvk":{"file":"../x.wcp","release":"Dxvk-Linux"}}}}""", + """{"version":1,"games":{"42":{"dxvk":{"file":"dxvk.tar","release":"Dxvk-Linux"}}}}""", + """{"version":1,"games":{"42":{"dxvk":{"file":"$dxvk","release":""}}}}""", + """{"version":1,"games":{"42":{"env":{"LD_PRELOAD":"/x.so"}}}}""", + """{"version":1,"games":{"42":{"env":{"DXVK_HUD":1}}}}""", + ) + for (text in bad) { + assertThrows(text, Exception::class.java) { DdProtonRecipes.parse(text) } + } + } + + @Test fun onlyTuningVariablesAreAllowed() { + for (name in listOf("DXVK_HUD", "VKD3D_CONFIG", "FEX_TSOENABLED", "MESA_SHADER_CACHE_MAX_SIZE", "PROTON_USE_WINED3D", "mesa_glthread")) { + assertTrue(name, DdProtonRecipes.envAllowed(name, "1")) + } + for (name in listOf("LD_PRELOAD", "LD_LIBRARY_PATH", "WINEDLLPATH", "PATH", "HOME", "PROTON_LOG_DIR", "DXVK_CONFIG_FILE", + "VKD3D_SHADER_CACHE_PATH", "MESA_SHADER_CACHE_DIR", "FEX_ROOTFS", "FEX_THUNKHOSTLIBS", "CUSTOM", "BAD-NAME")) { + assertFalse(name, DdProtonRecipes.envAllowed(name, "1")) + } + assertTrue(DdProtonRecipes.envAllowed("WINEDLLOVERRIDES", "a=n;b=b,n;c=d;d=")) + for (value in listOf("../evil.dll=n", "a=native", "a", "a=n;b=x")) { + assertFalse(value, DdProtonRecipes.envAllowed("WINEDLLOVERRIDES", value)) + } + assertFalse(DdProtonRecipes.envAllowed("DXVK_HUD", "a\u0000b")) + } + + @Test fun thePublishedFileNamesStorePathsAndLeavesOutWhatIsMissing() { + val recipes = DdProtonRecipes.parse( + """{"version":1,"games":{"42":{"dxvk":{"file":"$dxvk","release":"Dxvk-Linux"}, + "vkd3d":{"file":"$vkd3d","release":"Vkd3d-proton-Linux"},"env":{"DXVK_HUD":"version"},"note":"n"}}}""", + ) + val json = DdProtonRecipes.publishJson(recipes, mapOf(dxvk to "/root/.local/share/droiddeck-recipes/store/dxvk-2.4-linux")) + assertEquals(1, json.getInt("version")) + val game = json.getJSONObject("games").getJSONObject("42") + assertEquals("/root/.local/share/droiddeck-recipes/store/dxvk-2.4-linux", game.getString("dxvk")) + assertFalse(game.has("vkd3d")) + assertEquals("version", game.getJSONObject("env").getString("DXVK_HUD")) + assertEquals("n", game.getString("note")) + } + + @Test fun ensureFetchesUnpacksAndPublishes() { + File(context.filesDir, DdProtonRecipes.OVERRIDE).writeText( + """{"version":1,"games":{"42":{"dxvk":{"file":"$dxvk","release":"Dxvk-Linux"}, + "vkd3d":{"file":"$vkd3d","release":"Vkd3d-proton-Linux"}}}}""", + ) + val fetched = mutableListOf() + val line = DdProtonRecipes.ensure( + context, + { file, release -> if (file == dxvk) item(file, release) else null }, + { fetched += it.file }, + kind = { file -> if (file == dxvk) "dxvk" else null }, + unpack = ::fakeUnpack, + ) + assertEquals(listOf(dxvk), fetched) + assertTrue(line, line.startsWith("DdProtonRecipes: 1 recipe(s), 1 package(s) ready; $vkd3d is not in the Nightlies listing")) + val root = LinuxRuntime.rootDir(context) + val store = File(root, "root/.local/share/droiddeck-recipes/store/dxvk-2.4-linux") + assertTrue(File(store, ".complete").isFile) + assertEquals("d3d11", File(store, "files/lib/wine/dxvk/aarch64-windows/d3d11.dll").readText()) + val published = JSONObject(File(root, "root/.config/droiddeck/recipes.json").readText()) + val game = published.getJSONObject("games").getJSONObject("42") + assertEquals("/root/.local/share/droiddeck-recipes/store/dxvk-2.4-linux", game.getString("dxvk")) + assertFalse(game.has("vkd3d")) + + // A recipe that stops naming a package drops its unpacked copy; a package of the wrong kind is refused. + File(context.filesDir, DdProtonRecipes.OVERRIDE).writeText( + """{"version":1,"games":{"42":{"vkd3d":{"file":"$dxvk","release":"Dxvk-Linux"}}}}""", + ) + val second = DdProtonRecipes.ensure(context, { f, r -> item(f, r) }, {}, kind = { "dxvk" }, unpack = ::fakeUnpack) + assertTrue(second, second.endsWith("$dxvk is not a vkd3d package")) + assertFalse(store.exists()) + assertFalse(JSONObject(File(root, "root/.config/droiddeck/recipes.json").readText()) + .getJSONObject("games").getJSONObject("42").has("vkd3d")) + } + + @Test fun noRecipesPublishesAnEmptyFile() { + val line = DdProtonRecipes.ensure(context, { _, _ -> null }, {}, kind = { null }, unpack = ::fakeUnpack) + assertEquals("DdProtonRecipes: 0 recipe(s), 0 package(s) ready", line) + val published = JSONObject(File(LinuxRuntime.rootDir(context), "root/.config/droiddeck/recipes.json").readText()) + assertEquals(0, published.getJSONObject("games").length()) + } + + private fun item(file: String, release: String) = + ComponentsManager.CatalogItem(file, "dxvk", release, "https://example.invalid/$file", 1, "sha256:00") + + /** Stands in for ComponentsManager.unpack: zstd-jni has no host library for JVM tests. */ + private fun fakeUnpack(file: String, dir: File) { + File(dir, "files/lib/wine/dxvk/aarch64-windows").mkdirs() + File(dir, "files/lib/wine/dxvk/aarch64-windows/d3d11.dll").writeText("d3d11") + File(dir, ".complete").writeText("1\n") + } +} diff --git a/docs/development/game-environment.md b/docs/development/game-environment.md index 223915adc1..1558b8015d 100644 --- a/docs/development/game-environment.md +++ b/docs/development/game-environment.md @@ -56,6 +56,46 @@ requiring a different/custom component; custom entries remain allowed. The old `DXVK_FRAME_RATE` environment variable was removed from current DXVK, so it is not offered as a suggestion. Prefer the session frame limiter for ordinary play. +## DroidDeck Proton (Auto) + +A proof of concept for per-game Proton assembly. `steam-compatibility` registers +`droiddeck-proton-auto` ("DroidDeck Proton (Auto)") next to `droiddeck-proton-arm64`. It runs +the same Valve ARM64 depot, and its launcher also exports `DROIDDECK_RECIPES=1`. Choose it per +game in Steam's Properties → Compatibility. Every other tool behaves as before, including +Components swaps and the forced DXVK on Adreno 6xx. + +For a game launched through it, `droiddeck-game-env` asks `droiddeck-recipe` for the game's +recipe. A recipe can name a DXVK, VKD3D-Proton and/or FEX package, plus environment variables. +Proton is then started from `~/.local/share/droiddeck-recipes/dist//`: + +- real directories down to each directory a package replaces; +- every other entry a symlink to the depot; +- the component's own DLLs (the same set Components swaps) linked from the package. + +The depot is never written. The key covers the depot, its version line and the packages, so a +Steam update gives a fresh tree. Trees unused for 14 days are removed. esync builds its own tree +on top, as it does for a depot. A recipe never stops a launch: a missing package or an error +leaves that component, or the whole launch, on the stock Proton. The session log and +`droiddeck-recipes/launches.log` record what was applied. + +Recipe variables sit in the automatic-fixes layer: above shared entries, below the game's own +entries. They are limited to tuning names: the `DXVK_`, `VKD3D_`, `FEX_`, `MESA_` and `PROTON_` +families, `mesa_glthread` and `WINEDLLOVERRIDES`. Names that refer to a path, directory, file, +library or preload are refused. The app (`DdProtonRecipes`) and the guest both enforce this. + +Recipes come from the app's `files/recipes.json` when present, otherwise from the bundled +`assets/recipes/default.json`, which is empty for now. At session start the app fetches each +named package from the Nightlies "-Linux" releases, the same way Components does. It unpacks +them into `droiddeck-recipes/store/` and publishes `~/.config/droiddeck/recipes.json` with those +paths: + +```json +{"version": 1, "games": {"": { + "dxvk": {"file": "dxvk-2.4-linux.wcp", "release": "Dxvk-Linux"}, + "env": {"DXVK_HUD": "version"}, + "note": "why this game needs it"}}} +``` + ## Launch path and references WinNative's Linux session filters Android-only options, merges user variables @@ -87,7 +127,8 @@ Upstream references: [VKD3D capability parsing](https://github.com/HansKristian- [DXVK variables](https://github.com/doitsujin/dxvk#environment-variables), [DXVK frame-limiter changes](https://github.com/doitsujin/dxvk/releases). -Validation: `./gradlew testDebugUnitTest` and -`python3 -m unittest discover -s tools/tests -p test_game_environment.py`. +Validation: `./gradlew testDebugUnitTest`, +`python3 -m unittest discover -s tools/tests -p test_game_environment.py` and +`python3 -m unittest discover -s tools/tests -p test_recipe.py`. Actual game compatibility depends on the installed Proton, VKD3D and Vulkan driver; these checks do not establish that every device supports feature level 12_2 or SM 6_9. diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-game-env b/tools/linuxfs/overlay/usr/local/bin/droiddeck-game-env index e8f5605a4d..b34a2d7f71 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-game-env +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-game-env @@ -174,6 +174,39 @@ def sync_config(config, app_id): } +def load_recipes(): + try: + return runpy.run_path(str(Path(__file__).resolve().parent / "droiddeck-recipe")) + except Exception as error: + print("droiddeck-game-env: recipes not loaded (%s); stock Proton" % error, file=sys.stderr) + return None + + +def game_recipe(env, app_id, recipes=None): + """The game's recipe when DroidDeck Proton (Auto) launched it (see droiddeck-recipe), or None.""" + if env.get("DROIDDECK_RECIPES") != "1": + return None, None + recipes = recipes if recipes is not None else load_recipes() + if recipes is None: + return None, None + try: + return recipes, recipes["lookup"](env, app_id) + except Exception as error: + print("droiddeck-game-env: recipe not read (%s); stock Proton" % error, file=sys.stderr) + return None, None + + +def recipe_fixes(fixes, recipe_env): + """A recipe's variables over the engine fixes: both are automatic, the recipe is the one made for + this game. WINEDLLOVERRIDES entries are joined rather than replaced, the recipe's after.""" + result = dict(fixes) + for name, value in recipe_env.items(): + if name == "WINEDLLOVERRIDES" and result.get(name): + value = result[name] + ";" + value + result[name] = value + return result + + def load_sync(): try: return runpy.run_path(str(Path(__file__).resolve().parent / "droiddeck-esync")) @@ -265,6 +298,7 @@ def main(): env = os.environ.copy() mark_install_scripts(command, env) app_id = game_id(env) + recipes = recipe = None if app_id: waiting = len(command) > 1 and command[1] == "waitforexitandrun" fixes = {} @@ -273,6 +307,9 @@ def main(): for note in notes: print("droiddeck-game-env: " + note, file=sys.stderr) command += extra + recipes, recipe = game_recipe(env, app_id) + if recipe: + fixes = recipe_fixes(fixes, recipe["env"]) components = win_components(env.get("STEAM_COMPAT_DATA_PATH", ""), app_id) if components: own = fixes.get("WINEDLLOVERRIDES") @@ -290,6 +327,9 @@ def main(): env = apply_config(env, {"version": 1}, app_id, fixes) except (OSError, ValueError, TypeError, AttributeError): print("droiddeck-game-env: invalid configuration; using inherited environment", file=sys.stderr) + env.pop("DROIDDECK_RECIPES", None) + if recipe: + command = recipes["assemble"](command, recipe, app_id) env = force_ssbs(command, env) sync = load_sync() launch, launch_env = select_sync(command, env, sync) diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-recipe b/tools/linuxfs/overlay/usr/local/bin/droiddeck-recipe new file mode 100644 index 0000000000..2411acedcf --- /dev/null +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-recipe @@ -0,0 +1,338 @@ +#!/usr/bin/python3 +# droiddeck-recipe: the per-game Proton behind "DroidDeck Proton (Auto)". +# +# That tool runs Valve's ARM64 Proton like droiddeck-proton-arm64 does, and exports +# DROIDDECK_RECIPES=1. droiddeck-game-env then asks this module for the game's recipe, a DXVK, +# VKD3D-Proton and/or FEX package and a few environment variables the app published in +# ~/.config/droiddeck/recipes.json, and starts Proton from a tree assembled for that recipe: +# +# ~/.local/share/droiddeck-recipes/dist// +# real directories down to each directory a package replaces, every other entry a symlink to +# the depot, and in a replaced directory the component's own files linked from the package. +# +# Proton takes its base directory from dirname(sys.argv[0]) without resolving it (the same thing +# steam-compatibility and droiddeck-esync rely on), so it reads the package's DLLs from that tree +# and copies them into the game's prefix as if they were its own. The depot is never written: other +# tools, and this one for other games, keep Valve's files. A Steam update changes the depot's +# version line, which changes the key, so the next launch gets a fresh tree. +# +# A recipe never stops a launch. Anything wrong - a missing package, a bad file, an error here - +# is logged and the game starts on the stock Proton. +import contextlib +import fcntl +import hashlib +import json +import os +import re +import shutil +import sys +import time + +HOME = os.environ.get("HOME") or os.path.expanduser("~") +RECIPES = os.path.join(HOME, ".config", "droiddeck", "recipes.json") +ROOT = os.path.join(HOME, ".local", "share", "droiddeck-recipes") +FLAG = "DROIDDECK_RECIPES" +RECORD = ".droiddeck-recipe.json" +LOG_LINES = 200 +DIST_AGE = 14 * 24 * 3600 +TEMP_AGE = 3600 + +WINE = "files/lib/wine" +PE_ARCHES = ("aarch64-windows", "i386-windows") +COMPONENTS = ("dxvk", "vkd3d", "fex") +LABEL = {"dxvk": "DXVK", "vkd3d": "VKD3D", "fex": "FEX"} +# The files that ARE each component, as ComponentsManager swaps them: in a replaced directory these +# are dropped from the depot's side before the package's files are linked in, so no file of the +# depot's version is left mixed in. Anything else there (GE's openvr_api_dxvk.dll) stays. +COMP_DIRS = {"dxvk": WINE + "/dxvk", "vkd3d": WINE + "/vkd3d-proton"} +COMP_DLLS = { + "dxvk": frozenset(("d3d8.dll", "d3d9.dll", "d3d10.dll", "d3d10_1.dll", "d3d10core.dll", "d3d11.dll", "dxgi.dll")), + "vkd3d": frozenset(("d3d12.dll", "d3d12core.dll")), +} +# The directories droiddeck-esync rebuilds as real ones when it puts its own tree on top of this +# one (its REAL_DIRS). They are real here too, so it never has to list a directory through a +# symlink: a guest opendir() of a symlink, answered by proot's fast path, comes back ENOTDIR and +# esync would take that directory for empty. +ESYNC_DIRS = (".", "files", "files/bin-arm64", "files/lib", "files/lib/wine", "files/lib/wine/aarch64-unix") +FEX_FILES = { + WINE + "/aarch64-windows": frozenset(("libarm64ecfex.dll", "libwow64fex.dll")), + WINE + "/aarch64-unix": frozenset(("libarm64ecfex.so", "libwow64fex.so")), +} + +# Environment a recipe may set. Names only from the graphics, translation and Proton families, and +# never one that names a file, a directory or a library to load: a recipe can tune a game, not run +# code in it. +ENV_PREFIXES = ("DXVK_", "VKD3D_", "FEX_", "MESA_", "PROTON_") +ENV_EXACT = frozenset(("mesa_glthread", "WINEDLLOVERRIDES")) +ENV_DENIED_PARTS = ("PATH", "_DIR", "_FILE", "LIBRARY", "ROOTFS", "THUNK", "PRELOAD") +ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +DLL_OVERRIDE = re.compile(r"[A-Za-z0-9_.*-]+(?:,[A-Za-z0-9_.*-]+)*=(?:n|b|n,b|b,n|d)?") + + +def say(message): + with contextlib.suppress(Exception): + print("droiddeck-recipe: " + message, file=sys.stderr, flush=True) + + +def env_allowed(name, value): + if not isinstance(name, str) or not ENV_NAME.fullmatch(name) or name.startswith("LD_"): + return False + if any(part in name.upper() for part in ENV_DENIED_PARTS): + return False + if name not in ENV_EXACT and not name.startswith(ENV_PREFIXES): + return False + if not isinstance(value, str) or "\0" in value or len(value) > 8192: + return False + if name == "WINEDLLOVERRIDES": + return all(DLL_OVERRIDE.fullmatch(entry) for entry in value.split(";") if entry) + return True + + +def store_root(): + return os.path.join(ROOT, "store") + + +def package_ok(path): + """A package directory the app unpacked completely, inside the recipe store.""" + if not isinstance(path, str) or not os.path.isabs(path): + return False + store = os.path.realpath(store_root()) + real = os.path.realpath(path) + return (os.path.dirname(real) == store and os.path.isfile(os.path.join(real, ".complete")) + and os.path.isdir(os.path.join(real, "files"))) + + +def lookup(env, app_id, path=None): + """The game's recipe when this launch came through the Auto tool, else None. + + Returns {"components": {comp: package dir}, "skipped": [comp], "env": {...}, "note": str}. + Raises nothing: a bad file is logged and treated as no recipe.""" + if env.get(FLAG) != "1" or not app_id: + return None + try: + with open(path or RECIPES, encoding="utf-8") as source: + data = json.load(source) + except FileNotFoundError: + return None + except (OSError, ValueError) as error: + say("recipes not read (%s); stock Proton" % error) + return None + if not isinstance(data, dict) or data.get("version") != 1 or not isinstance(data.get("games"), dict): + say("recipes file not understood; stock Proton") + return None + entry = data["games"].get(app_id) + if not isinstance(entry, dict): + return None + components, skipped, environment = {}, [], {} + for comp in COMPONENTS: + if comp not in entry: + continue + if package_ok(entry[comp]): + components[comp] = os.path.realpath(entry[comp]) + else: + skipped.append(comp) + raw = entry.get("env", {}) + if isinstance(raw, dict): + for name, value in raw.items(): + if env_allowed(name, value): + environment[name] = value + else: + say("app %s: %s is not a variable a recipe may set; left out" % (app_id, name)) + note = entry.get("note") if isinstance(entry.get("note"), str) else "" + return {"components": components, "skipped": skipped, "env": environment, "note": note[:200]} + + +# ------------------------------------------------------------------ the tree + +def version_line(tool): + try: + with open(os.path.join(tool, "version"), encoding="utf-8", errors="replace") as source: + return source.readline().strip() + except OSError: + return "" + + +def overrides(components): + """{rel dir: (names dropped from the depot's side, {name: source file})} for these packages.""" + result = {} + + def entry(rel): + return result.setdefault(rel, (set(), {})) + + for comp, package in sorted(components.items()): + files = os.path.join(package, "files") + for folder, dirs, names in os.walk(files): + dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(folder, d))] + rel = os.path.relpath(folder, package).replace(os.sep, "/") + for name in names: + source = os.path.join(folder, name) + if os.path.isfile(source) and not os.path.islink(source): + entry(rel)[1][name] = source + if comp in COMP_DIRS: + for arch in PE_ARCHES: + rel = COMP_DIRS[comp] + "/" + arch + if rel in result: + result[rel][0].update(COMP_DLLS[comp]) + else: + for rel, names in FEX_FILES.items(): + entry(rel)[0].update(names) + return result + + +def ancestors(rel): + parts = rel.split("/") + return {"/".join(parts[:i]) for i in range(1, len(parts) + 1)} | {"."} + + +def fill(base, out, rel, real, replaced): + folder = os.path.join(out, rel) if rel != "." else out + os.makedirs(folder, exist_ok=True) + source = os.path.join(base, rel) if rel != "." else base + dropped, files = replaced.get(rel, (set(), {})) + try: + names = sorted(os.listdir(source)) + except OSError: + names = [] + for name in names: + child = name if rel == "." else rel + "/" + name + if rel == "." and name == RECORD: + continue + if child in real and os.path.isdir(os.path.join(source, name)): + fill(base, out, child, real, replaced) + elif name.lower() not in dropped and name not in files: + os.symlink(os.path.join(source, name), os.path.join(folder, name)) + for child in sorted(r for r in real if r != "." and os.path.dirname(r) == ("" if rel == "." else rel)): + if not os.path.lexists(os.path.join(out, child)): + fill(base, out, child, real, replaced) + for name, target in sorted(files.items()): + os.symlink(target, os.path.join(folder, name)) + + +def build(base, components, out): + replaced = overrides(components) + real = {rel for rel in ESYNC_DIRS if os.path.isdir(os.path.join(base, rel))} + for rel in replaced: + real |= ancestors(rel) + fill(base, out, ".", real, replaced) + + +def key_of(base, components): + stamp = {comp: [path, os.stat(os.path.join(path, ".complete")).st_mtime_ns] for comp, path in sorted(components.items())} + text = json.dumps({"base": os.path.realpath(base), "version": version_line(base), "components": stamp}, sort_keys=True) + return hashlib.sha256(text.encode("utf-8", "surrogateescape")).hexdigest()[:16] + + +@contextlib.contextmanager +def locked(path): + fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_CLOEXEC, 0o644) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) + + +def remove(path): + if os.path.islink(path) or (os.path.lexists(path) and not os.path.isdir(path)): + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) + + +def ensure(base, components, root=None): + """The assembled tree for `base` with these packages, built once and reused.""" + folder = os.path.join(root or ROOT, "dist") + os.makedirs(folder, exist_ok=True) + key = key_of(base, components) + dist = os.path.join(folder, key) + with locked(dist + ".lock"): + if not os.path.isfile(os.path.join(dist, RECORD)): + remove(dist) + temp = os.path.join(folder, ".%s.tmp-%d" % (key, os.getpid())) + remove(temp) + try: + build(base, components, temp) + with open(os.path.join(temp, RECORD), "w", encoding="utf-8") as target: + json.dump({"base": base, "version": version_line(base), "components": components}, target, sort_keys=True) + os.rename(temp, dist) + except BaseException: + remove(temp) + raise + os.utime(dist) + collect(folder, keep=key) + return dist + + +def collect(folder, keep=None, now=None): + """Drops trees not used for DIST_AGE and temp builds older than TEMP_AGE.""" + now = now if now is not None else time.time() + try: + names = os.listdir(folder) + except OSError: + return + for name in names: + path = os.path.join(folder, name) + try: + age = now - os.lstat(path).st_mtime + except OSError: + continue + if name.startswith("."): + if ".tmp-" in name and age > TEMP_AGE: + remove(path) + elif name.endswith(".lock"): + if age > DIST_AGE and not os.path.isdir(path[:-5]): + remove(path) + elif name != keep and age > DIST_AGE: + with locked(path + ".lock"): + remove(path) + + +def log_line(app_id, text, root=None): + path = os.path.join(root or ROOT, "launches.log") + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with locked(path + ".lock"): + try: + with open(path, encoding="utf-8", errors="replace") as source: + lines = [line for line in source.read().split("\n") if line] + except FileNotFoundError: + lines = [] + lines.append("%d\t%s\t%s" % (int(time.time()), app_id, re.sub(r"[\t\r\n]", " ", text))) + temp = "%s.tmp-%d" % (path, os.getpid()) + with open(temp, "w", encoding="utf-8") as target: + target.write("\n".join(lines[-LOG_LINES:]) + "\n") + os.replace(temp, path) + except Exception as error: + say("launch not logged (%s)" % error) + + +def summary(recipe): + parts = [] + for comp in COMPONENTS: + if comp in recipe["components"]: + parts.append("%s %s" % (LABEL[comp], os.path.basename(recipe["components"][comp]))) + elif comp in recipe["skipped"]: + parts.append("%s stock (package missing)" % LABEL[comp]) + else: + parts.append("%s stock" % LABEL[comp]) + parts.append("env %d" % len(recipe["env"])) + text = ", ".join(parts) + return text + (" (%s)" % recipe["note"] if recipe["note"] else "") + + +def assemble(command, recipe, app_id, root=None): + """`command` started from the recipe's tree, or unchanged when it has no packages or fails.""" + text = summary(recipe) + say("app %s: %s" % (app_id, text)) + log_line(app_id, text, root) + if not recipe["components"] or not command: + return list(command) + try: + base = os.path.dirname(command[0]) or "." + base = base if os.path.isabs(base) else os.path.join(os.getcwd(), base) + dist = ensure(base, recipe["components"], root) + return [os.path.join(dist, os.path.basename(command[0]))] + list(command[1:]) + except Exception as error: + say("app %s: tree not assembled (%s); stock Proton" % (app_id, error)) + return list(command) diff --git a/tools/linuxfs/overlay/usr/local/bin/steam-compatibility b/tools/linuxfs/overlay/usr/local/bin/steam-compatibility index 351e32b0c2..9cc651435c 100755 --- a/tools/linuxfs/overlay/usr/local/bin/steam-compatibility +++ b/tools/linuxfs/overlay/usr/local/bin/steam-compatibility @@ -43,7 +43,12 @@ LIBRARY = "/mnt/droiddeck" # the card library is LIBRARY + "-sd"; internal stor # unmistakable at a glance, or a user picks a neighbour and gets a launch that fails silently. # Newest first: whichever the client has downloaded is used. SOURCES = ("Proton Experimental (ARM64)", "Proton 11.0 (ARM64)") -OWN_TOOLS = ((TOOL, SOURCES), (TOOL_11, ("Proton 11.0 (ARM64)",))) +# DroidDeck Proton (Auto): the same Valve depot, started from a tree assembled per game out of the +# recipe the app published for it (see droiddeck-recipe). Listed last: the app's default request +# names a depot directory, and the first tool on that directory is the one it means. +RECIPE_TOOL = "droiddeck-proton-auto" +RECIPE_DISPLAY = "DroidDeck Proton (Auto)" +OWN_TOOLS = ((TOOL, SOURCES), (TOOL_11, ("Proton 11.0 (ARM64)",)), (RECIPE_TOOL, SOURCES)) FEX_TOOL = "droiddeck-fex" FEX_LAUNCHER = "droiddeck-fex-tool" FEX_FILES = ("toolmanifest.vdf", "compatibilitytool.vdf", FEX_LAUNCHER) @@ -340,15 +345,22 @@ exit 1 LAUNCHER_TEMPLATE = LAUNCHER_SH -def launcher_sh(sources=SOURCES): +def launcher_sh(sources=SOURCES, recipes=False): # Spliced in after the formatting, never through it: the function body contains ${major%%.*} # and a % format would eat that into a single %, leaving a shell expansion that silently does # something else. + flag = "export DROIDDECK_RECIPES=1\n" if recipes else "" return (LAUNCHER_TEMPLATE % " ".join('"%s"' % n for n in sources)).replace( - "verb=$1; shift", "verb=$1; shift\n" + BL_DIRECTAUDIO_SETUP + BL_GAME_CPUS_SETUP + BL_IDTECH3_SETUP + "verb=$1; shift", "verb=$1; shift\n" + flag + BL_DIRECTAUDIO_SETUP + BL_GAME_CPUS_SETUP + BL_IDTECH3_SETUP + BL_COMPONENTS_SETUP + BL_LIBPATH_SETUP, 1) +def tool_display(name, depot): + if name == RECIPE_TOOL: + return display_name(RECIPE_DISPLAY) + return display_name(os.path.basename(depot).replace(" (ARM64)", " ARM64") if depot else "Proton ARM64") + + LAUNCHER_SH = launcher_sh() # A third-party Proton dropped into compatibilitytools.d - GE-Proton, proton-cachyos, anything of @@ -935,8 +947,7 @@ def tool_catalog(steam_root, extras=None): depot = find_source(steam_root, sources) if depot is None and name != TOOL: continue - catalog.append(tool_entry(name, display_name( - os.path.basename(depot).replace(" (ARM64)", " ARM64") if depot else "Proton ARM64"), depot, True)) + catalog.append(tool_entry(name, tool_display(name, depot), depot, True)) if extras is None: extras = {} try: @@ -1127,9 +1138,8 @@ def build_tool(tool_dir, depot, sources=SOURCES, name=TOOL): shutil.rmtree(tool_dir) os.makedirs(tool_dir, exist_ok=True) write_changed(os.path.join(tool_dir, "toolmanifest.vdf"), TOOL_MANIFEST_VDF) - write_changed(os.path.join(tool_dir, "compatibilitytool.vdf"), COMPAT_TOOL_VDF % (name, display_name( - os.path.basename(depot).replace(" (ARM64)", " ARM64") if depot else "Proton ARM64"))) - write_changed(os.path.join(tool_dir, LAUNCHER), launcher_sh(sources), 0o755) + write_changed(os.path.join(tool_dir, "compatibilitytool.vdf"), COMPAT_TOOL_VDF % (name, tool_display(name, depot))) + write_changed(os.path.join(tool_dir, LAUNCHER), launcher_sh(sources, recipes=name == RECIPE_TOOL), 0o755) def build_fex_tool(tool_dir): diff --git a/tools/tests/test_game_environment.py b/tools/tests/test_game_environment.py index 85926c04f7..2e07076700 100644 --- a/tools/tests/test_game_environment.py +++ b/tools/tests/test_game_environment.py @@ -373,17 +373,50 @@ class ProtonDefaultTest(unittest.TestCase): self.assertEqual(COMPAT["find_source"](str(steam), COMPAT["OWN_TOOLS"][1][1]), str(steam / "steamapps/common/Proton 11.0 (ARM64)")) self.assertEqual(COMPAT["find_source"](str(steam)), str(steam / "steamapps/common/Proton Experimental (ARM64)")) self.assertEqual([(t["name"], t["dir"]) for t in COMPAT["tool_catalog"](str(steam), {})], - [(COMPAT["TOOL"], "Proton Experimental (ARM64)"), (COMPAT["TOOL_11"], "Proton 11.0 (ARM64)")]) + [(COMPAT["TOOL"], "Proton Experimental (ARM64)"), (COMPAT["TOOL_11"], "Proton 11.0 (ARM64)"), + (COMPAT["RECIPE_TOOL"], "Proton Experimental (ARM64)")]) for name, sources in COMPAT["OWN_TOOLS"]: COMPAT["build_tool"](str(steam / "compatibilitytools.d" / name), COMPAT["find_source"](str(steam), sources), sources, name) self.assertEqual(COMPAT["adopt_extras"](str(steam / "compatibilitytools.d")), {}) self.assertEqual(sorted(os.listdir(steam / "compatibilitytools.d" / COMPAT["TOOL_11"])), sorted(COMPAT["OWN_FILES"])) launcher = COMPAT["launcher_sh"](COMPAT["OWN_TOOLS"][1][1]) + self.assertNotIn("DROIDDECK_RECIPES", launcher) self.assertIn('for name in "Proton 11.0 (ARM64)"; do', launcher) self.assertIn("${major%%.*}", launcher) +class RecipeToolTest(unittest.TestCase): + def test_the_auto_tool_is_registered_beside_the_valve_one(self): + with tempfile.TemporaryDirectory() as tmp: + steam = Path(tmp) / "Steam" + (steam / "steamapps/common" / COMPAT["SOURCES"][0] / "files/bin-arm64").mkdir(parents=True) + tools = steam / "compatibilitytools.d" + for name, sources in COMPAT["OWN_TOOLS"]: + COMPAT["build_tool"](str(tools / name), COMPAT["find_source"](str(steam), sources), sources, name) + auto = tools / COMPAT["RECIPE_TOOL"] + self.assertEqual(sorted(os.listdir(auto)), sorted(COMPAT["OWN_FILES"])) + self.assertIn('"display_name" "DroidDeck Proton (Auto)', (auto / "compatibilitytool.vdf").read_text()) + launcher = (auto / COMPAT["LAUNCHER"]).read_text() + self.assertIn("verb=$1; shift\nexport DROIDDECK_RECIPES=1\n", launcher) + self.assertIn('for name in "Proton Experimental (ARM64)" "Proton 11.0 (ARM64)"; do', launcher) + self.assertNotIn("DROIDDECK_RECIPES", (tools / COMPAT["TOOL"] / COMPAT["LAUNCHER"]).read_text()) + catalog = COMPAT["tool_catalog"](str(steam), {}) + self.assertEqual(catalog[-1], {"name": COMPAT["RECIPE_TOOL"], "display": "DroidDeck Proton (Auto)", + "dir": "Proton Experimental (ARM64)", "valve": True}) + # The app's default request names a depot directory: it still means the plain tool. + self.assertEqual(COMPAT["requested_tool"]({"dir": "Proton Experimental (ARM64)", "valve": True}, catalog), COMPAT["TOOL"]) + + def test_a_game_set_to_the_auto_tool_keeps_it(self): + names = [COMPAT["TOOL"], COMPAT["RECIPE_TOOL"]] + current = {"0": COMPAT["TOOL"], "42": COMPAT["RECIPE_TOOL"]} + changes, kept = COMPAT["plan_mapping"](current, ["42", "43"], names, COMPAT["TOOL"], {}) + self.assertEqual(changes, {"43": COMPAT["TOOL"]}) + self.assertNotIn("42", kept) + changes, _ = COMPAT["plan_mapping"](current, ["42"], [COMPAT["TOOL"]], COMPAT["TOOL"], {}) + self.assertEqual(changes, {"42": COMPAT["TOOL"]}) + + class DirectAudioPrefixTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() diff --git a/tools/tests/test_recipe.py b/tools/tests/test_recipe.py new file mode 100644 index 0000000000..43e282044c --- /dev/null +++ b/tools/tests/test_recipe.py @@ -0,0 +1,332 @@ +import contextlib +import hashlib +import io +import json +import os +from pathlib import Path +import runpy +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +from test_esync_packs import make_pack, make_tool + +BIN = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin" +RECIPE = runpy.run_path(str(BIN / "droiddeck-recipe")) +SYNC = runpy.run_path(str(BIN / "droiddeck-esync")) +GAME_ENV = runpy.run_path(str(BIN / "droiddeck-game-env")) +COMPAT = runpy.run_path(str(BIN / "steam-compatibility")) +DXVK = "files/lib/wine/dxvk" +VKD3D = "files/lib/wine/vkd3d-proton" +PROTON = ("#!/usr/bin/python3\nimport json, os, sys\n" + "print(json.dumps([sys.argv[0], sys.argv[1:], {k: os.environ.get(k) for k in " + "('DXVK_HUD', 'CUSTOM', 'DROIDDECK_RECIPES', 'FEX_TSOENABLED')}]))\n") + + +def tree_hashes(path): + """Every entry under `path`, with symlink targets and file contents, for before/after checks.""" + result = {} + for folder, dirs, names in os.walk(path): + for name in dirs + names: + full = os.path.join(folder, name) + rel = os.path.relpath(full, path) + if os.path.islink(full): + result[rel] = "-> " + os.readlink(full) + elif os.path.isfile(full): + result[rel] = hashlib.sha256(Path(full).read_bytes()).hexdigest() + else: + result[rel] = "dir" + return result + + +class RecipeTestCase(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.home = self.tmp / "home" + self.root = self.home / ".local/share/droiddeck-recipes" + self.store = self.root / "store" + self.store.mkdir(parents=True) + self.recipes = self.home / ".config/droiddeck/recipes.json" + self.recipes.parent.mkdir(parents=True) + RECIPE["store_root"].__globals__["ROOT"] = str(self.root) + self.depot = self.make_depot(self.tmp / "steamapps/common/Proton Experimental (ARM64)") + + def make_depot(self, depot, line="1789159687 experimental-11.0-20260910b-arm64"): + for arch in ("aarch64-windows", "i386-windows"): + for comp, names in ((DXVK, ("d3d9.dll", "d3d11.dll", "dxgi.dll", "openvr_api_dxvk.dll")), + (VKD3D, ("d3d12.dll", "d3d12core.dll"))): + (depot / comp / arch).mkdir(parents=True, exist_ok=True) + for name in names: + (depot / comp / arch / name).write_bytes(b"valve " + name.encode() + arch.encode()) + windows = depot / "files/lib/wine/aarch64-windows" + unix = depot / "files/lib/wine/aarch64-unix" + windows.mkdir(parents=True, exist_ok=True) + unix.mkdir(parents=True, exist_ok=True) + for name in ("libarm64ecfex.dll", "libwow64fex.dll", "kernel32.dll"): + (windows / name).write_bytes(b"valve " + name.encode()) + for name in ("libarm64ecfex.so", "libwow64fex.so", "ntdll.so"): + (unix / name).write_bytes(b"valve " + name.encode()) + (depot / "files/bin-arm64").mkdir(parents=True, exist_ok=True) + (depot / "files/share/wine").mkdir(parents=True, exist_ok=True) + (depot / "files/share/wine/wine.inf").write_text("inf") + (depot / "version").write_text(line + "\n") + (depot / "proton").write_text(PROTON) + (depot / "proton").chmod(0o755) + return depot + + def make_package(self, name, files, complete=True): + package = self.store / name + for rel, data in files.items(): + (package / rel).parent.mkdir(parents=True, exist_ok=True) + (package / rel).write_bytes(data) + (package / "files").mkdir(parents=True, exist_ok=True) + if complete: + (package / ".complete").write_text("1\n") + return package + + def dxvk_package(self, name="dxvk-2.4-linux", with_i386=True): + files = {DXVK + "/aarch64-windows/d3d11.dll": b"recipe d3d11", DXVK + "/aarch64-windows/dxgi.dll": b"recipe dxgi"} + if with_i386: + files[DXVK + "/i386-windows/d3d11.dll"] = b"recipe d3d11 i386" + return self.make_package(name, files) + + def write(self, games, version=1): + self.recipes.write_text(json.dumps({"version": version, "games": games})) + + def lookup(self, app="42", env=None): + with contextlib.redirect_stderr(io.StringIO()) as said: + found = RECIPE["lookup"](env if env is not None else {"DROIDDECK_RECIPES": "1"}, app, str(self.recipes)) + return found, said.getvalue() + + def assemble(self, recipe, app="42"): + command = [str(self.depot / "proton"), "waitforexitandrun", "game.exe"] + with contextlib.redirect_stderr(io.StringIO()) as said: + result = RECIPE["assemble"](command, recipe, app, str(self.root)) + return result, said.getvalue() + + +class LookupTest(RecipeTestCase): + def test_only_the_auto_tool_and_a_listed_game_get_a_recipe(self): + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package)}}) + self.assertIsNone(self.lookup(env={})[0]) + self.assertIsNone(self.lookup(env={"DROIDDECK_RECIPES": "0"})[0]) + self.assertIsNone(self.lookup(app=None)[0]) + self.assertIsNone(self.lookup(app="43")[0]) + found, _ = self.lookup() + self.assertEqual(found["components"], {"dxvk": os.path.realpath(package)}) + self.recipes.unlink() + self.assertEqual(self.lookup(), (None, "")) + + def test_a_bad_file_is_no_recipe(self): + for text in ("{broken", json.dumps({"version": 2, "games": {}}), json.dumps([1]), json.dumps({"version": 1, "games": []})): + self.recipes.write_text(text) + found, said = self.lookup() + self.assertIsNone(found) + self.assertIn("stock Proton", said) + + def test_packages_must_be_complete_and_in_the_store(self): + outside = self.tmp / "elsewhere" + (outside / "files").mkdir(parents=True) + (outside / ".complete").write_text("1") + partial = self.make_package("vkd3d-partial", {VKD3D + "/aarch64-windows/d3d12.dll": b"x"}, complete=False) + link = self.store / "escape" + os.symlink(outside, link) + self.write({"42": {"dxvk": str(outside), "vkd3d": str(partial), "fex": str(link)}}) + found, _ = self.lookup() + self.assertEqual(found["components"], {}) + self.assertEqual(found["skipped"], ["dxvk", "vkd3d", "fex"]) + self.write({"42": {"dxvk": "relative/path", "fex": 7}}) + self.assertEqual(self.lookup()[0]["skipped"], ["dxvk", "fex"]) + + def test_environment_is_limited_to_tuning_variables(self): + allowed = {"DXVK_HUD": "version", "VKD3D_FEATURE_LEVEL": "12_1", "FEX_TSOENABLED": "1", "MESA_SHADER_CACHE_MAX_SIZE": "1G", + "PROTON_USE_WINED3D": "1", "mesa_glthread": "true", "WINEDLLOVERRIDES": "xinput1_3=n,b;d3dx9_43=n;libsentry=d;dinput8="} + denied = {"LD_PRELOAD": "/x.so", "WINEDLLPATH": "/x", "PROTON_LOG_DIR": "/x", "DXVK_CONFIG_FILE": "/x", + "VKD3D_SHADER_CACHE_PATH": "/x", "MESA_SHADER_CACHE_DIR": "/x", "FEX_ROOTFS": "/x", "FEX_THUNKHOSTLIBS": "/x", + "PATH": "/x", "HOME": "/x", "CUSTOM": "x", "DXVK_HUD2": "a\0b", "BAD-NAME": "x"} + self.write({"42": {"env": {**allowed, **denied}}}) + found, said = self.lookup() + self.assertEqual(found["env"], allowed) + for name in denied: + self.assertIn(name, said) + for value in ("../evil.dll=n", "a=n;b=x", "a=native", "a"): + self.assertFalse(RECIPE["env_allowed"]("WINEDLLOVERRIDES", value), value) + self.assertFalse(RECIPE["env_allowed"]("DXVK_HUD", "x" * 8193)) + self.assertFalse(RECIPE["env_allowed"]("DXVK_HUD", 1)) + + +class TreeTest(RecipeTestCase): + def test_the_tree_links_the_depot_and_replaces_only_the_component(self): + before = tree_hashes(self.depot) + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package), "note": "test"}}) + command, said = self.assemble(self.lookup()[0]) + dist = Path(command[0]).parent + self.assertEqual(command[1:], ["waitforexitandrun", "game.exe"]) + self.assertEqual(dist.parent, self.root / "dist") + self.assertEqual(os.readlink(dist / "proton"), str(self.depot / "proton")) + self.assertEqual(os.readlink(dist / "files/share"), str(self.depot / "files/share")) + self.assertTrue((dist / VKD3D).is_symlink()) + arm = dist / DXVK / "aarch64-windows" + self.assertFalse(arm.is_symlink()) + real = Path(os.path.realpath(package)) + self.assertEqual({p.name: os.readlink(p) for p in arm.iterdir()}, { + "d3d11.dll": str(real / DXVK / "aarch64-windows/d3d11.dll"), + "dxgi.dll": str(real / DXVK / "aarch64-windows/dxgi.dll"), + "openvr_api_dxvk.dll": str(self.depot / DXVK / "aarch64-windows/openvr_api_dxvk.dll"), + }) + self.assertNotIn("d3d9.dll", os.listdir(arm)) + self.assertEqual((dist / DXVK / "i386-windows/d3d11.dll").read_bytes(), b"recipe d3d11 i386") + self.assertEqual((dist / "files/lib/wine/aarch64-unix/ntdll.so").read_bytes(), b"valve ntdll.so") + self.assertIn("app 42: DXVK dxvk-2.4-linux, VKD3D stock, FEX stock, env 0 (test)", said) + self.assertEqual(tree_hashes(self.depot), before) + log = (self.root / "launches.log").read_text().splitlines() + self.assertEqual(log[-1].split("\t")[1:], ["42", "DXVK dxvk-2.4-linux, VKD3D stock, FEX stock, env 0 (test)"]) + + def test_fex_replaces_its_four_files_even_when_the_package_has_no_unix_half(self): + package = self.make_package("fex-2607", {"files/lib/wine/aarch64-windows/libarm64ecfex.dll": b"fex ec", + "files/lib/wine/aarch64-windows/libwow64fex.dll": b"fex wow"}) + self.write({"42": {"fex": str(package)}}) + dist = Path(self.assemble(self.lookup()[0])[0][0]).parent + windows = dist / "files/lib/wine/aarch64-windows" + unix = dist / "files/lib/wine/aarch64-unix" + self.assertEqual((windows / "libarm64ecfex.dll").read_bytes(), b"fex ec") + self.assertEqual(os.readlink(windows / "kernel32.dll"), str(self.depot / "files/lib/wine/aarch64-windows/kernel32.dll")) + self.assertEqual(sorted(os.listdir(unix)), ["ntdll.so"]) + self.assertTrue((dist / DXVK).is_symlink()) + + def test_the_same_inputs_reuse_the_tree_and_changes_make_a_new_one(self): + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package)}}) + recipe = self.lookup()[0] + first = self.assemble(recipe)[0][0] + marker = Path(first).parent / "marker" + marker.write_text("kept") + self.assertEqual(self.assemble(recipe)[0][0], first) + self.assertTrue(marker.exists()) + (self.depot / "version").write_text("1789999999 experimental-11.0-20261001-arm64\n") + updated = self.assemble(recipe)[0][0] + self.assertNotEqual(updated, first) + os.utime(package / ".complete", (time.time() + 5, time.time() + 5)) + self.assertNotEqual(self.assemble(recipe)[0][0], updated) + + def test_a_recipe_without_packages_or_with_a_failure_is_the_stock_command(self): + stock = [str(self.depot / "proton"), "waitforexitandrun", "game.exe"] + self.write({"42": {"env": {"DXVK_HUD": "fps"}}}) + command, said = self.assemble(self.lookup()[0]) + self.assertEqual(command, stock) + self.assertIn("env 1", said) + self.assertFalse((self.root / "dist").exists()) + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package), "vkd3d": str(self.store / "missing")}}) + recipe = self.lookup()[0] + self.assertIn("VKD3D stock (package missing)", self.assemble(recipe)[1]) + shutil.rmtree(package) + command, said = self.assemble(recipe) + self.assertEqual(command, stock) + self.assertIn("tree not assembled", said) + + def test_old_trees_are_collected(self): + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package)}}) + recipe = self.lookup()[0] + dist = Path(self.assemble(recipe)[0][0]).parent + folder = dist.parent + stale = folder / "0123456789abcdef" + stale.mkdir() + temp = folder / ".0123.tmp-1" + temp.mkdir() + fresh_temp = folder / ".4567.tmp-2" + fresh_temp.mkdir() + old = time.time() - RECIPE["DIST_AGE"] - 10 + for path in (stale, dist, temp): + os.utime(path, (old, old)) + RECIPE["collect"](str(folder), keep=dist.name) + self.assertFalse(stale.exists()) + self.assertFalse(temp.exists()) + self.assertTrue(fresh_temp.exists()) + self.assertTrue(dist.exists()) + + +class EsyncCompositionTest(RecipeTestCase): + def test_esync_builds_its_tree_on_top_of_the_recipes(self): + self.make_depot(self.tmp / "valve/Proton Experimental (ARM64)") + depot = make_tool(self.tmp / "valve", "Proton Experimental (ARM64)") + esync = self.home / ".local/share/droiddeck-esync" + esync.mkdir(parents=True) + pack = make_pack(esync, depot, "valve-experimental-r1") + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package)}}) + with contextlib.redirect_stderr(io.StringIO()): + command = RECIPE["assemble"]([str(depot / "proton"), "waitforexitandrun", "game.exe"], self.lookup()[0], "42", str(self.root)) + chosen, env = SYNC["select"](command, {"HOME": str(self.home), "STEAM_COMPAT_DATA_PATH": "/compatdata/42", + "BL_SYNC_FALLBACK": "1", "BL_SYNC_MIN_NOFILE": "1"}) + final = Path(chosen[0]).parent + self.assertEqual(final.parent, esync / "dist") + self.assertEqual(env["BL_SYNC_PACK"], pack["id"]) + self.assertEqual((final / DXVK / "aarch64-windows/d3d11.dll").read_bytes(), b"recipe d3d11") + self.assertEqual((final / SYNC["NTDLL"]).read_bytes(), b"patched ntdll " + pack["id"].encode()) + self.assertEqual((final / "files/lib/wine/aarch64-windows/libarm64ecfex.dll").read_bytes(), b"valve libarm64ecfex.dll") + self.assertIn("winevulkan.so", os.listdir(final / "files/lib/wine/aarch64-unix")) + self.assertIn("msidb", os.listdir(final / "files/bin-arm64")) + + def test_esyncs_directories_are_real_in_the_recipe_tree(self): + # On the device a guest opendir() through a directory symlink fails (proot's fast path), so + # esync must find every directory it lists already real in this tree. + self.assertEqual(RECIPE["ESYNC_DIRS"], SYNC["REAL_DIRS"]) + (self.depot / "files/bin-arm64/wineserver").write_bytes(b"server") + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package)}}) + dist = Path(self.assemble(self.lookup()[0])[0][0]).parent + for rel in SYNC["REAL_DIRS"]: + self.assertTrue((dist / rel).is_dir() and not (dist / rel).is_symlink(), rel) + self.assertEqual(os.readlink(dist / "files/bin-arm64/wineserver"), str(self.depot / "files/bin-arm64/wineserver")) + + +class LaunchTest(RecipeTestCase): + def test_the_auto_launcher_reaches_proton_through_the_recipe_tree(self): + steam = self.tmp / "Steam" + depot = self.make_depot(steam / "steamapps/common" / COMPAT["SOURCES"][0]) + tools = steam / "compatibilitytools.d" + package = self.dxvk_package() + self.write({"42": {"dxvk": str(package), "env": {"DXVK_HUD": "version", "FEX_TSOENABLED": "0"}}}) + config = self.home / ".config/droiddeck/game-environment.json" + config.write_text(json.dumps({"version": 1, "shared": {"CUSTOM": "shared", "DXVK_HUD": "fps"}, + "games": {"42": {"FEX_TSOENABLED": "1"}}})) + outputs = {} + for name in (COMPAT["TOOL"], COMPAT["RECIPE_TOOL"]): + COMPAT["build_tool"](str(tools / name), str(depot), COMPAT["SOURCES"], name) + wrapper = tools / name / COMPAT["LAUNCHER"] + wrapper.write_text(wrapper.read_text().replace("/usr/local/bin/droiddeck-game-env", str(BIN / "droiddeck-game-env"))) + for prefix in ("/compatdata/42", "/compatdata/0"): + result = subprocess.run([str(wrapper), "waitforexitandrun", "game.exe"], text=True, capture_output=True, + env={"PATH": os.defpath, "HOME": str(self.home), "STEAM_COMPAT_DATA_PATH": prefix, + "STEAM_COMPAT_CLIENT_INSTALL_PATH": str(steam)}) + self.assertEqual(result.returncode, 0, result.stderr) + outputs[name, prefix] = (json.loads(result.stdout), result.stderr) + (argv0, args, env), said = outputs[COMPAT["RECIPE_TOOL"], "/compatdata/42"] + self.assertEqual(Path(argv0).parent.parent, self.root / "dist") + self.assertEqual(args, ["waitforexitandrun", "game.exe"]) + self.assertEqual(env, {"DXVK_HUD": "version", "CUSTOM": "shared", "DROIDDECK_RECIPES": None, "FEX_TSOENABLED": "1"}) + self.assertIn("droiddeck-recipe: app 42: DXVK dxvk-2.4-linux", said) + for key in ((COMPAT["TOOL"], "/compatdata/42"), (COMPAT["TOOL"], "/compatdata/0"), (COMPAT["RECIPE_TOOL"], "/compatdata/0")): + (argv0, _, env), said = outputs[key] + self.assertEqual(argv0, str(depot / "proton"), key) + self.assertIsNone(env["DROIDDECK_RECIPES"], key) + self.assertNotIn("droiddeck-recipe", said, key) + self.assertEqual(outputs[COMPAT["TOOL"], "/compatdata/42"][0][2]["DXVK_HUD"], "fps") + + def test_recipe_variables_join_the_automatic_fixes(self): + fixes = {"WINEDLLOVERRIDES": "libsentry=d", "FEX_MULTIBLOCK": "0"} + merged = GAME_ENV["recipe_fixes"](fixes, {"WINEDLLOVERRIDES": "xinput1_3=n", "FEX_MULTIBLOCK": "1"}) + self.assertEqual(merged, {"WINEDLLOVERRIDES": "libsentry=d;xinput1_3=n", "FEX_MULTIBLOCK": "1"}) + self.assertEqual(fixes["FEX_MULTIBLOCK"], "0") + + +if __name__ == "__main__": + unittest.main()