From 292295ba30aba74639a9fca9d03e868d8b1acbd0 Mon Sep 17 00:00:00 2001 From: The412Banner <205237651+The412Banner@users.noreply.github.com> Date: Sun, 18 Oct 2026 03:40:17 -0400 Subject: [PATCH] feat: x86-64 client follows DroidDeck-Components' fex.json; fuller x86-64 session logs (#1053) FEX builds can reach devices without an app release: while the x86-64 client is selected the app reads DroidDeck-Components' fex.json (directly, on its own thread, not in Offline Mode) and a session passes that build to droiddeck-steam-x64, which installs it only once its sha256 matches and its programs load, keeping the replaced files aside until the installed copy loads too; it keeps a working installed build otherwise and falls back to the build it pins. FEX is built and published by Components' build-fex.yml on its single fex release; this repository's build-fex.yml only checks the build. x86-64 session logs name the FEX build and its source, the rootfs and the client's builds, report the rootfs download in tenths, and say in one line when the web helper's GPU process keeps exiting, with x86 glxinfo and vulkaninfo through FEX. Settings shows the FEX build installed. --- .github/workflows/build-fex.yml | 38 +---- .../com/droiddeck/launcher/MainActivity.kt | 3 + .../droiddeck/launcher/session/FexCatalog.kt | 71 ++++++++++ .../launcher/session/SessionService.kt | 1 + .../launcher/ui/ModeSettingsDialog.kt | 16 ++- .../launcher/session/FexCatalogTest.kt | 46 ++++++ .../droiddeck/launcher/ui/FexVersionTest.kt | 5 + docs/development/x86-64-steam-client.md | 36 +++-- docs/progress-log.md | 21 +++ .../overlay/usr/local/bin/droiddeck-steam-x64 | 133 ++++++++++++++++-- tools/tests/test_fex_install.py | 91 ++++++++++++ tools/tests/test_fex_select.py | 85 +++++++++++ 12 files changed, 488 insertions(+), 58 deletions(-) create mode 100644 app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt create mode 100644 app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt create mode 100644 tools/tests/test_fex_install.py create mode 100644 tools/tests/test_fex_select.py diff --git a/.github/workflows/build-fex.yml b/.github/workflows/build-fex.yml index 85ced9cfdc..8ec3369a1a 100644 --- a/.github/workflows/build-fex.yml +++ b/.github/workflows/build-fex.yml @@ -1,4 +1,7 @@ -name: Build FEX (x86-64 Steam client experiment) +name: Build FEX (x86-64 Steam client) + +# Checks tools/fex: the same build DroidDeck-Components' build-fex.yml runs from this repository to publish FEX +# (its single `fex` release and fex.json, which the app follows). Nothing is published from here. on: push: @@ -7,16 +10,12 @@ on: - '.github/workflows/build-fex.yml' workflow_dispatch: inputs: - tag: - description: 'Release tag to publish as (e.g. fex-2609-r1); empty = artifact only' - required: false - default: '' thunks: description: 'Build the host thunks (Vulkan/GL/X11/... to the runtime arm64 libraries)' required: false default: 'false' cxxflags: - description: 'Extra CXXFLAGS for a diagnostic build (e.g. -DDEBUG_STRACE); never published with a tag' + description: 'Extra CXXFLAGS for a diagnostic build (e.g. -DDEBUG_STRACE)' required: false default: '' @@ -48,30 +47,3 @@ jobs: fex.tzst.sha256 fex-needed.txt fex-files.txt - - # Only this job can write: the build above runs third-party images and toolchains with a read-only token. - publish: - needs: build - if: github.event_name == 'workflow_dispatch' && inputs.tag != '' && inputs.cxxflags == '' && inputs.thunks == 'true' - runs-on: ubuntu-24.04 - permissions: - contents: write - steps: - - uses: actions/download-artifact@v4 - with: - name: fex - - name: Publish - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - # droiddeck-steam-x64 pins each tag's sha256 and the build is not reproducible: never rebuild over a tag. - if gh release view "$TAG" >/dev/null 2>&1; then - echo "::error::release $TAG already exists; publish under a new tag" - exit 1 - fi - gh release create "$TAG" --target "$GITHUB_SHA" --latest=false --prerelease=false --title "$TAG" --notes "FEX for the runtime (tools/fex/build-in-arch.sh); fetched by droiddeck-steam-x64. sha256 $(cut -d' ' -f1 fex.tzst.sha256)" fex.tzst fex.tzst.sha256 - # Fetched by tag by the app, never installed by users: never the repo's Latest release nor a pre-release. - gh release edit "$TAG" --latest=false --prerelease=false diff --git a/app/src/main/java/com/droiddeck/launcher/MainActivity.kt b/app/src/main/java/com/droiddeck/launcher/MainActivity.kt index 4c41ea2693..6e704951d5 100644 --- a/app/src/main/java/com/droiddeck/launcher/MainActivity.kt +++ b/app/src/main/java/com/droiddeck/launcher/MainActivity.kt @@ -1906,6 +1906,9 @@ class MainActivity : ComponentActivity() { val scanGeneration = ++libraryScanGeneration val artRefreshesAtStart = HashMap(steamArtRefreshes) if (shortcutPicker) shortcutLibraryScanning = true + if (SessionPrefs.steamClientArch(this) == "x86_64" && !OfflineMode.enabled(this)) { + Thread({ com.droiddeck.launcher.session.FexCatalog.refresh(this) }, "fex-catalog").start() + } Thread({ val emus = Library.emulators(this) { id -> DesktopCatalog.installed(this, id) != null } ui.post { if (scanGeneration == libraryScanGeneration) emulatorList = emus } diff --git a/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt b/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt new file mode 100644 index 0000000000..73f42cb141 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt @@ -0,0 +1,71 @@ +package com.droiddeck.launcher.session + +import android.content.Context +import android.util.Log +import com.droiddeck.launcher.core.DownloadSource +import com.droiddeck.launcher.core.Downloader +import org.json.JSONObject +import java.io.File + +/** + * The FEX build DroidDeck-Components' fex.json names for the x86-64 client. A session uses the last copy read. + * Components' build-fex.yml replaces the archive on its single `fex` release and records the build's version; + * a release per build (fex-2610-r5) is still read. + */ +object FexCatalog { + private const val TAG = "FexCatalog" + const val CATALOG_URL = "https://raw.githubusercontent.com/Droid-Deck/DroidDeck-Components/main/fex.json" + private const val RELEASES = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/" + private const val REFRESH_MS = 10 * 60 * 1000L + private val TAG_FORMAT = Regex("""fex-\d+-r\d+""") + private val VERSION_FORMAT = Regex("""\d+-r\d+""") + private val SHA256 = Regex("[0-9a-f]{64}") + + data class Build(val tag: String, val url: String, val sha256: String) + + @Volatile private var refreshedAt = 0L + + private fun file(context: Context) = File(context.filesDir, "fex-catalog.json") + + internal fun parse(text: String): Build? = runCatching { + val o = JSONObject(text) + val release = o.getString("release") + val url = o.getString("url") + val sha = o.getString("sha256").lowercase() + val version = o.optString("version") + val tag = when { + TAG_FORMAT.matches(release) -> release + release == "fex" && VERSION_FORMAT.matches(version) -> "fex-$version" + else -> return@runCatching null + } + Build(tag, url, sha).takeIf { url == "$RELEASES$release/fex.tzst" && SHA256.matches(sha) } + }.getOrNull() + + fun cached(context: Context): Build? = runCatching { parse(file(context).readText()) }.getOrNull() + + /** Reads fex.json straight from GitHub (a digest never comes from a mirror), at most every ten minutes. */ + @Synchronized + fun refresh(context: Context, now: Long = System.currentTimeMillis()): Build? { + if (now - refreshedAt < REFRESH_MS) return cached(context) + refreshedAt = now + val text = Downloader.downloadDirect(CATALOG_URL) ?: return cached(context) + val build = parse(text) ?: run { + Log.w(TAG, "fex.json not usable; keeping the last one") + return cached(context) + } + runCatching { + val f = file(context) + val tmp = File(f.path + ".tmp") + tmp.writeText(text) + if (!tmp.renameTo(f)) tmp.delete() + } + return build + } + + /** For droiddeck-steam-x64: the build and every source its archive may come from (it checks the sha256). */ + fun environment(build: Build, sources: List = DownloadSource.candidates(build.url)): List = listOf( + "BL_FEX_TAG=${build.tag}", + "BL_FEX_SHA256=${build.sha256}", + "BL_FEX_URLS=" + sources.joinToString(" "), + ) +} diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 1d8db3d539..4d61f698da 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -590,6 +590,7 @@ class SessionService : Service() { SessionState.steamClientArch?.let { arch -> guest.add("BL_STEAM_CLIENT_ARCH=$arch") SessionPaths.current()?.let { SessionArtifacts.recordClient(it, arch) } + if (arch == "x86_64") FexCatalog.cached(this)?.let { guest.addAll(FexCatalog.environment(it)) } } if (SessionState.mode == MODE_STEAM) { guest.add("BL_GAMESCOPE_STRETCH_16X9=" + (if (SessionPrefs.stretch16x9(this)) "1" else "0")) diff --git a/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt b/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt index 2cd1c63b8b..0fb34d6634 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt @@ -152,13 +152,21 @@ class ModeSettingsActions( /** "2610 r3" from droiddeck-steam-x64's FEX_TAG line (fex-2610-r3), the build the x86-64 client downloads. */ internal fun fexVersionOf(script: String): String? = - Regex("""(?m)^FEX_TAG=fex-(\d+)-r(\d+)\s*$""").find(script)?.let { "${it.groupValues[1]} r${it.groupValues[2]}" } + Regex("""(?m)^FEX_TAG=(fex-\d+-r\d+)\s*$""").find(script)?.let { fexVersionOfTag(it.groupValues[1]) } +/** "2610 r5" from a tag such as fex-2610-r5. */ +internal fun fexVersionOfTag(tag: String): String? = + Regex("""fex-(\d+)-r(\d+)""").matchEntire(tag.trim())?.let { "${it.groupValues[1]} r${it.groupValues[2]}" } + +/** The build installed in the runtime, else the one fex.json names, else the one the launcher pins. */ @Composable private fun x64FexVersion(context: android.content.Context): String? = remember { - runCatching { - context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) } - }.getOrNull() + val root = com.droiddeck.launcher.runtime.LinuxRuntime.rootDir(context) + runCatching { java.io.File(root, "usr/local/share/fex-emu/.droiddeck-tag").readText() }.getOrNull()?.let(::fexVersionOfTag) + ?: com.droiddeck.launcher.session.FexCatalog.cached(context)?.tag?.let(::fexVersionOfTag) + ?: runCatching { + context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) } + }.getOrNull() } @Composable diff --git a/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt b/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt new file mode 100644 index 0000000000..3f4fa2f75d --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt @@ -0,0 +1,46 @@ +package com.droiddeck.launcher.session + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class FexCatalogTest { + private val sha = "0fc1ca2ba13817ad70d1ea157119a75c9c02446b57d166e6d676cce0807c19c4" + private val url = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/fex-2610-r5/fex.tzst" + + private fun json(release: String = "fex-2610-r5", url: String = this.url, sha256: String = sha) = + """{"schemaVersion": 1, "category": "fex", "release": "$release", "url": "$url", "sha256": "$sha256", "size": 4083539}""" + + @Test fun readsTheBuildComponentsNames() { + assertEquals(FexCatalog.Build("fex-2610-r5", url, sha), FexCatalog.parse(json())) + } + + @Test fun readsTheSingleReleaseByItsVersion() { + val single = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/fex/fex.tzst" + assertEquals(FexCatalog.Build("fex-2610-r6", single, sha), FexCatalog.parse(json(release = "fex", url = single).replace("}", ", \"version\": \"2610-r6\"}"))) + assertNull(FexCatalog.parse(json(release = "fex", url = single).replace("}", ", \"version\": \"latest\"}"))) + assertNull(FexCatalog.parse(json(release = "fex", url = single))) + } + + @Test fun refusesAnArchiveOutsideItsOwnRelease() { + assertNull(FexCatalog.parse(json(url = "https://example.com/fex.tzst"))) + assertNull(FexCatalog.parse(json(url = url.replace("fex-2610-r5/", "fex-2610-r4/")))) + assertNull(FexCatalog.parse(json(release = "latest", url = url.replace("fex-2610-r5", "latest")))) + } + + @Test fun refusesAMissingOrMalformedDigest() { + assertNull(FexCatalog.parse(json(sha256 = "abc"))) + assertNull(FexCatalog.parse("""{"release": "fex-2610-r5", "url": "$url"}""")) + assertNull(FexCatalog.parse("not json")) + } + + @Test fun theSessionGetsTheBuildAndEverySource() { + val env = FexCatalog.environment(FexCatalog.Build("fex-2610-r5", url, sha), listOf(url, "https://ghfast.top/$url")) + assertEquals(listOf("BL_FEX_TAG=fex-2610-r5", "BL_FEX_SHA256=$sha", "BL_FEX_URLS=$url https://ghfast.top/$url"), env) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt b/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt index 8fb2016365..3d4600f8c1 100644 --- a/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt @@ -14,6 +14,11 @@ class FexVersionTest { assertNull(fexVersionOf("# FEX_TAG=fex-2610-r3\n")) } + @Test fun readsAnInstalledOrCatalogTag() { + assertEquals("2610 r5", fexVersionOfTag("fex-2610-r5\n")) + assertNull(fexVersionOfTag("latest")) + } + @Test fun theShippedLauncherHasOne() { val script = java.io.File("../tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64").takeIf { it.isFile } ?: java.io.File("tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64") diff --git a/docs/development/x86-64-steam-client.md b/docs/development/x86-64-steam-client.md index aebfe51342..25e25a700c 100644 --- a/docs/development/x86-64-steam-client.md +++ b/docs/development/x86-64-steam-client.md @@ -186,27 +186,43 @@ alone would then read as an installed client. restarted from the top forever (found with `perf_event_open` IP sampling, since proot holds ptrace). Such a range is now skipped whole and a second exhausted scan returns `-ENOMEM`, as the kernel does (`fex-2610-r3`). -- **`build-fex.yml`.** The build job runs third-party images and toolchains with a read-only token, - and inputs reach the shell as environment variables. A separate publish job (contents: write) runs - only on a dispatch with a tag, host thunks on and no extra `cxxflags` (diagnostic builds such as - `-DDEBUG_STRACE` stay artifacts). `droiddeck-steam-x64` pins each tag's sha256 and the archive is not - reproducible, so a tag is never rebuilt over. The release is never the repo's Latest nor a - pre-release: it is a component fetched by tag, not something users install. +- **Building and publishing.** Droid-Deck/DroidDeck-Components' `build-fex.yml` builds `tools/fex` from + this repository at a chosen ref (default `main`) with host thunks, in the same Arch Linux ARM container + with a read-only token, then a separate job replaces `fex.tzst` on Components' single `fex` release, + checks the digest GitHub stored, and records the build's version, sha256 and size in its `fex.json` + (in that order: a device that reads the old `fex.json` meanwhile gets a mismatching archive and keeps + its FEX). This repository's `build-fex.yml` only builds `tools/fex` when it changes (and on dispatch, + for diagnostic builds such as `-DDEBUG_STRACE`); nothing is published from it. - Releases so far: `fex-2609-r2` (no thunks), `fex-2609-r3` (thunks), `fex-2610-r1`, `fex-2610-r2`, `fex-2610-r3`, `fex-2610-r4` (in-root resolution when openat2 is missing, exact MAP_32BIT marking), `fex-2610-r5` (current: bundles libxxhash, which the Linux runtime does not ship; the build fails when a program in the package needs a library that is neither bundled nor listed in `tools/fex/runtime-libs.txt`). They were built and published in The412Banner/DroidDeck; up to r3 their tags point at that - repository's `main`, from r4 at the commit each was built from. + repository's `main`, from r4 at the commit each was built from. Later builds are versions on the single + `fex` release; `fex-2610-r5` stays published as the build `droiddeck-steam-x64` pins. - **Where the app gets it.** The app downloads FEX from Droid-Deck/DroidDeck-Components, where nothing is - uploaded by hand: a new tag is published by `build-fex.yml`, recorded in that repository's `fex.json` - (sha256 and size), and brought in by its `mirror-release.yml`, which refuses an archive that does not - match. `FEX_TAG` and `FEX_SHA256` in `droiddeck-steam-x64` then move to the new tag. The rootfs is + uploaded by hand. The app follows its `fex.json`: while the x86-64 client is selected it reads it from + Components' `main` (directly, at most every ten minutes) and keeps the last copy, and a session passes + that build to `droiddeck-steam-x64` (`BL_FEX_TAG`, `BL_FEX_SHA256`, `BL_FEX_URLS` with the download + source's mirrors). The script installs a build only after its archive matches the sha256 and every + program in it loads (`ld-linux --list`); the files it replaces are kept aside until the installed copy + loads too and put back otherwise, and the tag is written last. Otherwise it keeps the installed one while that still loads, + then falls back to `FEX_TAG`/`FEX_SHA256` built in, which stay the first-start default when no copy + of `fex.json` has been read. Publishing a FEX build therefore takes a Components run only; a + replaced archive cannot be pointed back to, so undoing a build means building an earlier ref. The rootfs is FEX-Emu's own image, unmodified: `fex-rootfs.json` there names its rootfs.fex-emu.gg link, sha256 and size, and `mirror-upstream.yml` fetches it into the catalog's release only when both match. A newer image is a new catalog release; `ROOTFS_URL` and `ROOTFS_SHA256` then move to it, and a device that already unpacked one keeps it. +- **What a session log shows.** `FEX ready (, )` names the build and where it came from + (`catalog`, `built-in`, or `kept` when a newer one did not install, with the reason a line above); + `x86-64 rootfs:` names the image; `x86-64 client:` the client's ubuntu12 and steamrt64 builds at start + (it may update itself after). The rootfs download reports every tenth of its size as a step. When the + web helper's GPU process keeps exiting until Chromium gives up on the GPU (the menus then never appear), + one line says so with the exit codes, followed by x86 `glxinfo -B` and `vulkaninfo --summary` through + FEX's thunks. + ## DroidDeck direct for Linux games and runtimes - Steam runs native Linux games in the Steam Linux Runtime container (pressure-vessel), which needs diff --git a/docs/progress-log.md b/docs/progress-log.md index 65a26cbde3..81ec5bc3a3 100644 --- a/docs/progress-log.md +++ b/docs/progress-log.md @@ -139,6 +139,27 @@ the timeline, then lessons and backlog. Companion to the README (what the app *d FEX uses). With the device's libxxhash hidden, r4 failed with the reported error and r5 loaded. Mirrored in DroidDeck-Components (#5); the app moves to fex-2610-r5 and an installed r4 is replaced at the next x86-64 start. +- Merged as 7d21ba22. +- **FEX from the catalog**: FEX builds may need to follow user reports faster than app builds, so the + app now reads Components' `fex.json` (directly, kept between starts) and passes it to the session; a + build is installed only once its sha256 matches and its programs load, the installed one stays while it + loads, and the build pinned in the script remains the first-start default. **Session logs** for the + x86-64 client now name the FEX build and its source, the rootfs and the client's builds, report the + rootfs download in tenths instead of a progress bar (one report's log ended in hundreds of bar lines), + and, when the web helper's GPU process keeps exiting (a report from an Adreno 840: the menus never + appeared), say so in one line with x86 `glxinfo`/`vulkaninfo` through FEX. Settings shows the FEX + build installed. FEX is now built and published by DroidDeck-Components' `build-fex.yml` from this + repository's `tools/fex`, replacing the archive on its single `fex` release and recording the version + in `fex.json`; this repository's `build-fex.yml` only checks the build. Tests: `test_fex_select.py`, + `FexCatalogTest`. + Components' first build there published `2610-r6` (run 39104443888). Device (Pocket FIT, r4 + installed): the app read `fex.json`, and the next x86-64 start logged `downloading FEX (fex-2610-r6)` + then `FEX ready (fex-2610-r6, catalog)` a second later, with the rootfs and client lines; the client + came up on it. +- Review of #1053: an install keeps the files it replaces aside until the installed copy loads and puts + them back otherwise (a failed copy could leave a partly replaced FEX marked as the new one; + `test_fex_install.py`), and the catalog is read on its own thread, not before the library scan, and + not in Offline Mode. ## 2026-10-15 - `feat/x86-64-steam-client`: SD card library, ARM parity, org main, Counter-Strike 2 diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 index 4a2113fbf4..2c27c9db4d 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 @@ -319,24 +319,101 @@ print("droiddeck-steam-x64: %d Linux title(s) on DroidDeck direct (%d on the new PY } -x64_prepare() { -if [ ! -x /usr/local/bin/FEX ] || [ "$(cat /usr/local/share/fex-emu/.droiddeck-tag 2>/dev/null)" != "$FEX_TAG" ]; then - step "downloading FEX ($FEX_TAG)" +# Whether every program in the FEX tree at $1 finds its libraries (r4 did not without libxxhash). +fex_loads() { + local b out + for b in FEX FEXServer unsquashfs; do + out=$(/usr/lib/ld-linux-aarch64.so.1 --list "$1/usr/local/bin/$b" 2>&1) && ! grep -q "not found" <<<"$out" && continue + echo "== FEX check: $b: $(grep -E "not found|error" <<<"$out" | tr -s '\t ' ' ' | tr '\n' ';')" >&2 + return 1 + done +} + +# Installs one FEX build only once its archive matches the sha256 and it loads. The files it replaces are kept +# until the installed copy loads too, and put back otherwise; the tag is written last. +fex_install() { + local tag=$1 sha=$2 urls=$3 dst=/ tmp u f tmp=$(mktemp -d) - curl -fL --progress-bar "$FEX_URL" -o "$tmp/fex.tzst" - if [ "$FEX_SHA256" != PENDING ]; then echo "$FEX_SHA256 $tmp/fex.tzst" | sha256sum -c --quiet; fi - zstd -dc "$tmp/fex.tzst" | tar -C / -x + for u in $urls; do + curl -fsSL --retry 2 "$u" -o "$tmp/fex.tzst" && echo "$sha $tmp/fex.tzst" | sha256sum -c --quiet >/dev/null 2>&1 && break + rm -f "$tmp/fex.tzst" + done + if [ ! -s "$tmp/fex.tzst" ]; then + echo "== FEX $tag: no source gave the archive with sha256 ${sha:0:12}" >&2 + rm -rf "$tmp" + return 1 + fi + if ! { mkdir "$tmp/root" && zstd -dc "$tmp/fex.tzst" | tar -C "$tmp/root" -x && fex_loads "$tmp/root"; }; then + rm -rf "$tmp" + return 1 + fi + (cd "$tmp/root" && find . ! -type d) > "$tmp/files" + mkdir "$tmp/old" + while IFS= read -r f; do + f=${f#./} + if [ -e "$dst$f" ] || [ -L "$dst$f" ]; then + cp -a --parents "$dst$f" "$tmp/old" || { echo "== FEX $tag: could not keep $dst$f aside" >&2; rm -rf "$tmp"; return 1; } + fi + done < "$tmp/files" + if tar -C "$tmp/root" -c . | tar -C "$dst" -x && fex_loads "$dst" && + mkdir -p "${dst}usr/local/share/fex-emu" && echo "$tag" > "${dst}usr/local/share/fex-emu/.droiddeck-tag"; then + rm -rf "$tmp" + return 0 + fi + echo "== FEX $tag: the installed copy did not load or its tag was not written; putting the previous files back" >&2 + while IFS= read -r f; do + f=${f#./} + [ -e "$tmp/old/$dst$f" ] || [ -L "$tmp/old/$dst$f" ] || rm -f "$dst$f" + done < "$tmp/files" + tar -C "$tmp/old$dst" -c . | tar -C "$dst" -x || echo "== FEX $tag: could not put the previous files back" >&2 rm -rf "$tmp" - mkdir -p /usr/local/share/fex-emu && echo "$FEX_TAG" > /usr/local/share/fex-emu/.droiddeck-tag -fi -step "FEX ready ($FEX_TAG)" + return 1 +} + +# The build DroidDeck-Components' fex.json names (the app passes it as BL_FEX_*) comes first, then a working +# installed one, then FEX_TAG above. Sets FEX_NOW and FEX_FROM. +x64_fex() { + local have ok= + have=$(cat /usr/local/share/fex-emu/.droiddeck-tag 2>/dev/null || true) + [ -x /usr/local/bin/FEX ] && fex_loads / && ok=1 + if [ -n "${BL_FEX_TAG:-}" ]; then + if [ -n "$ok" ] && [ "$have" = "$BL_FEX_TAG" ]; then FEX_NOW=$have FEX_FROM=catalog; return 0; fi + step "downloading FEX ($BL_FEX_TAG)" + if fex_install "$BL_FEX_TAG" "${BL_FEX_SHA256:-}" "${BL_FEX_URLS:-}"; then FEX_NOW=$BL_FEX_TAG FEX_FROM=catalog; return 0; fi + if [ -n "$ok" ]; then FEX_NOW=$have FEX_FROM="kept; $BL_FEX_TAG did not install"; return 0; fi + fi + if [ -n "$ok" ] && [ "$have" = "$FEX_TAG" ]; then FEX_NOW=$have FEX_FROM=built-in; return 0; fi + step "downloading FEX ($FEX_TAG)" + if fex_install "$FEX_TAG" "$FEX_SHA256" "$FEX_URL"; then FEX_NOW=$FEX_TAG FEX_FROM=built-in; return 0; fi + if [ -n "$ok" ]; then FEX_NOW=$have FEX_FROM="kept; $FEX_TAG did not install"; return 0; fi + return 1 +} + +# A resumable download that reports every tenth of its size as a step. +x64_fetch() { + local url=$1 out=$2 label=$3 total pid rc=0 last=-1 pct + total=$(curl -fsSIL "$url" 2>/dev/null | tr -d '\r' | awk 'tolower($1) == "content-length:" { v = $2 } END { print v + 0 }') + curl -fL -sS -C - "$url" -o "$out" & + pid=$! + while kill -0 "$pid" 2>/dev/null; do + sleep 3 + [ "$total" -gt 0 ] || continue + pct=$(( $(stat -c %s "$out" 2>/dev/null || echo 0) * 10 / total * 10 )) + if [ "$pct" -gt "$last" ]; then last=$pct; step "$label ($pct%)"; fi + done + wait "$pid" || rc=$? + return "$rc" +} + +x64_prepare() { +x64_fex || { step "FEX could not be installed; see $LOG"; exit 1; } +step "FEX ready ($FEX_NOW, $FEX_FROM)" # The client's opt-in to its SteamRT3 build; without it, it re-executes through steam.sh, whose checks fail under proot. mkdir -p "$STEAM_X64" && : > "$STEAM_X64/.steam-enable-steamrt64-client" if [ ! -f "$ROOTFS/.droiddeck-complete" ]; then - step "downloading FEX's Ubuntu 24.04 x86 rootfs" # Resumes a part-done download; a finished one answers 416, which is not a failure here. - curl -fL --progress-bar -C - "$ROOTFS_URL" -o "$X64_HOME/rootfs.sqsh" || { + x64_fetch "$ROOTFS_URL" "$X64_HOME/rootfs.sqsh" "downloading FEX's Ubuntu 24.04 x86 rootfs" || { rc=$? [ "$rc" = 22 ] && [ -s "$X64_HOME/rootfs.sqsh" ] || { step "rootfs download failed (curl $rc)"; exit 1; } } @@ -350,6 +427,7 @@ if [ ! -f "$ROOTFS/.droiddeck-complete" ]; then rm -f "$X64_HOME/rootfs.sqsh" touch "$ROOTFS/.droiddeck-complete" fi +echo "== x86-64 rootfs: $(sed -n 's|.*/download/\([^/]*\)/.*|\1|p' <<<"$ROOTFS_URL") (sha256 ${ROOTFS_SHA256:0:12})" >&2 # FEX falls back to the guest's own /etc/ld.so.preload (arm64 libraries) when the rootfs has none. The x86 # session shim provides System V IPC, which Android kernels lack and the client's tier0 needs to start. : > "$ROOTFS/etc/ld.so.preload" @@ -792,6 +870,37 @@ export STEAM_STEAMRT_RECURSION_GUARD=1 export BL_WEBHELPER_HIDE_TRACER=1 } +x64_manifest_version() { + local chan + chan=$(cat "$STEAM_X64/package/beta" 2>/dev/null || echo "$X64_CHANNEL") + awk -F'"' '$2 == "version" { print $4; exit }' "$STEAM_X64/package/steam_client_${chan}_$1.manifest" 2>/dev/null || echo "?" +} + +# What x86 GL and Vulkan look like through FEX's thunks, for a session whose menus failed on the GPU. +x64_gpu_probe() { + local fex=${BL_FEX:-/usr/local/bin/FEX} + echo "== x86-64 GPU probe (x86 glxinfo and vulkaninfo through FEX):" + ( + unset LD_LIBRARY_PATH + export DISABLE_GAMESCOPE_WSI=1 vk_xwayland_wait_ready=true LIBGL_KOPPER_DISABLE=true + timeout 30 "$fex" /usr/bin/glxinfo -B 2>&1 | grep -E "OpenGL (renderer|core profile version|version) string|direct rendering|rror" | head -5 + timeout 30 "$fex" /usr/bin/vulkaninfo --summary 2>&1 | grep -E "deviceName|driverName|driverInfo|apiVersion|rror" | head -6 + ) | sed 's/^/ /' +} + +# Steam's menus come from the web helper; when its GPU process keeps exiting, Chromium gives up on the GPU and +# the menus never appear. Says so once in the session log, with the probe above. +x64_gpu_watch() { + local log=$STEAM_X64/logs/console-linux.txt from n codes + from=$(( $(wc -l 2>/dev/null < "$log" || echo 0) + 1 )) + tail -n0 -F --pid=$$ "$log" 2>/dev/null | grep -q -m1 "GPU process isn't usable" || return 0 + n=$(tail -n +"$from" "$log" | grep -c "GPU process exited unexpectedly" || true) + codes=$(tail -n +"$from" "$log" | grep -oE "GPU process exited unexpectedly: exit_code=[0-9]+" | sed 's/.*=//' | + sort | uniq -c | awk '{ printf "%s%s x%s", s, $2, $1; s = ", " }') + echo "== x86-64: Steam's menu renderer gave up on the GPU after $n GPU process exits (exit code ${codes:-?}); the menus stay blank. See steam/console-linux.txt and steam/webhelper_gpu.txt" + x64_gpu_probe +} + x64_exec() { x64_env cd "$STEAM_X64" @@ -828,6 +937,8 @@ x64_exec() { touch "$STEAM_X64/logs/bootstrap_log.txt" ( tail -n0 -F --pid=$$ "$STEAM_X64/logs/bootstrap_log.txt" 2>/dev/null | grep --line-buffered -oE 'Downloading update \([0-9,]+ of [0-9,]+ KB\)' | sed -u 's/,//g' >&2 ) & + echo "== x86-64 client: ubuntu12 $(x64_manifest_version ubuntu12), steamrt64 $(x64_manifest_version steamrt64) (the client may update itself)" >&2 + x64_webhelper_gpu && { x64_gpu_watch >&2 & } exec "${BL_FEX:-/usr/local/bin/FEX}" "$STEAM_X64/steamrt64/steam" "$@" } diff --git a/tools/tests/test_fex_install.py b/tools/tests/test_fex_install.py new file mode 100644 index 0000000000..355b540381 --- /dev/null +++ b/tools/tests/test_fex_install.py @@ -0,0 +1,91 @@ +import hashlib +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin/droiddeck-steam-x64" +SOURCE = SCRIPT.read_text() +START = SOURCE.index("fex_install() {") +END = SOURCE.index("\n}\n", START) + 3 +FEX_INSTALL = SOURCE[START:END] + +HARNESS = textwrap.dedent("""\ + set -eu + # The archive comes from PKG; the installed tree loads when INSTALLED_LOADS=1, any staged tree always does. + curl() { local out=; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done; cp "$PKG" "$out"; } + fex_loads() { case $1 in "$ROOT"/) [ "${INSTALLED_LOADS:-1}" = 1 ] ;; *) true ;; esac; } + """) + + +class FexInstallTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + base = Path(self.temp.name) + self.root = base / "root" + old = self.root / "usr/local" + (old / "bin").mkdir(parents=True) + (old / "bin/FEX").write_text("old FEX\n") + (old / "share/fex-emu").mkdir(parents=True) + (old / "share/fex-emu/.droiddeck-tag").write_text("fex-2610-r5\n") + new = base / "pkg/usr/local" + (new / "bin").mkdir(parents=True) + (new / "bin/FEX").write_text("new FEX\n") + (new / "lib").mkdir() + (new / "lib/libxxhash.so.0").write_text("lib\n") + self.pkg = base / "fex.tzst" + subprocess.run(f"tar -C '{base / 'pkg'}' -c . | zstd -q -o '{self.pkg}'", shell=True, check=True) + self.sha = hashlib.sha256(self.pkg.read_bytes()).hexdigest() + + def install(self, installed_loads, sha=None): + body = FEX_INSTALL.replace("local tag=$1 sha=$2 urls=$3 dst=/ ", "local tag=$1 sha=$2 urls=$3 dst=$ROOT/ ") + self.assertNotEqual(body, FEX_INSTALL) + script = HARNESS + body + 'if fex_install fex-2610-r6 "$SHA" u; then echo OK; else echo FAILED; fi\n' + env = dict(os.environ, ROOT=str(self.root), PKG=str(self.pkg), SHA=sha or self.sha, + INSTALLED_LOADS="1" if installed_loads else "0") + out = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True, timeout=30) + self.assertEqual(0, out.returncode, out.stderr) + return out.stdout.strip().splitlines()[-1], out.stderr + + def tag(self): + return (self.root / "usr/local/share/fex-emu/.droiddeck-tag").read_text().strip() + + def test_a_build_that_loads_once_installed_replaces_the_old_one(self): + result, _ = self.install(installed_loads=True) + self.assertEqual("OK", result) + self.assertEqual("new FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertTrue((self.root / "usr/local/lib/libxxhash.so.0").is_file()) + self.assertEqual("fex-2610-r6", self.tag()) + + def test_an_installed_copy_that_does_not_load_puts_the_old_files_back(self): + result, err = self.install(installed_loads=False) + self.assertEqual("FAILED", result) + self.assertIn("putting the previous files back", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertFalse((self.root / "usr/local/lib/libxxhash.so.0").exists()) + self.assertEqual("fex-2610-r5", self.tag()) + + def test_a_tag_that_cannot_be_written_puts_the_old_files_back(self): + share = self.root / "usr/local/share/fex-emu" + (share / ".droiddeck-tag").unlink() + share.rmdir() + share.write_text("not a folder\n") + result, err = self.install(installed_loads=True) + self.assertEqual("FAILED", result) + self.assertIn("putting the previous files back", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertFalse((self.root / "usr/local/lib/libxxhash.so.0").exists()) + + def test_an_archive_that_does_not_match_installs_nothing(self): + result, err = self.install(installed_loads=True, sha="0" * 64) + self.assertEqual("FAILED", result) + self.assertIn("no source gave the archive", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertEqual("fex-2610-r5", self.tag()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_fex_select.py b/tools/tests/test_fex_select.py new file mode 100644 index 0000000000..94008a1c70 --- /dev/null +++ b/tools/tests/test_fex_select.py @@ -0,0 +1,85 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin/droiddeck-steam-x64" +SOURCE = SCRIPT.read_text() +START = SOURCE.index("x64_fex() {") +END = SOURCE.index("\n}\n", START) + 3 +X64_FEX = SOURCE[START:END] + +HARNESS = textwrap.dedent("""\ + set -eu + FEX_TAG=fex-2610-r5 FEX_SHA256=builtin FEX_URL=builtin-url + step() { echo "STEP $*"; } + # Installed FEX loads when LOADS=1; a build installs when its tag is in INSTALLABLE. + fex_loads() { [ "${LOADS:-0}" = 1 ]; } + fex_install() { + echo "INSTALL $1 $2 $3" + case " ${INSTALLABLE:-} " in *" $1 "*) echo "$1" > "$ROOT/tag"; : > "$ROOT/FEX"; chmod +x "$ROOT/FEX"; return 0 ;; esac + return 1 + } + """) + + +class FexSelectTest(unittest.TestCase): + def run_select(self, installed=None, loads=False, catalog=None, installable=()): + with tempfile.TemporaryDirectory() as root: + if installed: + Path(root, "tag").write_text(installed + "\n") + fex = Path(root, "FEX") + fex.write_text("") + fex.chmod(0o755) + body = (X64_FEX.replace("/usr/local/share/fex-emu/.droiddeck-tag", "$ROOT/tag") + .replace("/usr/local/bin/FEX", "$ROOT/FEX")) + script = HARNESS + body + 'if x64_fex; then echo "NOW $FEX_NOW|$FEX_FROM"; else echo FAILED; fi\n' + env = dict(os.environ, ROOT=root, LOADS="1" if loads else "0", INSTALLABLE=" ".join(installable)) + env.pop("BL_FEX_TAG", None) + if catalog: + env.update(BL_FEX_TAG=catalog, BL_FEX_SHA256="catsha", BL_FEX_URLS="a b") + out = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True, timeout=10) + self.assertEqual(0, out.returncode, out.stderr) + lines = out.stdout.splitlines() + return [l for l in lines if l.startswith("INSTALL")], lines[-1] + + def test_the_catalog_build_already_installed_is_used(self): + installs, now = self.run_select(installed="fex-2610-r6", loads=True, catalog="fex-2610-r6") + self.assertEqual([], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_newer_catalog_build_is_installed(self): + installs, now = self.run_select(installed="fex-2610-r5", loads=True, catalog="fex-2610-r6", installable=["fex-2610-r6"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_catalog_build_that_does_not_install_keeps_the_working_one(self): + installs, now = self.run_select(installed="fex-2610-r5", loads=True, catalog="fex-2610-r6") + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r5|kept; fex-2610-r6 did not install", now) + + def test_without_a_catalog_the_built_in_build_replaces_an_older_one(self): + installs, now = self.run_select(installed="fex-2610-r4", loads=True, installable=["fex-2610-r5"]) + self.assertEqual(["INSTALL fex-2610-r5 builtin builtin-url"], installs) + self.assertEqual("NOW fex-2610-r5|built-in", now) + + def test_an_installed_build_that_does_not_load_is_replaced(self): + installs, now = self.run_select(installed="fex-2610-r6", loads=False, catalog="fex-2610-r6", installable=["fex-2610-r6"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_first_install_falls_back_to_the_built_in_build(self): + installs, now = self.run_select(catalog="fex-2610-r6", installable=["fex-2610-r5"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b", "INSTALL fex-2610-r5 builtin builtin-url"], installs) + self.assertEqual("NOW fex-2610-r5|built-in", now) + + def test_nothing_usable_fails(self): + installs, now = self.run_select(catalog="fex-2610-r6") + self.assertEqual(2, len(installs)) + self.assertEqual("FAILED", now) + + +if __name__ == "__main__": + unittest.main()