diff --git a/.github/workflows/build-fex.yml b/.github/workflows/build-fex.yml index 85ced9cfdc..8ec3369a1a 100644 --- a/.github/workflows/build-fex.yml +++ b/.github/workflows/build-fex.yml @@ -1,4 +1,7 @@ -name: Build FEX (x86-64 Steam client experiment) +name: Build FEX (x86-64 Steam client) + +# Checks tools/fex: the same build DroidDeck-Components' build-fex.yml runs from this repository to publish FEX +# (its single `fex` release and fex.json, which the app follows). Nothing is published from here. on: push: @@ -7,16 +10,12 @@ on: - '.github/workflows/build-fex.yml' workflow_dispatch: inputs: - tag: - description: 'Release tag to publish as (e.g. fex-2609-r1); empty = artifact only' - required: false - default: '' thunks: description: 'Build the host thunks (Vulkan/GL/X11/... to the runtime arm64 libraries)' required: false default: 'false' cxxflags: - description: 'Extra CXXFLAGS for a diagnostic build (e.g. -DDEBUG_STRACE); never published with a tag' + description: 'Extra CXXFLAGS for a diagnostic build (e.g. -DDEBUG_STRACE)' required: false default: '' @@ -48,30 +47,3 @@ jobs: fex.tzst.sha256 fex-needed.txt fex-files.txt - - # Only this job can write: the build above runs third-party images and toolchains with a read-only token. - publish: - needs: build - if: github.event_name == 'workflow_dispatch' && inputs.tag != '' && inputs.cxxflags == '' && inputs.thunks == 'true' - runs-on: ubuntu-24.04 - permissions: - contents: write - steps: - - uses: actions/download-artifact@v4 - with: - name: fex - - name: Publish - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - TAG: ${{ inputs.tag }} - run: | - set -euo pipefail - # droiddeck-steam-x64 pins each tag's sha256 and the build is not reproducible: never rebuild over a tag. - if gh release view "$TAG" >/dev/null 2>&1; then - echo "::error::release $TAG already exists; publish under a new tag" - exit 1 - fi - gh release create "$TAG" --target "$GITHUB_SHA" --latest=false --prerelease=false --title "$TAG" --notes "FEX for the runtime (tools/fex/build-in-arch.sh); fetched by droiddeck-steam-x64. sha256 $(cut -d' ' -f1 fex.tzst.sha256)" fex.tzst fex.tzst.sha256 - # Fetched by tag by the app, never installed by users: never the repo's Latest release nor a pre-release. - gh release edit "$TAG" --latest=false --prerelease=false diff --git a/app/src/main/java/com/droiddeck/launcher/MainActivity.kt b/app/src/main/java/com/droiddeck/launcher/MainActivity.kt index 4c41ea2693..6e704951d5 100644 --- a/app/src/main/java/com/droiddeck/launcher/MainActivity.kt +++ b/app/src/main/java/com/droiddeck/launcher/MainActivity.kt @@ -1906,6 +1906,9 @@ class MainActivity : ComponentActivity() { val scanGeneration = ++libraryScanGeneration val artRefreshesAtStart = HashMap(steamArtRefreshes) if (shortcutPicker) shortcutLibraryScanning = true + if (SessionPrefs.steamClientArch(this) == "x86_64" && !OfflineMode.enabled(this)) { + Thread({ com.droiddeck.launcher.session.FexCatalog.refresh(this) }, "fex-catalog").start() + } Thread({ val emus = Library.emulators(this) { id -> DesktopCatalog.installed(this, id) != null } ui.post { if (scanGeneration == libraryScanGeneration) emulatorList = emus } diff --git a/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt b/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt new file mode 100644 index 0000000000..73f42cb141 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/session/FexCatalog.kt @@ -0,0 +1,71 @@ +package com.droiddeck.launcher.session + +import android.content.Context +import android.util.Log +import com.droiddeck.launcher.core.DownloadSource +import com.droiddeck.launcher.core.Downloader +import org.json.JSONObject +import java.io.File + +/** + * The FEX build DroidDeck-Components' fex.json names for the x86-64 client. A session uses the last copy read. + * Components' build-fex.yml replaces the archive on its single `fex` release and records the build's version; + * a release per build (fex-2610-r5) is still read. + */ +object FexCatalog { + private const val TAG = "FexCatalog" + const val CATALOG_URL = "https://raw.githubusercontent.com/Droid-Deck/DroidDeck-Components/main/fex.json" + private const val RELEASES = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/" + private const val REFRESH_MS = 10 * 60 * 1000L + private val TAG_FORMAT = Regex("""fex-\d+-r\d+""") + private val VERSION_FORMAT = Regex("""\d+-r\d+""") + private val SHA256 = Regex("[0-9a-f]{64}") + + data class Build(val tag: String, val url: String, val sha256: String) + + @Volatile private var refreshedAt = 0L + + private fun file(context: Context) = File(context.filesDir, "fex-catalog.json") + + internal fun parse(text: String): Build? = runCatching { + val o = JSONObject(text) + val release = o.getString("release") + val url = o.getString("url") + val sha = o.getString("sha256").lowercase() + val version = o.optString("version") + val tag = when { + TAG_FORMAT.matches(release) -> release + release == "fex" && VERSION_FORMAT.matches(version) -> "fex-$version" + else -> return@runCatching null + } + Build(tag, url, sha).takeIf { url == "$RELEASES$release/fex.tzst" && SHA256.matches(sha) } + }.getOrNull() + + fun cached(context: Context): Build? = runCatching { parse(file(context).readText()) }.getOrNull() + + /** Reads fex.json straight from GitHub (a digest never comes from a mirror), at most every ten minutes. */ + @Synchronized + fun refresh(context: Context, now: Long = System.currentTimeMillis()): Build? { + if (now - refreshedAt < REFRESH_MS) return cached(context) + refreshedAt = now + val text = Downloader.downloadDirect(CATALOG_URL) ?: return cached(context) + val build = parse(text) ?: run { + Log.w(TAG, "fex.json not usable; keeping the last one") + return cached(context) + } + runCatching { + val f = file(context) + val tmp = File(f.path + ".tmp") + tmp.writeText(text) + if (!tmp.renameTo(f)) tmp.delete() + } + return build + } + + /** For droiddeck-steam-x64: the build and every source its archive may come from (it checks the sha256). */ + fun environment(build: Build, sources: List = DownloadSource.candidates(build.url)): List = listOf( + "BL_FEX_TAG=${build.tag}", + "BL_FEX_SHA256=${build.sha256}", + "BL_FEX_URLS=" + sources.joinToString(" "), + ) +} diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 1d8db3d539..4d61f698da 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -590,6 +590,7 @@ class SessionService : Service() { SessionState.steamClientArch?.let { arch -> guest.add("BL_STEAM_CLIENT_ARCH=$arch") SessionPaths.current()?.let { SessionArtifacts.recordClient(it, arch) } + if (arch == "x86_64") FexCatalog.cached(this)?.let { guest.addAll(FexCatalog.environment(it)) } } if (SessionState.mode == MODE_STEAM) { guest.add("BL_GAMESCOPE_STRETCH_16X9=" + (if (SessionPrefs.stretch16x9(this)) "1" else "0")) diff --git a/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt b/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt index 2cd1c63b8b..0fb34d6634 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/ModeSettingsDialog.kt @@ -152,13 +152,21 @@ class ModeSettingsActions( /** "2610 r3" from droiddeck-steam-x64's FEX_TAG line (fex-2610-r3), the build the x86-64 client downloads. */ internal fun fexVersionOf(script: String): String? = - Regex("""(?m)^FEX_TAG=fex-(\d+)-r(\d+)\s*$""").find(script)?.let { "${it.groupValues[1]} r${it.groupValues[2]}" } + Regex("""(?m)^FEX_TAG=(fex-\d+-r\d+)\s*$""").find(script)?.let { fexVersionOfTag(it.groupValues[1]) } +/** "2610 r5" from a tag such as fex-2610-r5. */ +internal fun fexVersionOfTag(tag: String): String? = + Regex("""fex-(\d+)-r(\d+)""").matchEntire(tag.trim())?.let { "${it.groupValues[1]} r${it.groupValues[2]}" } + +/** The build installed in the runtime, else the one fex.json names, else the one the launcher pins. */ @Composable private fun x64FexVersion(context: android.content.Context): String? = remember { - runCatching { - context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) } - }.getOrNull() + val root = com.droiddeck.launcher.runtime.LinuxRuntime.rootDir(context) + runCatching { java.io.File(root, "usr/local/share/fex-emu/.droiddeck-tag").readText() }.getOrNull()?.let(::fexVersionOfTag) + ?: com.droiddeck.launcher.session.FexCatalog.cached(context)?.tag?.let(::fexVersionOfTag) + ?: runCatching { + context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) } + }.getOrNull() } @Composable diff --git a/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt b/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt new file mode 100644 index 0000000000..3f4fa2f75d --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/FexCatalogTest.kt @@ -0,0 +1,46 @@ +package com.droiddeck.launcher.session + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class FexCatalogTest { + private val sha = "0fc1ca2ba13817ad70d1ea157119a75c9c02446b57d166e6d676cce0807c19c4" + private val url = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/fex-2610-r5/fex.tzst" + + private fun json(release: String = "fex-2610-r5", url: String = this.url, sha256: String = sha) = + """{"schemaVersion": 1, "category": "fex", "release": "$release", "url": "$url", "sha256": "$sha256", "size": 4083539}""" + + @Test fun readsTheBuildComponentsNames() { + assertEquals(FexCatalog.Build("fex-2610-r5", url, sha), FexCatalog.parse(json())) + } + + @Test fun readsTheSingleReleaseByItsVersion() { + val single = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/fex/fex.tzst" + assertEquals(FexCatalog.Build("fex-2610-r6", single, sha), FexCatalog.parse(json(release = "fex", url = single).replace("}", ", \"version\": \"2610-r6\"}"))) + assertNull(FexCatalog.parse(json(release = "fex", url = single).replace("}", ", \"version\": \"latest\"}"))) + assertNull(FexCatalog.parse(json(release = "fex", url = single))) + } + + @Test fun refusesAnArchiveOutsideItsOwnRelease() { + assertNull(FexCatalog.parse(json(url = "https://example.com/fex.tzst"))) + assertNull(FexCatalog.parse(json(url = url.replace("fex-2610-r5/", "fex-2610-r4/")))) + assertNull(FexCatalog.parse(json(release = "latest", url = url.replace("fex-2610-r5", "latest")))) + } + + @Test fun refusesAMissingOrMalformedDigest() { + assertNull(FexCatalog.parse(json(sha256 = "abc"))) + assertNull(FexCatalog.parse("""{"release": "fex-2610-r5", "url": "$url"}""")) + assertNull(FexCatalog.parse("not json")) + } + + @Test fun theSessionGetsTheBuildAndEverySource() { + val env = FexCatalog.environment(FexCatalog.Build("fex-2610-r5", url, sha), listOf(url, "https://ghfast.top/$url")) + assertEquals(listOf("BL_FEX_TAG=fex-2610-r5", "BL_FEX_SHA256=$sha", "BL_FEX_URLS=$url https://ghfast.top/$url"), env) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt b/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt index 8fb2016365..3d4600f8c1 100644 --- a/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/ui/FexVersionTest.kt @@ -14,6 +14,11 @@ class FexVersionTest { assertNull(fexVersionOf("# FEX_TAG=fex-2610-r3\n")) } + @Test fun readsAnInstalledOrCatalogTag() { + assertEquals("2610 r5", fexVersionOfTag("fex-2610-r5\n")) + assertNull(fexVersionOfTag("latest")) + } + @Test fun theShippedLauncherHasOne() { val script = java.io.File("../tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64").takeIf { it.isFile } ?: java.io.File("tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64") diff --git a/docs/development/x86-64-steam-client.md b/docs/development/x86-64-steam-client.md index aebfe51342..25e25a700c 100644 --- a/docs/development/x86-64-steam-client.md +++ b/docs/development/x86-64-steam-client.md @@ -186,27 +186,43 @@ alone would then read as an installed client. restarted from the top forever (found with `perf_event_open` IP sampling, since proot holds ptrace). Such a range is now skipped whole and a second exhausted scan returns `-ENOMEM`, as the kernel does (`fex-2610-r3`). -- **`build-fex.yml`.** The build job runs third-party images and toolchains with a read-only token, - and inputs reach the shell as environment variables. A separate publish job (contents: write) runs - only on a dispatch with a tag, host thunks on and no extra `cxxflags` (diagnostic builds such as - `-DDEBUG_STRACE` stay artifacts). `droiddeck-steam-x64` pins each tag's sha256 and the archive is not - reproducible, so a tag is never rebuilt over. The release is never the repo's Latest nor a - pre-release: it is a component fetched by tag, not something users install. +- **Building and publishing.** Droid-Deck/DroidDeck-Components' `build-fex.yml` builds `tools/fex` from + this repository at a chosen ref (default `main`) with host thunks, in the same Arch Linux ARM container + with a read-only token, then a separate job replaces `fex.tzst` on Components' single `fex` release, + checks the digest GitHub stored, and records the build's version, sha256 and size in its `fex.json` + (in that order: a device that reads the old `fex.json` meanwhile gets a mismatching archive and keeps + its FEX). This repository's `build-fex.yml` only builds `tools/fex` when it changes (and on dispatch, + for diagnostic builds such as `-DDEBUG_STRACE`); nothing is published from it. - Releases so far: `fex-2609-r2` (no thunks), `fex-2609-r3` (thunks), `fex-2610-r1`, `fex-2610-r2`, `fex-2610-r3`, `fex-2610-r4` (in-root resolution when openat2 is missing, exact MAP_32BIT marking), `fex-2610-r5` (current: bundles libxxhash, which the Linux runtime does not ship; the build fails when a program in the package needs a library that is neither bundled nor listed in `tools/fex/runtime-libs.txt`). They were built and published in The412Banner/DroidDeck; up to r3 their tags point at that - repository's `main`, from r4 at the commit each was built from. + repository's `main`, from r4 at the commit each was built from. Later builds are versions on the single + `fex` release; `fex-2610-r5` stays published as the build `droiddeck-steam-x64` pins. - **Where the app gets it.** The app downloads FEX from Droid-Deck/DroidDeck-Components, where nothing is - uploaded by hand: a new tag is published by `build-fex.yml`, recorded in that repository's `fex.json` - (sha256 and size), and brought in by its `mirror-release.yml`, which refuses an archive that does not - match. `FEX_TAG` and `FEX_SHA256` in `droiddeck-steam-x64` then move to the new tag. The rootfs is + uploaded by hand. The app follows its `fex.json`: while the x86-64 client is selected it reads it from + Components' `main` (directly, at most every ten minutes) and keeps the last copy, and a session passes + that build to `droiddeck-steam-x64` (`BL_FEX_TAG`, `BL_FEX_SHA256`, `BL_FEX_URLS` with the download + source's mirrors). The script installs a build only after its archive matches the sha256 and every + program in it loads (`ld-linux --list`); the files it replaces are kept aside until the installed copy + loads too and put back otherwise, and the tag is written last. Otherwise it keeps the installed one while that still loads, + then falls back to `FEX_TAG`/`FEX_SHA256` built in, which stay the first-start default when no copy + of `fex.json` has been read. Publishing a FEX build therefore takes a Components run only; a + replaced archive cannot be pointed back to, so undoing a build means building an earlier ref. The rootfs is FEX-Emu's own image, unmodified: `fex-rootfs.json` there names its rootfs.fex-emu.gg link, sha256 and size, and `mirror-upstream.yml` fetches it into the catalog's release only when both match. A newer image is a new catalog release; `ROOTFS_URL` and `ROOTFS_SHA256` then move to it, and a device that already unpacked one keeps it. +- **What a session log shows.** `FEX ready (, )` names the build and where it came from + (`catalog`, `built-in`, or `kept` when a newer one did not install, with the reason a line above); + `x86-64 rootfs:` names the image; `x86-64 client:` the client's ubuntu12 and steamrt64 builds at start + (it may update itself after). The rootfs download reports every tenth of its size as a step. When the + web helper's GPU process keeps exiting until Chromium gives up on the GPU (the menus then never appear), + one line says so with the exit codes, followed by x86 `glxinfo -B` and `vulkaninfo --summary` through + FEX's thunks. + ## DroidDeck direct for Linux games and runtimes - Steam runs native Linux games in the Steam Linux Runtime container (pressure-vessel), which needs diff --git a/docs/progress-log.md b/docs/progress-log.md index 65a26cbde3..81ec5bc3a3 100644 --- a/docs/progress-log.md +++ b/docs/progress-log.md @@ -139,6 +139,27 @@ the timeline, then lessons and backlog. Companion to the README (what the app *d FEX uses). With the device's libxxhash hidden, r4 failed with the reported error and r5 loaded. Mirrored in DroidDeck-Components (#5); the app moves to fex-2610-r5 and an installed r4 is replaced at the next x86-64 start. +- Merged as 7d21ba22. +- **FEX from the catalog**: FEX builds may need to follow user reports faster than app builds, so the + app now reads Components' `fex.json` (directly, kept between starts) and passes it to the session; a + build is installed only once its sha256 matches and its programs load, the installed one stays while it + loads, and the build pinned in the script remains the first-start default. **Session logs** for the + x86-64 client now name the FEX build and its source, the rootfs and the client's builds, report the + rootfs download in tenths instead of a progress bar (one report's log ended in hundreds of bar lines), + and, when the web helper's GPU process keeps exiting (a report from an Adreno 840: the menus never + appeared), say so in one line with x86 `glxinfo`/`vulkaninfo` through FEX. Settings shows the FEX + build installed. FEX is now built and published by DroidDeck-Components' `build-fex.yml` from this + repository's `tools/fex`, replacing the archive on its single `fex` release and recording the version + in `fex.json`; this repository's `build-fex.yml` only checks the build. Tests: `test_fex_select.py`, + `FexCatalogTest`. + Components' first build there published `2610-r6` (run 39104443888). Device (Pocket FIT, r4 + installed): the app read `fex.json`, and the next x86-64 start logged `downloading FEX (fex-2610-r6)` + then `FEX ready (fex-2610-r6, catalog)` a second later, with the rootfs and client lines; the client + came up on it. +- Review of #1053: an install keeps the files it replaces aside until the installed copy loads and puts + them back otherwise (a failed copy could leave a partly replaced FEX marked as the new one; + `test_fex_install.py`), and the catalog is read on its own thread, not before the library scan, and + not in Offline Mode. ## 2026-10-15 - `feat/x86-64-steam-client`: SD card library, ARM parity, org main, Counter-Strike 2 diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 index 4a2113fbf4..2c27c9db4d 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-x64 @@ -319,24 +319,101 @@ print("droiddeck-steam-x64: %d Linux title(s) on DroidDeck direct (%d on the new PY } -x64_prepare() { -if [ ! -x /usr/local/bin/FEX ] || [ "$(cat /usr/local/share/fex-emu/.droiddeck-tag 2>/dev/null)" != "$FEX_TAG" ]; then - step "downloading FEX ($FEX_TAG)" +# Whether every program in the FEX tree at $1 finds its libraries (r4 did not without libxxhash). +fex_loads() { + local b out + for b in FEX FEXServer unsquashfs; do + out=$(/usr/lib/ld-linux-aarch64.so.1 --list "$1/usr/local/bin/$b" 2>&1) && ! grep -q "not found" <<<"$out" && continue + echo "== FEX check: $b: $(grep -E "not found|error" <<<"$out" | tr -s '\t ' ' ' | tr '\n' ';')" >&2 + return 1 + done +} + +# Installs one FEX build only once its archive matches the sha256 and it loads. The files it replaces are kept +# until the installed copy loads too, and put back otherwise; the tag is written last. +fex_install() { + local tag=$1 sha=$2 urls=$3 dst=/ tmp u f tmp=$(mktemp -d) - curl -fL --progress-bar "$FEX_URL" -o "$tmp/fex.tzst" - if [ "$FEX_SHA256" != PENDING ]; then echo "$FEX_SHA256 $tmp/fex.tzst" | sha256sum -c --quiet; fi - zstd -dc "$tmp/fex.tzst" | tar -C / -x + for u in $urls; do + curl -fsSL --retry 2 "$u" -o "$tmp/fex.tzst" && echo "$sha $tmp/fex.tzst" | sha256sum -c --quiet >/dev/null 2>&1 && break + rm -f "$tmp/fex.tzst" + done + if [ ! -s "$tmp/fex.tzst" ]; then + echo "== FEX $tag: no source gave the archive with sha256 ${sha:0:12}" >&2 + rm -rf "$tmp" + return 1 + fi + if ! { mkdir "$tmp/root" && zstd -dc "$tmp/fex.tzst" | tar -C "$tmp/root" -x && fex_loads "$tmp/root"; }; then + rm -rf "$tmp" + return 1 + fi + (cd "$tmp/root" && find . ! -type d) > "$tmp/files" + mkdir "$tmp/old" + while IFS= read -r f; do + f=${f#./} + if [ -e "$dst$f" ] || [ -L "$dst$f" ]; then + cp -a --parents "$dst$f" "$tmp/old" || { echo "== FEX $tag: could not keep $dst$f aside" >&2; rm -rf "$tmp"; return 1; } + fi + done < "$tmp/files" + if tar -C "$tmp/root" -c . | tar -C "$dst" -x && fex_loads "$dst" && + mkdir -p "${dst}usr/local/share/fex-emu" && echo "$tag" > "${dst}usr/local/share/fex-emu/.droiddeck-tag"; then + rm -rf "$tmp" + return 0 + fi + echo "== FEX $tag: the installed copy did not load or its tag was not written; putting the previous files back" >&2 + while IFS= read -r f; do + f=${f#./} + [ -e "$tmp/old/$dst$f" ] || [ -L "$tmp/old/$dst$f" ] || rm -f "$dst$f" + done < "$tmp/files" + tar -C "$tmp/old$dst" -c . | tar -C "$dst" -x || echo "== FEX $tag: could not put the previous files back" >&2 rm -rf "$tmp" - mkdir -p /usr/local/share/fex-emu && echo "$FEX_TAG" > /usr/local/share/fex-emu/.droiddeck-tag -fi -step "FEX ready ($FEX_TAG)" + return 1 +} + +# The build DroidDeck-Components' fex.json names (the app passes it as BL_FEX_*) comes first, then a working +# installed one, then FEX_TAG above. Sets FEX_NOW and FEX_FROM. +x64_fex() { + local have ok= + have=$(cat /usr/local/share/fex-emu/.droiddeck-tag 2>/dev/null || true) + [ -x /usr/local/bin/FEX ] && fex_loads / && ok=1 + if [ -n "${BL_FEX_TAG:-}" ]; then + if [ -n "$ok" ] && [ "$have" = "$BL_FEX_TAG" ]; then FEX_NOW=$have FEX_FROM=catalog; return 0; fi + step "downloading FEX ($BL_FEX_TAG)" + if fex_install "$BL_FEX_TAG" "${BL_FEX_SHA256:-}" "${BL_FEX_URLS:-}"; then FEX_NOW=$BL_FEX_TAG FEX_FROM=catalog; return 0; fi + if [ -n "$ok" ]; then FEX_NOW=$have FEX_FROM="kept; $BL_FEX_TAG did not install"; return 0; fi + fi + if [ -n "$ok" ] && [ "$have" = "$FEX_TAG" ]; then FEX_NOW=$have FEX_FROM=built-in; return 0; fi + step "downloading FEX ($FEX_TAG)" + if fex_install "$FEX_TAG" "$FEX_SHA256" "$FEX_URL"; then FEX_NOW=$FEX_TAG FEX_FROM=built-in; return 0; fi + if [ -n "$ok" ]; then FEX_NOW=$have FEX_FROM="kept; $FEX_TAG did not install"; return 0; fi + return 1 +} + +# A resumable download that reports every tenth of its size as a step. +x64_fetch() { + local url=$1 out=$2 label=$3 total pid rc=0 last=-1 pct + total=$(curl -fsSIL "$url" 2>/dev/null | tr -d '\r' | awk 'tolower($1) == "content-length:" { v = $2 } END { print v + 0 }') + curl -fL -sS -C - "$url" -o "$out" & + pid=$! + while kill -0 "$pid" 2>/dev/null; do + sleep 3 + [ "$total" -gt 0 ] || continue + pct=$(( $(stat -c %s "$out" 2>/dev/null || echo 0) * 10 / total * 10 )) + if [ "$pct" -gt "$last" ]; then last=$pct; step "$label ($pct%)"; fi + done + wait "$pid" || rc=$? + return "$rc" +} + +x64_prepare() { +x64_fex || { step "FEX could not be installed; see $LOG"; exit 1; } +step "FEX ready ($FEX_NOW, $FEX_FROM)" # The client's opt-in to its SteamRT3 build; without it, it re-executes through steam.sh, whose checks fail under proot. mkdir -p "$STEAM_X64" && : > "$STEAM_X64/.steam-enable-steamrt64-client" if [ ! -f "$ROOTFS/.droiddeck-complete" ]; then - step "downloading FEX's Ubuntu 24.04 x86 rootfs" # Resumes a part-done download; a finished one answers 416, which is not a failure here. - curl -fL --progress-bar -C - "$ROOTFS_URL" -o "$X64_HOME/rootfs.sqsh" || { + x64_fetch "$ROOTFS_URL" "$X64_HOME/rootfs.sqsh" "downloading FEX's Ubuntu 24.04 x86 rootfs" || { rc=$? [ "$rc" = 22 ] && [ -s "$X64_HOME/rootfs.sqsh" ] || { step "rootfs download failed (curl $rc)"; exit 1; } } @@ -350,6 +427,7 @@ if [ ! -f "$ROOTFS/.droiddeck-complete" ]; then rm -f "$X64_HOME/rootfs.sqsh" touch "$ROOTFS/.droiddeck-complete" fi +echo "== x86-64 rootfs: $(sed -n 's|.*/download/\([^/]*\)/.*|\1|p' <<<"$ROOTFS_URL") (sha256 ${ROOTFS_SHA256:0:12})" >&2 # FEX falls back to the guest's own /etc/ld.so.preload (arm64 libraries) when the rootfs has none. The x86 # session shim provides System V IPC, which Android kernels lack and the client's tier0 needs to start. : > "$ROOTFS/etc/ld.so.preload" @@ -792,6 +870,37 @@ export STEAM_STEAMRT_RECURSION_GUARD=1 export BL_WEBHELPER_HIDE_TRACER=1 } +x64_manifest_version() { + local chan + chan=$(cat "$STEAM_X64/package/beta" 2>/dev/null || echo "$X64_CHANNEL") + awk -F'"' '$2 == "version" { print $4; exit }' "$STEAM_X64/package/steam_client_${chan}_$1.manifest" 2>/dev/null || echo "?" +} + +# What x86 GL and Vulkan look like through FEX's thunks, for a session whose menus failed on the GPU. +x64_gpu_probe() { + local fex=${BL_FEX:-/usr/local/bin/FEX} + echo "== x86-64 GPU probe (x86 glxinfo and vulkaninfo through FEX):" + ( + unset LD_LIBRARY_PATH + export DISABLE_GAMESCOPE_WSI=1 vk_xwayland_wait_ready=true LIBGL_KOPPER_DISABLE=true + timeout 30 "$fex" /usr/bin/glxinfo -B 2>&1 | grep -E "OpenGL (renderer|core profile version|version) string|direct rendering|rror" | head -5 + timeout 30 "$fex" /usr/bin/vulkaninfo --summary 2>&1 | grep -E "deviceName|driverName|driverInfo|apiVersion|rror" | head -6 + ) | sed 's/^/ /' +} + +# Steam's menus come from the web helper; when its GPU process keeps exiting, Chromium gives up on the GPU and +# the menus never appear. Says so once in the session log, with the probe above. +x64_gpu_watch() { + local log=$STEAM_X64/logs/console-linux.txt from n codes + from=$(( $(wc -l 2>/dev/null < "$log" || echo 0) + 1 )) + tail -n0 -F --pid=$$ "$log" 2>/dev/null | grep -q -m1 "GPU process isn't usable" || return 0 + n=$(tail -n +"$from" "$log" | grep -c "GPU process exited unexpectedly" || true) + codes=$(tail -n +"$from" "$log" | grep -oE "GPU process exited unexpectedly: exit_code=[0-9]+" | sed 's/.*=//' | + sort | uniq -c | awk '{ printf "%s%s x%s", s, $2, $1; s = ", " }') + echo "== x86-64: Steam's menu renderer gave up on the GPU after $n GPU process exits (exit code ${codes:-?}); the menus stay blank. See steam/console-linux.txt and steam/webhelper_gpu.txt" + x64_gpu_probe +} + x64_exec() { x64_env cd "$STEAM_X64" @@ -828,6 +937,8 @@ x64_exec() { touch "$STEAM_X64/logs/bootstrap_log.txt" ( tail -n0 -F --pid=$$ "$STEAM_X64/logs/bootstrap_log.txt" 2>/dev/null | grep --line-buffered -oE 'Downloading update \([0-9,]+ of [0-9,]+ KB\)' | sed -u 's/,//g' >&2 ) & + echo "== x86-64 client: ubuntu12 $(x64_manifest_version ubuntu12), steamrt64 $(x64_manifest_version steamrt64) (the client may update itself)" >&2 + x64_webhelper_gpu && { x64_gpu_watch >&2 & } exec "${BL_FEX:-/usr/local/bin/FEX}" "$STEAM_X64/steamrt64/steam" "$@" } diff --git a/tools/tests/test_fex_install.py b/tools/tests/test_fex_install.py new file mode 100644 index 0000000000..355b540381 --- /dev/null +++ b/tools/tests/test_fex_install.py @@ -0,0 +1,91 @@ +import hashlib +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin/droiddeck-steam-x64" +SOURCE = SCRIPT.read_text() +START = SOURCE.index("fex_install() {") +END = SOURCE.index("\n}\n", START) + 3 +FEX_INSTALL = SOURCE[START:END] + +HARNESS = textwrap.dedent("""\ + set -eu + # The archive comes from PKG; the installed tree loads when INSTALLED_LOADS=1, any staged tree always does. + curl() { local out=; while [ $# -gt 0 ]; do [ "$1" = -o ] && out=$2; shift; done; cp "$PKG" "$out"; } + fex_loads() { case $1 in "$ROOT"/) [ "${INSTALLED_LOADS:-1}" = 1 ] ;; *) true ;; esac; } + """) + + +class FexInstallTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + base = Path(self.temp.name) + self.root = base / "root" + old = self.root / "usr/local" + (old / "bin").mkdir(parents=True) + (old / "bin/FEX").write_text("old FEX\n") + (old / "share/fex-emu").mkdir(parents=True) + (old / "share/fex-emu/.droiddeck-tag").write_text("fex-2610-r5\n") + new = base / "pkg/usr/local" + (new / "bin").mkdir(parents=True) + (new / "bin/FEX").write_text("new FEX\n") + (new / "lib").mkdir() + (new / "lib/libxxhash.so.0").write_text("lib\n") + self.pkg = base / "fex.tzst" + subprocess.run(f"tar -C '{base / 'pkg'}' -c . | zstd -q -o '{self.pkg}'", shell=True, check=True) + self.sha = hashlib.sha256(self.pkg.read_bytes()).hexdigest() + + def install(self, installed_loads, sha=None): + body = FEX_INSTALL.replace("local tag=$1 sha=$2 urls=$3 dst=/ ", "local tag=$1 sha=$2 urls=$3 dst=$ROOT/ ") + self.assertNotEqual(body, FEX_INSTALL) + script = HARNESS + body + 'if fex_install fex-2610-r6 "$SHA" u; then echo OK; else echo FAILED; fi\n' + env = dict(os.environ, ROOT=str(self.root), PKG=str(self.pkg), SHA=sha or self.sha, + INSTALLED_LOADS="1" if installed_loads else "0") + out = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True, timeout=30) + self.assertEqual(0, out.returncode, out.stderr) + return out.stdout.strip().splitlines()[-1], out.stderr + + def tag(self): + return (self.root / "usr/local/share/fex-emu/.droiddeck-tag").read_text().strip() + + def test_a_build_that_loads_once_installed_replaces_the_old_one(self): + result, _ = self.install(installed_loads=True) + self.assertEqual("OK", result) + self.assertEqual("new FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertTrue((self.root / "usr/local/lib/libxxhash.so.0").is_file()) + self.assertEqual("fex-2610-r6", self.tag()) + + def test_an_installed_copy_that_does_not_load_puts_the_old_files_back(self): + result, err = self.install(installed_loads=False) + self.assertEqual("FAILED", result) + self.assertIn("putting the previous files back", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertFalse((self.root / "usr/local/lib/libxxhash.so.0").exists()) + self.assertEqual("fex-2610-r5", self.tag()) + + def test_a_tag_that_cannot_be_written_puts_the_old_files_back(self): + share = self.root / "usr/local/share/fex-emu" + (share / ".droiddeck-tag").unlink() + share.rmdir() + share.write_text("not a folder\n") + result, err = self.install(installed_loads=True) + self.assertEqual("FAILED", result) + self.assertIn("putting the previous files back", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertFalse((self.root / "usr/local/lib/libxxhash.so.0").exists()) + + def test_an_archive_that_does_not_match_installs_nothing(self): + result, err = self.install(installed_loads=True, sha="0" * 64) + self.assertEqual("FAILED", result) + self.assertIn("no source gave the archive", err) + self.assertEqual("old FEX\n", (self.root / "usr/local/bin/FEX").read_text()) + self.assertEqual("fex-2610-r5", self.tag()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_fex_select.py b/tools/tests/test_fex_select.py new file mode 100644 index 0000000000..94008a1c70 --- /dev/null +++ b/tools/tests/test_fex_select.py @@ -0,0 +1,85 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import textwrap +import unittest + +SCRIPT = Path(__file__).resolve().parents[1] / "linuxfs/overlay/usr/local/bin/droiddeck-steam-x64" +SOURCE = SCRIPT.read_text() +START = SOURCE.index("x64_fex() {") +END = SOURCE.index("\n}\n", START) + 3 +X64_FEX = SOURCE[START:END] + +HARNESS = textwrap.dedent("""\ + set -eu + FEX_TAG=fex-2610-r5 FEX_SHA256=builtin FEX_URL=builtin-url + step() { echo "STEP $*"; } + # Installed FEX loads when LOADS=1; a build installs when its tag is in INSTALLABLE. + fex_loads() { [ "${LOADS:-0}" = 1 ]; } + fex_install() { + echo "INSTALL $1 $2 $3" + case " ${INSTALLABLE:-} " in *" $1 "*) echo "$1" > "$ROOT/tag"; : > "$ROOT/FEX"; chmod +x "$ROOT/FEX"; return 0 ;; esac + return 1 + } + """) + + +class FexSelectTest(unittest.TestCase): + def run_select(self, installed=None, loads=False, catalog=None, installable=()): + with tempfile.TemporaryDirectory() as root: + if installed: + Path(root, "tag").write_text(installed + "\n") + fex = Path(root, "FEX") + fex.write_text("") + fex.chmod(0o755) + body = (X64_FEX.replace("/usr/local/share/fex-emu/.droiddeck-tag", "$ROOT/tag") + .replace("/usr/local/bin/FEX", "$ROOT/FEX")) + script = HARNESS + body + 'if x64_fex; then echo "NOW $FEX_NOW|$FEX_FROM"; else echo FAILED; fi\n' + env = dict(os.environ, ROOT=root, LOADS="1" if loads else "0", INSTALLABLE=" ".join(installable)) + env.pop("BL_FEX_TAG", None) + if catalog: + env.update(BL_FEX_TAG=catalog, BL_FEX_SHA256="catsha", BL_FEX_URLS="a b") + out = subprocess.run(["bash", "-c", script], env=env, text=True, capture_output=True, timeout=10) + self.assertEqual(0, out.returncode, out.stderr) + lines = out.stdout.splitlines() + return [l for l in lines if l.startswith("INSTALL")], lines[-1] + + def test_the_catalog_build_already_installed_is_used(self): + installs, now = self.run_select(installed="fex-2610-r6", loads=True, catalog="fex-2610-r6") + self.assertEqual([], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_newer_catalog_build_is_installed(self): + installs, now = self.run_select(installed="fex-2610-r5", loads=True, catalog="fex-2610-r6", installable=["fex-2610-r6"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_catalog_build_that_does_not_install_keeps_the_working_one(self): + installs, now = self.run_select(installed="fex-2610-r5", loads=True, catalog="fex-2610-r6") + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r5|kept; fex-2610-r6 did not install", now) + + def test_without_a_catalog_the_built_in_build_replaces_an_older_one(self): + installs, now = self.run_select(installed="fex-2610-r4", loads=True, installable=["fex-2610-r5"]) + self.assertEqual(["INSTALL fex-2610-r5 builtin builtin-url"], installs) + self.assertEqual("NOW fex-2610-r5|built-in", now) + + def test_an_installed_build_that_does_not_load_is_replaced(self): + installs, now = self.run_select(installed="fex-2610-r6", loads=False, catalog="fex-2610-r6", installable=["fex-2610-r6"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b"], installs) + self.assertEqual("NOW fex-2610-r6|catalog", now) + + def test_a_first_install_falls_back_to_the_built_in_build(self): + installs, now = self.run_select(catalog="fex-2610-r6", installable=["fex-2610-r5"]) + self.assertEqual(["INSTALL fex-2610-r6 catsha a b", "INSTALL fex-2610-r5 builtin builtin-url"], installs) + self.assertEqual("NOW fex-2610-r5|built-in", now) + + def test_nothing_usable_fails(self): + installs, now = self.run_select(catalog="fex-2610-r6") + self.assertEqual(2, len(installs)) + self.assertEqual("FAILED", now) + + +if __name__ == "__main__": + unittest.main()