diff --git a/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt b/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt index a23f23bebf..f69983f0db 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt @@ -87,11 +87,10 @@ class LoadingState(context: Context, private val steam: Boolean = true) { if (line.startsWith("== STEP ")) { stepAt = at stepText = line.substringAfter("== STEP ").substringAfter(' ') - val m = INSTALL_COUNT.find(line) - if (m != null) { + clientDownload(line)?.let { (what, percent) -> clientDownloadAt = at - clientDownload = m.groupValues[1] - clientPercent = m.groupValues[2].toInt() * 100 / maxOf(1, m.groupValues[3].toInt()) + clientDownload = what + clientPercent = percent } } else { val m = UPDATE_PROGRESS.find(line) @@ -132,6 +131,19 @@ class LoadingState(context: Context, private val steam: Boolean = true) { } } + internal fun clientDownload(line: String): Pair? { + val count = INSTALL_COUNT.find(line) ?: return null + val mb = INSTALL_MEGABYTES.find(line.substring(count.range.last + 1)) + val done = mb?.groupValues?.get(1)?.toLongOrNull() + val total = mb?.groupValues?.get(2)?.toLongOrNull() + if (done != null && total != null && total > 0) { + return "$done / $total MB" to (done * 100 / total).toInt().coerceIn(0, 100) + } + val part = count.groupValues[2].toIntOrNull() ?: return null + val all = count.groupValues[3].toIntOrNull() ?: return null + return count.groupValues[1] to (part.toLong() * 100 / maxOf(1, all)).coerceIn(0L, 100L).toInt() + } + internal fun readableScriptStep(scriptStep: String): Boolean { val t = scriptStep.lowercase() return t.startsWith("download") || t.startsWith("checking the linux") || t.startsWith("unpacking") || t.startsWith("installing") @@ -140,5 +152,6 @@ class LoadingState(context: Context, private val steam: Boolean = true) { private const val TAIL_BYTES = 48L * 1024 private val UPDATE_PROGRESS = Regex("""Downloading update \((\d+) of (\d+) KB\)""") private val INSTALL_COUNT = Regex("""downloading Steam: (\S+) \((\d+)/(\d+)\)""") + private val INSTALL_MEGABYTES = Regex("""^, (\d+) of (\d+) MB""") } } diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt index 86643fc15f..762de4f93b 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionService.kt @@ -71,6 +71,7 @@ class SessionService : Service() { private val steamDownloadMonitor = SteamDownloadMonitor() private var downloadMonitorTask: Runnable? = null private var downloadActive = false + private var clientInstallActive = false @Volatile private var steamLibraryMounts = emptyMap() private var suspendDownloadNow = false private var suspendPolicy = SessionPrefs.SUSPEND_MANUAL @@ -321,6 +322,7 @@ class SessionService : Service() { suspendDownloadNow = false steamDownloadMonitor.reset() downloadActive = false + clientInstallActive = false steamSleepToken = null completeNativeSleep() suspendOperationPending = false @@ -640,8 +642,8 @@ class SessionService : Service() { } }, ) - mainHandler.post { updateSuspendPolicy() } updateDownloadMonitoring() + mainHandler.post { updateSuspendPolicy() } } } @@ -1413,8 +1415,7 @@ class SessionService : Service() { private fun updateSuspendPolicy() { if (!SessionState.running) return val hidden = !hasVisibleSurface - val downloadsBlockingSuspend = steamDownloadsInBackground && SessionState.mode == MODE_STEAM && - suspendPolicy != SessionPrefs.SUSPEND_NEVER && hidden && downloadActive && !suspendDownloadNow + val downloadsBlockingSuspend = downloadsKeepAwake() && hidden && !suspendDownloadNow if (suspendPolicy == SessionPrefs.SUSPEND_MANUAL && hidden && !downloadsBlockingSuspend) { manualPauseRequested = true } else if (!hidden && !SessionState.suspended) { @@ -1828,8 +1829,7 @@ class SessionService : Service() { .setContentText(getString( when { SessionState.suspended -> R.string.session_notification_paused - downloadActive && steamDownloadsInBackground && - suspendPolicy != SessionPrefs.SUSPEND_NEVER -> R.string.session_notification_downloads + downloadsKeepAwake() -> R.string.session_notification_downloads else -> R.string.session_notification } )) @@ -1837,8 +1837,7 @@ class SessionService : Service() { .apply { if (SessionState.suspended) { addAction(Notification.Action.Builder(null, getString(R.string.resume_session), resume).build()) - } else if (downloadActive && steamDownloadsInBackground && - suspendPolicy != SessionPrefs.SUSPEND_NEVER) { + } else if (downloadsKeepAwake()) { val suspend = PendingIntent.getService( this@SessionService, 3, Intent(this@SessionService, SessionService::class.java).setAction(ACTION_SUSPEND_NOW), @@ -1858,33 +1857,50 @@ class SessionService : Service() { getSystemService(NotificationManager::class.java)?.notify(NOTIFICATION_ID, buildNotification()) } + private fun downloadsKeepAwake(): Boolean = + downloadActive && (steamDownloadsInBackground || clientInstallActive) && + SessionState.mode == MODE_STEAM && suspendPolicy != SessionPrefs.SUSPEND_NEVER + + private fun downloadWatchWanted(): Boolean = + SessionState.running && SessionState.mode == MODE_STEAM && suspendPolicy != SessionPrefs.SUSPEND_NEVER && + (steamDownloadsInBackground || !SessionState.firstFrameSeen) + private fun updateDownloadMonitoring() { - val enabled = SessionState.running && SessionState.mode == MODE_STEAM && - steamDownloadsInBackground && suspendPolicy != SessionPrefs.SUSPEND_NEVER - if (!enabled) { + if (!downloadWatchWanted()) { val wasActive = downloadActive downloadMonitorTask?.let(mainHandler::removeCallbacks) downloadMonitorTask = null steamDownloadMonitor.reset() downloadActive = false + if (clientInstallActive) SessionEvents.record("steam.client_download", mapOf("active" to false)) + clientInstallActive = false if (wasActive) refreshNotification() return } if (downloadMonitorTask != null) return val poll = object : Runnable { override fun run() { - if (downloadMonitorTask !== this || !SessionState.running || - SessionState.mode != MODE_STEAM || !steamDownloadsInBackground || - suspendPolicy == SessionPrefs.SUSPEND_NEVER) { + if (downloadMonitorTask !== this) return + if (!downloadWatchWanted()) { + downloadMonitorTask = null + updateDownloadMonitoring() + updateSuspendPolicy() return } + val root = LinuxRuntime.rootDir(this@SessionService) val wasActive = downloadActive - downloadActive = steamDownloadMonitor.poll( - LinuxRuntime.rootDir(this@SessionService), + val wasInstalling = clientInstallActive + clientInstallActive = !SessionState.firstFrameSeen && + (clientInstallActive || SteamDownloadMonitor.clientInstallPending(root)) + downloadActive = clientInstallActive || steamDownloadsInBackground && steamDownloadMonitor.poll( + root, receivedBytes = TrafficStats.getUidRxBytes(android.os.Process.myUid()), client = SessionState.steamClientArch, mounts = steamLibraryMounts, ) + if (wasInstalling != clientInstallActive) { + SessionEvents.record("steam.client_download", mapOf("active" to clientInstallActive)) + } if (wasActive != downloadActive) refreshNotification() updateSuspendPolicy() mainHandler.postDelayed(this, DOWNLOAD_POLL_MS) diff --git a/app/src/main/java/com/droiddeck/launcher/session/SteamDownloadMonitor.kt b/app/src/main/java/com/droiddeck/launcher/session/SteamDownloadMonitor.kt index 37f7c1fb90..c125e59b80 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SteamDownloadMonitor.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SteamDownloadMonitor.kt @@ -138,13 +138,21 @@ internal class SteamDownloadMonitor { private fun value(text: String, key: String): String? = Regex("\"$key\"\\s+\"([^\"]*)\"").find(text)?.groupValues?.get(1) - private companion object { - const val MANIFEST_PROGRESS_GRACE_MS = 8_000L - const val CONTENT_ACTIVITY_GRACE_MS = 90_000L - const val MIN_RECEIVE_BYTES_PER_SEC = 64L * 1024L - val PATH = Regex("\"path\"\\s+\"([^\"]+)\"") - val STEAM_HOMES = listOf("root/.local/share/Steam", "root/.droiddeck-x64/.local/share/Steam") - val CONTENT_UPDATE = Regex("""AppID (\d+) (App|Workshop|Shader) update changed : (.*)""") - val CURRENT_RATE = Regex("""Current download rate: ([\d.]+) Mbps""") + companion object { + fun clientInstallPending(runtimeRoot: File): Boolean { + val home = File(runtimeRoot, "root") + val steam = File(home, ".local/share/Steam") + val stamped = File(steam, "package/droiddeck-installed").isFile || + File(steam, "package/bannerlator-installed").isFile + return !(stamped && File(steam, "steamrtarm64/steam").isFile) || File(home, ".bl-steam-repair").exists() + } + + private const val MANIFEST_PROGRESS_GRACE_MS = 8_000L + private const val CONTENT_ACTIVITY_GRACE_MS = 90_000L + private const val MIN_RECEIVE_BYTES_PER_SEC = 64L * 1024L + private val PATH = Regex("\"path\"\\s+\"([^\"]+)\"") + private val STEAM_HOMES = listOf("root/.local/share/Steam", "root/.droiddeck-x64/.local/share/Steam") + private val CONTENT_UPDATE = Regex("""AppID (\d+) (App|Workshop|Shader) update changed : (.*)""") + private val CURRENT_RATE = Regex("""Current download rate: ([\d.]+) Mbps""") } } diff --git a/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt b/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt new file mode 100644 index 0000000000..c062823494 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt @@ -0,0 +1,36 @@ +package com.droiddeck.launcher.session + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class LoadingStateTest { + @Test fun clientDownloadCountsMegabytesWhenTheRuntimeSaysThem() { + assertEquals( + "45 / 360 MB" to 12, + LoadingState.clientDownload("== STEP 17:38:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB"), + ) + } + + @Test fun aStalledDownloadStillShowsWhereItIs() { + assertEquals( + "45 / 360 MB" to 12, + LoadingState.clientDownload( + "== STEP 17:39:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB" + + " - interrupted (curl exit 28), trying again in 5 s", + ), + ) + } + + @Test fun anOlderRuntimeShowsThePartAndItsNumber() { + assertEquals( + "bins_linuxarm64_linuxarm64" to 11, + LoadingState.clientDownload("== STEP 17:38:17 downloading Steam: bins_linuxarm64_linuxarm64 (2/17)"), + ) + assertEquals("a_all" to 50, LoadingState.clientDownload("== STEP 00:00:00 downloading Steam: a_all (1/2), 0 of 0 MB")) + } + + @Test fun otherStepsAreNotClientDownloads() { + assertNull(LoadingState.clientDownload("== STEP 17:38:14 checking the Steam client (a first run downloads it)")) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/session/SteamDownloadMonitorTest.kt b/app/src/test/java/com/droiddeck/launcher/session/SteamDownloadMonitorTest.kt index e60c6dcd9f..4da943020d 100644 --- a/app/src/test/java/com/droiddeck/launcher/session/SteamDownloadMonitorTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/session/SteamDownloadMonitorTest.kt @@ -203,6 +203,38 @@ class SteamDownloadMonitorTest { } } + @Test fun theClientIsPendingUntilItIsInstalledAndStamped() { + val root = Files.createTempDirectory("droiddeck-steam-client-").toFile() + try { + val steam = File(root, "root/.local/share/Steam") + assertTrue(SteamDownloadMonitor.clientInstallPending(root)) + File(steam, "steamrtarm64").mkdirs() + File(steam, "steamrtarm64/steam").writeText("client") + assertTrue(SteamDownloadMonitor.clientInstallPending(root)) + File(steam, "package").mkdirs() + File(steam, "package/droiddeck-installed").writeText("1791686101\n") + assertFalse(SteamDownloadMonitor.clientInstallPending(root)) + } finally { + root.deleteRecursively() + } + } + + @Test fun anOlderStampCountsAndARepairRequestIsPending() { + val root = Files.createTempDirectory("droiddeck-steam-client-").toFile() + try { + val steam = File(root, "root/.local/share/Steam") + File(steam, "steamrtarm64").mkdirs() + File(steam, "steamrtarm64/steam").writeText("client") + File(steam, "package").mkdirs() + File(steam, "package/bannerlator-installed").writeText("1\n") + assertFalse(SteamDownloadMonitor.clientInstallPending(root)) + File(root, "root/.bl-steam-repair").writeText("1\n") + assertTrue(SteamDownloadMonitor.clientInstallPending(root)) + } finally { + root.deleteRecursively() + } + } + private fun manifest(root: File, appId: String, downloaded: Long, total: Long, flags: Int): File { val file = File(root, "steamapps/appmanifest_$appId.acf") check(file.parentFile?.mkdirs() != false) diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install index 8ebef7ea5d..45bd96c259 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install @@ -25,6 +25,7 @@ stamp=$steam_root/package/droiddeck-installed legacy_stamp=$steam_root/package/bannerlator-installed unconfirmed=$steam_root/package/droiddeck-unconfirmed repair_request=$HOME/.bl-steam-repair +parts=$steam_root/package/droiddeck-download if [ -f "$legacy_stamp" ] && [ ! -f "$stamp" ]; then mv -f "$legacy_stamp" "$stamp" || true fi @@ -47,6 +48,10 @@ if [ -n "$fallback" ]; then fi if [ -n "$repair" ]; then echo "== STEP $(date -u +%H:%M:%S) repairing the Steam client ($repair)" + if [ ! -f "$parts/.repair" ]; then + rm -rf "$parts" + mkdir -p "$parts" && : > "$parts/.repair" + fi rm -f "$stamp" "$legacy_stamp" "$steam_root/.crash" "$steam_root/steam.pid" \ "$steam_root"/package/steam_client_*.manifest "$steam_root"/package/steam_client_*.installed for part in "$steam_root"/package/*.zip.*; do @@ -69,6 +74,62 @@ fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT + +nap() { + [ -n "${nap_fd:-}" ] || exec {nap_fd}<> <(:) + read -r -t "$1" -u "$nap_fd" _ 2>/dev/null || : +} +size_of() { + local n=0 + [ -f "$1" ] && n=$(wc -c < "$1") + echo $((n)) +} +limits=(--connect-timeout 30 --speed-limit 4096 --speed-time 60) +read -r -a delays <<< "${BL_STEAM_RETRY_DELAYS:-5 10 20 30 30}" + +step() { echo "== STEP $(date -u +%H:%M:%S) $*"; } +progress() { + local mb=$(( (done_bytes + $(size_of "$1")) / 1000000 )) + [ -z "${3:-}" ] && [ "$mb" = "${shown_mb:-}" ] && [ "$2" = "${shown_part:-}" ] && return 0 + shown_mb=$mb shown_part=$2 + step "downloading Steam: $2 ($i/$total), $mb of $((total_bytes / 1000000)) MB${3:+ - $3}" +} + +fetch() { + local url=$1 dest=$2 size=$3 sum=$4 part=$5 failures=0 bad=0 finished=0 before rc fd wait + while :; do + if [ -f "$dest" ] && { [ "$finished" = 1 ] || { [ "$size" -gt 0 ] && [ "$(size_of "$dest")" -ge "$size" ]; }; }; then + echo "$sum $dest" | sha256sum -c --quiet 2>/dev/null && return 0 + rm -f "$dest" + finished=0 bad=$((bad + 1)) + [ "$bad" -lt 3 ] || { echo "droiddeck-steam-install: $part does not match its checksum" >&2; return 1; } + fi + before=$(size_of "$dest") + exec {fd}< <(code=$(curl -fsSL "${limits[@]}" -w '%{http_code}' -C - -o "$dest" "$url") && echo "0 $code" || echo "$? $code") + until read -r -t 5 -u "$fd" rc http; do + if [ $? -le 128 ]; then rc=1; break; fi + progress "$dest" "$part" + done + exec {fd}<&- + if [ "$rc" = 0 ]; then finished=1; continue; fi + case "$rc:${http:-}" in + 22:416 | 33:*) rm -f "$dest" ;; + 22:4??) + echo "droiddeck-steam-install: $part: HTTP $http from ${url%/*}" >&2 + return 1 + ;; + esac + if [ "$(size_of "$dest")" -gt "$before" ]; then failures=0; else failures=$((failures + 1)); fi + if [ "$failures" -gt "${#delays[@]}" ]; then + echo "droiddeck-steam-install: could not download $part (curl exit $rc); what arrived is kept for the next try" >&2 + return 1 + fi + wait=${delays[$((failures > 0 ? failures - 1 : 0))]} + progress "$dest" "$part" "interrupted (curl exit $rc), trying again in ${wait} s" + nap "$wait" + done +} + fetch_manifest() { cdn="" for candidate in $cdn_list; do @@ -88,7 +149,8 @@ fetch_manifest() { /^\t"/ && NF >= 3 && $4 == "" { block = $2 } $2 == "file" { file[block] = $4 } $2 == "sha2" { sha[block] = $4 } - END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b] } + $2 == "size" { size[block] = $4 } + END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b], size[b] + 0 } ' "$work/manifest" | sort > "$work/parts" [ -s "$work/parts" ] && return 0 echo "droiddeck-steam-install: no client components in $manifest; its blocks are:" \ @@ -114,23 +176,35 @@ if [ -n "$sync" ]; then echo "== STEP $(date -u +%H:%M:%S) updating the arm64 Steam client to $version (was ${installed:-none}) for the x86-64 client" fi total=$(grep -c . "$work/parts" 2>/dev/null || echo 0) +total_bytes=$(awk '{ n += $4 } END { print n + 0 }' "$work/parts") -mkdir -p "$steam_root/package" -while read -r block file sum; do +mkdir -p "$parts" +for left in "$parts"/*; do + [ -e "$left" ] || continue + name=${left##*/} + name=${name%.unpacked} + grep -qF " $name " "$work/parts" || rm -rf "$left" +done +done_bytes=0 +while read -r block file sum size; do i=$((${i:-0} + 1)) - echo "== STEP $(date -u +%H:%M:%S) downloading Steam: $block ($i/$total)" + if [ -f "$parts/$file.unpacked" ]; then + done_bytes=$((done_bytes + size)) + continue + fi + progress "$parts/$file" "$block" fetched="" for host in "${cdn_order[@]}"; do - if curl -fsSL --connect-timeout 10 --speed-limit 1024 --speed-time 60 --max-time 1800 \ - "$host/$file" -o "$work/$file" && echo "$sum $work/$file" | sha256sum -c --quiet; then + if fetch "$host/$file" "$parts/$file" "$size" "$sum" "$block"; then fetched=$host break fi - rm -f "$work/$file" done [ -n "$fetched" ] || { echo "droiddeck-steam-install: $file failed on every CDN" >&2; exit 1; } - unzip -qo "$work/$file" -d "$steam_root" - rm -f "$work/$file" + unzip -qo "$parts/$file" -d "$steam_root" + : > "$parts/$file.unpacked" + rm -f "$parts/$file" + done_bytes=$((done_bytes + size)) done < "$work/parts" find "$steam_root" -depth -name '*\\*' | while read -r odd; do fixed=${odd//\\//} @@ -159,6 +233,7 @@ ln -sfn "$steam_root/linuxarm64" "$HOME/.steam/sdkarm64" ln -sfn "$steam_root/steamrtarm64" "$HOME/.steam/binarm64" ln -sfn "$steam_root/linux32" "$HOME/.steam/sdk32" echo "$version" > "$stamp" +rm -rf "$parts" if [ -n "$fallback" ]; then echo "$stable_channel" > "$channel_state" echo "== STEP $(date -u +%H:%M:%S) the Steam client now follows $stable_channel (beta in ~/.config/droiddeck/steam-channel returns to $beta_channel)" diff --git a/tools/tests/test_steam_install.py b/tools/tests/test_steam_install.py index 256e711332..6a02bcb0ae 100644 --- a/tools/tests/test_steam_install.py +++ b/tools/tests/test_steam_install.py @@ -26,25 +26,48 @@ class SteamInstallTest(unittest.TestCase): self.bin = self.root / 'bin' self.bin.mkdir() curl = self.bin / 'curl' - curl.write_text('''#!/bin/bash -set -eu -url="" -out="" -while [ $# -gt 0 ]; do - case $1 in - -o) out=$2; shift 2 ;; - --connect-timeout|--max-time|--speed-limit|--speed-time|--retry|--retry-delay) shift 2 ;; - -*) shift ;; - *) url=$1; shift ;; - esac -done -printf '%s\\n' "$url" >> "$TEST_REQUESTS" -if [ -n "${TEST_FAIL_HOST:-}" ]; then - case $url in *"$TEST_FAIL_HOST"*) exit 22 ;; esac -fi -cp "$TEST_CDN/${url##*/}" "$out" + curl.write_text('''#!/usr/bin/env python3 +import os, sys +args = sys.argv[1:] +url = next(a for a in args if a.startswith('https://')) +out = args[args.index('-o') + 1] +name = url.rsplit('/', 1)[1] +offset = os.path.getsize(out) if '-C' in args and os.path.exists(out) else 0 +with open(os.environ['TEST_REQUESTS'], 'a') as log: + log.write(url + '\\n') +with open(os.environ['TEST_REQUESTS'] + '.offsets', 'a') as log: + log.write(f'{name} {offset}\\n') +def done(status, code='200'): + if '-w' in args: + sys.stdout.write(code) + sys.exit(status) +host_fail = os.environ.get('TEST_FAIL_HOST') +if host_fail and host_fail in url: + done(22, '000') +action = [] +plan = os.environ.get('TEST_PLAN') +if plan and os.path.exists(plan): + lines = open(plan).read().splitlines() + for n, line in enumerate(lines): + if line.split()[0] == name: + action = line.split()[1:] + del lines[n] + open(plan, 'w').write(''.join(l + '\\n' for l in lines)) + break +if action[:1] == ['fail']: + done(int(action[1]), '000') +if action[:1] == ['http']: + done(22, action[1]) +if action[:1] == ['norange'] and offset: + done(33) +data = open(os.path.join(os.environ['TEST_CDN'], name), 'rb').read() +end = int(action[1]) if action[:1] == ['cut'] else len(data) +with open(out, 'ab' if offset else 'wb') as target: + target.write(data[offset:end]) +done(28 if end < len(data) else 0) ''') curl.chmod(0o755) + self.plan = self.root / 'plan' if shutil.which('sha256sum') is None: sha = self.bin / 'sha256sum' sha.write_text('#!/bin/bash\nexec shasum -a 256 "$@"\n') @@ -55,7 +78,8 @@ cp "$TEST_CDN/${url##*/}" "$out" sha1.chmod(0o755) self.env = dict(os.environ, HOME=str(self.home), XDG_DATA_HOME=str(self.data), PATH=str(self.bin) + os.pathsep + os.environ['PATH'], - TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests)) + TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests), + TEST_PLAN=str(self.plan), BL_STEAM_RETRY_DELAYS='0 0 0') for channel in ('steamdeck_publicbeta', 'steamdeck_stable'): archive = self.cdn / (channel + '.zip') with zipfile.ZipFile(archive, 'w') as package: @@ -255,6 +279,143 @@ cp "$TEST_CDN/${url##*/}" "$out" self.assertEqual(0, self.install().returncode) self.assertFalse(self.requests.exists()) + def publish(self, **parts): + """The CDN's client as these parts (block name -> byte count of incompressible content), + listed with sizes as Valve's manifest lists them.""" + files = {} + blocks = '' + for block, length in parts.items(): + archive = self.cdn / f'{block}.zip.{hashlib.sha1(block.encode()).hexdigest()}' + with zipfile.ZipFile(archive, 'w', zipfile.ZIP_STORED) as package: + package.writestr(f'parts/{block}', os.urandom(length)) + if block.endswith('_linuxarm64_linuxarm64'): + package.writestr('steamrtarm64/steam', 'steamdeck_publicbeta') + data = archive.read_bytes() + files[block] = archive.name + blocks += (f'\t"{block}"\n\t{{\n\t\t"file"\t\t"{archive.name}"\n\t\t"size"\t\t"{len(data)}"\n' + f'\t\t"sha2"\t\t"{hashlib.sha256(data).hexdigest()}"\n\t}}\n') + (self.cdn / 'steam_client_steamdeck_publicbeta_linuxarm64').write_text( + '"linuxarm64"\n{\n\t"version"\t\t"123456"\n' + blocks + '}\n') + return files + + def two_parts(self): + return self.publish(a_all=1_200_000, client_linuxarm64_linuxarm64=1_300_000) + + def plan_for(self, name, *actions): + with self.plan.open('a') as plan: + plan.writelines(f'{name} {action}\n' for action in actions) + + def offsets(self, name): + log = Path(str(self.requests) + '.offsets') + return [int(line.split()[1]) for line in log.read_text().splitlines() if line.split()[0] == name] + + def requested(self): + return [line.rsplit('/', 1)[1] for line in self.requests.read_text().splitlines()] + + @property + def download(self): + return self.steam / 'package/droiddeck-download' + + def test_progress_is_counted_in_megabytes_across_parts(self): + self.two_parts() + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn('downloading Steam: a_all (1/2), 0 of 2 MB\n', result.stdout) + self.assertIn('downloading Steam: client_linuxarm64_linuxarm64 (2/2), 1 of 2 MB\n', result.stdout) + self.assertTrue((self.steam / 'parts/a_all').is_file()) + self.assertFalse(self.download.exists()) + + def test_a_dropped_connection_resumes_from_what_arrived(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 500000') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0, 500000], self.offsets(client)) + self.assertIn('interrupted (curl exit 28), trying again in 0 s', result.stdout) + self.assertEqual('123456\n', (self.steam / 'package/droiddeck-installed').read_text()) + + def test_a_failed_session_keeps_what_it_finished_and_fetched(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 400000', *['fail 6'] * 12) + result = self.install() + self.assertNotEqual(0, result.returncode) + self.assertIn('could not download client_linuxarm64_linuxarm64 (curl exit 6)', result.stderr) + self.assertFalse((self.steam / 'package/droiddeck-installed').exists()) + self.assertTrue((self.download / (files['a_all'] + '.unpacked')).is_file()) + self.assertFalse((self.download / files['a_all']).exists()) + self.assertEqual(400000, (self.download / client).stat().st_size) + self.requests.unlink() + Path(str(self.requests) + '.offsets').unlink() + self.assertEqual(0, self.install().returncode) + self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', client], self.requested()) + self.assertEqual([400000], self.offsets(client)) + self.assertFalse(self.download.exists()) + + def test_a_server_that_will_not_resume_starts_the_part_over(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 400000', 'norange') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0, 400000, 0], self.offsets(client)) + + def test_a_missing_part_hands_over_to_the_next_cdn_at_once(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'http 404') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(['https://client-update.steamstatic.com/' + client, + 'https://client-update.akamai.steamstatic.com/' + client], + [line for line in self.requests.read_text().splitlines() if line.endswith(client)]) + self.assertIn('HTTP 404 from https://client-update.steamstatic.com', result.stderr) + self.assertNotIn('trying again', result.stdout) + + def test_a_server_error_is_retried_on_the_same_cdn(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'http 503') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(['https://client-update.steamstatic.com/' + client] * 2, + [line for line in self.requests.read_text().splitlines() if line.endswith(client)]) + self.assertIn('interrupted (curl exit 22), trying again in 0 s', result.stdout) + + def test_bytes_that_do_not_check_out_are_fetched_again(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.download.mkdir(parents=True) + (self.download / client).write_bytes(b'x' * (self.cdn / client).stat().st_size) + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0], self.offsets(client)) + + def test_parts_left_for_another_client_version_are_dropped(self): + files = self.two_parts() + self.download.mkdir(parents=True) + (self.download / 'a_all.zip.0123').write_bytes(b'older') + (self.download / 'b_all.zip.4567.unpacked').write_bytes(b'') + self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 12) + self.assertNotEqual(0, self.install().returncode) + self.assertEqual([files['a_all'] + '.unpacked'], sorted(p.name for p in self.download.iterdir())) + + def test_a_repair_unpacks_every_part_again_once(self): + files = self.two_parts() + self.assertEqual(0, self.install().returncode) + (self.home / '.bl-steam-repair').write_text('1\n') + self.requests.unlink() + self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 12) + self.assertNotEqual(0, self.install().returncode) + self.assertIn(files['a_all'], self.requested()) + self.requests.unlink() + self.assertEqual(0, self.install().returncode) + self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', files['client_linuxarm64_linuxarm64']], + self.requested()) + self.assertFalse((self.home / '.bl-steam-repair').exists()) + self.assertFalse(self.download.exists()) + if __name__ == '__main__': unittest.main()