diff --git a/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt b/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt index 26799d0036..a190cbaef8 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/LoadingState.kt @@ -102,11 +102,10 @@ class LoadingState(context: Context, private val steam: Boolean = true) { if (line.startsWith("== STEP ")) { stepAt = at stepText = line.substringAfter("== STEP ").substringAfter(' ') - val m = INSTALL_COUNT.find(line) - if (m != null) { + clientDownload(line)?.let { (what, percent) -> clientDownloadAt = at - clientDownload = m.groupValues[1] - clientPercent = m.groupValues[2].toInt() * 100 / maxOf(1, m.groupValues[3].toInt()) + clientDownload = what + clientPercent = percent } } else { val m = UPDATE_PROGRESS.find(line) @@ -152,6 +151,21 @@ class LoadingState(context: Context, private val steam: Boolean = true) { } } + /** + * What a Steam client download line says to show and how far it is: megabytes when the + * runtime counts them (" (2/17), 45 of 360 MB"), else the part and its number. + */ + internal fun clientDownload(line: String): Pair? { + val count = INSTALL_COUNT.find(line) ?: return null + val mb = INSTALL_MEGABYTES.find(line.substring(count.range.last + 1)) + val done = mb?.groupValues?.get(1)?.toLongOrNull() + val total = mb?.groupValues?.get(2)?.toLongOrNull() + if (done != null && total != null && total > 0) { + return "$done / $total MB" to (done * 100 / total).toInt().coerceIn(0, 100) + } + return count.groupValues[1] to count.groupValues[2].toInt() * 100 / maxOf(1, count.groupValues[3].toInt()) + } + /** The script's install lines are worth reading as they are; its other notes are not. */ internal fun readableScriptStep(scriptStep: String): Boolean { val t = scriptStep.lowercase() @@ -161,5 +175,6 @@ class LoadingState(context: Context, private val steam: Boolean = true) { private const val TAIL_BYTES = 48L * 1024 private val UPDATE_PROGRESS = Regex("""Downloading update \((\d+) of (\d+) KB\)""") private val INSTALL_COUNT = Regex("""downloading Steam: (\S+) \((\d+)/(\d+)\)""") + private val INSTALL_MEGABYTES = Regex("""^, (\d+) of (\d+) MB""") } } diff --git a/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt b/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt new file mode 100644 index 0000000000..dd545f1f51 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/session/LoadingStateTest.kt @@ -0,0 +1,37 @@ +package com.droiddeck.launcher.session + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class LoadingStateTest { + @Test fun clientDownloadCountsMegabytesWhenTheRuntimeSaysThem() { + assertEquals( + "45 / 360 MB" to 12, + LoadingState.clientDownload("== STEP 17:38:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB"), + ) + } + + @Test fun aStalledDownloadStillShowsWhereItIs() { + assertEquals( + "45 / 360 MB" to 12, + LoadingState.clientDownload( + "== STEP 17:39:20 downloading Steam: bins_linuxarm64_linuxarm64 (2/17), 45 of 360 MB" + + " - interrupted (curl exit 28), trying again in 5 s", + ), + ) + } + + @Test fun anOlderRuntimeShowsThePartAndItsNumber() { + assertEquals( + "bins_linuxarm64_linuxarm64" to 11, + LoadingState.clientDownload("== STEP 17:38:17 downloading Steam: bins_linuxarm64_linuxarm64 (2/17)"), + ) + // Under a megabyte in all: nothing to count yet. + assertEquals("a_all" to 50, LoadingState.clientDownload("== STEP 00:00:00 downloading Steam: a_all (1/2), 0 of 0 MB")) + } + + @Test fun otherStepsAreNotClientDownloads() { + assertNull(LoadingState.clientDownload("== STEP 17:38:14 checking the Steam client (a first run downloads it)")) + } +} diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-session b/tools/linuxfs/overlay/usr/local/bin/droiddeck-session index 94d9a69370..5822ef0beb 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-session +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-session @@ -368,7 +368,9 @@ case $mode in esac ;; steam) - step "checking the Steam client (first run downloads it, ~1-2 min with nothing on screen)" + # A first run downloads the client (about 360 MB in 2026): minutes on a slow connection, with + # the progress in megabytes on the loading screen. + step "checking the Steam client (a first run downloads it)" droiddeck-steam-install || exit $? step "Steam client ready" steam_root=$XDG_DATA_HOME/Steam diff --git a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install index 0eae6d0d27..85404bd620 100755 --- a/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install +++ b/tools/linuxfs/overlay/usr/local/bin/droiddeck-steam-install @@ -12,6 +12,9 @@ stamp=$steam_root/package/droiddeck-installed legacy_stamp=$steam_root/package/bannerlator-installed unconfirmed=$steam_root/package/droiddeck-unconfirmed repair_request=$HOME/.bl-steam-repair +# The parts of an unfinished download: a session that ends, or loses its connection, part way +# through picks up where it stopped instead of fetching the whole client again. +parts=$steam_root/package/droiddeck-download if [ -f "$legacy_stamp" ] && [ ! -f "$stamp" ]; then mv -f "$legacy_stamp" "$stamp" || true fi @@ -24,6 +27,11 @@ elif [ -f "$unconfirmed" ] && ! grep -qx repaired "$unconfirmed" \ fi if [ -n "$repair" ]; then echo "== STEP $(date -u +%H:%M:%S) repairing the Steam client ($repair)" + # A repair unpacks every part again - once: an interrupted one keeps what it has redone. + if [ ! -f "$parts/.repair" ]; then + rm -rf "$parts" + mkdir -p "$parts" && : > "$parts/.repair" + fi rm -f "$stamp" "$legacy_stamp" "$steam_root/.crash" "$steam_root/steam.pid" \ "$steam_root"/package/steam_client_*.manifest "$steam_root"/package/steam_client_*.installed for part in "$steam_root"/package/*.zip.*; do @@ -41,26 +49,105 @@ fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT -curl -fsSL "$cdn/$manifest" -o "$work/manifest" + +# A sleep that starts no process (droiddeck-session's nap): read -t on a pipe nothing writes to. +nap() { + [ -n "${nap_fd:-}" ] || exec {nap_fd}<> <(:) + read -r -t "$1" -u "$nap_fd" _ 2>/dev/null || : +} +size_of() { + local n=0 + [ -f "$1" ] && n=$(wc -c < "$1") + echo $((n)) +} +# A connection can stall without closing (Wi-Fi power save, a phone changing networks), and +# curl's default then is to wait for ever with nothing on screen. A transfer under 4 KB/s for a +# minute is dropped and resumed from the bytes already on disk. +limits=(--connect-timeout 30 --speed-limit 4096 --speed-time 60) +# Seconds between attempts that made no progress; the download fails after the last one. +read -r -a delays <<< "${BL_STEAM_RETRY_DELAYS:-5 10 20 30 30}" + +step() { echo "== STEP $(date -u +%H:%M:%S) $*"; } +# The line the app's loading screen reads: the part, its number, and megabytes overall. A third +# argument is said after it (a stall), and is always said; otherwise only a change is. +progress() { + local mb=$(( (done_bytes + $(size_of "$1")) / 1000000 )) + [ -z "${3:-}" ] && [ "$mb" = "${shown_mb:-}" ] && [ "$2" = "${shown_part:-}" ] && return 0 + shown_mb=$mb shown_part=$2 + step "downloading Steam: $2 ($i/$total), $mb of $((total_bytes / 1000000)) MB${3:+ - $3}" +} + +# fetch URL FILE SIZE SHA256 PART: FILE complete and checked, resuming whatever is already there. +fetch() { + local url=$1 dest=$2 size=$3 sum=$4 part=$5 failures=0 bad=0 finished=0 before rc fd wait + while :; do + # Whole once it is as long as the manifest says, or (no size given) once curl finished it. + if [ -f "$dest" ] && { [ "$finished" = 1 ] || { [ "$size" -gt 0 ] && [ "$(size_of "$dest")" -ge "$size" ]; }; }; then + echo "$sum $dest" | sha256sum -c --quiet 2>/dev/null && return 0 + # Bytes that do not check out are fetched whole again, a few times at most. + rm -f "$dest" + finished=0 bad=$((bad + 1)) + [ "$bad" -lt 3 ] || { echo "droiddeck-steam-install: $part does not match its checksum" >&2; return 1; } + fi + before=$(size_of "$dest") + # curl's status comes back through the pipe (|| keeps set -e from ending the subshell before it + # is said); until then a progress line every 5 s. + exec {fd}< <(curl -fsSL "${limits[@]}" -C - -o "$dest" "$url" && echo 0 || echo $?) + until read -r -t 5 -u "$fd" rc; do + if [ $? -le 128 ]; then rc=1; break; fi + progress "$dest" "$part" + done + exec {fd}<&- + if [ "$rc" = 0 ]; then finished=1; continue; fi + # 33: the server would not resume. Start the part over. + [ "$rc" = 33 ] && rm -f "$dest" + if [ "$(size_of "$dest")" -gt "$before" ]; then failures=0; else failures=$((failures + 1)); fi + if [ "$failures" -gt "${#delays[@]}" ]; then + echo "droiddeck-steam-install: could not download $part (curl exit $rc); what arrived is kept for the next try" >&2 + return 1 + fi + wait=${delays[$((failures > 0 ? failures - 1 : 0))]} + progress "$dest" "$part" "interrupted (curl exit $rc), trying again in ${wait} s" + nap "$wait" + done +} + +curl -fsSL "${limits[@]}" --retry 5 --retry-connrefused --retry-delay 5 "$cdn/$manifest" -o "$work/manifest" version=$(awk -F'"' '$2 == "version" { print $4; exit }' "$work/manifest") -# Every component block: its name, then the plain zip's file name and sha256. +# Every component block: its name, then the plain zip's file name, sha256 and size. awk -F'"' ' /^\t"/ && NF >= 3 && $4 == "" { block = $2 } $2 == "file" { file[block] = $4 } $2 == "sha2" { sha[block] = $4 } - END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b] } + $2 == "size" { size[block] = $4 } + END { for (b in file) if (b ~ /_all$/ || b ~ /_linuxarm64_linuxarm64$/) print b, file[b], sha[b], size[b] + 0 } ' "$work/manifest" | sort > "$work/parts" [ -s "$work/parts" ] || { echo "droiddeck-steam-install: no client components in $manifest" >&2; exit 1; } total=$(grep -c . "$work/parts" 2>/dev/null || echo 0) +total_bytes=$(awk '{ n += $4 } END { print n + 0 }' "$work/parts") -mkdir -p "$steam_root/package" -while read -r block file sum; do +mkdir -p "$parts" +# What an earlier attempt left for another client version is no use to this one. A part's file +# name carries its own hash, so a name still listed is the same part. +for left in "$parts"/*; do + [ -e "$left" ] || continue + name=${left##*/} + name=${name%.unpacked} + grep -qF " $name " "$work/parts" || rm -rf "$left" +done +done_bytes=0 +while read -r block file sum size; do i=$((${i:-0} + 1)) - echo "== STEP $(date -u +%H:%M:%S) downloading Steam: $block ($i/$total)" - curl -fsSL "$cdn/$file" -o "$work/$file" - echo "$sum $work/$file" | sha256sum -c --quiet - unzip -qo "$work/$file" -d "$steam_root" - rm -f "$work/$file" + if [ -f "$parts/$file.unpacked" ]; then + done_bytes=$((done_bytes + size)) + continue + fi + progress "$parts/$file" "$block" + fetch "$cdn/$file" "$parts/$file" "$size" "$sum" "$block" || exit 1 + unzip -qo "$parts/$file" -d "$steam_root" + : > "$parts/$file.unpacked" + rm -f "$parts/$file" + done_bytes=$((done_bytes + size)) done < "$work/parts" # Some entries are zipped with Windows separators; unzip keeps the backslash in the file name. find "$steam_root" -depth -name '*\\*' | while read -r odd; do @@ -90,6 +177,7 @@ ln -sfn "$steam_root/linuxarm64" "$HOME/.steam/sdkarm64" ln -sfn "$steam_root/steamrtarm64" "$HOME/.steam/binarm64" ln -sfn "$steam_root/linux32" "$HOME/.steam/sdk32" echo "$version" > "$stamp" +rm -rf "$parts" if [ -n "$repair" ]; then echo repaired > "$unconfirmed" rm -f "$repair_request" diff --git a/tools/tests/test_steam_install.py b/tools/tests/test_steam_install.py index f3e29bbe26..d71dd7b4db 100644 --- a/tools/tests/test_steam_install.py +++ b/tools/tests/test_steam_install.py @@ -26,13 +26,43 @@ class SteamInstallTest(unittest.TestCase): self.bin = self.root / 'bin' self.bin.mkdir() # Only the network is faked: exercise real manifest parsing, checksums and zip extraction. + # TEST_PLAN lines " " are used up in order by requests for that file: + # "cut " delivers up to that offset then drops the connection, "fail " + # delivers nothing, "norange" refuses to resume. curl = self.bin / 'curl' - curl.write_text('''#!/bin/bash -set -eu -printf '%s\\n' "$2" >> "$TEST_REQUESTS" -cp "$TEST_CDN/${2##*/}" "$4" + curl.write_text('''#!/usr/bin/env python3 +import os, sys +args = sys.argv[1:] +url = next(a for a in args if a.startswith('https://')) +out = args[args.index('-o') + 1] +name = url.rsplit('/', 1)[1] +offset = os.path.getsize(out) if '-C' in args and os.path.exists(out) else 0 +with open(os.environ['TEST_REQUESTS'], 'a') as log: + log.write(url + '\\n') +with open(os.environ['TEST_REQUESTS'] + '.offsets', 'a') as log: + log.write(f'{name} {offset}\\n') +action = [] +plan = os.environ.get('TEST_PLAN') +if plan and os.path.exists(plan): + lines = open(plan).read().splitlines() + for n, line in enumerate(lines): + if line.split()[0] == name: + action = line.split()[1:] + del lines[n] + open(plan, 'w').write(''.join(l + '\\n' for l in lines)) + break +if action[:1] == ['fail']: + sys.exit(int(action[1])) +if action[:1] == ['norange'] and offset: + sys.exit(33) +data = open(os.path.join(os.environ['TEST_CDN'], name), 'rb').read() +end = int(action[1]) if action[:1] == ['cut'] else len(data) +with open(out, 'ab' if offset else 'wb') as target: + target.write(data[offset:end]) +sys.exit(28 if end < len(data) else 0) ''') curl.chmod(0o755) + self.plan = self.root / 'plan' # macOS has shasum; the guest and Linux CI have sha256sum. if shutil.which('sha256sum') is None: sha = self.bin / 'sha256sum' @@ -44,7 +74,8 @@ cp "$TEST_CDN/${2##*/}" "$4" sha1.chmod(0o755) self.env = dict(os.environ, HOME=str(self.home), XDG_DATA_HOME=str(self.data), PATH=str(self.bin) + os.pathsep + os.environ['PATH'], - TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests)) + TEST_CDN=str(self.cdn), TEST_REQUESTS=str(self.requests), + TEST_PLAN=str(self.plan), BL_STEAM_RETRY_DELAYS='0 0 0') for channel in ('steamdeck_publicbeta',): archive = self.cdn / (channel + '.zip') with zipfile.ZipFile(archive, 'w') as package: @@ -157,5 +188,123 @@ cp "$TEST_CDN/${2##*/}" "$4" self.assertFalse(self.requests.exists()) + def publish(self, **parts): + """The CDN's client as these parts (block name -> byte count of incompressible content), + listed with sizes as Valve's manifest lists them.""" + files = {} + blocks = '' + for block, length in parts.items(): + archive = self.cdn / f'{block}.zip.{hashlib.sha1(block.encode()).hexdigest()}' + with zipfile.ZipFile(archive, 'w', zipfile.ZIP_STORED) as package: + package.writestr(f'parts/{block}', os.urandom(length)) + if block.endswith('_linuxarm64_linuxarm64'): + package.writestr('steamrtarm64/steam', 'steamdeck_publicbeta') + data = archive.read_bytes() + files[block] = archive.name + blocks += (f'\t"{block}"\n\t{{\n\t\t"file"\t\t"{archive.name}"\n\t\t"size"\t\t"{len(data)}"\n' + f'\t\t"sha2"\t\t"{hashlib.sha256(data).hexdigest()}"\n\t}}\n') + (self.cdn / 'steam_client_steamdeck_publicbeta_linuxarm64').write_text( + '"linuxarm64"\n{\n\t"version"\t\t"123456"\n' + blocks + '}\n') + return files + + def two_parts(self): + return self.publish(a_all=1_200_000, client_linuxarm64_linuxarm64=1_300_000) + + def plan_for(self, name, *actions): + with self.plan.open('a') as plan: + plan.writelines(f'{name} {action}\n' for action in actions) + + def offsets(self, name): + log = Path(str(self.requests) + '.offsets') + return [int(line.split()[1]) for line in log.read_text().splitlines() if line.split()[0] == name] + + def requested(self): + return [line.rsplit('/', 1)[1] for line in self.requests.read_text().splitlines()] + + @property + def download(self): + return self.steam / 'package/droiddeck-download' + + def test_progress_is_counted_in_megabytes_across_parts(self): + self.two_parts() + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn('downloading Steam: a_all (1/2), 0 of 2 MB\n', result.stdout) + self.assertIn('downloading Steam: client_linuxarm64_linuxarm64 (2/2), 1 of 2 MB\n', result.stdout) + self.assertTrue((self.steam / 'parts/a_all').is_file()) + self.assertFalse(self.download.exists()) + + def test_a_dropped_connection_resumes_from_what_arrived(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 500000') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0, 500000], self.offsets(client)) + self.assertIn('interrupted (curl exit 28), trying again in 0 s', result.stdout) + self.assertEqual('123456\n', (self.steam / 'package/droiddeck-installed').read_text()) + + def test_a_failed_session_keeps_what_it_finished_and_fetched(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 400000', *['fail 6'] * 4) + result = self.install() + self.assertNotEqual(0, result.returncode) + self.assertIn('could not download client_linuxarm64_linuxarm64 (curl exit 6)', result.stderr) + self.assertFalse((self.steam / 'package/droiddeck-installed').exists()) + self.assertTrue((self.download / (files['a_all'] + '.unpacked')).is_file()) + self.assertFalse((self.download / files['a_all']).exists()) + self.assertEqual(400000, (self.download / client).stat().st_size) + self.requests.unlink() + Path(str(self.requests) + '.offsets').unlink() + self.assertEqual(0, self.install().returncode) + # The next session asks only for the rest of the part it did not finish. + self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', client], self.requested()) + self.assertEqual([400000], self.offsets(client)) + self.assertFalse(self.download.exists()) + + def test_a_server_that_will_not_resume_starts_the_part_over(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.plan_for(client, 'cut 400000', 'norange') + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0, 400000, 0], self.offsets(client)) + + def test_bytes_that_do_not_check_out_are_fetched_again(self): + files = self.two_parts() + client = files['client_linuxarm64_linuxarm64'] + self.download.mkdir(parents=True) + (self.download / client).write_bytes(b'x' * (self.cdn / client).stat().st_size) + result = self.install() + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual([0], self.offsets(client)) + + def test_parts_left_for_another_client_version_are_dropped(self): + files = self.two_parts() + self.download.mkdir(parents=True) + (self.download / 'a_all.zip.0123').write_bytes(b'older') + (self.download / 'b_all.zip.4567.unpacked').write_bytes(b'') + self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 4) + self.assertNotEqual(0, self.install().returncode) + self.assertEqual([files['a_all'] + '.unpacked'], sorted(p.name for p in self.download.iterdir())) + + def test_a_repair_unpacks_every_part_again_once(self): + files = self.two_parts() + self.assertEqual(0, self.install().returncode) + (self.home / '.bl-steam-repair').write_text('1\n') + self.requests.unlink() + self.plan_for(files['client_linuxarm64_linuxarm64'], *['fail 7'] * 4) + self.assertNotEqual(0, self.install().returncode) + self.assertIn(files['a_all'], self.requested()) + self.requests.unlink() + # Still a repair, but what the interrupted one redid is not done a third time. + self.assertEqual(0, self.install().returncode) + self.assertEqual(['steam_client_steamdeck_publicbeta_linuxarm64', files['client_linuxarm64_linuxarm64']], + self.requested()) + self.assertFalse((self.home / '.bl-steam-repair').exists()) + self.assertFalse(self.download.exists()) + + if __name__ == '__main__': unittest.main()