mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 12:56:06 +00:00
fix: extract GOG installer fallback natively on ARM (#1174)
This commit is contained in:
17 files changed
+435
-65
No files matched your search
@@ -9,10 +9,12 @@ import com.droiddeck.launcher.session.SessionFiles
|
||||
import com.droiddeck.launcher.session.SessionPaths
|
||||
import com.droiddeck.launcher.session.SessionPrefs
|
||||
import java.io.File
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
object GuestCommand {
|
||||
fun run(context: Context, argv: List<String>, fakeRoot: Boolean = false, logName: String? = null,
|
||||
linkDir: File? = null, extraBinds: List<String>? = null, fastPath: Boolean = false, onLine: (String) -> Unit): Int {
|
||||
linkDir: File? = null, extraBinds: List<String>? = null, fastPath: Boolean = false, cancelled: AtomicBoolean? = null,
|
||||
onLine: (String) -> Unit): Int {
|
||||
val log = logName?.let {
|
||||
try {
|
||||
File(File(LinuxRuntime.logDir(context), SessionPaths.TOOLS_DIR).apply { mkdirs() }, "$it.log").printWriter()
|
||||
@@ -20,14 +22,14 @@ object GuestCommand {
|
||||
}
|
||||
log?.println("== ${java.util.Date()} ${argv.joinToString(" ")}")
|
||||
try {
|
||||
return runLogged(context, argv, fakeRoot, linkDir, extraBinds, fastPath) { line -> log?.println(line); log?.flush(); onLine(line) }
|
||||
return runLogged(context, argv, fakeRoot, linkDir, extraBinds, fastPath, cancelled) { line -> log?.println(line); log?.flush(); onLine(line) }
|
||||
} finally {
|
||||
log?.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun runLogged(context: Context, argv: List<String>, fakeRoot: Boolean, linkDir: File?, extraBinds: List<String>?,
|
||||
fastPath: Boolean, onLine: (String) -> Unit): Int {
|
||||
fastPath: Boolean, cancelled: AtomicBoolean?, onLine: (String) -> Unit): Int {
|
||||
val root = LinuxRuntime.rootDir(context)
|
||||
LinuxRuntime.writeAccounts(context)
|
||||
SessionFiles.stage(context, root)
|
||||
@@ -66,13 +68,32 @@ object GuestCommand {
|
||||
fastPathKey?.let { ProotFastPath.hostEnv(it).let { (k, v) -> put(k, v) } }
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { put("LD_LIBRARY_PATH", it) }
|
||||
}
|
||||
if (cancelled?.get() == true) return 130
|
||||
val process = builder.start()
|
||||
val pid = HostProcess.pidOf(process)
|
||||
OrphanReaper.keep(pid)
|
||||
val watcher = cancelled?.let { flag ->
|
||||
Thread({
|
||||
try {
|
||||
while (process.isAlive && !flag.get()) Thread.sleep(200)
|
||||
if (flag.get()) {
|
||||
// PRoot ignores SIGTERM; SIGQUIT makes it kill its tracees and exit cleanly.
|
||||
if (pid > 0) {
|
||||
runCatching { android.system.Os.kill(pid, android.system.OsConstants.SIGQUIT) }
|
||||
} else {
|
||||
process.destroy()
|
||||
}
|
||||
if (!process.waitFor(3, java.util.concurrent.TimeUnit.SECONDS)) process.destroyForcibly()
|
||||
}
|
||||
} catch (_: InterruptedException) {
|
||||
}
|
||||
}, "guest-cancel").apply { isDaemon = true; start() }
|
||||
}
|
||||
try {
|
||||
process.inputStream.bufferedReader().useLines { lines -> lines.forEach(onLine) }
|
||||
return process.waitFor()
|
||||
} finally {
|
||||
watcher?.interrupt()
|
||||
OrphanReaper.release(pid)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,6 +98,15 @@ object SessionFiles {
|
||||
"usr/local/bin/droiddeck-script-run" to "usr/local/bin/droiddeck-script-run",
|
||||
"usr/local/bin/droiddeck-open" to "usr/local/bin/droiddeck-open",
|
||||
"usr/local/bin/droiddeck-proton-run" to "usr/local/bin/droiddeck-proton-run",
|
||||
"usr/local/bin/droiddeck-innoextract" to "usr/local/bin/droiddeck-innoextract",
|
||||
"usr/local/lib/droiddeck-inno/innoextract" to "usr/local/lib/droiddeck-inno/innoextract",
|
||||
"usr/local/lib/droiddeck-inno/libboost_container.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_container.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_filesystem.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_filesystem.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_iostreams.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_iostreams.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_program_options.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_program_options.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_random.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_random.so.1.92.0",
|
||||
"usr/local/share/licenses/droiddeck-inno/LICENSE.innoextract" to "usr/local/share/licenses/droiddeck-inno/LICENSE.innoextract",
|
||||
"usr/local/share/licenses/droiddeck-inno/LICENSE.boost" to "usr/local/share/licenses/droiddeck-inno/LICENSE.boost",
|
||||
"usr/bin/steamos-update" to "usr/bin/steamos-update",
|
||||
"usr/bin/steamos-select-branch" to "usr/bin/steamos-select-branch",
|
||||
"usr/bin/steamos-session-select" to "usr/bin/steamos-session-select",
|
||||
|
||||
@@ -102,24 +102,31 @@ object GogBackend : StoreBackend {
|
||||
StoreInstalls.begin(folder, Store.GOG, game.gameId, game.title, item.tallImageUrl ?: item.imageUrl, item.imageUrl)
|
||||
var downloading = false
|
||||
val language = item.extra[StoreLanguage.EXTRA] ?: StoreLanguage.preferredCode(app)
|
||||
val result = GogDownloadManager.install(app, game, folder, object : GogDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {
|
||||
when {
|
||||
message.startsWith("Finishing") -> handle.stage(DownloadStage.INSTALL, message)
|
||||
message.startsWith("Downloading") || message.startsWith("Verified") || message.startsWith("Resuming") -> {
|
||||
if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message)
|
||||
val result = try {
|
||||
GogDownloadManager.install(app, game, folder, object : GogDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {
|
||||
when {
|
||||
message.startsWith("Finishing") || message.startsWith("Extracting") -> handle.stage(DownloadStage.INSTALL, message)
|
||||
message.startsWith("Downloading") || message.startsWith("Verified") || message.startsWith("Resuming") -> {
|
||||
if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message)
|
||||
}
|
||||
else -> handle.stage(DownloadStage.MANIFEST, message)
|
||||
}
|
||||
else -> handle.stage(DownloadStage.MANIFEST, message)
|
||||
}
|
||||
override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) }
|
||||
override fun onLog(line: String) { handle.log(line) }
|
||||
override fun onSizes(downloadBytes: Long, diskBytes: Long) { handle.diskSize(diskBytes) }
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL })
|
||||
handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install")
|
||||
}
|
||||
}, cancelled, language)
|
||||
} catch (t: Throwable) {
|
||||
if (t is GogDownloadManager.InstallerException && !cancelled.get()) {
|
||||
StoreInstalls.discard(app, folder, listOf(File(folder, ".gog_chunks")))
|
||||
}
|
||||
override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) }
|
||||
override fun onLog(line: String) { handle.log(line) }
|
||||
override fun onSizes(downloadBytes: Long, diskBytes: Long) { handle.diskSize(diskBytes) }
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL })
|
||||
handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install")
|
||||
}
|
||||
}, cancelled, language) ?: return null
|
||||
throw t
|
||||
} ?: return null
|
||||
if (cancelled.get()) return null
|
||||
handle.stage(DownloadStage.INSTALL, app.getString(com.droiddeck.launcher.R.string.stores_registering))
|
||||
val installedLanguage = result.language?.takeIf { it.isNotEmpty() } ?: language
|
||||
|
||||
@@ -3,6 +3,8 @@ package com.droiddeck.launcher.stores.gog;
|
||||
import android.content.Context;
|
||||
import android.util.Log;
|
||||
|
||||
import com.droiddeck.launcher.frontend.AddedGames;
|
||||
import com.droiddeck.launcher.runtime.GuestCommand;
|
||||
import com.droiddeck.launcher.stores.StoreLanguage;
|
||||
import com.droiddeck.launcher.stores.StoreLog;
|
||||
import com.droiddeck.launcher.stores.download.StoreDownloadTier;
|
||||
@@ -21,7 +23,9 @@ import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
@@ -63,10 +67,15 @@ public final class GogDownloadManager {
|
||||
Result(String exeRelative, String buildId, long bytes) { this.exeRelative = exeRelative; this.buildId = buildId; this.bytes = bytes; }
|
||||
}
|
||||
|
||||
public static final class InstallException extends Exception {
|
||||
public static class InstallException extends Exception {
|
||||
InstallException(String message) { super(message); }
|
||||
}
|
||||
|
||||
/** A failure of the offline-installer fallback; its partial files can be discarded. */
|
||||
public static final class InstallerException extends InstallException {
|
||||
InstallerException(String message) { super(message); }
|
||||
}
|
||||
|
||||
private GogDownloadManager() {}
|
||||
|
||||
public static Result install(Context ctx, GogGame game, File installPath, Callback cb, AtomicBoolean cancelled) throws InstallException {
|
||||
@@ -424,41 +433,149 @@ public final class GogDownloadManager {
|
||||
String productJson = httpGet("https://api.gog.com/products/" + game.gameId + "?expand=downloads", token);
|
||||
if (productJson == null) throw new InstallException("No downloadable builds for this game");
|
||||
JSONObject downloads = new JSONObject(productJson).optJSONObject("downloads");
|
||||
JSONArray installers = downloads == null ? null : downloads.optJSONArray("installers");
|
||||
if (installers == null || installers.length() == 0) throw new InstallException("No downloadable builds for this game");
|
||||
String manualUrl = null, fileName = null;
|
||||
for (int i = 0; i < installers.length(); i++) {
|
||||
JSONObject inst = installers.getJSONObject(i);
|
||||
if (!"windows".equals(inst.optString("os"))) continue;
|
||||
JSONArray files = inst.optJSONArray("files");
|
||||
if (files != null && files.length() > 0) {
|
||||
JSONObject f = files.getJSONObject(0);
|
||||
manualUrl = f.optString("downlink", null);
|
||||
if (manualUrl == null) manualUrl = f.optString("manualUrl", null);
|
||||
fileName = f.optString("filename", null);
|
||||
}
|
||||
if (manualUrl == null) manualUrl = inst.optString("manualUrl", null);
|
||||
if (fileName == null || fileName.isEmpty()) fileName = "setup_" + game.gameId + ".exe";
|
||||
break;
|
||||
}
|
||||
if (manualUrl == null) throw new InstallException("No Windows installer for this game");
|
||||
String downloadUrl = resolveRedirect(manualUrl, token);
|
||||
if (downloadUrl == null) throw new InstallException("The installer link could not be resolved");
|
||||
List<InstallerPart> parts = installerParts(downloads, game.gameId);
|
||||
if (parts.isEmpty()) throw new InstallException("No downloadable builds for this game");
|
||||
installPath.mkdirs();
|
||||
File outFile = new File(installPath, fileName);
|
||||
cb.onProgress("Downloading installer: " + fileName, 15);
|
||||
long size = downloadWithProgress(downloadUrl, outFile, cb, cancelled);
|
||||
long planned = 0;
|
||||
boolean sized = true;
|
||||
for (InstallerPart part : parts) { planned += part.size; if (part.size <= 0) sized = false; }
|
||||
if (sized) {
|
||||
long usable = com.droiddeck.launcher.stores.StoreInstallRoot.availableBytes(installPath);
|
||||
if (com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(planned, usable)) {
|
||||
throw new InstallException("Not enough free space: need " + formatBytes(planned) + ", only " + formatBytes(usable) + " free");
|
||||
}
|
||||
}
|
||||
long total = 0;
|
||||
for (InstallerPart part : parts) {
|
||||
if (cancelled.get()) return null;
|
||||
String downloadUrl = resolveRedirect(part.url, token);
|
||||
if (downloadUrl == null) throw new InstallException("The installer link could not be resolved: " + part.filename);
|
||||
File outFile;
|
||||
try {
|
||||
outFile = installerDestination(installPath, part.filename);
|
||||
} catch (IOException e) {
|
||||
throw new InstallException("The installer part is not inside the game folder: " + part.filename);
|
||||
}
|
||||
cb.onProgress("Downloading installer: " + part.filename, 15 + (int) ((total / (float) Math.max(1, planned)) * 80));
|
||||
long size = downloadWithProgress(downloadUrl, outFile, cb, cancelled, total, sized ? planned : 0);
|
||||
if (cancelled.get()) return null;
|
||||
if (size <= 0) throw new InstallException("The installer download failed: " + part.filename);
|
||||
total += size;
|
||||
}
|
||||
if (cancelled.get()) return null;
|
||||
if (size <= 0) throw new InstallException("The installer download failed");
|
||||
cb.onLog("gog: installer downloaded; the game installs itself on first launch");
|
||||
return new Result(fileName, "", size);
|
||||
} catch (InstallException e) {
|
||||
cb.onLog("gog: installer downloaded (" + parts.size() + " file(s)); extracting natively");
|
||||
extractInstaller(ctx, installPath, parts.get(0).filename, cb, cancelled);
|
||||
if (cancelled.get()) return null;
|
||||
for (InstallerPart part : parts) new File(installPath, part.filename).delete();
|
||||
deleteDir(new File(installPath, "tmp"));
|
||||
deleteDir(new File(installPath, "commonappdata"));
|
||||
String exe = pickExe(installPath, null, game.title);
|
||||
if (exe.isEmpty()) throw new InstallException("The installer was extracted but no game executable was found");
|
||||
cb.onLog("gog: installer extracted natively; game exe " + exe);
|
||||
return new Result(exe, "", total);
|
||||
} catch (InstallerException e) {
|
||||
throw e;
|
||||
} catch (InstallException e) {
|
||||
throw new InstallerException(e.getMessage());
|
||||
} catch (Exception e) {
|
||||
throw new InstallException("Installer download error: " + e.getClass().getSimpleName());
|
||||
throw new InstallerException("Installer download error: " + e.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GOG's Windows installer is a 32-bit Inno Setup stub. On ARM64 it runs through
|
||||
* Proton/FEX, where its path strings are corrupted before CreateFileW (observed on
|
||||
* Wine 11 + FEX: "Path not found"). Extracting the same files with innoextract in
|
||||
* the Linux guest avoids that path entirely.
|
||||
*/
|
||||
private static void extractInstaller(Context ctx, File installPath, String setupName, Callback cb, AtomicBoolean cancelled) throws InstallException {
|
||||
String guestDir = AddedGames.INSTANCE.guestPath(ctx, installPath);
|
||||
if (guestDir == null || guestDir.isEmpty()) {
|
||||
throw new InstallException("The game folder is not visible to the Linux session");
|
||||
}
|
||||
String guestSetup = guestDir + "/" + setupName;
|
||||
cb.onProgress("Extracting installer…", 95);
|
||||
int rc;
|
||||
try {
|
||||
rc = GuestCommand.INSTANCE.run(ctx, Arrays.asList(
|
||||
"/usr/local/bin/droiddeck-innoextract", "--gog", "--silent",
|
||||
"-d", guestDir, guestSetup),
|
||||
false, "gog-innoextract", null, null, false, cancelled,
|
||||
line -> { cb.onLog("gog innoextract: " + line); return kotlin.Unit.INSTANCE; });
|
||||
} catch (Exception e) {
|
||||
throw new InstallException("The installer could not be extracted: " + e.getClass().getSimpleName());
|
||||
}
|
||||
if (cancelled.get()) return;
|
||||
if (rc != 0) throw new InstallException("The installer could not be extracted (innoextract exit " + rc + ")");
|
||||
}
|
||||
|
||||
static final class InstallerPart {
|
||||
final String url;
|
||||
final String filename;
|
||||
final long size;
|
||||
InstallerPart(String url, String filename, long size) { this.url = url; this.filename = filename; this.size = size; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file of the first usable Windows installer. GOG lists a setup stub and its data parts as
|
||||
* separate files, often without a filename; the parts are named after the stub so the installer
|
||||
* finds them when launched (setup_<id>.exe, setup_<id>-1.bin, ...).
|
||||
*/
|
||||
static List<InstallerPart> installerParts(JSONObject downloads, String gameId) {
|
||||
List<InstallerPart> out = new ArrayList<>();
|
||||
if (downloads == null) return out;
|
||||
JSONArray installers = downloads.optJSONArray("installers");
|
||||
if (installers == null) return out;
|
||||
for (int i = 0; i < installers.length(); i++) {
|
||||
JSONObject inst = installers.optJSONObject(i);
|
||||
if (inst == null || !"windows".equals(inst.optString("os"))) continue;
|
||||
JSONArray files = inst.optJSONArray("files");
|
||||
String base = null;
|
||||
if (files != null) {
|
||||
for (int j = 0; j < files.length(); j++) {
|
||||
JSONObject f = files.optJSONObject(j);
|
||||
if (f == null) continue;
|
||||
String url = f.optString("downlink", "");
|
||||
if (url.isEmpty()) url = f.optString("manualUrl", "");
|
||||
if (url.isEmpty()) continue;
|
||||
String name = partName(f.optString("filename", ""));
|
||||
if (name.isEmpty()) {
|
||||
if (base == null) base = "setup_" + safeName(gameId);
|
||||
name = out.isEmpty() ? base + ".exe" : base + "-" + out.size() + ".bin";
|
||||
} else if (base == null) {
|
||||
base = name.toLowerCase(java.util.Locale.US).endsWith(".exe") ? name.substring(0, name.length() - 4) : name;
|
||||
}
|
||||
out.add(new InstallerPart(url, name, f.optLong("size", 0)));
|
||||
}
|
||||
}
|
||||
if (out.isEmpty()) {
|
||||
String url = inst.optString("manualUrl", "");
|
||||
if (url.isEmpty()) url = inst.optString("downlink", "");
|
||||
if (!url.isEmpty()) out.add(new InstallerPart(url, "setup_" + safeName(gameId) + ".exe", inst.optLong("total_size", 0)));
|
||||
}
|
||||
if (!out.isEmpty()) return out;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static File installerDestination(File installPath, String filename) throws IOException {
|
||||
File root = installPath.getCanonicalFile();
|
||||
File candidate = new File(root, filename);
|
||||
if (Files.isSymbolicLink(candidate.toPath())) throw new IOException("installer part is a link");
|
||||
File out = candidate.getCanonicalFile();
|
||||
if (out.equals(root) || !out.toPath().startsWith(root.toPath())) throw new IOException("installer part leaves the game folder");
|
||||
return out;
|
||||
}
|
||||
|
||||
private static String partName(String raw) {
|
||||
return raw == null ? "" : new File(raw.replace('\\', '/')).getName();
|
||||
}
|
||||
|
||||
private static String safeName(String raw) {
|
||||
String cleaned = raw == null ? "" : raw.replaceAll("[^A-Za-z0-9._-]", "_");
|
||||
return cleaned.isEmpty() ? "game" : cleaned;
|
||||
}
|
||||
|
||||
private static String resolveRedirect(String url, String token) {
|
||||
try {
|
||||
if (url.startsWith("/")) url = "https://www.gog.com" + url;
|
||||
@@ -491,7 +608,7 @@ public final class GogDownloadManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
private static long downloadWithProgress(String url, File out, Callback cb, AtomicBoolean cancelled) {
|
||||
private static long downloadWithProgress(String url, File out, Callback cb, AtomicBoolean cancelled, long doneBefore, long planned) {
|
||||
try {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
conn.setConnectTimeout(TIMEOUT); conn.setReadTimeout(60_000);
|
||||
@@ -507,7 +624,11 @@ public final class GogDownloadManager {
|
||||
downloaded += n; windowBytes += n;
|
||||
long elapsed = System.currentTimeMillis() - windowStart;
|
||||
if (elapsed >= 500) { speed = windowBytes * 1000L / elapsed; windowStart = System.currentTimeMillis(); windowBytes = 0; }
|
||||
if (total > 0) {
|
||||
if (planned > 0) {
|
||||
long shown = doneBefore + downloaded;
|
||||
cb.onProgress("Downloading: " + out.getName() + speedSuffix(speed), 15 + (int) ((shown / (float) planned) * 80));
|
||||
cb.onBytes(shown, planned, speed);
|
||||
} else if (total > 0) {
|
||||
cb.onProgress("Downloading: " + out.getName() + speedSuffix(speed), 15 + (int) ((downloaded / (float) total) * 80));
|
||||
cb.onBytes(downloaded, total, speed);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use crate::fetch_tuning::{
|
||||
budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError,
|
||||
FetchFailKind, BOOTSTRAP_WINDOW, MAX_CHUNK_ATTEMPTS, NOMINAL_CHUNK_RESERVE_BYTES,
|
||||
RATE_LIMIT_COOLDOWN_MS, SERVER_EXPLORE_EVERY, USER_AGENT, WINDOW_BPS_EWMA_ALPHA,
|
||||
RATE_LIMIT_COOLDOWN_MS, SERVER_EXPLORE_EVERY, SERVER_FAULT_COOLDOWN_MS, USER_AGENT,
|
||||
WINDOW_BPS_EWMA_ALPHA,
|
||||
WINDOW_COOLDOWN_MS, WINDOW_DECLINE_EPS, WINDOW_ERR_BURST_IMMEDIATE, WINDOW_ERR_RATE_HIGH,
|
||||
WINDOW_ERR_RATE_LOW, WINDOW_HARD_CAP, WINDOW_IMPROVE_EPS, WINDOW_LOG_EVERY_PROBES,
|
||||
WINDOW_MIN_FLOOR, WINDOW_PLATEAU_PATIENCE, WINDOW_PLATEAU_REARM_MAX_MS,
|
||||
@@ -607,6 +608,10 @@ fn classify_reqwest_error(err: &reqwest::Error) -> FetchFailKind {
|
||||
FetchFailKind::Timeout
|
||||
} else if err.is_connect() {
|
||||
FetchFailKind::Connect
|
||||
} else if err.is_body() || err.is_decode() {
|
||||
// A truncated or undecodable body comes from the CDN, not the request. Rest that host so
|
||||
// the retry goes to another mirror instead of reading the same bad object again.
|
||||
FetchFailKind::ServerFault
|
||||
} else {
|
||||
FetchFailKind::Other
|
||||
}
|
||||
@@ -663,14 +668,9 @@ async fn fetch_once(
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(err) => {
|
||||
let kind = if err.is_timeout() {
|
||||
FetchFailKind::Timeout
|
||||
} else {
|
||||
FetchFailKind::Other
|
||||
};
|
||||
return Err(AsyncFetchError {
|
||||
message: format!("http body: {err}"),
|
||||
kind,
|
||||
kind: classify_reqwest_error(&err),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1150,7 +1150,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
Ok(Fetched::Body(body)) => {
|
||||
let raw_len = body.len() as u64;
|
||||
meter.record(done.server_idx, raw_len);
|
||||
sched.on_success(done.server_idx, raw_len, done.elapsed);
|
||||
sched.on_success(done.server_idx, raw_len, done.elapsed, now);
|
||||
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
|
||||
if raw_len >= done.reserve {
|
||||
in_flight.fetch_add(raw_len - done.reserve, Ordering::Relaxed);
|
||||
@@ -1172,7 +1172,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
}
|
||||
Ok(Fetched::Streamed(total_len)) => {
|
||||
meter.record(done.server_idx, total_len);
|
||||
sched.on_success(done.server_idx, total_len, done.elapsed);
|
||||
sched.on_success(done.server_idx, total_len, done.elapsed, now);
|
||||
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
|
||||
outstanding += 1;
|
||||
if txs[done.idx % txs.len()]
|
||||
@@ -1335,6 +1335,7 @@ struct ServerHealth {
|
||||
samples: u32,
|
||||
consecutive_errors: u32,
|
||||
cooldown_until: Option<Instant>,
|
||||
fault_until: Option<Instant>,
|
||||
}
|
||||
|
||||
struct FetchScheduler {
|
||||
@@ -1412,7 +1413,7 @@ impl FetchScheduler {
|
||||
})
|
||||
}
|
||||
|
||||
fn on_success(&mut self, server_idx: usize, bytes: u64, elapsed: Duration) {
|
||||
fn on_success(&mut self, server_idx: usize, bytes: u64, elapsed: Duration, now: Instant) {
|
||||
let secs = elapsed.as_secs_f64().max(0.001);
|
||||
let bps = bytes as f64 / secs;
|
||||
let h = &mut self.health[server_idx];
|
||||
@@ -1423,7 +1424,15 @@ impl FetchScheduler {
|
||||
};
|
||||
h.samples = h.samples.saturating_add(1);
|
||||
h.consecutive_errors = 0;
|
||||
h.cooldown_until = None;
|
||||
// A success can belong to a request that was already in flight when another response from
|
||||
// this host failed. Keep a server-fault cooldown until it expires instead of clearing it
|
||||
// early and sending the retry straight back to the same mirror.
|
||||
if let Some(fault) = h.fault_until.filter(|until| *until > now) {
|
||||
h.cooldown_until = Some(h.cooldown_until.map_or(fault, |current| current.max(fault)));
|
||||
} else {
|
||||
h.fault_until = None;
|
||||
h.cooldown_until = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn rest(&mut self, server_idx: usize, now: Instant, rest: Duration) {
|
||||
@@ -1436,10 +1445,18 @@ impl FetchScheduler {
|
||||
let h = &mut self.health[server_idx];
|
||||
h.consecutive_errors = h.consecutive_errors.saturating_add(1);
|
||||
let mut backoff = retry_backoff_millis(h.consecutive_errors);
|
||||
if kind == FetchFailKind::RateLimited {
|
||||
backoff = backoff.max(RATE_LIMIT_COOLDOWN_MS);
|
||||
match kind {
|
||||
FetchFailKind::RateLimited => backoff = backoff.max(RATE_LIMIT_COOLDOWN_MS),
|
||||
FetchFailKind::ServerFault => backoff = backoff.max(SERVER_FAULT_COOLDOWN_MS),
|
||||
_ => {}
|
||||
}
|
||||
h.cooldown_until = Some(now + Duration::from_millis(backoff));
|
||||
let until = now + Duration::from_millis(backoff);
|
||||
if kind == FetchFailKind::ServerFault {
|
||||
h.fault_until = Some(now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS));
|
||||
}
|
||||
let current = h.cooldown_until.unwrap_or(until);
|
||||
let fault = h.fault_until.unwrap_or(until);
|
||||
h.cooldown_until = Some(current.max(fault).max(until));
|
||||
h.ewma_bps *= 0.5;
|
||||
}
|
||||
}
|
||||
@@ -1806,6 +1823,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::fetch_tuning::{
|
||||
FETCH_INFLIGHT_BUDGET_FLOOR_BYTES, FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES, PER_HOST_CAP,
|
||||
SERVER_FAULT_COOLDOWN_MS,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Read, Write};
|
||||
@@ -1900,9 +1918,9 @@ mod tests {
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
assert_eq!(s.pick(now), Some(0));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(500));
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(500), now);
|
||||
assert_eq!(s.pick(now), Some(0));
|
||||
s.on_error(0, now, FetchFailKind::Timeout);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
@@ -1911,6 +1929,57 @@ mod tests {
|
||||
assert_eq!(s.pick(now + Duration::from_secs(6)), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_fault_rests_the_host_for_the_other_mirror() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now);
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(100), now);
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
assert_eq!(
|
||||
s.pick(now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS - 1)),
|
||||
Some(1)
|
||||
);
|
||||
assert!(!s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS - 1)));
|
||||
assert!(s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_success_from_an_earlier_request_does_not_clear_a_server_fault_cooldown() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now + Duration::from_millis(100));
|
||||
assert!(!s.eligible(0, now + Duration::from_millis(100)));
|
||||
assert!(s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_success_keeps_a_longer_cooldown_than_the_server_fault() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
s.rest(0, now, Duration::from_secs(30));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now + Duration::from_millis(100));
|
||||
assert!(!s.eligible(0, now + Duration::from_secs(29)));
|
||||
assert!(s.eligible(0, now + Duration::from_secs(30)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_fault_keeps_a_longer_host_stall_cooldown() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.rest(0, now, Duration::from_secs(30));
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
assert!(!s.eligible(0, now + Duration::from_secs(29)));
|
||||
assert!(s.eligible(0, now + Duration::from_secs(30)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_shrinks_only_on_error_bursts() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -68,6 +68,10 @@ pub const SERVER_EXPLORE_EVERY: u64 = 8;
|
||||
pub const NOMINAL_CHUNK_RESERVE_BYTES: u64 = 1024 * 1024;
|
||||
pub const RATE_LIMIT_COOLDOWN_MS: u64 = 5_000;
|
||||
|
||||
// Long enough that every retry of the item (5 attempts, at most ~4.5 s of backoff) goes to another
|
||||
// mirror after this one served a truncated or undecodable body.
|
||||
pub const SERVER_FAULT_COOLDOWN_MS: u64 = 10_000;
|
||||
|
||||
pub fn retry_backoff_millis(attempt: u32) -> u64 {
|
||||
if attempt == 0 {
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package com.droiddeck.launcher.stores.gog
|
||||
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import org.json.JSONObject
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertThrows
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [33])
|
||||
class GogInstallerPartsTest {
|
||||
private fun downloads(json: String) = JSONObject("""{"downloads":$json}""").getJSONObject("downloads")
|
||||
|
||||
@Test fun everyFileOfTheWindowsInstallerIsKeptAndThePartsAreNamedAfterItsSetup() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads(
|
||||
"""{"installers":[
|
||||
{"os":"windows","language":"en","files":[
|
||||
{"id":"en1installer0","size":1048576,"downlink":"https://api.gog.com/products/1947927225/downlink/installer/en1installer0"},
|
||||
{"id":"en1installer1","size":3786407936,"downlink":"https://api.gog.com/products/1947927225/downlink/installer/en1installer1"}
|
||||
]},
|
||||
{"os":"linux","files":[{"downlink":"https://example.invalid/linux"}]}
|
||||
]}""",
|
||||
),
|
||||
"1947927225",
|
||||
)
|
||||
assertEquals(listOf("setup_1947927225.exe", "setup_1947927225-1.bin"), parts.map { it.filename })
|
||||
assertEquals(listOf(1048576L, 3786407936L), parts.map { it.size })
|
||||
assertTrue(parts[1].url.endsWith("en1installer1"))
|
||||
}
|
||||
|
||||
@Test fun explicitNamesAreKeptAndUnnamedPartsFollowTheFirstSetupExe() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads(
|
||||
"""{"installers":[{"os":"windows","files":[
|
||||
{"filename":"setup_witcher_2.1.0.exe","downlink":"https://example.invalid/setup"},
|
||||
{"downlink":"https://example.invalid/data"},
|
||||
{"filename":"readme.bin","downlink":"https://example.invalid/readme"}
|
||||
]}]}""",
|
||||
),
|
||||
"1207658924",
|
||||
)
|
||||
assertEquals(listOf("setup_witcher_2.1.0.exe", "setup_witcher_2.1.0-1.bin", "readme.bin"), parts.map { it.filename })
|
||||
}
|
||||
|
||||
@Test fun aSingleManualInstallerIsNamedForTheGameAndSanitised() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads("""{"installers":[{"os":"windows","manualUrl":"https://example.invalid/setup","total_size":42}]}"""),
|
||||
"../game id",
|
||||
)
|
||||
assertEquals(1, parts.size)
|
||||
assertEquals("setup_.._game_id.exe", parts[0].filename)
|
||||
assertEquals(42L, parts[0].size)
|
||||
}
|
||||
|
||||
@Test fun installerPartsStayInsideTheGameFolder() {
|
||||
val root = kotlin.io.path.createTempDirectory("gog-installer").toFile()
|
||||
try {
|
||||
assertEquals(
|
||||
File(root, "setup_1.exe").canonicalFile,
|
||||
GogDownloadManager.installerDestination(root, "setup_1.exe"),
|
||||
)
|
||||
assertThrows(IOException::class.java) {
|
||||
GogDownloadManager.installerDestination(root, "../outside.exe")
|
||||
}
|
||||
val outside = File(root.parentFile, "outside-${System.nanoTime()}.bin").apply { writeText("x") }
|
||||
try {
|
||||
Files.createSymbolicLink(File(root, "setup_1-1.bin").toPath(), outside.toPath())
|
||||
assertThrows(IOException::class.java) {
|
||||
GogDownloadManager.installerDestination(root, "setup_1-1.bin")
|
||||
}
|
||||
} finally {
|
||||
outside.delete()
|
||||
}
|
||||
} finally {
|
||||
root.deleteRecursively()
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun anotherPlatformOrNoDownloadsAddsNothing() {
|
||||
assertEquals(0, GogDownloadManager.installerParts(downloads("""{"installers":[{"os":"linux","manualUrl":"https://example.invalid/x"}]}"""), "1").size)
|
||||
assertEquals(0, GogDownloadManager.installerParts(downloads("{}"), "1").size)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user