mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 21:06:09 +00:00
fix: extract GOG installers natively on ARM
The GOG installer fallback downloaded only the setup stub and then launched it through Proton/FEX, where the 32-bit Inno Setup stub corrupts its own path strings before CreateFileW. Download every installer part and extract them with the bundled ARM64 innoextract in the Linux guest instead, registering the real game executable. Also classify truncated response bodies as server faults so the native engine retries on the other CDN mirror.
This commit is contained in:
1 parent
a5c587bd71
commit
b9d703db34
17 files changed
+435
-65
No files matched your search
@@ -8,10 +8,12 @@ import com.droiddeck.launcher.session.SessionFiles
|
||||
import com.droiddeck.launcher.session.SessionPaths
|
||||
import com.droiddeck.launcher.session.SessionPrefs
|
||||
import java.io.File
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
|
||||
object GuestCommand {
|
||||
fun run(context: Context, argv: List<String>, fakeRoot: Boolean = false, logName: String? = null,
|
||||
linkDir: File? = null, extraBinds: List<String>? = null, fastPath: Boolean = false, onLine: (String) -> Unit): Int {
|
||||
linkDir: File? = null, extraBinds: List<String>? = null, fastPath: Boolean = false, cancelled: AtomicBoolean? = null,
|
||||
onLine: (String) -> Unit): Int {
|
||||
val log = logName?.let {
|
||||
try {
|
||||
File(File(LinuxRuntime.logDir(context), SessionPaths.TOOLS_DIR).apply { mkdirs() }, "$it.log").printWriter()
|
||||
@@ -19,14 +21,14 @@ object GuestCommand {
|
||||
}
|
||||
log?.println("== ${java.util.Date()} ${argv.joinToString(" ")}")
|
||||
try {
|
||||
return runLogged(context, argv, fakeRoot, linkDir, extraBinds, fastPath) { line -> log?.println(line); log?.flush(); onLine(line) }
|
||||
return runLogged(context, argv, fakeRoot, linkDir, extraBinds, fastPath, cancelled) { line -> log?.println(line); log?.flush(); onLine(line) }
|
||||
} finally {
|
||||
log?.close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun runLogged(context: Context, argv: List<String>, fakeRoot: Boolean, linkDir: File?, extraBinds: List<String>?,
|
||||
fastPath: Boolean, onLine: (String) -> Unit): Int {
|
||||
fastPath: Boolean, cancelled: AtomicBoolean?, onLine: (String) -> Unit): Int {
|
||||
val root = LinuxRuntime.rootDir(context)
|
||||
LinuxRuntime.writeAccounts(context)
|
||||
SessionFiles.stage(context, root)
|
||||
@@ -56,13 +58,32 @@ object GuestCommand {
|
||||
fastPathKey?.let { ProotFastPath.hostEnv(it).let { (k, v) -> put(k, v) } }
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { put("LD_LIBRARY_PATH", it) }
|
||||
}
|
||||
if (cancelled?.get() == true) return 130
|
||||
val process = builder.start()
|
||||
val pid = HostProcess.pidOf(process)
|
||||
OrphanReaper.keep(pid)
|
||||
val watcher = cancelled?.let { flag ->
|
||||
Thread({
|
||||
try {
|
||||
while (process.isAlive && !flag.get()) Thread.sleep(200)
|
||||
if (flag.get()) {
|
||||
// PRoot ignores SIGTERM; SIGQUIT makes it kill its tracees and exit cleanly.
|
||||
if (pid > 0) {
|
||||
runCatching { android.system.Os.kill(pid, android.system.OsConstants.SIGQUIT) }
|
||||
} else {
|
||||
process.destroy()
|
||||
}
|
||||
if (!process.waitFor(3, java.util.concurrent.TimeUnit.SECONDS)) process.destroyForcibly()
|
||||
}
|
||||
} catch (_: InterruptedException) {
|
||||
}
|
||||
}, "guest-cancel").apply { isDaemon = true; start() }
|
||||
}
|
||||
try {
|
||||
process.inputStream.bufferedReader().useLines { lines -> lines.forEach(onLine) }
|
||||
return process.waitFor()
|
||||
} finally {
|
||||
watcher?.interrupt()
|
||||
OrphanReaper.release(pid)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -88,6 +88,15 @@ object SessionFiles {
|
||||
"usr/local/bin/droiddeck-script-run" to "usr/local/bin/droiddeck-script-run",
|
||||
"usr/local/bin/droiddeck-open" to "usr/local/bin/droiddeck-open",
|
||||
"usr/local/bin/droiddeck-proton-run" to "usr/local/bin/droiddeck-proton-run",
|
||||
"usr/local/bin/droiddeck-innoextract" to "usr/local/bin/droiddeck-innoextract",
|
||||
"usr/local/lib/droiddeck-inno/innoextract" to "usr/local/lib/droiddeck-inno/innoextract",
|
||||
"usr/local/lib/droiddeck-inno/libboost_container.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_container.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_filesystem.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_filesystem.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_iostreams.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_iostreams.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_program_options.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_program_options.so.1.92.0",
|
||||
"usr/local/lib/droiddeck-inno/libboost_random.so.1.92.0" to "usr/local/lib/droiddeck-inno/libboost_random.so.1.92.0",
|
||||
"usr/local/share/licenses/droiddeck-inno/LICENSE.innoextract" to "usr/local/share/licenses/droiddeck-inno/LICENSE.innoextract",
|
||||
"usr/local/share/licenses/droiddeck-inno/LICENSE.boost" to "usr/local/share/licenses/droiddeck-inno/LICENSE.boost",
|
||||
"usr/bin/steamos-update" to "usr/bin/steamos-update",
|
||||
"usr/bin/steamos-select-branch" to "usr/bin/steamos-select-branch",
|
||||
"usr/bin/steamos-session-select" to "usr/bin/steamos-session-select",
|
||||
|
||||
@@ -82,24 +82,31 @@ object GogBackend : StoreBackend {
|
||||
handle.stage(DownloadStage.MANIFEST)
|
||||
StoreInstalls.begin(folder, Store.GOG, game.gameId, game.title, item.tallImageUrl ?: item.imageUrl, item.imageUrl)
|
||||
var downloading = false
|
||||
val result = GogDownloadManager.install(app, game, folder, object : GogDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {
|
||||
when {
|
||||
message.startsWith("Finishing") -> handle.stage(DownloadStage.INSTALL, message)
|
||||
message.startsWith("Downloading") || message.startsWith("Verified") || message.startsWith("Resuming") -> {
|
||||
if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message)
|
||||
val result = try {
|
||||
GogDownloadManager.install(app, game, folder, object : GogDownloadManager.Callback {
|
||||
override fun onProgress(message: String, pct: Int) {
|
||||
when {
|
||||
message.startsWith("Finishing") || message.startsWith("Extracting") -> handle.stage(DownloadStage.INSTALL, message)
|
||||
message.startsWith("Downloading") || message.startsWith("Verified") || message.startsWith("Resuming") -> {
|
||||
if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message)
|
||||
}
|
||||
else -> handle.stage(DownloadStage.MANIFEST, message)
|
||||
}
|
||||
else -> handle.stage(DownloadStage.MANIFEST, message)
|
||||
}
|
||||
override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) }
|
||||
override fun onLog(line: String) { handle.log(line) }
|
||||
override fun onSizes(downloadBytes: Long, diskBytes: Long) { handle.diskSize(diskBytes) }
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL })
|
||||
handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install")
|
||||
}
|
||||
}, cancelled)
|
||||
} catch (t: Throwable) {
|
||||
if (t is GogDownloadManager.InstallerException && !cancelled.get()) {
|
||||
StoreInstalls.discard(app, folder, listOf(File(folder, ".gog_chunks")))
|
||||
}
|
||||
override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) }
|
||||
override fun onLog(line: String) { handle.log(line) }
|
||||
override fun onSizes(downloadBytes: Long, diskBytes: Long) { handle.diskSize(diskBytes) }
|
||||
override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) {
|
||||
handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL })
|
||||
handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install")
|
||||
}
|
||||
}, cancelled) ?: return null
|
||||
throw t
|
||||
} ?: return null
|
||||
if (cancelled.get()) return null
|
||||
handle.stage(DownloadStage.INSTALL, app.getString(com.droiddeck.launcher.R.string.stores_registering))
|
||||
val sidecar = StoreGameSidecar(
|
||||
|
||||
@@ -3,6 +3,8 @@ package com.droiddeck.launcher.stores.gog;
|
||||
import android.content.Context;
|
||||
import android.util.Log;
|
||||
|
||||
import com.droiddeck.launcher.frontend.AddedGames;
|
||||
import com.droiddeck.launcher.runtime.GuestCommand;
|
||||
import com.droiddeck.launcher.stores.StoreLog;
|
||||
import com.droiddeck.launcher.stores.download.StoreDownloadTier;
|
||||
|
||||
@@ -20,7 +22,9 @@ import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Callable;
|
||||
@@ -60,10 +64,15 @@ public final class GogDownloadManager {
|
||||
Result(String exeRelative, String buildId, long bytes) { this.exeRelative = exeRelative; this.buildId = buildId; this.bytes = bytes; }
|
||||
}
|
||||
|
||||
public static final class InstallException extends Exception {
|
||||
public static class InstallException extends Exception {
|
||||
InstallException(String message) { super(message); }
|
||||
}
|
||||
|
||||
/** A failure of the offline-installer fallback; its partial files can be discarded. */
|
||||
public static final class InstallerException extends InstallException {
|
||||
InstallerException(String message) { super(message); }
|
||||
}
|
||||
|
||||
private GogDownloadManager() {}
|
||||
|
||||
public static Result install(Context ctx, GogGame game, File installPath, Callback cb, AtomicBoolean cancelled) throws InstallException {
|
||||
@@ -385,41 +394,149 @@ public final class GogDownloadManager {
|
||||
String productJson = httpGet("https://api.gog.com/products/" + game.gameId + "?expand=downloads", token);
|
||||
if (productJson == null) throw new InstallException("No downloadable builds for this game");
|
||||
JSONObject downloads = new JSONObject(productJson).optJSONObject("downloads");
|
||||
JSONArray installers = downloads == null ? null : downloads.optJSONArray("installers");
|
||||
if (installers == null || installers.length() == 0) throw new InstallException("No downloadable builds for this game");
|
||||
String manualUrl = null, fileName = null;
|
||||
for (int i = 0; i < installers.length(); i++) {
|
||||
JSONObject inst = installers.getJSONObject(i);
|
||||
if (!"windows".equals(inst.optString("os"))) continue;
|
||||
JSONArray files = inst.optJSONArray("files");
|
||||
if (files != null && files.length() > 0) {
|
||||
JSONObject f = files.getJSONObject(0);
|
||||
manualUrl = f.optString("downlink", null);
|
||||
if (manualUrl == null) manualUrl = f.optString("manualUrl", null);
|
||||
fileName = f.optString("filename", null);
|
||||
}
|
||||
if (manualUrl == null) manualUrl = inst.optString("manualUrl", null);
|
||||
if (fileName == null || fileName.isEmpty()) fileName = "setup_" + game.gameId + ".exe";
|
||||
break;
|
||||
}
|
||||
if (manualUrl == null) throw new InstallException("No Windows installer for this game");
|
||||
String downloadUrl = resolveRedirect(manualUrl, token);
|
||||
if (downloadUrl == null) throw new InstallException("The installer link could not be resolved");
|
||||
List<InstallerPart> parts = installerParts(downloads, game.gameId);
|
||||
if (parts.isEmpty()) throw new InstallException("No downloadable builds for this game");
|
||||
installPath.mkdirs();
|
||||
File outFile = new File(installPath, fileName);
|
||||
cb.onProgress("Downloading installer: " + fileName, 15);
|
||||
long size = downloadWithProgress(downloadUrl, outFile, cb, cancelled);
|
||||
long planned = 0;
|
||||
boolean sized = true;
|
||||
for (InstallerPart part : parts) { planned += part.size; if (part.size <= 0) sized = false; }
|
||||
if (sized) {
|
||||
long usable = com.droiddeck.launcher.stores.StoreInstallRoot.availableBytes(installPath);
|
||||
if (com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(planned, usable)) {
|
||||
throw new InstallException("Not enough free space: need " + formatBytes(planned) + ", only " + formatBytes(usable) + " free");
|
||||
}
|
||||
}
|
||||
long total = 0;
|
||||
for (InstallerPart part : parts) {
|
||||
if (cancelled.get()) return null;
|
||||
String downloadUrl = resolveRedirect(part.url, token);
|
||||
if (downloadUrl == null) throw new InstallException("The installer link could not be resolved: " + part.filename);
|
||||
File outFile;
|
||||
try {
|
||||
outFile = installerDestination(installPath, part.filename);
|
||||
} catch (IOException e) {
|
||||
throw new InstallException("The installer part is not inside the game folder: " + part.filename);
|
||||
}
|
||||
cb.onProgress("Downloading installer: " + part.filename, 15 + (int) ((total / (float) Math.max(1, planned)) * 80));
|
||||
long size = downloadWithProgress(downloadUrl, outFile, cb, cancelled, total, sized ? planned : 0);
|
||||
if (cancelled.get()) return null;
|
||||
if (size <= 0) throw new InstallException("The installer download failed: " + part.filename);
|
||||
total += size;
|
||||
}
|
||||
if (cancelled.get()) return null;
|
||||
if (size <= 0) throw new InstallException("The installer download failed");
|
||||
cb.onLog("gog: installer downloaded; the game installs itself on first launch");
|
||||
return new Result(fileName, "", size);
|
||||
} catch (InstallException e) {
|
||||
cb.onLog("gog: installer downloaded (" + parts.size() + " file(s)); extracting natively");
|
||||
extractInstaller(ctx, installPath, parts.get(0).filename, cb, cancelled);
|
||||
if (cancelled.get()) return null;
|
||||
for (InstallerPart part : parts) new File(installPath, part.filename).delete();
|
||||
deleteDir(new File(installPath, "tmp"));
|
||||
deleteDir(new File(installPath, "commonappdata"));
|
||||
String exe = pickExe(installPath, null, game.title);
|
||||
if (exe.isEmpty()) throw new InstallException("The installer was extracted but no game executable was found");
|
||||
cb.onLog("gog: installer extracted natively; game exe " + exe);
|
||||
return new Result(exe, "", total);
|
||||
} catch (InstallerException e) {
|
||||
throw e;
|
||||
} catch (InstallException e) {
|
||||
throw new InstallerException(e.getMessage());
|
||||
} catch (Exception e) {
|
||||
throw new InstallException("Installer download error: " + e.getClass().getSimpleName());
|
||||
throw new InstallerException("Installer download error: " + e.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GOG's Windows installer is a 32-bit Inno Setup stub. On ARM64 it runs through
|
||||
* Proton/FEX, where its path strings are corrupted before CreateFileW (observed on
|
||||
* Wine 11 + FEX: "Path not found"). Extracting the same files with innoextract in
|
||||
* the Linux guest avoids that path entirely.
|
||||
*/
|
||||
private static void extractInstaller(Context ctx, File installPath, String setupName, Callback cb, AtomicBoolean cancelled) throws InstallException {
|
||||
String guestDir = AddedGames.INSTANCE.guestPath(ctx, installPath);
|
||||
if (guestDir == null || guestDir.isEmpty()) {
|
||||
throw new InstallException("The game folder is not visible to the Linux session");
|
||||
}
|
||||
String guestSetup = guestDir + "/" + setupName;
|
||||
cb.onProgress("Extracting installer…", 95);
|
||||
int rc;
|
||||
try {
|
||||
rc = GuestCommand.INSTANCE.run(ctx, Arrays.asList(
|
||||
"/usr/local/bin/droiddeck-innoextract", "--gog", "--silent",
|
||||
"-d", guestDir, guestSetup),
|
||||
false, "gog-innoextract", null, null, false, cancelled,
|
||||
line -> { cb.onLog("gog innoextract: " + line); return kotlin.Unit.INSTANCE; });
|
||||
} catch (Exception e) {
|
||||
throw new InstallException("The installer could not be extracted: " + e.getClass().getSimpleName());
|
||||
}
|
||||
if (cancelled.get()) return;
|
||||
if (rc != 0) throw new InstallException("The installer could not be extracted (innoextract exit " + rc + ")");
|
||||
}
|
||||
|
||||
static final class InstallerPart {
|
||||
final String url;
|
||||
final String filename;
|
||||
final long size;
|
||||
InstallerPart(String url, String filename, long size) { this.url = url; this.filename = filename; this.size = size; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Every file of the first usable Windows installer. GOG lists a setup stub and its data parts as
|
||||
* separate files, often without a filename; the parts are named after the stub so the installer
|
||||
* finds them when launched (setup_<id>.exe, setup_<id>-1.bin, ...).
|
||||
*/
|
||||
static List<InstallerPart> installerParts(JSONObject downloads, String gameId) {
|
||||
List<InstallerPart> out = new ArrayList<>();
|
||||
if (downloads == null) return out;
|
||||
JSONArray installers = downloads.optJSONArray("installers");
|
||||
if (installers == null) return out;
|
||||
for (int i = 0; i < installers.length(); i++) {
|
||||
JSONObject inst = installers.optJSONObject(i);
|
||||
if (inst == null || !"windows".equals(inst.optString("os"))) continue;
|
||||
JSONArray files = inst.optJSONArray("files");
|
||||
String base = null;
|
||||
if (files != null) {
|
||||
for (int j = 0; j < files.length(); j++) {
|
||||
JSONObject f = files.optJSONObject(j);
|
||||
if (f == null) continue;
|
||||
String url = f.optString("downlink", "");
|
||||
if (url.isEmpty()) url = f.optString("manualUrl", "");
|
||||
if (url.isEmpty()) continue;
|
||||
String name = partName(f.optString("filename", ""));
|
||||
if (name.isEmpty()) {
|
||||
if (base == null) base = "setup_" + safeName(gameId);
|
||||
name = out.isEmpty() ? base + ".exe" : base + "-" + out.size() + ".bin";
|
||||
} else if (base == null) {
|
||||
base = name.toLowerCase(java.util.Locale.US).endsWith(".exe") ? name.substring(0, name.length() - 4) : name;
|
||||
}
|
||||
out.add(new InstallerPart(url, name, f.optLong("size", 0)));
|
||||
}
|
||||
}
|
||||
if (out.isEmpty()) {
|
||||
String url = inst.optString("manualUrl", "");
|
||||
if (url.isEmpty()) url = inst.optString("downlink", "");
|
||||
if (!url.isEmpty()) out.add(new InstallerPart(url, "setup_" + safeName(gameId) + ".exe", inst.optLong("total_size", 0)));
|
||||
}
|
||||
if (!out.isEmpty()) return out;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static File installerDestination(File installPath, String filename) throws IOException {
|
||||
File root = installPath.getCanonicalFile();
|
||||
File candidate = new File(root, filename);
|
||||
if (Files.isSymbolicLink(candidate.toPath())) throw new IOException("installer part is a link");
|
||||
File out = candidate.getCanonicalFile();
|
||||
if (out.equals(root) || !out.toPath().startsWith(root.toPath())) throw new IOException("installer part leaves the game folder");
|
||||
return out;
|
||||
}
|
||||
|
||||
private static String partName(String raw) {
|
||||
return raw == null ? "" : new File(raw.replace('\\', '/')).getName();
|
||||
}
|
||||
|
||||
private static String safeName(String raw) {
|
||||
String cleaned = raw == null ? "" : raw.replaceAll("[^A-Za-z0-9._-]", "_");
|
||||
return cleaned.isEmpty() ? "game" : cleaned;
|
||||
}
|
||||
|
||||
private static String resolveRedirect(String url, String token) {
|
||||
try {
|
||||
if (url.startsWith("/")) url = "https://www.gog.com" + url;
|
||||
@@ -452,7 +569,7 @@ public final class GogDownloadManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
private static long downloadWithProgress(String url, File out, Callback cb, AtomicBoolean cancelled) {
|
||||
private static long downloadWithProgress(String url, File out, Callback cb, AtomicBoolean cancelled, long doneBefore, long planned) {
|
||||
try {
|
||||
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
|
||||
conn.setConnectTimeout(TIMEOUT); conn.setReadTimeout(60_000);
|
||||
@@ -468,7 +585,11 @@ public final class GogDownloadManager {
|
||||
downloaded += n; windowBytes += n;
|
||||
long elapsed = System.currentTimeMillis() - windowStart;
|
||||
if (elapsed >= 500) { speed = windowBytes * 1000L / elapsed; windowStart = System.currentTimeMillis(); windowBytes = 0; }
|
||||
if (total > 0) {
|
||||
if (planned > 0) {
|
||||
long shown = doneBefore + downloaded;
|
||||
cb.onProgress("Downloading: " + out.getName() + speedSuffix(speed), 15 + (int) ((shown / (float) planned) * 80));
|
||||
cb.onBytes(shown, planned, speed);
|
||||
} else if (total > 0) {
|
||||
cb.onProgress("Downloading: " + out.getName() + speedSuffix(speed), 15 + (int) ((downloaded / (float) total) * 80));
|
||||
cb.onBytes(downloaded, total, speed);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use crate::fetch_tuning::{
|
||||
budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError,
|
||||
FetchFailKind, BOOTSTRAP_WINDOW, MAX_CHUNK_ATTEMPTS, NOMINAL_CHUNK_RESERVE_BYTES,
|
||||
RATE_LIMIT_COOLDOWN_MS, SERVER_EXPLORE_EVERY, USER_AGENT, WINDOW_BPS_EWMA_ALPHA,
|
||||
RATE_LIMIT_COOLDOWN_MS, SERVER_EXPLORE_EVERY, SERVER_FAULT_COOLDOWN_MS, USER_AGENT,
|
||||
WINDOW_BPS_EWMA_ALPHA,
|
||||
WINDOW_COOLDOWN_MS, WINDOW_DECLINE_EPS, WINDOW_ERR_BURST_IMMEDIATE, WINDOW_ERR_RATE_HIGH,
|
||||
WINDOW_ERR_RATE_LOW, WINDOW_HARD_CAP, WINDOW_IMPROVE_EPS, WINDOW_LOG_EVERY_PROBES,
|
||||
WINDOW_MIN_FLOOR, WINDOW_PLATEAU_PATIENCE, WINDOW_PLATEAU_REARM_MAX_MS,
|
||||
@@ -607,6 +608,10 @@ fn classify_reqwest_error(err: &reqwest::Error) -> FetchFailKind {
|
||||
FetchFailKind::Timeout
|
||||
} else if err.is_connect() {
|
||||
FetchFailKind::Connect
|
||||
} else if err.is_body() || err.is_decode() {
|
||||
// A truncated or undecodable body comes from the CDN, not the request. Rest that host so
|
||||
// the retry goes to another mirror instead of reading the same bad object again.
|
||||
FetchFailKind::ServerFault
|
||||
} else {
|
||||
FetchFailKind::Other
|
||||
}
|
||||
@@ -663,14 +668,9 @@ async fn fetch_once(
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(err) => {
|
||||
let kind = if err.is_timeout() {
|
||||
FetchFailKind::Timeout
|
||||
} else {
|
||||
FetchFailKind::Other
|
||||
};
|
||||
return Err(AsyncFetchError {
|
||||
message: format!("http body: {err}"),
|
||||
kind,
|
||||
kind: classify_reqwest_error(&err),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1150,7 +1150,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
Ok(Fetched::Body(body)) => {
|
||||
let raw_len = body.len() as u64;
|
||||
meter.record(done.server_idx, raw_len);
|
||||
sched.on_success(done.server_idx, raw_len, done.elapsed);
|
||||
sched.on_success(done.server_idx, raw_len, done.elapsed, now);
|
||||
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
|
||||
if raw_len >= done.reserve {
|
||||
in_flight.fetch_add(raw_len - done.reserve, Ordering::Relaxed);
|
||||
@@ -1172,7 +1172,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
}
|
||||
Ok(Fetched::Streamed(total_len)) => {
|
||||
meter.record(done.server_idx, total_len);
|
||||
sched.on_success(done.server_idx, total_len, done.elapsed);
|
||||
sched.on_success(done.server_idx, total_len, done.elapsed, now);
|
||||
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
|
||||
outstanding += 1;
|
||||
if txs[done.idx % txs.len()]
|
||||
@@ -1335,6 +1335,7 @@ struct ServerHealth {
|
||||
samples: u32,
|
||||
consecutive_errors: u32,
|
||||
cooldown_until: Option<Instant>,
|
||||
fault_until: Option<Instant>,
|
||||
}
|
||||
|
||||
struct FetchScheduler {
|
||||
@@ -1412,7 +1413,7 @@ impl FetchScheduler {
|
||||
})
|
||||
}
|
||||
|
||||
fn on_success(&mut self, server_idx: usize, bytes: u64, elapsed: Duration) {
|
||||
fn on_success(&mut self, server_idx: usize, bytes: u64, elapsed: Duration, now: Instant) {
|
||||
let secs = elapsed.as_secs_f64().max(0.001);
|
||||
let bps = bytes as f64 / secs;
|
||||
let h = &mut self.health[server_idx];
|
||||
@@ -1423,7 +1424,15 @@ impl FetchScheduler {
|
||||
};
|
||||
h.samples = h.samples.saturating_add(1);
|
||||
h.consecutive_errors = 0;
|
||||
h.cooldown_until = None;
|
||||
// A success can belong to a request that was already in flight when another response from
|
||||
// this host failed. Keep a server-fault cooldown until it expires instead of clearing it
|
||||
// early and sending the retry straight back to the same mirror.
|
||||
if let Some(fault) = h.fault_until.filter(|until| *until > now) {
|
||||
h.cooldown_until = Some(h.cooldown_until.map_or(fault, |current| current.max(fault)));
|
||||
} else {
|
||||
h.fault_until = None;
|
||||
h.cooldown_until = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn rest(&mut self, server_idx: usize, now: Instant, rest: Duration) {
|
||||
@@ -1436,10 +1445,18 @@ impl FetchScheduler {
|
||||
let h = &mut self.health[server_idx];
|
||||
h.consecutive_errors = h.consecutive_errors.saturating_add(1);
|
||||
let mut backoff = retry_backoff_millis(h.consecutive_errors);
|
||||
if kind == FetchFailKind::RateLimited {
|
||||
backoff = backoff.max(RATE_LIMIT_COOLDOWN_MS);
|
||||
match kind {
|
||||
FetchFailKind::RateLimited => backoff = backoff.max(RATE_LIMIT_COOLDOWN_MS),
|
||||
FetchFailKind::ServerFault => backoff = backoff.max(SERVER_FAULT_COOLDOWN_MS),
|
||||
_ => {}
|
||||
}
|
||||
h.cooldown_until = Some(now + Duration::from_millis(backoff));
|
||||
let until = now + Duration::from_millis(backoff);
|
||||
if kind == FetchFailKind::ServerFault {
|
||||
h.fault_until = Some(now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS));
|
||||
}
|
||||
let current = h.cooldown_until.unwrap_or(until);
|
||||
let fault = h.fault_until.unwrap_or(until);
|
||||
h.cooldown_until = Some(current.max(fault).max(until));
|
||||
h.ewma_bps *= 0.5;
|
||||
}
|
||||
}
|
||||
@@ -1806,6 +1823,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::fetch_tuning::{
|
||||
FETCH_INFLIGHT_BUDGET_FLOOR_BYTES, FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES, PER_HOST_CAP,
|
||||
SERVER_FAULT_COOLDOWN_MS,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::io::{Read, Write};
|
||||
@@ -1900,9 +1918,9 @@ mod tests {
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
assert_eq!(s.pick(now), Some(0));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(500));
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(500), now);
|
||||
assert_eq!(s.pick(now), Some(0));
|
||||
s.on_error(0, now, FetchFailKind::Timeout);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
@@ -1911,6 +1929,57 @@ mod tests {
|
||||
assert_eq!(s.pick(now + Duration::from_secs(6)), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_fault_rests_the_host_for_the_other_mirror() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now);
|
||||
s.on_success(1, 1_000_000, Duration::from_millis(100), now);
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
assert_eq!(s.pick(now), Some(1));
|
||||
assert_eq!(
|
||||
s.pick(now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS - 1)),
|
||||
Some(1)
|
||||
);
|
||||
assert!(!s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS - 1)));
|
||||
assert!(s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_success_from_an_earlier_request_does_not_clear_a_server_fault_cooldown() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now + Duration::from_millis(100));
|
||||
assert!(!s.eligible(0, now + Duration::from_millis(100)));
|
||||
assert!(s.eligible(0, now + Duration::from_millis(SERVER_FAULT_COOLDOWN_MS)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_success_keeps_a_longer_cooldown_than_the_server_fault() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
s.rest(0, now, Duration::from_secs(30));
|
||||
s.on_success(0, 1_000_000, Duration::from_millis(100), now + Duration::from_millis(100));
|
||||
assert!(!s.eligible(0, now + Duration::from_secs(29)));
|
||||
assert!(s.eligible(0, now + Duration::from_secs(30)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_fault_keeps_a_longer_host_stall_cooldown() {
|
||||
let hosts = vec!["a".to_string(), "b".to_string()];
|
||||
let mut s = FetchScheduler::new(&hosts, 2);
|
||||
let now = Instant::now();
|
||||
s.rest(0, now, Duration::from_secs(30));
|
||||
s.on_error(0, now, FetchFailKind::ServerFault);
|
||||
assert!(!s.eligible(0, now + Duration::from_secs(29)));
|
||||
assert!(s.eligible(0, now + Duration::from_secs(30)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_shrinks_only_on_error_bursts() {
|
||||
let now = Instant::now();
|
||||
|
||||
@@ -68,6 +68,10 @@ pub const SERVER_EXPLORE_EVERY: u64 = 8;
|
||||
pub const NOMINAL_CHUNK_RESERVE_BYTES: u64 = 1024 * 1024;
|
||||
pub const RATE_LIMIT_COOLDOWN_MS: u64 = 5_000;
|
||||
|
||||
// Long enough that every retry of the item (5 attempts, at most ~4.5 s of backoff) goes to another
|
||||
// mirror after this one served a truncated or undecodable body.
|
||||
pub const SERVER_FAULT_COOLDOWN_MS: u64 = 10_000;
|
||||
|
||||
pub fn retry_backoff_millis(attempt: u32) -> u64 {
|
||||
if attempt == 0 {
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package com.droiddeck.launcher.stores.gog
|
||||
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import org.json.JSONObject
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertThrows
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import org.junit.runner.RunWith
|
||||
import org.robolectric.RobolectricTestRunner
|
||||
import org.robolectric.annotation.Config
|
||||
|
||||
@RunWith(RobolectricTestRunner::class)
|
||||
@Config(sdk = [33])
|
||||
class GogInstallerPartsTest {
|
||||
private fun downloads(json: String) = JSONObject("""{"downloads":$json}""").getJSONObject("downloads")
|
||||
|
||||
@Test fun everyFileOfTheWindowsInstallerIsKeptAndThePartsAreNamedAfterItsSetup() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads(
|
||||
"""{"installers":[
|
||||
{"os":"windows","language":"en","files":[
|
||||
{"id":"en1installer0","size":1048576,"downlink":"https://api.gog.com/products/1947927225/downlink/installer/en1installer0"},
|
||||
{"id":"en1installer1","size":3786407936,"downlink":"https://api.gog.com/products/1947927225/downlink/installer/en1installer1"}
|
||||
]},
|
||||
{"os":"linux","files":[{"downlink":"https://example.invalid/linux"}]}
|
||||
]}""",
|
||||
),
|
||||
"1947927225",
|
||||
)
|
||||
assertEquals(listOf("setup_1947927225.exe", "setup_1947927225-1.bin"), parts.map { it.filename })
|
||||
assertEquals(listOf(1048576L, 3786407936L), parts.map { it.size })
|
||||
assertTrue(parts[1].url.endsWith("en1installer1"))
|
||||
}
|
||||
|
||||
@Test fun explicitNamesAreKeptAndUnnamedPartsFollowTheFirstSetupExe() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads(
|
||||
"""{"installers":[{"os":"windows","files":[
|
||||
{"filename":"setup_witcher_2.1.0.exe","downlink":"https://example.invalid/setup"},
|
||||
{"downlink":"https://example.invalid/data"},
|
||||
{"filename":"readme.bin","downlink":"https://example.invalid/readme"}
|
||||
]}]}""",
|
||||
),
|
||||
"1207658924",
|
||||
)
|
||||
assertEquals(listOf("setup_witcher_2.1.0.exe", "setup_witcher_2.1.0-1.bin", "readme.bin"), parts.map { it.filename })
|
||||
}
|
||||
|
||||
@Test fun aSingleManualInstallerIsNamedForTheGameAndSanitised() {
|
||||
val parts = GogDownloadManager.installerParts(
|
||||
downloads("""{"installers":[{"os":"windows","manualUrl":"https://example.invalid/setup","total_size":42}]}"""),
|
||||
"../game id",
|
||||
)
|
||||
assertEquals(1, parts.size)
|
||||
assertEquals("setup_.._game_id.exe", parts[0].filename)
|
||||
assertEquals(42L, parts[0].size)
|
||||
}
|
||||
|
||||
@Test fun installerPartsStayInsideTheGameFolder() {
|
||||
val root = kotlin.io.path.createTempDirectory("gog-installer").toFile()
|
||||
try {
|
||||
assertEquals(
|
||||
File(root, "setup_1.exe").canonicalFile,
|
||||
GogDownloadManager.installerDestination(root, "setup_1.exe"),
|
||||
)
|
||||
assertThrows(IOException::class.java) {
|
||||
GogDownloadManager.installerDestination(root, "../outside.exe")
|
||||
}
|
||||
val outside = File(root.parentFile, "outside-${System.nanoTime()}.bin").apply { writeText("x") }
|
||||
try {
|
||||
Files.createSymbolicLink(File(root, "setup_1-1.bin").toPath(), outside.toPath())
|
||||
assertThrows(IOException::class.java) {
|
||||
GogDownloadManager.installerDestination(root, "setup_1-1.bin")
|
||||
}
|
||||
} finally {
|
||||
outside.delete()
|
||||
}
|
||||
} finally {
|
||||
root.deleteRecursively()
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun anotherPlatformOrNoDownloadsAddsNothing() {
|
||||
assertEquals(0, GogDownloadManager.installerParts(downloads("""{"installers":[{"os":"linux","manualUrl":"https://example.invalid/x"}]}"""), "1").size)
|
||||
assertEquals(0, GogDownloadManager.installerParts(downloads("{}"), "1").size)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user