diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudSavePaths.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudSavePaths.kt index 62133fbbdc..194448f0c8 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudSavePaths.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudSavePaths.kt @@ -25,22 +25,17 @@ object CloudSavePaths { } fun epic(template: String, prefix: File, install: File, accountId: String, appName: String): File? { - val norm = template.replace('\\', '/').trim().trimStart('/') - val match = Regex("^\\{([^}]+)\\}").find(norm) ?: return null + val parsed = com.droiddeck.launcher.stores.epic.EpicSaveTemplate.parse(template, accountId, appName) ?: return null val profile = profile(prefix) - val (base, boundary) = when (match.groupValues[1].lowercase()) { - "installdir" -> install to install - "userprofile" -> profile to profile - "userdir" -> File(profile, "Documents") to profile - "usersavedgames" -> File(profile, "Saved Games") to profile - "appdata", "localappdata" -> File(profile, "AppData/Local") to profile - "roamingappdata" -> File(profile, "AppData/Roaming") to profile - else -> return null + val (base, boundary) = when (parsed.root) { + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.INSTALL_DIR -> install to install + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.USER_PROFILE -> profile to profile + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.DOCUMENTS -> File(profile, "Documents") to profile + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.SAVED_GAMES -> File(profile, "Saved Games") to profile + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.LOCAL_APPDATA -> File(profile, "AppData/Local") to profile + com.droiddeck.launcher.stores.epic.EpicSaveTemplate.Root.ROAMING_APPDATA -> File(profile, "AppData/Roaming") to profile } - val rest = norm.substring(match.value.length) - .replace("{epicid}", accountId, ignoreCase = true) - .replace("{appname}", appName, ignoreCase = true) - return walk(base, boundary, rest, fuzzy = true) + return walk(base, boundary, parsed.rest, fuzzy = true) } private fun walk(base: File, boundary: File, rest: String, fuzzy: Boolean): File? { diff --git a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt index becf14e21c..33c34196ff 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/CloudTransport.kt @@ -112,62 +112,10 @@ class GogCloud(private val userId: String, private val clientId: String, private } class EpicCloud(private val accountId: String, private val appName: String, private val token: String) : CloudTransport { - private val base = "https://datastorage-public-service-liveegs.live.use1a.on.epicgames.com/api/v1/access/egstore/savesync/$accountId/$appName/" - private val ua = "EpicGamesLauncher/15.17.1-22692490" + override fun list(): List = com.droiddeck.launcher.stores.epic.EpicSaveSync.list(token, accountId, appName) - override fun list(): List { - val c = open(base, "GET", token, ua) - val code = c.responseCode - if (code == 404) { c.disconnect(); return emptyList() } - if (code !in 200..299) { c.disconnect(); throw java.io.IOException("Epic cloud list HTTP $code") } - val root = JSONObject(c.body().decodeToString().ifEmpty { "{}" }) - c.disconnect() - val files = root.optJSONObject("files") ?: return emptyList() - return files.keys().asSequence().mapNotNull { k -> - val e = files.optJSONObject(k) ?: return@mapNotNull null - CloudFile(strip(k), isoDate(e.optString("lastModified", null)), e.optString("hash", "").ifEmpty { null }, e.optString("readLink", "").ifEmpty { null }) - }.toList() - } + override fun get(file: CloudFile): Pair = com.droiddeck.launcher.stores.epic.EpicSaveSync.read(file) - private fun strip(key: String): String { - val marker = "/$appName/" - val i = key.indexOf(marker) - if (i >= 0) return key.substring(i + marker.length) - return key.removePrefix("$appName/") - } - - override fun get(file: CloudFile): Pair { - val link = file.readLink ?: throw java.io.IOException("Epic cloud: no read link") - val c = open(link, "GET", null, ua) - if (c.responseCode !in 200..299) { val code = c.responseCode; c.disconnect(); throw java.io.IOException("Epic cloud get HTTP $code") } - return c.body().also { c.disconnect() } to file.modifiedMs - } - - override fun put(files: Map): Map { - if (files.isEmpty()) return emptyMap() - val c = open(base, "POST", token, ua) - c.doOutput = true - c.setRequestProperty("Content-Type", "application/json") - val req = JSONObject().put("files", JSONArray(files.keys.toList())).toString().toByteArray() - c.outputStream.use { it.write(req) } - if (c.responseCode !in 200..299) { val code = c.responseCode; c.disconnect(); throw java.io.IOException("Epic cloud write links HTTP $code") } - val links = JSONObject(c.body().decodeToString()).optJSONObject("files") ?: JSONObject() - c.disconnect() - val out = HashMap() - for (k in links.keys()) { - val link = links.optJSONObject(k)?.optString("writeLink", "")?.ifEmpty { null } ?: continue - val name = strip(k) - val data = files[name] ?: files[k] ?: continue - val p = open(link, "PUT", null, ua) - p.doOutput = true - p.setRequestProperty("Content-Type", "application/octet-stream") - p.setFixedLengthStreamingMode(data.size) - p.outputStream.use { it.write(data) } - val code = p.responseCode - p.disconnect() - if (code !in 200..299) throw java.io.IOException("Epic cloud put HTTP $code") - out[name] = -1L - } - return out - } + override fun put(files: Map): Map = + com.droiddeck.launcher.stores.epic.EpicSaveSync.write(token, accountId, appName, files) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.java deleted file mode 100644 index a17b6a7ccd..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public final class AmazonApiClient { - - private static final String TAG = "AmazonApi"; - private static final String ENTITLEMENTS_URL = "https://gaming.amazon.com/api/distribution/entitlements"; - private static final String DISTRIBUTION_URL = "https://gaming.amazon.com/api/distribution/v2/public"; - private static final String GAMING_USER_AGENT = "AGSLauncher/1.0.0"; - private static final String LAUNCHER_USER_AGENT = "com.amazon.agslauncher.win/3.0.9202.1"; - private static final String DOWNLOAD_USER_AGENT = "nile/0.1 Amazon"; - private static final String KEY_ID = "d5dc8b8b-86c8-4fc4-ae93-18c0def5314d"; - - private AmazonApiClient() {} - - public static List getEntitlements(String accessToken, String deviceSerial) { - Map seen = new HashMap<>(); - String nextToken = null; - String hardwareHash; - try { - hardwareHash = AmazonPkce.sha256Upper(deviceSerial); - } catch (Exception e) { - Log.e(TAG, "hardware hash failed", e); - return null; - } - boolean answered = false; - do { - try { - JSONObject body = new JSONObject(); - body.put("Operation", "GetEntitlements"); - body.put("clientId", "Sonic"); - body.put("syncPoint", JSONObject.NULL); - body.put("nextToken", nextToken != null ? nextToken : JSONObject.NULL); - body.put("maxResults", 50); - body.put("productIdFilter", JSONObject.NULL); - body.put("keyId", KEY_ID); - body.put("hardwareHash", hardwareHash); - String resp = postGaming(ENTITLEMENTS_URL, "com.amazon.animusdistributionservice.entitlement.AnimusEntitlementsService.GetEntitlements", accessToken, body.toString()); - if (resp == null) break; - JSONObject json = new JSONObject(resp); - answered = true; - JSONArray entitlements = json.optJSONArray("entitlements"); - if (entitlements != null) for (int i = 0; i < entitlements.length(); i++) { - AmazonGame game = parseEntitlement(entitlements.getJSONObject(i)); - if (game != null && !game.productId.isEmpty()) seen.put(game.productId, game); - } - nextToken = json.isNull("nextToken") ? null : json.optString("nextToken", ""); - if (nextToken != null && nextToken.isEmpty()) nextToken = null; - } catch (Exception e) { - Log.e(TAG, "entitlements page failed: " + e.getClass().getSimpleName()); - break; - } - } while (nextToken != null); - if (!answered) return null; - Log.i(TAG, "entitlements: " + seen.size()); - return new ArrayList<>(seen.values()); - } - - static AmazonGame parseEntitlement(JSONObject e) { - try { - JSONObject product = e.optJSONObject("product"); - if (product == null) return null; - AmazonGame game = new AmazonGame(); - game.entitlementId = e.optString("id", ""); - game.productId = product.optString("id", ""); - game.title = product.optString("title", "Unknown"); - game.productSku = product.optString("sku", ""); - String productType = product.optString("productType", ""); - if (productType.isEmpty()) productType = product.optString("type", ""); - String parentId = product.optString("parentProductId", ""); - if (parentId.isEmpty()) parentId = product.optString("baseProductId", ""); - if (parentId.isEmpty()) parentId = product.optString("parentId", ""); - JSONObject detail = product.optJSONObject("productDetail"); - if (detail != null) { - game.artUrl = detail.optString("iconUrl", ""); - JSONObject details = detail.optJSONObject("details"); - if (details != null) { - if (game.artUrl.isEmpty()) game.artUrl = details.optString("pgCrownImageUrl", ""); - game.heroUrl = details.optString("backgroundUrl2", ""); - if (game.heroUrl.isEmpty()) game.heroUrl = details.optString("backgroundUrl1", ""); - if (game.heroUrl.isEmpty()) game.heroUrl = details.optString("pgCrownImageUrl", ""); - game.developer = details.optString("developer", ""); - game.publisher = details.optString("publisher", ""); - if (productType.isEmpty()) productType = details.optString("productType", ""); - if (parentId.isEmpty()) parentId = details.optString("parentProductId", ""); - } - } - boolean typeIsDlc = !productType.isEmpty() && !productType.equalsIgnoreCase("GAME"); - if (typeIsDlc || !parentId.isEmpty()) { game.isDLC = true; game.parentProductId = parentId; } - return game; - } catch (Exception ex) { - Log.e(TAG, "entitlement parse failed: " + ex.getClass().getSimpleName()); - return null; - } - } - - public static final class GameDownloadSpec { - public String downloadUrl; - public String versionId; - } - - public static GameDownloadSpec getGameDownload(String accessToken, String entitlementId) { - try { - JSONObject body = new JSONObject().put("entitlementId", entitlementId).put("Operation", "GetGameDownload"); - String resp = postGaming(DISTRIBUTION_URL, "com.amazon.animusdistributionservice.external.AnimusDistributionService.GetGameDownload", accessToken, body.toString()); - if (resp == null) return null; - JSONObject json = new JSONObject(resp); - GameDownloadSpec spec = new GameDownloadSpec(); - spec.downloadUrl = json.optString("downloadUrl", ""); - spec.versionId = json.optString("versionId", ""); - return spec.downloadUrl.isEmpty() ? null : spec; - } catch (Exception e) { - Log.e(TAG, "GetGameDownload failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static String getLiveVersionId(String accessToken, String productId) { - try { - JSONObject body = new JSONObject().put("adgProductIds", new JSONArray().put(productId)).put("Operation", "GetLiveVersionIds"); - String resp = postGaming(DISTRIBUTION_URL, "com.amazon.animusdistributionservice.external.AnimusDistributionService.GetLiveVersionIds", accessToken, body.toString()); - if (resp == null) return null; - JSONObject json = new JSONObject(resp); - JSONObject versions = json.optJSONObject("adgProductIdToVersionIdMap"); - if (versions == null) versions = json.optJSONObject("versionIds"); - return versions == null ? null : versions.optString(productId, null); - } catch (Exception e) { - Log.e(TAG, "GetLiveVersionIds failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static String appendPath(String baseUrl, String segment) { - int q = baseUrl.indexOf('?'); - if (q >= 0) return baseUrl.substring(0, q) + "/" + segment + baseUrl.substring(q); - return baseUrl + "/" + segment; - } - - static String postGaming(String urlStr, String target, String accessToken, String body) { - try { - HttpURLConnection conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("POST"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(30000); - conn.setDoOutput(true); - conn.setRequestProperty("X-Amz-Target", target); - conn.setRequestProperty("x-amzn-token", accessToken); - conn.setRequestProperty("User-Agent", GAMING_USER_AGENT); - conn.setRequestProperty("UserAgent", LAUNCHER_USER_AGENT); - conn.setRequestProperty("Content-Type", "application/json"); - conn.setRequestProperty("Content-Encoding", "amz-1.0"); - try (OutputStream os = conn.getOutputStream()) { os.write(body.getBytes(StandardCharsets.UTF_8)); } - int code = conn.getResponseCode(); - String resp = AmazonAuthClient.readStream(code < 400 ? conn.getInputStream() : conn.getErrorStream()); - conn.disconnect(); - if (code < 200 || code >= 300) { - Log.e(TAG, "HTTP " + code + " from " + StoreLog.redactUrl(urlStr) + ": " + (resp.length() > 200 ? resp.substring(0, 200) : resp)); - return null; - } - return resp; - } catch (Exception e) { - Log.e(TAG, "POST failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } - } - - public static byte[] getBytes(String urlStr, String accessToken) { - try { - HttpURLConnection conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(120000); - if (accessToken != null) conn.setRequestProperty("x-amzn-token", accessToken); - conn.setRequestProperty("User-Agent", DOWNLOAD_USER_AGENT); - int code = conn.getResponseCode(); - if (code < 200 || code >= 300) { Log.e(TAG, "bytes HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); conn.disconnect(); return null; } - java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); - try (java.io.InputStream in = conn.getInputStream()) { - byte[] buf = new byte[131072]; - int n; - while ((n = in.read(buf)) != -1) out.write(buf, 0, n); - } - conn.disconnect(); - return out.toByteArray(); - } catch (Exception e) { - Log.e(TAG, "bytes failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.kt new file mode 100644 index 0000000000..bc8c68cb53 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonApiClient.kt @@ -0,0 +1,31 @@ +package com.droiddeck.launcher.stores.amazon + +import org.json.JSONObject + +class GameDownloadSpec(@JvmField val downloadUrl: String, @JvmField val versionId: String) + +object AmazonApiClient { + @JvmStatic + fun getGameDownload(accessToken: String, entitlementId: String): GameDownloadSpec? { + val answer = AmazonNative.call( + "download.lookup", + JSONObject().put("accessToken", accessToken).put("entitlementId", entitlementId), + ) ?: return null + if (answer.has("error")) return null + val url = answer.optString("downloadUrl").ifEmpty { return null } + return GameDownloadSpec(url, answer.optString("versionId")) + } + + @JvmStatic + fun getBytes(url: String, accessToken: String?): ByteArray? { + val request = JSONObject().put("url", url) + if (accessToken != null) request.put("accessToken", accessToken) + return AmazonNative.fetch("cdn.get", request) + } + + @JvmStatic + fun appendPath(baseUrl: String, segment: String): String { + val q = baseUrl.indexOf('?') + return if (q < 0) "$baseUrl/$segment" else baseUrl.substring(0, q) + "/" + segment + baseUrl.substring(q) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonAuthClient.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonAuthClient.java deleted file mode 100644 index 39dce7ca36..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonAuthClient.java +++ /dev/null @@ -1,171 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -public final class AmazonAuthClient { - - private static final String TAG = "AmazonAuth"; - static final String REGISTER_URL = "https://api.amazon.com/auth/register"; - static final String REFRESH_URL = "https://api.amazon.com/auth/token"; - static final String DEREGISTER_URL = "https://api.amazon.com/auth/deregister"; - public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0"; - static final String APP_NAME = "AGSLauncher for Windows"; - static final String APP_VERSION = "1.0.0"; - static final String DEVICE_TYPE = "A2UMVHOX7UP4V7"; - static final String OS_VERSION = "10.0.19044.0"; - - public static final class RegisterResult { - public String accessToken; - public String refreshToken; - public long expiresIn; - public String name; - public String deviceSerial; - } - - private AmazonAuthClient() {} - - public static RegisterResult registerDevice(String authCode, String codeVerifier, String deviceSerial, String clientId) { - try { - JSONObject authData = new JSONObject(); - authData.put("authorization_code", authCode); - authData.put("client_domain", "DeviceLegacy"); - authData.put("client_id", clientId); - authData.put("code_algorithm", "SHA-256"); - authData.put("code_verifier", codeVerifier); - authData.put("use_global_authentication", false); - JSONObject regData = new JSONObject(); - regData.put("app_name", APP_NAME); - regData.put("app_version", APP_VERSION); - regData.put("device_model", "Windows"); - regData.put("device_name", JSONObject.NULL); - regData.put("device_serial", deviceSerial); - regData.put("device_type", DEVICE_TYPE); - regData.put("domain", "Device"); - regData.put("os_version", OS_VERSION); - JSONArray tokenTypes = new JSONArray().put("bearer").put("mac_dms"); - JSONArray extensions = new JSONArray().put("customer_info").put("device_info"); - JSONObject body = new JSONObject(); - body.put("auth_data", authData); - body.put("registration_data", regData); - body.put("requested_extensions", extensions); - body.put("requested_token_type", tokenTypes); - body.put("user_context_map", new JSONObject()); - String response = postJson(REGISTER_URL, body.toString(), null, null, null); - if (response == null) return null; - JSONObject success = new JSONObject(response).getJSONObject("response").getJSONObject("success"); - JSONObject bearer = success.getJSONObject("tokens").getJSONObject("bearer"); - RegisterResult result = new RegisterResult(); - result.accessToken = bearer.getString("access_token"); - result.refreshToken = bearer.getString("refresh_token"); - result.expiresIn = bearer.optLong("expires_in", 3600L); - JSONObject ext = success.optJSONObject("extensions"); - JSONObject customer = ext != null ? ext.optJSONObject("customer_info") : null; - result.name = customer != null ? customer.optString("name", "") : ""; - JSONObject device = ext != null ? ext.optJSONObject("device_info") : null; - String registered = device != null ? device.optString("device_serial_number", "") : ""; - result.deviceSerial = registered.isEmpty() ? deviceSerial : registered; - return result; - } catch (Exception e) { - Log.e(TAG, "registerDevice failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static RegisterResult refreshAccessToken(String refreshToken) { - try { - JSONObject body = new JSONObject(); - body.put("source_token", refreshToken); - body.put("source_token_type", "refresh_token"); - body.put("requested_token_type", "access_token"); - body.put("app_name", APP_NAME); - body.put("app_version", APP_VERSION); - String response = postJson(REFRESH_URL, body.toString(), null, "x-amzn-identity-auth-domain", "api.amazon.com"); - if (response == null) return null; - JSONObject json = new JSONObject(response); - RegisterResult result = new RegisterResult(); - result.accessToken = json.getString("access_token"); - result.refreshToken = refreshToken; - result.expiresIn = json.optLong("expires_in", 3600L); - return result; - } catch (Exception e) { - Log.e(TAG, "refresh failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static void deregisterDevice(String accessToken) { - try { - JSONObject body = new JSONObject().put("requested_extensions", new JSONArray().put("device_info").put("customer_info")); - postJson(DEREGISTER_URL, body.toString(), accessToken, null, null); - } catch (Exception e) { - Log.w(TAG, "deregister failed (ignored): " + e.getClass().getSimpleName()); - } - } - - static String postJson(String urlStr, String body, String bearerToken, String extraKey, String extraVal) { - try { - HttpURLConnection conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("POST"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(30000); - conn.setDoOutput(true); - conn.setRequestProperty("User-Agent", USER_AGENT); - conn.setRequestProperty("Content-Type", "application/json"); - conn.setRequestProperty("Accept", "application/json"); - if (bearerToken != null) conn.setRequestProperty("Authorization", "Bearer " + bearerToken); - if (extraKey != null) conn.setRequestProperty(extraKey, extraVal); - try (OutputStream os = conn.getOutputStream()) { os.write(body.getBytes(StandardCharsets.UTF_8)); } - int code = conn.getResponseCode(); - String resp = readStream(code < 400 ? conn.getInputStream() : conn.getErrorStream()); - conn.disconnect(); - if (code < 200 || code >= 300) { Log.e(TAG, "HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); return null; } - return resp; - } catch (Exception e) { - Log.e(TAG, "POST failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } - } - - static String getRequest(String urlStr, String token, String tokenHeader, String extraKey, String extraVal) { - try { - HttpURLConnection conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(120000); - if (token != null && tokenHeader != null) conn.setRequestProperty(tokenHeader, token); - if (extraKey != null) conn.setRequestProperty(extraKey, extraVal); - int code = conn.getResponseCode(); - String resp = readStream(code < 400 ? conn.getInputStream() : conn.getErrorStream()); - conn.disconnect(); - if (code < 200 || code >= 300) { Log.e(TAG, "GET HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); return null; } - return resp; - } catch (Exception e) { - Log.e(TAG, "GET failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } - } - - static String readStream(InputStream is) throws IOException { - if (is == null) return ""; - StringBuilder sb = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) sb.append(line); - } - return sb.toString(); - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonBackend.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonBackend.kt index 10b74d0df3..ec251355f0 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonBackend.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonBackend.kt @@ -2,155 +2,74 @@ package com.droiddeck.launcher.stores.amazon import android.content.Context import android.content.Intent -import android.net.Uri -import android.util.Log -import android.webkit.WebView import com.droiddeck.launcher.stores.CatalogItem import com.droiddeck.launcher.stores.InstalledStoreGame import com.droiddeck.launcher.stores.LoginFlows import com.droiddeck.launcher.stores.Store import com.droiddeck.launcher.stores.StoreBackend -import com.droiddeck.launcher.stores.StoreExe -import com.droiddeck.launcher.stores.StoreGameSidecar import com.droiddeck.launcher.stores.StoreInstallRoot -import com.droiddeck.launcher.stores.StoreInstalls import com.droiddeck.launcher.stores.StoreLoginActivity -import com.droiddeck.launcher.stores.StoreShelves import com.droiddeck.launcher.stores.StoresState import com.droiddeck.launcher.stores.download.DownloadEntry import com.droiddeck.launcher.stores.download.DownloadQueue -import com.droiddeck.launcher.stores.download.DownloadStage -import java.io.File -import java.util.concurrent.atomic.AtomicBoolean +import org.json.JSONObject object AmazonBackend : StoreBackend { - private const val TAG = "AmazonBackend" override val store: Store = Store.AMAZON - init { LoginFlows.amazonFlow = { LoginFlow() } } + init { + LoginFlows.amazonFlow = { AmazonLogin.create() } + } + + private fun engineMissing(): Boolean { + if (AmazonNative.ready()) return false + StoresState.logLine(AmazonNative.ENGINE_MISSING) + StoresState.post { StoresState.problems[store] = AmazonNative.ENGINE_MISSING } + return true + } override fun signIn(context: Context) { - context.startActivity(Intent(context, StoreLoginActivity::class.java).putExtra(StoreLoginActivity.EXTRA_STORE, store.id).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + if (engineMissing()) return + context.startActivity( + Intent(context, StoreLoginActivity::class.java) + .putExtra(StoreLoginActivity.EXTRA_STORE, store.id) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK), + ) } override fun signOut(context: Context) { - AmazonCredentialStore.load(context)?.let { Thread({ AmazonAuthClient.deregisterDevice(it.accessToken) }, "amazon-deregister").start() } - AmazonPrefs.get(context).edit().remove("library_cache").remove("library_synced_at").apply() + val app = context.applicationContext + runCatching { + val token = AmazonCredentialStore.load(app)?.accessToken + if (!token.isNullOrEmpty()) { + Thread({ + runCatching { AmazonNative.call("auth.deregister", JSONObject().put("accessToken", token)) } + }, "amazon-deregister").start() + } + } + runCatching { AmazonLibrary.forget(app) } } override fun syncLibrary(context: Context, force: Boolean) { val app = context.applicationContext - Thread({ - val cached = AmazonLibrary.cached(app) - val quiet = cached.isNotEmpty() && !force - StoresState.post { - if (cached.isNotEmpty()) publish(app, cached) - if (!quiet) StoresState.status[store] = app.getString(com.droiddeck.launcher.R.string.stores_sync_library) - } - val result = AmazonLibrary.sync(app, force) { line -> if (!quiet) StoresState.post { StoresState.status[store] = line } } - StoresState.post { - StoresState.status.remove(store) - when (result) { - is AmazonLibrary.SyncResult.Ok -> { publish(app, result.games); StoresState.problems.remove(store) } - is AmazonLibrary.SyncResult.Failed -> StoresState.problems[store] = result.message - AmazonLibrary.SyncResult.NotLoggedIn -> StoresState.problems[store] = StoresState.notSignedInLine(store) - AmazonLibrary.SyncResult.Throttled -> publish(app, AmazonLibrary.cached(app)) - AmazonLibrary.SyncResult.Busy -> {} - } - } - }, "amazon-sync").start() - } - - private fun publish(app: Context, games: List) { - val items = games.map { AmazonLibrary.toCatalogItem(app, it) } - StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], items) - val installed = StoresState.installed.filter { it.sidecar.store == store }.map { it.sidecar.id }.toSet() - StoresState.shelves[store] = StoreShelves(trending = items.filter { it.id !in installed }, free = emptyList(), deals = emptyList(), whatsNew = emptyList()) + Thread({ AmazonLibrary.sync(app, force) }, "amazon-sync").start() } override fun loadShelves(context: Context, force: Boolean) {} - override fun install(context: Context, item: CatalogItem, root: File) { + override fun install(context: Context, item: CatalogItem, root: java.io.File) { val app = context.applicationContext - val entitlementId = item.extra["entitlementId"] ?: "" - if (entitlementId.isEmpty()) { StoresState.logLine("Amazon: \"${item.title}\" has no entitlement to download with"); return } - val game = AmazonGame().apply { productId = item.id; this.entitlementId = entitlementId; title = item.title; productSku = item.extra["sku"] ?: ""; heroUrl = item.imageUrl ?: ""; artUrl = item.tallImageUrl ?: "" } - val folder = StoreInstallRoot.folderFor(app, Store.AMAZON, game.productId, game.title, root) - DownloadQueue.enqueue(app, DownloadEntry(store, item.id, item.title, cover = item.imageUrl, bytesTotal = item.sizeBytes, location = StoreInstallRoot.labelFor(app, folder))) { InstallJob(app, game, item, folder) } + if (item.extra["entitlementId"].isNullOrEmpty()) { + StoresState.logLine("amazon: \"${item.title}\" has no entitlement to download with") + return + } + val folder = StoreInstallRoot.folderFor(app, Store.AMAZON, item.id, item.title, root) + val entry = DownloadEntry( + Store.AMAZON, item.id, item.title, cover = item.imageUrl, bytesTotal = item.sizeBytes, + location = StoreInstallRoot.labelFor(app, folder), + ) + DownloadQueue.enqueue(app, entry) { AmazonInstaller(app, item, folder) } } override fun uninstall(context: Context, game: InstalledStoreGame) {} - - private class InstallJob(val app: Context, val game: AmazonGame, val item: CatalogItem, val folder: File) : DownloadQueue.DownloadJob { - private val cancelled = AtomicBoolean(false) - - override fun run(handle: DownloadQueue.JobHandle): String? { - handle.stage(DownloadStage.MANIFEST, app.getString(com.droiddeck.launcher.R.string.stores_sync_sign_in)) - val token = AmazonCredentialStore.getValidAccessToken(app) ?: throw AmazonDownloadManager.InstallException("Not signed in to Amazon Games") - StoreInstalls.begin(folder, Store.AMAZON, game.productId, game.title, item.tallImageUrl ?: item.imageUrl, item.imageUrl) - var downloading = false - val result = AmazonDownloadManager.install(app, game, token, folder, cancelled, object : AmazonDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) { - when { - message.startsWith("Finishing") -> handle.stage(DownloadStage.INSTALL, message) - message.startsWith("Downloading") && !message.startsWith("Downloading manifest") -> { if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message) } - else -> handle.stage(DownloadStage.MANIFEST, message) - } - } - override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) } - override fun onLog(line: String) { handle.log(line) } - override fun onSizes(downloadBytes: Long, diskBytes: Long) { handle.diskSize(diskBytes) } - }) ?: return null - if (cancelled.get()) return null - handle.stage(DownloadStage.INSTALL, app.getString(com.droiddeck.launcher.R.string.stores_registering)) - AmazonLibrary.rememberSize(app, game.productId, result.bytes) - AmazonPrefs.get(app).edit().putString("version_${game.productId}", result.versionId).apply() - val spec = AmazonLaunchHelper.buildLaunchSpec(folder, game.title) - val exe = spec.exeRelative.ifEmpty { StoreExe.pick(folder, null, game.title) } - if (exe.isEmpty()) handle.log("amazon: no exe found in ${folder.name}; pick one in Steam settings › Added games") - val env = LinkedHashMap() - for (kv in AmazonLaunchHelper.buildFuelEnv(game.entitlementId, game.productSku, com.droiddeck.launcher.stores.StoreAccounts.signedInAs(app, Store.AMAZON)?.takeIf { it != Store.AMAZON.label })) env[kv.substringBefore('=')] = kv.substringAfter('=') - val sidecar = StoreGameSidecar( - Store.AMAZON, game.productId, game.title, exe = exe.ifEmpty { "game.exe" }, args = spec.args, env = env, - installVersion = result.versionId ?: "", installedAt = System.currentTimeMillis(), - cover = item.tallImageUrl ?: item.imageUrl, hero = item.imageUrl, - extra = mapOf("entitlementId" to game.entitlementId, "sku" to game.productSku), - ) - StoreInstalls.complete(app, folder, onStep = { done, steps -> handle.stageProgress(done.toLong(), steps.toLong()) }, sidecar = sidecar) - return folder.path - } - - override fun cancel(deleteFiles: Boolean) { - cancelled.set(true) - if (deleteFiles) StoreInstalls.discard(app, folder, emptyList()) - } - } - - private class LoginFlow : StoreLoginActivity.Flow { - private val serial = AmazonPkce.generateDeviceSerial() - private val clientId = AmazonPkce.generateClientId(serial) - private val verifier = AmazonPkce.generateCodeVerifier() - override val startUrl: String = "https://www.amazon.com/ap/signin" + - "?openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0" + - "&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select" + - "&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select" + - "&openid.mode=checkid_setup&openid.oa2.scope=device_auth_access" + - "&openid.ns.oa2=http%3A%2F%2Fwww.amazon.com%2Fap%2Fext%2Foauth%2F2&openid.oa2.response_type=code" + - "&openid.oa2.code_challenge_method=S256&openid.oa2.client_id=device%3A$clientId" + - "&language=en_US&marketPlaceId=ATVPDKIKX0DER&openid.return_to=https%3A%2F%2Fwww.amazon.com" + - "&openid.pape.max_auth_age=0&openid.ns.pape=http%3A%2F%2Fspecs.openid.net%2Fextensions%2Fpape%2F1.0" + - "&openid.assoc_handle=amzn_sonic_games_launcher&pageId=amzn_sonic_games_launcher" + - "&openid.oa2.code_challenge=${AmazonPkce.generateCodeChallenge(verifier)}" - override val userAgent: String get() = AmazonAuthClient.USER_AGENT - override fun isRedirect(url: String): Boolean = url.contains("openid.oa2.authorization_code=") - - override fun finish(activity: StoreLoginActivity, view: WebView, url: String): String? { - val code = Uri.parse(url).getQueryParameter("openid.oa2.authorization_code") - if (code.isNullOrEmpty()) { Log.w(TAG, "login: return URL without a code"); return activity.getString(com.droiddeck.launcher.R.string.stores_login_error_verification) } - val result = AmazonAuthClient.registerDevice(code, verifier, serial, clientId) ?: return activity.getString(com.droiddeck.launcher.R.string.stores_login_error_generic) - AmazonCredentialStore.save(activity, result.accessToken, result.refreshToken, result.deviceSerial ?: serial, clientId, System.currentTimeMillis() + result.expiresIn * 1000L, result.name) - Log.i(TAG, "signed in") - return null - } - } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonCredentialStore.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonCredentialStore.kt index 1f05c13f13..f4b00de67f 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonCredentialStore.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonCredentialStore.kt @@ -7,50 +7,98 @@ import com.droiddeck.launcher.stores.StoreAccounts import org.json.JSONObject object AmazonCredentialStore { - private const val TAG = "AmazonAuth" + private const val TAG = "AmazonCredentials" + private const val REFRESH_MARGIN_MS = 5 * 60 * 1000L + private val refreshLock = Any() - class Credentials(val accessToken: String, val refreshToken: String?, val deviceSerial: String, val clientId: String, val expiresAt: Long) + const val ACCESS_TOKEN = "access_token" + const val REFRESH_TOKEN = "refresh_token" + const val DEVICE_SERIAL = "device_serial" + const val CLIENT_ID = "client_id" + const val EXPIRES_AT = "expires_at" + const val DISPLAY_NAME = "display_name" - @JvmStatic - fun save(context: Context, accessToken: String, refreshToken: String?, deviceSerial: String, clientId: String, expiresAt: Long, displayName: String?) { - val json = StoreAccounts.read(context, Store.AMAZON) ?: JSONObject() - json.put("access_token", accessToken).put("device_serial", deviceSerial).put("client_id", clientId).put("expires_at", expiresAt) - if (!refreshToken.isNullOrEmpty()) json.put("refresh_token", refreshToken) - if (!displayName.isNullOrEmpty()) json.put("display_name", displayName) - StoreAccounts.write(context, Store.AMAZON, json) + class Account( + val accessToken: String, + val refreshToken: String, + val deviceSerial: String, + val clientId: String, + val expiresAt: Long, + val displayName: String, + ) + + internal fun fromJson(o: JSONObject): Account? { + val access = o.optString(ACCESS_TOKEN, "").ifEmpty { return null } + return Account( + accessToken = access, + refreshToken = o.optString(REFRESH_TOKEN, ""), + deviceSerial = o.optString(DEVICE_SERIAL, ""), + clientId = o.optString(CLIENT_ID, ""), + expiresAt = o.optLong(EXPIRES_AT, 0L), + displayName = o.optString(DISPLAY_NAME, ""), + ) } @JvmStatic - fun load(context: Context): Credentials? { - val json = StoreAccounts.read(context, Store.AMAZON) ?: return null - val token = json.optString("access_token", "").ifEmpty { return null } - return Credentials(token, json.optString("refresh_token", "").ifEmpty { null }, json.optString("device_serial", ""), json.optString("client_id", ""), json.optLong("expires_at", 0L)) - } + fun load(context: Context): Account? = StoreAccounts.read(context.applicationContext, Store.AMAZON)?.let(::fromJson) - @JvmStatic fun isLoggedIn(context: Context): Boolean = load(context) != null - @JvmStatic - fun refresh(context: Context): String? { - val creds = load(context) ?: return null - val result = AmazonAuthClient.refreshAccessToken(creds.refreshToken ?: return null) ?: return null - save(context, result.accessToken, creds.refreshToken, creds.deviceSerial, creds.clientId, System.currentTimeMillis() + result.expiresIn * 1000L, null) - Log.i(TAG, "token refreshed (forced)") - return result.accessToken + internal fun saveSignIn(context: Context, answer: JSONObject) { + val access = answer.optString(ACCESS_TOKEN, "") + require(access.isNotEmpty()) { "no access token" } + val record = JSONObject() + .put(ACCESS_TOKEN, access) + .put(DEVICE_SERIAL, answer.optString(DEVICE_SERIAL, "")) + .put(CLIENT_ID, answer.optString(CLIENT_ID, "")) + .put(EXPIRES_AT, answer.optLong(EXPIRES_AT, 0L)) + answer.optString(REFRESH_TOKEN, "").takeIf { it.isNotEmpty() }?.let { record.put(REFRESH_TOKEN, it) } + answer.optString(DISPLAY_NAME, "").takeIf { it.isNotEmpty() }?.let { record.put(DISPLAY_NAME, it) } + StoreAccounts.write(context.applicationContext, Store.AMAZON, record) + } + + private fun saveRefreshed(context: Context, before: Account, access: String, expiresAt: Long) { + val app = context.applicationContext + val record = StoreAccounts.read(app, Store.AMAZON) ?: return + record.put(ACCESS_TOKEN, access) + .put(DEVICE_SERIAL, before.deviceSerial) + .put(CLIENT_ID, before.clientId) + .put(EXPIRES_AT, expiresAt) + StoreAccounts.write(app, Store.AMAZON, record) + } + + private fun fresh(a: Account): Boolean = + a.expiresAt - System.currentTimeMillis() > REFRESH_MARGIN_MS || a.refreshToken.isEmpty() + + private fun refreshLocked(context: Context, a: Account): String? { + if (a.refreshToken.isEmpty()) return null + val answer = AmazonNative.call("auth.refresh", JSONObject().put("refreshToken", a.refreshToken)) + val access = answer?.takeIf { !it.has("error") }?.optString(ACCESS_TOKEN, "").orEmpty() + if (access.isEmpty()) { + Log.w(TAG, "token refresh failed: ${answer?.optString("error").orEmpty().ifEmpty { "no answer" }}") + return null + } + val expiresAt = answer!!.optLong(EXPIRES_AT, System.currentTimeMillis() + 3_600_000L) + saveRefreshed(context, a, access, expiresAt) + return access } @JvmStatic fun getValidAccessToken(context: Context): String? { - val creds = load(context) ?: return null - if (creds.expiresAt - System.currentTimeMillis() < 5L * 60L * 1000L && creds.refreshToken != null) { - val result = AmazonAuthClient.refreshAccessToken(creds.refreshToken) - if (result != null) { - save(context, result.accessToken, creds.refreshToken, creds.deviceSerial, creds.clientId, System.currentTimeMillis() + result.expiresIn * 1000L, null) - Log.i(TAG, "token refreshed") - return result.accessToken - } - Log.w(TAG, "token refresh failed; using the stored token") + val current = load(context) ?: return null + if (fresh(current)) return current.accessToken + synchronized(refreshLock) { + val again = load(context) ?: return null + if (fresh(again)) return again.accessToken + return refreshLocked(context, again) ?: again.accessToken + } + } + + fun refreshNow(context: Context, staleToken: String?): String? { + synchronized(refreshLock) { + val current = load(context) ?: return null + if (staleToken != null && current.accessToken != staleToken) return current.accessToken + return refreshLocked(context, current) } - return creds.accessToken } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java deleted file mode 100644 index 6145f0250b..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java +++ /dev/null @@ -1,293 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import android.content.Context; -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; -import com.droiddeck.launcher.stores.download.StoreDownloadTier; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.security.MessageDigest; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -public final class AmazonDownloadManager { - - private static final String TAG = "AmazonDownload"; - private static final int MAX_PARALLEL = 8; - private static final int MAX_RETRIES = 3; - private static final long PROGRESS_INTERVAL = 512L * 1024L; - private static final String DOWNLOAD_USER_AGENT = "nile/0.1 Amazon"; - private static final String IN_PROGRESS_MARKER = ".amazon_download_in_progress"; - private static final String COMPLETE_MARKER = ".amazon_download_complete"; - - public interface Callback { - void onProgress(String message, int pct); - default void onBytes(long done, long total, long speedBps) {} - default void onLog(String line) {} - default void onSizes(long downloadBytes, long diskBytes) {} - } - - public static final class Result { - public final String versionId; - public final long bytes; - Result(String versionId, long bytes) { this.versionId = versionId; this.bytes = bytes; } - } - - public static final class InstallException extends Exception { - InstallException(String message) { super(message); } - } - - private AmazonDownloadManager() {} - - public static Result install(Context ctx, AmazonGame game, String accessToken, File installDir, AtomicBoolean cancel, Callback cb) throws InstallException { - if (game.entitlementId == null || game.entitlementId.isEmpty()) throw new InstallException("This game has no entitlement to download with"); - try { - installDir.mkdirs(); - new File(installDir, IN_PROGRESS_MARKER).createNewFile(); - cb.onProgress("Asking Amazon for the download…", 0); - AmazonApiClient.GameDownloadSpec spec = AmazonApiClient.getGameDownload(accessToken, game.entitlementId); - if (spec == null) throw new InstallException("Amazon gave no download for this game"); - cb.onLog("amazon: version " + spec.versionId); - cb.onProgress("Downloading manifest…", 1); - byte[] manifestBytes = AmazonApiClient.getBytes(AmazonApiClient.appendPath(spec.downloadUrl, "manifest.proto"), accessToken); - if (manifestBytes == null) throw new InstallException("The manifest could not be downloaded"); - AmazonManifest.ParsedManifest manifest = AmazonManifest.parse(manifestBytes); - cb.onLog("amazon: " + manifest.allFiles.size() + " files, " + fmt(manifest.totalInstallSize)); - long remainingBytes = remainingInstallBytes(manifest.allFiles, installDir); - long usable = com.droiddeck.launcher.stores.StoreInstallRoot.availableBytes(installDir); - if (com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(remainingBytes, usable)) - throw new InstallException("Not enough free space: need " + fmt(remainingBytes) + ", only " + fmt(usable) + " free"); - cb.onSizes(manifest.totalInstallSize, manifest.totalInstallSize); - cb.onBytes(0, manifest.totalInstallSize, 0); - if (cancel.get()) return null; - - AtomicLong downloaded = new AtomicLong(0); - AtomicLong lastEmit = new AtomicLong(0); - AtomicLong lastSpeedMs = new AtomicLong(System.currentTimeMillis()); - AtomicLong lastSpeedBytes = new AtomicLong(0); - AtomicLong speedBps = new AtomicLong(0); - List files = manifest.allFiles; - final long total = manifest.totalInstallSize; - - if (AmazonNative.isAvailable()) { - if (!downloadAllNative(ctx, spec.downloadUrl, files, total, installDir, downloaded, lastEmit, lastSpeedMs, lastSpeedBytes, speedBps, cancel, cb)) { - if (cancel.get()) return null; - throw new InstallException("A file failed to download"); - } - } else { - cb.onLog("amazon: engine=built-in (" + MAX_PARALLEL + " threads)"); - ExecutorService pool = Executors.newFixedThreadPool(MAX_PARALLEL, com.droiddeck.launcher.stores.download.DownloadQueue.workerFactory("amazon-dl")); - List> futures = new ArrayList<>(); - for (AmazonManifest.ManifestFile file : files) { - final String dlUrl = spec.downloadUrl; - futures.add(pool.submit(() -> downloadFileWithRetry(file, dlUrl, installDir, downloaded, lastEmit, total, lastSpeedMs, lastSpeedBytes, speedBps, cb, cancel))); - } - pool.shutdown(); - try { - for (Future f : futures) { - if (!f.get()) { - pool.shutdownNow(); - if (cancel.get()) return null; - throw new InstallException("A file failed to download"); - } - } - } catch (InstallException e) { - throw e; - } catch (Exception e) { - pool.shutdownNow(); - throw new InstallException("download pool error: " + e.getClass().getSimpleName()); - } - } - if (cancel.get()) return null; - cb.onProgress("Finishing…", 98); - new File(installDir, IN_PROGRESS_MARKER).delete(); - new File(installDir, COMPLETE_MARKER).createNewFile(); - return new Result(spec.versionId, total); - } catch (InstallException e) { - throw e; - } catch (Exception e) { - Log.w(TAG, "install failed", e); - throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "")); - } - } - - static String buildPlan(List files, String baseUrl) { - JSONArray arr = new JSONArray(); - try { - for (AmazonManifest.ManifestFile file : files) { - String hashHex = file.hashHex(); - arr.put(new JSONObject().put("relPath", file.unixPath()).put("url", AmazonApiClient.appendPath(baseUrl, "files/" + hashHex)) - .put("size", file.size).put("sha256hex", (file.hashAlgorithm == 0 && file.hashBytes.length > 0) ? hashHex : "")); - } - } catch (Exception e) { - Log.e(TAG, "plan failed", e); - return ""; - } - return arr.toString(); - } - - private static boolean downloadAllNative(Context ctx, String baseUrl, List files, long totalSize, File installDir, - AtomicLong totalDownloaded, AtomicLong lastEmit, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps, - AtomicBoolean cancel, Callback cb) { - String plan = buildPlan(files, baseUrl); - if (plan.isEmpty()) return false; - StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); - int maxWorkers = Math.max(1, Math.min(128, tier.getNetworkWindow())); - int processWorkers = Math.max(Math.max(1, Runtime.getRuntime().availableProcessors() / 2), tier.getProcessWorkers()); - cb.onLog("amazon: engine=native workers=" + maxWorkers + " process_workers=" + processWorkers + " tier=" + tier.getId()); - AmazonNative.Listener listener = new AmazonNative.Listener() { - @Override public void onProgress(long bytesDone, long bytesTotal, long filesDone, long filesTotal) { - totalDownloaded.set(bytesDone); - long emit = lastEmit.get(); - if (bytesDone - emit >= PROGRESS_INTERVAL && lastEmit.compareAndSet(emit, bytesDone)) { - sampleSpeed(bytesDone, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading (" + filesDone + "/" + filesTotal + " files)" + speedSuffix(speedBps.get()), (int) (bytesDone * 95L / Math.max(1, totalSize))); - cb.onBytes(bytesDone, totalSize, speedBps.get()); - } - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytesWritten) { - cb.onLog("amazon: complete success=" + success + " bytes=" + bytesWritten + (error.isEmpty() ? "" : " error=" + error)); - } - }; - AmazonNative.RunResult result = AmazonNative.runBlocking(plan, installDir.getAbsolutePath(), "", maxWorkers, processWorkers, cancel::get, listener); - if (result.cancelled) return false; - if (!result.success) { cb.onLog("amazon: engine failed: " + result.error); return false; } - return true; - } - - private static boolean downloadFileWithRetry(AmazonManifest.ManifestFile file, String baseUrl, File installDir, AtomicLong totalDownloaded, AtomicLong lastEmit, long totalSize, - AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps, Callback cb, AtomicBoolean cancel) { - File destFile = new File(installDir, file.unixPath()); - File tmpFile = new File(installDir, file.unixPath() + ".tmp"); - if (fileComplete(file, destFile)) { totalDownloaded.addAndGet(file.size); return true; } - String fileUrl = AmazonApiClient.appendPath(baseUrl, "files/" + file.hashHex()); - for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { - destFile.getParentFile().mkdirs(); - try { - if (!downloadFile(fileUrl, tmpFile, totalDownloaded, lastEmit, totalSize, lastSpeedMs, lastSpeedBytes, speedBps, cb, cancel)) { tmpFile.delete(); return false; } - if (file.hashAlgorithm == 0 && file.hashBytes.length > 0 && !Arrays.equals(sha256(tmpFile), file.hashBytes)) { - cb.onLog("amazon: SHA-256 mismatch for " + file.unixPath()); - tmpFile.delete(); - if (attempt < MAX_RETRIES) { Thread.sleep(1000L << (attempt - 1)); continue; } - return false; - } - if (destFile.exists()) destFile.delete(); - return tmpFile.renameTo(destFile); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - tmpFile.delete(); - return false; - } catch (Exception e) { - cb.onLog("amazon: attempt " + attempt + " failed for " + file.unixPath() + ": " + e.getClass().getSimpleName()); - tmpFile.delete(); - if (attempt < MAX_RETRIES) { try { Thread.sleep(1000L << (attempt - 1)); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return false; } } - } - } - return false; - } - - private static boolean downloadFile(String urlStr, File tmpFile, AtomicLong totalDownloaded, AtomicLong lastEmit, long totalSize, - AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps, Callback cb, AtomicBoolean cancel) throws IOException { - HttpURLConnection conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setConnectTimeout(30000); - conn.setReadTimeout(120000); - conn.setRequestProperty("User-Agent", DOWNLOAD_USER_AGENT); - int code = conn.getResponseCode(); - if (code < 200 || code >= 300) { conn.disconnect(); throw new IOException("HTTP " + code + " " + StoreLog.redactUrl(urlStr)); } - try (InputStream in = conn.getInputStream(); FileOutputStream out = new FileOutputStream(tmpFile)) { - byte[] buf = new byte[65536]; - int n; - while ((n = in.read(buf)) >= 0) { - if (cancel.get()) { conn.disconnect(); return false; } - out.write(buf, 0, n); - long dl = totalDownloaded.addAndGet(n); - long emit = lastEmit.get(); - if (dl - emit >= PROGRESS_INTERVAL && lastEmit.compareAndSet(emit, dl)) { - sampleSpeed(dl, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading: " + tmpFile.getName().replace(".tmp", "") + speedSuffix(speedBps.get()), (int) (dl * 95L / Math.max(1, totalSize))); - cb.onBytes(dl, totalSize, speedBps.get()); - } - } - } finally { - conn.disconnect(); - } - return true; - } - - public static long fetchInstallSizeBytes(String accessToken, String entitlementId) { - try { - AmazonApiClient.GameDownloadSpec spec = AmazonApiClient.getGameDownload(accessToken, entitlementId); - if (spec == null) return -1; - byte[] bytes = AmazonApiClient.getBytes(AmazonApiClient.appendPath(spec.downloadUrl, "manifest.proto"), accessToken); - if (bytes == null) return -1; - AmazonManifest.ParsedManifest m = AmazonManifest.parse(bytes); - return m.totalInstallSize > 0 ? m.totalInstallSize : -1; - } catch (Exception e) { - return -1; - } - } - - public static boolean isInstalled(File installDir) { return new File(installDir, COMPLETE_MARKER).exists(); } - - static long remainingInstallBytes(List files, File installDir) { - long remaining = 0; - for (AmazonManifest.ManifestFile file : files) { - if (!fileComplete(file, new File(installDir, file.unixPath()))) remaining += file.size; - } - return remaining; - } - - private static boolean fileComplete(AmazonManifest.ManifestFile file, File output) { - if (!output.isFile() || output.length() != file.size) return false; - if (file.hashAlgorithm != 0 || file.hashBytes.length == 0) return true; - try { return Arrays.equals(sha256(output), file.hashBytes); } - catch (Exception e) { return false; } - } - - private static byte[] sha256(File file) throws Exception { - MessageDigest digest = MessageDigest.getInstance("SHA-256"); - try (FileInputStream fis = new FileInputStream(file)) { - byte[] buf = new byte[65536]; - int n; - while ((n = fis.read(buf)) >= 0) digest.update(buf, 0, n); - } - return digest.digest(); - } - - private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); - if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { - long prevB = lastSpeedBytes.getAndSet(done); - long dt = nowMs - prevMs; - if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); - } - } - - private static String speedSuffix(long bps) { - if (bps <= 0) return ""; - return " " + (bps >= 1048576 ? String.format("%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); - } - - private static String fmt(long bytes) { - if (bytes >= 1073741824L) return String.format("%.1f GB", bytes / 1073741824.0); - if (bytes >= 1048576L) return String.format("%.1f MB", bytes / 1048576.0); - return String.format("%.0f KB", bytes / 1024.0); - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonGame.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonGame.java deleted file mode 100644 index 3ce769ade0..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonGame.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -public class AmazonGame { - public String productId = ""; - public String entitlementId = ""; - public String title = ""; - public String artUrl = ""; - public String heroUrl = ""; - public String developer = ""; - public String publisher = ""; - public String productSku = ""; - public String versionId = ""; - public boolean isDLC = false; - public String parentProductId = ""; - - public String shortId() { - int dot = productId.lastIndexOf('.'); - return (dot >= 0 && dot < productId.length() - 1) ? productId.substring(dot + 1) : productId; - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonInstaller.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonInstaller.kt new file mode 100644 index 0000000000..7eff5fbbe9 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonInstaller.kt @@ -0,0 +1,136 @@ +package com.droiddeck.launcher.stores.amazon + +import android.content.Context +import com.droiddeck.launcher.R +import com.droiddeck.launcher.stores.CatalogItem +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreAccounts +import com.droiddeck.launcher.stores.StoreExe +import com.droiddeck.launcher.stores.StoreGameSidecar +import com.droiddeck.launcher.stores.StoreInstalls +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadStage +import com.droiddeck.launcher.stores.download.StoreDownloadTier +import org.json.JSONObject +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong + +internal class AmazonInstaller( + private val app: Context, + private val item: CatalogItem, + private val folder: File, +) : DownloadQueue.DownloadJob { + private val cancelled = AtomicBoolean(false) + + override fun run(handle: DownloadQueue.JobHandle): String? { + handle.stage(DownloadStage.MANIFEST, app.getString(R.string.stores_sync_sign_in)) + val token = AmazonCredentialStore.getValidAccessToken(app) ?: throw IllegalStateException("Not signed in to Amazon Games") + val entitlementId = item.extra["entitlementId"].orEmpty() + val sku = item.extra["sku"].orEmpty() + val cover = item.tallImageUrl ?: item.imageUrl + StoreInstalls.begin(folder, Store.AMAZON, item.id, item.title, cover, item.imageUrl) + if (cancelled.get()) return null + + val tier = StoreDownloadTier.current(app) + handle.log("amazon: \"${item.title}\" tier=${tier.id} window=${tier.networkWindow} process_workers=${tier.processWorkers}") + val request = JSONObject() + .put("accessToken", token) + .put("entitlementId", entitlementId) + .put("gameDir", folder.absolutePath) + .put("maxWorkers", tier.networkWindow.coerceIn(1, 128)) + .put("processWorkers", tier.processWorkers) + .put("caBundle", "") + + val shown = AtomicLong(-1) + val downloading = AtomicBoolean(false) + val prefs = AmazonPrefs.get(app) + val listener = object : AmazonNative.InstallListener { + override fun onStatus(text: String) { handle.stage(DownloadStage.MANIFEST, text) } + + override fun onCheck(doneBytes: Long, totalBytes: Long, files: Int, filesTotal: Int) { + handle.stage(DownloadStage.MANIFEST, "Checking files…") + handle.stageProgress(doneBytes, totalBytes, files, filesTotal, bytes = true) + } + + override fun onSizes(pendingBytes: Long, totalBytes: Long) { + handle.diskSize(totalBytes) + } + + override fun onProgress(done: Long, total: Long, files: Int, filesTotal: Int) { + if (downloading.compareAndSet(false, true)) handle.stage(DownloadStage.DOWNLOAD) + while (true) { + val before = shown.get() + if (done < before) return + if (shown.compareAndSet(before, done)) break + } + handle.progress(done, total, "Downloading ($files/$filesTotal files)") + } + + override fun onLog(line: String) { handle.log("amazon: $line") } + + override fun beforeReplace(): Boolean = try { + StoreInstalls.markUpdating(folder) + true + } catch (e: Exception) { + handle.log("amazon: the install record could not be switched to updating (${e.javaClass.simpleName})") + false + } + + override fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) = Unit + } + + val outcome = AmazonNative.install(request, listener, cancelled) + if (outcome.cancelled || cancelled.get() && !outcome.success) return null + if (!outcome.success) throw IllegalStateException(outcome.error.ifEmpty { "The Amazon install failed" }) + val result = outcome.result ?: JSONObject() + + handle.stage(DownloadStage.INSTALL, "Finishing…") + val versionId = result.optString("versionId") + val total = result.optLong("totalBytes", 0L) + prefs.edit().apply { + if (total > 0) putLong("size_${item.id}", total) + putString("version_${item.id}", versionId) + }.apply() + + handle.stage(DownloadStage.INSTALL, app.getString(R.string.stores_registering)) + val spec = AmazonNative.call("launch.spec", JSONObject().put("gameDir", folder.absolutePath).put("title", item.title)) + ?: throw IllegalStateException(AmazonNative.ENGINE_MISSING) + spec.optString("warning").takeIf { it.isNotEmpty() && it != "null" }?.let { handle.log("amazon: $it") } + var exe = spec.optString("exe") + if (exe.isEmpty()) exe = StoreExe.pick(folder, null, item.title) + if (exe.isEmpty()) { + handle.log("amazon: no exe found in ${folder.name}; pick one in Steam settings › Added games") + exe = "game.exe" + } else { + handle.log("amazon: exe=$exe (${spec.optString("source").ifEmpty { "picker" }})") + } + val argList = spec.optJSONArray("args") + val args = List(argList?.length() ?: 0) { argList!!.optString(it) }.filter { it.isNotEmpty() } + + val display = StoreAccounts.signedInAs(app, Store.AMAZON).orEmpty() + val envAnswer = AmazonNative.call( + "launch.env", + JSONObject().put("entitlementId", entitlementId).put("sku", sku).put("displayName", display), + )?.optJSONArray("env") ?: throw IllegalStateException(AmazonNative.ENGINE_MISSING) + val env = LinkedHashMap() + for (i in 0 until envAnswer.length()) { + val pair = envAnswer.optJSONArray(i) ?: continue + env[pair.optString(0)] = pair.optString(1) + } + + val sidecar = StoreGameSidecar( + Store.AMAZON, item.id, item.title, exe = exe, args = args, env = env, + installVersion = versionId, installedAt = System.currentTimeMillis(), + cover = cover, hero = item.imageUrl, + extra = mapOf("entitlementId" to entitlementId, "sku" to sku), + ) + StoreInstalls.complete(app, folder, sidecar, onStep = { done, steps -> handle.stageProgress(done.toLong(), steps.toLong()) }) + return folder.path + } + + override fun cancel(deleteFiles: Boolean) { + cancelled.set(true) + if (deleteFiles) StoreInstalls.discard(app, folder, emptyList()) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLaunchHelper.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLaunchHelper.java deleted file mode 100644 index c95bb74f50..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLaunchHelper.java +++ /dev/null @@ -1,131 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import android.util.Log; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -public final class AmazonLaunchHelper { - - private static final String TAG = "AmazonLaunch"; - - public static final class LaunchSpec { - public String exeRelative = ""; - public List args = new ArrayList<>(); - } - - private AmazonLaunchHelper() {} - - public static LaunchSpec buildLaunchSpec(File installDir, String gameTitle) { - LaunchSpec spec = new LaunchSpec(); - String fuelCommand = null; - File fuelFile = new File(installDir, "fuel.json"); - if (fuelFile.exists()) { - try { - StringBuilder sb = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(fuelFile))) { - String line; - while ((line = br.readLine()) != null) sb.append(line); - } - JSONObject main = new JSONObject(sb.toString()).optJSONObject("Main"); - if (main != null) { - String cmd = main.optString("Command", "").trim(); - if (!cmd.isEmpty()) fuelCommand = cmd; - JSONArray argsArr = main.optJSONArray("Args"); - if (argsArr != null) for (int i = 0; i < argsArr.length(); i++) { - String a = argsArr.optString(i); - if (a != null && !a.isEmpty()) spec.args.add(a); - } - } - } catch (Exception e) { - Log.w(TAG, "fuel.json unreadable; choosing the exe by name"); - } - } - if (fuelCommand != null && new File(installDir, fuelCommand.replace('\\', '/')).isFile()) { - spec.exeRelative = fuelCommand.replace('\\', '/'); - } else { - File chosen = choosePrimaryExe(installDir, gameTitle); - if (chosen != null) spec.exeRelative = relativePath(installDir, chosen); - } - return spec; - } - - public static String[] buildFuelEnv(String entitlementId, String productSku, String displayName) { - return new String[]{ - "FUEL_DIR=C:\\ProgramData\\Amazon Games Services\\Legacy", - "AMAZON_GAMES_SDK_PATH=C:\\ProgramData\\Amazon Games Services\\AmazonGamesSDK", - "AMAZON_GAMES_FUEL_ENTITLEMENT_ID=" + entitlementId, - "AMAZON_GAMES_FUEL_PRODUCT_SKU=" + productSku, - "AMAZON_GAMES_FUEL_DISPLAY_NAME=" + (displayName == null || displayName.isEmpty() ? "Player" : displayName), - }; - } - - private static final Pattern UE_SHIPPING = Pattern.compile(".*-win(32|64)(-shipping)?\\.exe$", Pattern.CASE_INSENSITIVE); - private static final Pattern UE_BINARIES = Pattern.compile(".*/binaries/win(32|64)/.*\\.exe$", Pattern.CASE_INSENSITIVE); - private static final Pattern GENERIC_NAME = Pattern.compile("^[a-z]\\d{1,3}\\.exe$", Pattern.CASE_INSENSITIVE); - private static final String[] NEGATIVE_KEYWORDS = { "crash", "handler", "viewer", "compiler", "tool", "setup", "unins", "eac", "launcher", "steam" }; - - private static boolean isLikelyStub(File f) { - String n = f.getName().toLowerCase(); - if (GENERIC_NAME.matcher(n).matches()) return true; - if (f.length() < 1_000_000L) return true; - for (String kw : NEGATIVE_KEYWORDS) if (n.contains(kw)) return true; - return false; - } - - static int scoreExe(File f, String gameNameLower) { - int score = 50; - String path = f.getAbsolutePath().replace('\\', '/').toLowerCase(); - if (UE_SHIPPING.matcher(path).matches()) score += 300; - if (UE_BINARIES.matcher(path).find()) score += 250; - String fn = f.getName().toLowerCase(); - String cleanGame = gameNameLower.replaceAll("[^a-z]", ""); - String cleanFile = fn.replaceAll("[^a-z]", ""); - boolean nameMatch = path.contains(gameNameLower) || (cleanGame.length() >= 5 && cleanFile.length() >= 5 && cleanGame.substring(0, 5).equals(cleanFile.substring(0, 5))); - if (nameMatch) score += 100; - for (String kw : NEGATIVE_KEYWORDS) if (path.contains(kw)) { score -= 150; break; } - if (GENERIC_NAME.matcher(fn).matches()) score -= 200; - return score; - } - - static File choosePrimaryExe(File installDir, String gameTitle) { - if (!installDir.isDirectory()) return null; - List all = new ArrayList<>(); - collectExe(installDir, all); - if (all.isEmpty()) return null; - List pool = new ArrayList<>(); - for (File f : all) if (!isLikelyStub(f)) pool.add(f); - if (pool.isEmpty()) pool = all; - String lowerTitle = gameTitle.toLowerCase(); - File best = null; - int bestScore = Integer.MIN_VALUE; - for (File f : pool) { - int s = scoreExe(f, lowerTitle); - if (best == null || s > bestScore || (s == bestScore && f.length() > best.length())) { best = f; bestScore = s; } - } - return best; - } - - private static void collectExe(File dir, List out) { - File[] entries = dir.listFiles(); - if (entries == null) return; - for (File f : entries) { - if (f.isDirectory()) collectExe(f, out); - else if (f.getName().toLowerCase().endsWith(".exe")) out.add(f); - } - } - - private static String relativePath(File base, File absolute) { - String b = base.getAbsolutePath(); - if (!b.endsWith("/")) b += "/"; - String a = absolute.getAbsolutePath(); - return a.startsWith(b) ? a.substring(b.length()) : absolute.getName(); - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLibrary.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLibrary.kt index 93249b7d26..cae508c2ef 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLibrary.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLibrary.kt @@ -1,125 +1,208 @@ package com.droiddeck.launcher.stores.amazon import android.content.Context -import android.content.SharedPreferences -import android.util.Log import com.droiddeck.launcher.R import com.droiddeck.launcher.stores.CatalogItem import com.droiddeck.launcher.stores.Store import com.droiddeck.launcher.stores.StoreNet +import com.droiddeck.launcher.stores.StoreShelves import com.droiddeck.launcher.stores.StoresState +import com.droiddeck.launcher.stores.mergeItems import org.json.JSONArray import org.json.JSONObject -import java.util.concurrent.Callable import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean +data class AmazonGame( + val productId: String, + val entitlementId: String, + val title: String, + val artUrl: String = "", + val heroUrl: String = "", + val developer: String = "", + val publisher: String = "", + val productSku: String = "", + val versionId: String = "", +) + object AmazonLibrary { - private const val TAG = "AmazonLibrary" - private const val CACHE_KEY = "library_cache" - private const val LAST_SYNC_KEY = "library_synced_at" - private const val THROTTLE_MS = 6L * 60L * 60L * 1000L - private const val POSTER_BUDGET = 40 + const val KEY_CACHE = "library_cache" + const val KEY_SYNCED_AT = "library_synced_at" + private const val THROTTLE_MS = 6 * 60 * 60 * 1000L + private const val POSTERS_PER_SYNC = 40 + private const val POSTER_THREADS = 4 + private const val STORE_URL = "https://gaming.amazon.com/home" - private val syncing = AtomicBoolean(false) + private val running = AtomicBoolean(false) + private val store = Store.AMAZON - sealed interface SyncResult { - class Ok(val games: List) : SyncResult - class Failed(val message: String) : SyncResult - object NotLoggedIn : SyncResult - object Busy : SyncResult - object Throttled : SyncResult + fun toCacheJson(games: List): String = JSONArray().apply { + for (g in games) put( + JSONObject() + .put("productId", g.productId).put("entitlementId", g.entitlementId).put("title", g.title) + .put("artUrl", g.artUrl).put("heroUrl", g.heroUrl).put("developer", g.developer) + .put("publisher", g.publisher).put("productSku", g.productSku).put("versionId", g.versionId), + ) + }.toString() + + fun fromCacheJson(text: String?): List { + if (text.isNullOrBlank()) return emptyList() + val array = runCatching { JSONArray(text) }.getOrNull() ?: return emptyList() + val games = (0 until array.length()).mapNotNull { array.optJSONObject(it) }.mapNotNull { o -> + AmazonGame( + productId = o.optString("productId", "").ifEmpty { return@mapNotNull null }, + entitlementId = o.optString("entitlementId", ""), + title = o.optString("title", ""), + artUrl = o.optString("artUrl", ""), + heroUrl = o.optString("heroUrl", ""), + developer = o.optString("developer", ""), + publisher = o.optString("publisher", ""), + productSku = o.optString("productSku", ""), + versionId = o.optString("versionId", ""), + ) + } + return tidy(games) } - fun cached(context: Context): List { - val json = AmazonPrefs.get(context).getString(CACHE_KEY, null) ?: return emptyList() - return runCatching { - val arr = JSONArray(json) - val out = ArrayList(arr.length()) - for (i in 0 until arr.length()) { - val j = arr.optJSONObject(i) ?: continue - val g = AmazonGame() - g.productId = j.optString("productId", ""); g.entitlementId = j.optString("entitlementId", ""); g.title = j.optString("title", "") - g.artUrl = j.optString("artUrl", ""); g.heroUrl = j.optString("heroUrl", ""); g.developer = j.optString("developer", "") - g.publisher = j.optString("publisher", ""); g.productSku = j.optString("productSku", ""); g.versionId = j.optString("versionId", "") - if (g.productId.isNotEmpty()) out.add(g) - } - out.distinctBy { it.productId }.sortedBy { it.title.lowercase() } - }.getOrDefault(emptyList()) + private fun tidy(games: List): List = + games.distinctBy { it.productId }.sortedBy { it.title.lowercase() } + + fun cached(context: Context): List = fromCacheJson(AmazonPrefs.get(context).getString(KEY_CACHE, null)) + + fun forget(context: Context) { + AmazonPrefs.get(context).edit().remove(KEY_CACHE).remove(KEY_SYNCED_AT).apply() } - fun poster(context: Context, productId: String): String? = AmazonPrefs.get(context).getString("vcover_$productId", null)?.ifBlank { null } - - fun rememberSize(context: Context, productId: String, bytes: Long) { - if (bytes > 0) AmazonPrefs.get(context).edit().putLong("size_$productId", bytes).apply() + fun toCatalogItem(context: Context, game: AmazonGame): CatalogItem { + val prefs = AmazonPrefs.get(context) + val poster = prefs.getString("vcover_${game.productId}", null)?.takeIf { it.isNotBlank() } + val art = game.artUrl.ifEmpty { null } + val hero = game.heroUrl.ifEmpty { null } + val tags = listOf(game.developer, game.publisher.takeIf { it != game.developer }.orEmpty()) + .filter { it.isNotBlank() } + .joinToString(", ") + return CatalogItem( + store = store, + id = game.productId, + title = game.title.ifBlank { game.productId.substringAfterLast('.') }, + imageUrl = hero ?: art, + tallImageUrl = poster ?: art ?: hero, + tags = tags, + developer = game.developer, + owned = true, + sizeBytes = prefs.getLong("size_${game.productId}", 0L).takeIf { it > 0 } ?: 0L, + storeUrl = STORE_URL, + extra = mapOf("entitlementId" to game.entitlementId, "sku" to game.productSku, "version" to ""), + ) } - fun sync(context: Context, force: Boolean = false, onStatus: (String) -> Unit = {}): SyncResult { - if (!syncing.compareAndSet(false, true)) return SyncResult.Busy - try { - val app = context.applicationContext - val p = AmazonPrefs.get(app) - val cachedList = cached(app) - if (!force && cachedList.isNotEmpty() && System.currentTimeMillis() - p.getLong(LAST_SYNC_KEY, 0L) < THROTTLE_MS) { - backfillPosters(app, p, cachedList, onStatus) - return SyncResult.Throttled - } - onStatus(app.getString(R.string.stores_sync_sign_in)) - val creds = AmazonCredentialStore.load(app) ?: return SyncResult.NotLoggedIn - val token = AmazonCredentialStore.getValidAccessToken(app) ?: return SyncResult.Failed(StoresState.notSignedInLine(Store.AMAZON)) - onStatus(app.getString(R.string.stores_sync_game_list)) - val all = AmazonApiClient.getEntitlements(token, creds.deviceSerial) - ?: AmazonCredentialStore.refresh(app)?.let { AmazonApiClient.getEntitlements(it, creds.deviceSerial) } - if (all == null) return if (cachedList.isEmpty()) SyncResult.Failed(app.getString(R.string.stores_sync_amazon_no_answer)) else SyncResult.Ok(cachedList) - if (all.isEmpty()) return if (cachedList.isEmpty()) SyncResult.Failed(app.getString(R.string.stores_sync_no_games, Store.AMAZON.label)) else SyncResult.Ok(cachedList) - val games = all.filter { !(it.isDLC && it.parentProductId.isNotEmpty()) }.ifEmpty { all } - .sortedWith { a, b -> a.title.compareTo(b.title, ignoreCase = true) } - for (fresh in games) cachedList.firstOrNull { it.productId == fresh.productId }?.let { fresh.versionId = it.versionId } - saveCache(p, games) - p.edit().putLong(LAST_SYNC_KEY, System.currentTimeMillis()).apply() - backfillPosters(app, p, games, onStatus) - Log.i(TAG, "sync: entitlements=${all.size} games=${games.size}") - return SyncResult.Ok(games) - } catch (e: Exception) { - Log.w(TAG, "sync failed: ${e.message}") - return SyncResult.Failed(e.message ?: context.getString(R.string.stores_sync_failed)) - } finally { - syncing.set(false) + private fun publish(app: Context, games: List) { + val items = games.map { toCatalogItem(app, it) } + StoresState.post { + StoresState.library[store] = mergeItems(StoresState.library[store], items) + val installed = StoresState.installed.filter { it.sidecar.store == store }.map { it.sidecar.id }.toSet() + StoresState.shelves[store] = StoreShelves(trending = items.filterNot { it.id in installed }) } } - private fun backfillPosters(app: Context, p: SharedPreferences, games: List, onStatus: (String) -> Unit) { - val missing = games.filter { it.productId.isNotEmpty() && !p.contains("vcover_${it.productId}") }.take(POSTER_BUDGET) - if (missing.isEmpty()) return - onStatus(app.getString(R.string.stores_sync_cover_art)) - val pool = Executors.newFixedThreadPool(4) - val futures = missing.map { g -> pool.submit(Callable { g.productId to StoreNet.sgdbPoster(g.title) }) } - pool.shutdown() - val ed = p.edit() - for (f in futures) { - val (pid, url) = runCatching { f.get() }.getOrNull() ?: continue - ed.putString("vcover_$pid", url) - } - ed.apply() - } - - private fun saveCache(p: SharedPreferences, games: List) { - runCatching { - val arr = JSONArray() - for (g in games) arr.put(JSONObject().put("productId", g.productId).put("entitlementId", g.entitlementId).put("title", g.title).put("artUrl", g.artUrl) - .put("heroUrl", g.heroUrl).put("developer", g.developer).put("publisher", g.publisher).put("productSku", g.productSku).put("versionId", g.versionId)) - p.edit().putString(CACHE_KEY, arr.toString()).apply() - } - } - - fun toCatalogItem(context: Context, g: AmazonGame): CatalogItem = CatalogItem( - store = Store.AMAZON, id = g.productId, title = g.title.ifBlank { g.shortId() }, - imageUrl = g.heroUrl.ifBlank { null } ?: g.artUrl.ifBlank { null }, - tallImageUrl = poster(context, g.productId) ?: g.artUrl.ifBlank { null } ?: g.heroUrl.ifBlank { null }, - tags = listOfNotNull(g.developer.ifBlank { null }, g.publisher.ifBlank { null }.takeIf { it != g.developer }).joinToString(", "), - developer = g.developer, owned = true, - sizeBytes = AmazonPrefs.get(context).getLong("size_${g.productId}", 0L), - storeUrl = "https://gaming.amazon.com/home", - extra = mapOf("entitlementId" to g.entitlementId, "sku" to g.productSku, "version" to (g.versionId ?: "")), + private fun gameFrom(o: JSONObject, before: AmazonGame?): AmazonGame = AmazonGame( + productId = o.optString("productId"), + entitlementId = o.optString("entitlementId"), + title = o.optString("title"), + artUrl = o.optString("art"), + heroUrl = o.optString("hero"), + developer = o.optString("developer"), + publisher = o.optString("publisher"), + productSku = o.optString("sku"), + versionId = before?.versionId.orEmpty(), ) + + private fun backfillPosters(app: Context, games: List) { + val prefs = AmazonPrefs.get(app) + val todo = games.filter { !prefs.contains("vcover_${it.productId}") }.take(POSTERS_PER_SYNC) + if (todo.isEmpty()) return + val pool = Executors.newFixedThreadPool(POSTER_THREADS) + try { + for (g in todo) pool.execute { + val poster = runCatching { StoreNet.sgdbPoster(g.title) }.getOrDefault("") + prefs.edit().putString("vcover_${g.productId}", poster).apply() + } + } finally { + pool.shutdown() + pool.awaitTermination(5, TimeUnit.MINUTES) + } + } + + private fun entitlements(token: String, serial: String): JSONObject? = + AmazonNative.call("library.entitlements", JSONObject().put("accessToken", token).put("deviceSerial", serial)) + + private fun failed(answer: JSONObject?): Boolean = answer == null || answer.has("error") + + fun sync(context: Context, force: Boolean) { + if (!running.compareAndSet(false, true)) return + val app = context.applicationContext + val prefs = AmazonPrefs.get(app) + val cache = cached(app) + val quiet = cache.isNotEmpty() && !force + fun status(id: Int) { if (!quiet) StoresState.post { StoresState.status[store] = app.getString(id) } } + fun problem(text: String?) { + StoresState.post { + StoresState.status.remove(store) + if (text == null) { + StoresState.problems.remove(store) + } else { + StoresState.problems[store] = text + } + } + } + try { + if (cache.isNotEmpty()) publish(app, cache) + if (!AmazonNative.ready()) return problem(AmazonNative.ENGINE_MISSING) + val syncedAt = prefs.getLong(KEY_SYNCED_AT, 0L) + if (!force && cache.isNotEmpty() && System.currentTimeMillis() - syncedAt in 0 until THROTTLE_MS) { + backfillPosters(app, cache) + publish(app, cache) + return problem(null) + } + + status(R.string.stores_sync_sign_in) + val account = AmazonCredentialStore.load(app) ?: return problem(StoresState.notSignedInLine(store)) + val token = AmazonCredentialStore.getValidAccessToken(app) ?: return problem(StoresState.notSignedInLine(store)) + status(R.string.stores_sync_game_list) + var answer = entitlements(token, account.deviceSerial) + if (failed(answer)) { + StoresState.logLine("amazon: library call failed (${answer?.optString("error").orEmpty().ifEmpty { "no answer" }}); refreshing the session") + AmazonCredentialStore.refreshNow(app, token)?.let { answer = entitlements(it, account.deviceSerial) } + } + val result = answer + when { + result != null && result.optBoolean("signIn", false) -> return problem(StoresState.notSignedInLine(store)) + failed(result) -> { + StoresState.logLine("amazon: library sync failed: ${result?.optString("error").orEmpty().ifEmpty { "no answer" }}") + return problem(if (cache.isEmpty()) app.getString(R.string.stores_sync_amazon_no_answer) else null) + } + } + val list = result!!.optJSONArray("games") ?: JSONArray() + if (list.length() == 0) { + return problem(if (cache.isEmpty()) app.getString(R.string.stores_sync_no_games, store.label) else null) + } + val before = cache.associateBy { it.productId } + val games = tidy( + (0 until list.length()).mapNotNull { list.optJSONObject(it) } + .filter { it.optString("productId").isNotEmpty() } + .map { gameFrom(it, before[it.optString("productId")]) }, + ) + prefs.edit().putString(KEY_CACHE, toCacheJson(games)).putLong(KEY_SYNCED_AT, System.currentTimeMillis()).apply() + StoresState.logLine("amazon: library has ${games.size} games (${result.optInt("total", games.size)} entitlements)") + status(R.string.stores_sync_cover_art) + backfillPosters(app, games) + publish(app, games) + problem(null) + } catch (e: Exception) { + problem(e.message ?: app.getString(R.string.stores_sync_failed)) + } finally { + running.set(false) + } + } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLogin.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLogin.kt new file mode 100644 index 0000000000..1cc985df3a --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonLogin.kt @@ -0,0 +1,57 @@ +package com.droiddeck.launcher.stores.amazon + +import android.util.Log +import android.webkit.WebView +import com.droiddeck.launcher.R +import com.droiddeck.launcher.stores.StoreLoginActivity +import org.json.JSONObject + +internal class AmazonLogin private constructor( + override val startUrl: String, + override val userAgent: String, + private val marker: String, + private val serial: String, + private val clientId: String, + private val verifier: String, +) : StoreLoginActivity.Flow { + override fun isRedirect(url: String): Boolean = url.contains(marker) + + override fun finish(activity: StoreLoginActivity, view: WebView, url: String): String? { + val answer = AmazonNative.call( + "auth.register", + JSONObject().put("url", url).put("serial", serial).put("clientId", clientId).put("verifier", verifier), + ) + if (answer != null && answer.optBoolean("noCode", false)) { + Log.w(TAG, "the return URL carried no authorization code") + return activity.getString(R.string.stores_login_error_verification) + } + if (answer == null || answer.has("error")) { + Log.w(TAG, "registration failed: ${answer?.optString("error").orEmpty().ifEmpty { "no answer" }}") + return activity.getString(R.string.stores_login_error_generic) + } + return try { + AmazonCredentialStore.saveSignIn(activity, answer) + null + } catch (e: Exception) { + Log.w(TAG, "saving the sign-in failed: ${e.javaClass.simpleName}") + activity.getString(R.string.stores_login_error_generic) + } + } + + companion object { + private const val TAG = "AmazonLogin" + + fun create(): StoreLoginActivity.Flow? { + val start = AmazonNative.call("auth.start") ?: return null + if (start.has("error")) return null + return AmazonLogin( + startUrl = start.optString("startUrl").ifEmpty { return null }, + userAgent = start.optString("userAgent"), + marker = start.optString("redirectMarker").ifEmpty { return null }, + serial = start.optString("serial"), + clientId = start.optString("clientId"), + verifier = start.optString("verifier"), + ) + } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.java deleted file mode 100644 index f40db9c772..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.java +++ /dev/null @@ -1,151 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import org.tukaani.xz.LZMAInputStream; -import org.tukaani.xz.XZInputStream; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public final class AmazonManifest { - - public static final class ManifestFile { - public final String path; - public final long size; - public final int hashAlgorithm; - public final byte[] hashBytes; - - ManifestFile(String path, long size, int hashAlgorithm, byte[] hashBytes) { - this.path = path; this.size = size; this.hashAlgorithm = hashAlgorithm; this.hashBytes = hashBytes; - } - - public String unixPath() { return path.replace('\\', '/'); } - - public String hashHex() { - StringBuilder sb = new StringBuilder(); - for (byte b : hashBytes) sb.append(String.format("%02x", b & 0xFF)); - return sb.toString(); - } - } - - public static final class ParsedManifest { - public final List allFiles; - public final long totalInstallSize; - - ParsedManifest(List files) { - this.allFiles = files; - long total = 0; - for (ManifestFile f : files) total += f.size; - this.totalInstallSize = total; - } - } - - private AmazonManifest() {} - - public static ParsedManifest parse(byte[] data) throws IOException { - if (data == null || data.length < 4) throw new IOException("Manifest too short"); - int headerSize = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.BIG_ENDIAN).getInt(); - if (headerSize < 0 || 4 + headerSize > data.length) throw new IOException("Invalid header size " + headerSize); - byte[] body = Arrays.copyOfRange(data, 4 + headerSize, data.length); - return parseManifest(decompress(body)); - } - - private static byte[] decompress(byte[] body) throws IOException { - if (body.length < 2) return body; - boolean isXz = (body[0] & 0xFF) == 0xFD && (body[1] & 0xFF) == 0x37; - try (InputStream in = isXz ? new XZInputStream(new ByteArrayInputStream(body), -1) : new LZMAInputStream(new ByteArrayInputStream(body), -1)) { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - byte[] buf = new byte[65536]; - int n; - while ((n = in.read(buf)) >= 0) bos.write(buf, 0, n); - return bos.toByteArray(); - } - } - - private static ParsedManifest parseManifest(byte[] bytes) { - List files = new ArrayList<>(); - ProtoReader r = new ProtoReader(bytes); - while (r.hasMore()) { - int tag = (int) r.readVarint(); - if ((tag >>> 3) == 1 && (tag & 7) == 2) parsePackage(r.readBytes(), files); else r.skip(tag & 7); - } - return new ParsedManifest(files); - } - - private static void parsePackage(byte[] bytes, List out) { - ProtoReader r = new ProtoReader(bytes); - while (r.hasMore()) { - int tag = (int) r.readVarint(); - int field = tag >>> 3, wire = tag & 7; - if (field == 2 && wire == 2) { ManifestFile f = parseFile(r.readBytes()); if (f != null) out.add(f); } - else r.skip(wire); - } - } - - private static ManifestFile parseFile(byte[] bytes) { - String path = ""; - long size = 0L; - int hashAlgo = 0; - byte[] hashValue = new byte[0]; - ProtoReader r = new ProtoReader(bytes); - while (r.hasMore()) { - int tag = (int) r.readVarint(); - int field = tag >>> 3, wire = tag & 7; - if (field == 1 && wire == 2) path = r.readString(); - else if (field == 3 && wire == 0) size = r.readVarint(); - else if (field == 5 && wire == 2) { - ProtoReader hr = new ProtoReader(r.readBytes()); - while (hr.hasMore()) { - int ht = (int) hr.readVarint(); - int hf = ht >>> 3, hw = ht & 7; - if (hf == 1 && hw == 0) hashAlgo = (int) hr.readVarint(); - else if (hf == 2 && hw == 2) hashValue = hr.readBytes(); - else hr.skip(hw); - } - } else r.skip(wire); - } - if (path.isEmpty()) return null; - return new ManifestFile(path, size, hashAlgo, hashValue); - } - - private static final class ProtoReader { - private final byte[] buf; - private int pos; - ProtoReader(byte[] buf) { this.buf = buf; } - boolean hasMore() { return pos < buf.length; } - long readVarint() { - long result = 0; - int shift = 0; - while (pos < buf.length) { - int b = buf[pos++] & 0xFF; - result |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) break; - shift += 7; - } - return result; - } - byte[] readBytes() { - int len = (int) readVarint(); - if (len < 0 || pos + len > buf.length) return new byte[0]; - byte[] out = Arrays.copyOfRange(buf, pos, pos + len); - pos += len; - return out; - } - String readString() { return new String(readBytes(), java.nio.charset.StandardCharsets.UTF_8); } - void skip(int wireType) { - switch (wireType) { - case 0: readVarint(); break; - case 1: pos += 8; break; - case 2: readBytes(); break; - case 5: pos += 4; break; - default: pos = buf.length; break; - } - } - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.kt new file mode 100644 index 0000000000..522c6fd220 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonManifest.kt @@ -0,0 +1,15 @@ +package com.droiddeck.launcher.stores.amazon + +import java.io.IOException + +class ParsedManifest(@JvmField val totalInstallSize: Long, @JvmField val fileCount: Int) + +object AmazonManifest { + @JvmStatic + @Throws(IOException::class) + fun parse(data: ByteArray): ParsedManifest { + val answer = AmazonNative.inspect(data) ?: throw IOException(AmazonNative.ENGINE_MISSING) + if (answer.has("error")) throw IOException(answer.optString("error")) + return ParsedManifest(answer.optLong("totalBytes", 0L), answer.optInt("files", 0)) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonNative.kt index 165ef455c6..cbab62b842 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonNative.kt @@ -1,70 +1,119 @@ package com.droiddeck.launcher.stores.amazon import android.util.Log +import com.droiddeck.launcher.BuildConfig import com.droiddeck.launcher.stores.StoresNative +import org.json.JSONObject import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean object AmazonNative { - const val TAG = "AmazonNative" - private const val CANCEL_POLL_MS = 100L + private const val TAG = "AmazonNative" + private const val CANCEL_POLL_MS = 250L - interface Listener { - fun onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Long, filesTotal: Long) - fun onLog(line: String) - fun onComplete(success: Boolean, error: String, bytesWritten: Long) + const val ENGINE_MISSING = "Amazon Games: the store engine is not available" + + @Volatile private var configured = false + + fun ready(): Boolean { + if (!StoresNative.ensureLoaded()) return false + if (!configured) synchronized(this) { + if (!configured) { + runCatching { nativeConfigure(BuildConfig.VERSION_NAME) } + .onFailure { Log.w(TAG, "configure: ${it.javaClass.simpleName}") } + configured = true + } + } + return true } - fun interface CancelCheck { fun isCancelled(): Boolean } + fun call(op: String, request: JSONObject = JSONObject()): JSONObject? { + if (!ready()) return null + val text = try { + nativeCall(op, request.toString()) + } catch (t: Throwable) { + Log.w(TAG, "$op: ${t.javaClass.simpleName}") + null + } ?: return null + return runCatching { JSONObject(text) }.getOrNull() + } - class RunResult(@JvmField val success: Boolean, @JvmField val cancelled: Boolean, @JvmField val error: String, @JvmField val bytesWritten: Long) + fun fetch(op: String, request: JSONObject): ByteArray? { + if (!ready()) return null + return try { + nativeFetch(op, request.toString()) + } catch (t: Throwable) { + Log.w(TAG, "$op: ${t.javaClass.simpleName}") + null + } + } - @JvmStatic - fun isAvailable(): Boolean = StoresNative.available + fun inspect(manifest: ByteArray, request: JSONObject = JSONObject()): JSONObject? { + if (!ready()) return null + val text = try { + nativeInspect(manifest, request.toString()) + } catch (t: Throwable) { + Log.w(TAG, "inspect: ${t.javaClass.simpleName}") + null + } ?: return null + return runCatching { JSONObject(text) }.getOrNull() + } - @JvmStatic - fun runBlocking(planJson: String, installDir: String, caBundlePath: String, maxWorkers: Int, processWorkers: Int, isCancelled: CancelCheck?, listener: Listener): RunResult { - if (!StoresNative.ensureLoaded()) return RunResult(false, false, "engine not built", 0L) - val latch = CountDownLatch(1) - var outcome: RunResult? = null - val bridge = object : Listener { - override fun onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Long, filesTotal: Long) = listener.onProgress(bytesDone, bytesTotal, filesDone, filesTotal) - override fun onLog(line: String) = listener.onLog(line) - override fun onComplete(success: Boolean, error: String, bytesWritten: Long) { - outcome = RunResult(success, false, error, bytesWritten) - listener.onComplete(success, error, bytesWritten) - latch.countDown() + interface InstallListener { + fun onStatus(text: String) + fun onCheck(doneBytes: Long, totalBytes: Long, files: Int, filesTotal: Int) + fun onSizes(pendingBytes: Long, totalBytes: Long) + fun onProgress(done: Long, total: Long, files: Int, filesTotal: Int) + fun onLog(line: String) + fun beforeReplace(): Boolean + fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) + } + + class InstallOutcome(val success: Boolean, val cancelled: Boolean, val error: String, val result: JSONObject?) + + fun install(request: JSONObject, events: InstallListener, cancelled: AtomicBoolean): InstallOutcome { + if (!ready()) return InstallOutcome(false, false, ENGINE_MISSING, null) + val done = CountDownLatch(1) + val holder = arrayOfNulls(1) + val bridge = object : InstallListener by events { + override fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) { + holder[0] = InstallOutcome(success, cancelled, error, result.takeIf { it.isNotEmpty() }?.let { runCatching { JSONObject(it) }.getOrNull() }) + done.countDown() } } val handle = try { - nativeStart(planJson, installDir, caBundlePath, maxWorkers, processWorkers, bridge) + nativeInstall(request.toString(), bridge) } catch (t: Throwable) { - Log.w(TAG, "nativeStart threw: ${t.javaClass.simpleName}: ${t.message}") - return RunResult(false, false, "nativeStart: ${t.javaClass.simpleName}", 0L) + return InstallOutcome(false, false, "The Amazon install could not start: ${t.javaClass.simpleName}", null) } - if (handle == 0L) return outcome ?: RunResult(false, false, "native start failed", 0L) - var cancelSent = false + if (handle == 0L) return holder[0] ?: InstallOutcome(false, false, "The Amazon install could not start", null) try { - while (!latch.await(CANCEL_POLL_MS, TimeUnit.MILLISECONDS)) { - if (!cancelSent && isCancelled?.isCancelled() == true) { nativeCancel(handle); cancelSent = true } + var forwarded = false + while (true) { + val finished = try { + done.await(CANCEL_POLL_MS, TimeUnit.MILLISECONDS) + } catch (e: InterruptedException) { + cancelled.set(true) + false + } + if (finished) break + if (!forwarded && cancelled.get()) { + nativeCancel(handle) + forwarded = true + } } - } catch (_: InterruptedException) { - Thread.currentThread().interrupt() - if (!cancelSent) { nativeCancel(handle); cancelSent = true } - try { latch.await(30, TimeUnit.SECONDS) } catch (_: InterruptedException) {} } finally { nativeRelease(handle) } - val r = outcome ?: RunResult(false, cancelSent, "no completion", 0L) - return RunResult(r.success && !cancelSent, cancelSent, r.error, r.bytesWritten) + return holder[0] ?: InstallOutcome(false, false, "The Amazon install ended without a result", null) } - @JvmStatic - private external fun nativeStart(planJson: String, installDir: String, caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: Listener): Long - - @JvmStatic - private external fun nativeCancel(handle: Long) - - @JvmStatic - private external fun nativeRelease(handle: Long) + @JvmStatic private external fun nativeConfigure(appVersion: String) + @JvmStatic private external fun nativeCall(op: String, request: String): String? + @JvmStatic private external fun nativeFetch(op: String, request: String): ByteArray? + @JvmStatic private external fun nativeInspect(manifest: ByteArray, request: String): String? + @JvmStatic private external fun nativeInstall(request: String, listener: InstallListener): Long + @JvmStatic private external fun nativeCancel(handle: Long) + @JvmStatic private external fun nativeRelease(handle: Long) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPkce.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPkce.java deleted file mode 100644 index ff7804782c..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPkce.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.droiddeck.launcher.stores.amazon; - -import android.util.Base64; - -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.SecureRandom; -import java.util.UUID; - -public final class AmazonPkce { - private static final String DEVICE_TYPE = "A2UMVHOX7UP4V7"; - - private AmazonPkce() {} - - public static String generateDeviceSerial() { return UUID.randomUUID().toString().replace("-", "").toUpperCase(); } - - public static String generateClientId(String serial) { - byte[] bytes = (serial + "#" + DEVICE_TYPE).getBytes(StandardCharsets.UTF_8); - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) sb.append(String.format("%02x", b & 0xFF)); - return sb.toString(); - } - - public static String generateCodeVerifier() { - byte[] random = new byte[32]; - new SecureRandom().nextBytes(random); - return Base64.encodeToString(random, Base64.URL_SAFE | Base64.NO_PADDING | Base64.NO_WRAP); - } - - public static String generateCodeChallenge(String verifier) { - try { - byte[] hash = MessageDigest.getInstance("SHA-256").digest(verifier.getBytes(StandardCharsets.UTF_8)); - return Base64.encodeToString(hash, Base64.URL_SAFE | Base64.NO_PADDING | Base64.NO_WRAP); - } catch (Exception e) { - throw new RuntimeException("SHA-256 unavailable", e); - } - } - - public static String sha256Upper(String input) { - try { - byte[] hash = MessageDigest.getInstance("SHA-256").digest(input.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(); - for (byte b : hash) sb.append(String.format("%02x", b & 0xFF)); - return sb.toString().toUpperCase(); - } catch (Exception e) { - throw new RuntimeException("SHA-256 unavailable", e); - } - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPrefs.kt b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPrefs.kt index ef291535cb..fd7f3094a1 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPrefs.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonPrefs.kt @@ -4,6 +4,8 @@ import android.content.Context import android.content.SharedPreferences object AmazonPrefs { + const val FILE = "stores.amazon" + @JvmStatic - fun get(context: Context): SharedPreferences = context.applicationContext.getSharedPreferences("stores.amazon", Context.MODE_PRIVATE) + fun get(context: Context): SharedPreferences = context.applicationContext.getSharedPreferences(FILE, Context.MODE_PRIVATE) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAchievements.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAchievements.kt index 7d278e04d3..db85925f18 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAchievements.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAchievements.kt @@ -3,87 +3,31 @@ package com.droiddeck.launcher.stores.epic import android.content.Context import com.droiddeck.launcher.stores.Store import com.droiddeck.launcher.stores.StoreAchievements -import com.droiddeck.launcher.stores.StoreNet import org.json.JSONObject object EpicAchievements : StoreAchievements(Store.EPIC.id, Store.EPIC.shortLabel) { - private const val GRAPHQL = "https://launcher.store.epicgames.com/graphql" - private const val USER_AGENT = "EpicGamesLauncher/14.0.8-22004686+++Portal+Release-Live" - override fun request(context: Context, game: Game): Result { - val namespace = game.extra["namespace"].orEmpty() - if (namespace.isEmpty()) return Result.Failed(Reason.NO_SERVICE) + val namespace = game.extra["namespace"].orEmpty().ifEmpty { return Result.Failed(Reason.NO_SERVICE) } val token = EpicCredentialStore.getValidAccessToken(context) ?: return Result.Failed(Reason.SIGNED_OUT) val account = EpicCredentialStore.load(context)?.accountId.orEmpty() - val defs = query(token, GAME_ACHIEVEMENTS, JSONObject().put("sandboxId", namespace).put("locale", locale())) - ?: return Result.Failed(Reason.UNREACHABLE) - val list = defs.optJSONObject("data")?.optJSONObject("Achievement")?.optJSONObject("productAchievementsRecordBySandbox") - ?.optJSONArray("achievements") ?: return Result.Found(emptyList()) - val unlocked = if (account.isEmpty()) emptyMap() else unlockTimes(token, account, namespace) - val out = ArrayList() - for (i in 0 until list.length()) { - val a = list.optJSONObject(i)?.optJSONObject("achievement") ?: continue - val id = a.optString("name") - if (id.isEmpty()) continue - val at = unlocked[id] - val done = at != null - out += Achievement( - id = id, - name = (if (done) a.optString("unlockedDisplayName") else a.optString("lockedDisplayName")) - .ifEmpty { a.optString("unlockedDisplayName") }.ifEmpty { id }, - description = (if (done) a.optString("unlockedDescription") else a.optString("lockedDescription")) - .ifEmpty { a.optString("unlockedDescription") }, - hidden = a.optBoolean("hidden"), - unlocked = done, - unlockedAt = at ?: 0, - icon = a.optString("unlockedIconLink").ifEmpty { null }, - iconLocked = a.optString("lockedIconLink").ifEmpty { null }, - ) - } - return Result.Found(out) + val answer = EpicNative.call( + "achievements", + JSONObject().put("accessToken", token).put("namespace", namespace).put("accountId", account).put("locale", locale()), + ) ?: return Result.Failed(Reason.UNREACHABLE) + val list = answer.optJSONArray("list") ?: return Result.Failed(Reason.UNREACHABLE) + return Result.Found( + (0 until list.length()).mapNotNull { list.optJSONObject(it) }.map { a -> + Achievement( + id = a.optString("id"), + name = a.optString("name"), + description = a.optString("description"), + hidden = a.optBoolean("hidden", false), + unlocked = a.optBoolean("unlocked", false), + unlockedAt = a.optLong("unlockedAt", 0L), + icon = a.optString("icon").ifEmpty { null }, + iconLocked = a.optString("iconLocked").ifEmpty { null }, + ) + }, + ) } - - private fun unlockTimes(token: String, account: String, namespace: String): Map { - val out = HashMap() - val records = query(token, PLAYER_ACHIEVEMENTS, JSONObject().put("epicAccountId", account).put("sandboxId", namespace)) - ?.optJSONObject("data")?.optJSONObject("PlayerAchievement")?.optJSONObject("playerAchievementGameRecordsBySandbox") - ?.optJSONArray("records") ?: return out - for (r in 0 until records.length()) { - val player = records.optJSONObject(r)?.optJSONArray("playerAchievements") ?: continue - for (i in 0 until player.length()) { - val pa = player.optJSONObject(i)?.optJSONObject("playerAchievement") ?: continue - if (pa.optBoolean("unlocked")) out[pa.optString("achievementName")] = epochSeconds(pa.optString("unlockDate")) - } - } - return out - } - - private fun query(token: String, query: String, variables: JSONObject): JSONObject? = - StoreNet.postJson(GRAPHQL, JSONObject().put("query", query).put("variables", variables).toString(), bearer = token, userAgent = USER_AGENT) - ?.let(::JSONObject) - - private val GAME_ACHIEVEMENTS = """ - query Achievement(${'$'}sandboxId: String!, ${'$'}locale: String!) { - Achievement { - productAchievementsRecordBySandbox(sandboxId: ${'$'}sandboxId, locale: ${'$'}locale) { - achievements { - achievement { - name hidden unlockedDisplayName lockedDisplayName unlockedDescription lockedDescription - unlockedIconLink lockedIconLink XP - } - } - } - } - } - """.trimIndent() - - private val PLAYER_ACHIEVEMENTS = """ - query PlayerAchievement(${'$'}epicAccountId: String!, ${'$'}sandboxId: String!) { - PlayerAchievement { - playerAchievementGameRecordsBySandbox(epicAccountId: ${'$'}epicAccountId, sandboxId: ${'$'}sandboxId) { - records { playerAchievements { playerAchievement { achievementName unlocked unlockDate XP } } } - } - } - } - """.trimIndent() } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.java deleted file mode 100644 index 9ae3216714..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.java +++ /dev/null @@ -1,184 +0,0 @@ -package com.droiddeck.launcher.stores.epic; - -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.List; - -public final class EpicApiClient { - - private static final String TAG = "EpicApi"; - private static final String LIBRARY_URL = "https://library-service.live.use1a.on.epicgames.com/library/api/public/items?includeMetadata=true"; - private static final String CATALOG_BASE = "https://catalog-public-service-prod06.ol.epicgames.com/catalog/api/shared/namespace"; - private static final String MANIFEST_BASE = "https://launcher-public-service-prod06.ol.epicgames.com/launcher/api/public/assets/v2/platform/Windows/namespace"; - private static final String LEGENDARY_UA = "Legendary/0.1.0 (GameNative)"; - - private EpicApiClient() {} - - public static List getLibraryItems(String accessToken) { - List result = new ArrayList<>(); - String cursor = null; - do { - try { - String url = LIBRARY_URL + (cursor != null ? "&cursor=" + cursor : ""); - String resp = EpicAuthClient.getRequest(url, accessToken); - if (resp == null) break; - JSONObject json = new JSONObject(resp); - JSONArray records = json.optJSONArray("records"); - if (records == null) break; - for (int i = 0; i < records.length(); i++) { - JSONObject rec = records.getJSONObject(i); - String appName = rec.optString("appName", ""); - if (appName.isEmpty() || appName.equals("1")) continue; - String namespace = rec.optString("namespace", ""); - if ("ue".equals(namespace) || "89efe5924d3d467c839449ab6ab52e7f".equals(namespace)) continue; - if ("PRIVATE".equals(rec.optString("sandboxType", ""))) continue; - JSONArray platforms = rec.optJSONArray("platform"); - if (platforms != null && platforms.length() > 0) { - boolean windows = false; - for (int j = 0; j < platforms.length(); j++) { - String p = platforms.optString(j, ""); - if ("Windows".equals(p) || "Win32".equals(p)) { windows = true; break; } - } - if (!windows) continue; - } - EpicGame game = new EpicGame(); - game.appName = appName; - game.namespace = namespace; - game.catalogItemId = rec.optString("catalogItemId", ""); - result.add(game); - } - JSONObject meta = json.optJSONObject("responseMetadata"); - String next = meta != null ? meta.optString("nextCursor", null) : null; - cursor = (next == null || next.isEmpty() || next.equals(cursor)) ? null : next; - } catch (Exception e) { - Log.e(TAG, "library page failed: " + e.getClass().getSimpleName()); - break; - } - } while (cursor != null); - return result; - } - - public static boolean enrichFromCatalog(String accessToken, EpicGame game) { - if (game.namespace.isEmpty() || game.catalogItemId.isEmpty()) return false; - try { - String url = CATALOG_BASE + "/" + game.namespace + "/bulk/items?id=" + game.catalogItemId + "&includeDLCDetails=true&includeMainGameDetails=true&country=US"; - String resp = getWithLegendaryUA(url, accessToken); - if (resp == null) return false; - JSONObject root = new JSONObject(resp); - JSONObject item = root.optJSONObject(game.catalogItemId); - if (item == null && root.length() > 0) item = root.optJSONObject(root.keys().next()); - if (item == null) return false; - game.title = item.optString("title", game.title.isEmpty() ? game.appName : game.title); - game.developer = item.optString("developer", ""); - game.description = item.optString("description", ""); - game.isDLC = item.has("mainGameItem"); - if (game.isDLC) { - JSONObject main = item.optJSONObject("mainGameItem"); - if (main != null) game.baseGameCatalogItemId = main.optString("id", ""); - } - JSONArray keyImages = item.optJSONArray("keyImages"); - if (keyImages != null) for (int i = 0; i < keyImages.length(); i++) { - JSONObject img = keyImages.getJSONObject(i); - String type = img.optString("type", ""), imgUrl = img.optString("url", ""); - if (imgUrl.isEmpty()) continue; - if ("DieselGameBoxTall".equals(type)) game.artCover = imgUrl; - else if (("DieselGameBox".equals(type) || "Thumbnail".equals(type)) && game.artSquare.isEmpty()) game.artSquare = imgUrl; - } - JSONObject attrs = item.optJSONObject("customAttributes"); - if (attrs != null) { - JSONObject offline = attrs.optJSONObject("CanRunOffline"); - if (offline != null) game.canRunOffline = !"false".equalsIgnoreCase(offline.optString("value", "true")); - JSONObject publisher = attrs.optJSONObject("publisherName"); - if (publisher != null) game.publisher = publisher.optString("value", "").trim(); - } - JSONObject seller = item.optJSONObject("seller"); - if (game.publisher.isEmpty() && seller != null) game.publisher = seller.optString("name", "").trim(); - String rd = item.optString("viewableDate", ""); - if (rd.isEmpty()) rd = item.optString("effectiveDate", ""); - if (!rd.isEmpty()) game.releaseDate = rd; - return true; - } catch (Exception e) { - Log.e(TAG, "catalog enrich failed for " + game.appName + ": " + e.getClass().getSimpleName()); - return false; - } - } - - public static String getManifestApiJson(String accessToken, String namespace, String catalogItemId, String appName) { - try { - String url = MANIFEST_BASE + "/" + namespace + "/catalogItem/" + catalogItemId + "/app/" + appName + "/label/Live"; - String resp = getWithLegendaryUA(url, accessToken); - if (resp == null) return null; - JSONObject root = new JSONObject(resp); - JSONArray elements = root.optJSONArray("elements"); - if (elements == null || elements.length() == 0) return null; - JSONObject first = elements.getJSONObject(0); - JSONObject wrapper = new JSONObject(); - wrapper.put("manifests", first.optJSONArray("manifests")); - String ver = first.optString("buildVersion", ""); - if (ver.isEmpty()) ver = first.optString("versionId", ""); - wrapper.put("versionId", ver); - JSONObject sidecar = first.optJSONObject("sidecar"); - if (sidecar != null) { - String config = sidecar.optString("config", ""); - if (!config.isEmpty()) { - try { wrapper.put("deploymentId", new JSONObject(config).optString("deploymentId", "")); } catch (Exception ignored) {} - } - } - return wrapper.toString(); - } catch (Exception e) { - Log.e(TAG, "manifest API failed for " + appName + ": " + e.getClass().getSimpleName()); - return null; - } - } - - public static String getAdditionalCommandLine(String accessToken, String namespace, String catalogItemId) { - return getCustomAttribute(accessToken, namespace, catalogItemId, "AdditionalCommandLine"); - } - - public static String getCustomAttribute(String accessToken, String namespace, String catalogItemId, String name) { - try { - String url = CATALOG_BASE + "/" + namespace + "/bulk/items?id=" + catalogItemId + "&country=US"; - String resp = getWithLegendaryUA(url, accessToken); - if (resp == null) return null; - JSONObject root = new JSONObject(resp); - JSONObject item = root.optJSONObject(catalogItemId); - if (item == null && root.length() > 0) item = root.optJSONObject(root.keys().next()); - if (item == null) return ""; - JSONObject attrs = item.optJSONObject("customAttributes"); - if (attrs == null) return ""; - JSONObject attr = attrs.optJSONObject(name); - return attr == null ? "" : attr.optString("value", ""); - } catch (Exception e) { - Log.w(TAG, "catalog attribute " + name + ": " + e.getClass().getSimpleName()); - return null; - } - } - - static String getWithLegendaryUA(String urlStr, String accessToken) { - java.net.HttpURLConnection conn = null; - try { - conn = (java.net.HttpURLConnection) new java.net.URL(urlStr).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(30000); - conn.setRequestProperty("Authorization", "Bearer " + accessToken); - conn.setRequestProperty("User-Agent", LEGENDARY_UA); - int code = conn.getResponseCode(); - java.io.InputStream is = (code < 400) ? conn.getInputStream() : conn.getErrorStream(); - String body = EpicAuthClient.readStream(is); - if (code < 200 || code >= 300) { Log.e(TAG, "GET HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); return null; } - return body; - } catch (Exception e) { - Log.e(TAG, "GET failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } finally { - if (conn != null) conn.disconnect(); - } - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.kt new file mode 100644 index 0000000000..2f0df5eb2d --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicApiClient.kt @@ -0,0 +1,26 @@ +package com.droiddeck.launcher.stores.epic + +import org.json.JSONObject + +object EpicApiClient { + private fun request(token: String, namespace: String, catalogItemId: String) = + JSONObject().put("accessToken", token).put("namespace", namespace).put("catalogItemId", catalogItemId) + + @JvmStatic + fun getManifestApiJson(token: String, namespace: String, catalogItemId: String, appName: String): String? = + EpicNative.call("build.info", request(token, namespace, catalogItemId).put("appName", appName)) + ?.optJSONObject("info")?.toString() + + @JvmStatic + fun getCustomAttribute(token: String, namespace: String, catalogItemId: String, name: String): String? { + val answer = EpicNative.call("catalog.attribute", request(token, namespace, catalogItemId).put("name", name)) ?: return null + if (answer.optBoolean("unknown", false) || !answer.has("value")) return null + return answer.optString("value", "") + } + + internal fun catalogDetails(token: String, namespace: String, catalogItemId: String): JSONObject? = + EpicNative.call("catalog.details", request(token, namespace, catalogItemId))?.optJSONObject("details") + + internal fun exchangeCode(token: String): String? = + EpicNative.call("auth.exchange", JSONObject().put("accessToken", token))?.optString("code", "")?.ifEmpty { null } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAuthClient.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAuthClient.java deleted file mode 100644 index 2303d06d1f..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicAuthClient.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.droiddeck.launcher.stores.epic; - -import android.util.Base64; -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; - -import org.json.JSONObject; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; - -public final class EpicAuthClient { - - private static final String TAG = "EpicAuth"; - - static final String CLIENT_ID = "34a02cf8f4414e29b15921876da36f9a"; - static final String CLIENT_SECRET = "daafbccc737745039dffe53d94fc76cf"; - private static final String TOKEN_URL = "https://account-public-service-prod03.ol.epicgames.com/account/api/oauth/token"; - private static final String EXCHANGE_URL = "https://account-public-service-prod03.ol.epicgames.com/account/api/oauth/exchange"; - public static final String USER_AGENT = "UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; - - public static final class TokenResult { - public String accessToken; - public String refreshToken; - public String accountId; - public String displayName; - public long expiresAt; - } - - private EpicAuthClient() {} - - public static TokenResult exchangeCode(String authCode) { - return postToken("grant_type=authorization_code&code=" + authCode + "&token_type=eg1"); - } - - public static TokenResult refreshToken(String refreshToken) { - return postToken("grant_type=refresh_token&refresh_token=" + refreshToken + "&token_type=eg1"); - } - - private static TokenResult postToken(String formBody) { - try { - String creds64 = Base64.encodeToString((CLIENT_ID + ":" + CLIENT_SECRET).getBytes(StandardCharsets.UTF_8), Base64.NO_WRAP); - HttpURLConnection conn = (HttpURLConnection) new URL(TOKEN_URL).openConnection(); - conn.setRequestMethod("POST"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(30000); - conn.setDoOutput(true); - conn.setRequestProperty("Authorization", "Basic " + creds64); - conn.setRequestProperty("User-Agent", USER_AGENT); - conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); - byte[] bodyBytes = formBody.getBytes(StandardCharsets.UTF_8); - conn.setRequestProperty("Content-Length", String.valueOf(bodyBytes.length)); - try (OutputStream os = conn.getOutputStream()) { os.write(bodyBytes); } - int code = conn.getResponseCode(); - InputStream is = (code < 400) ? conn.getInputStream() : conn.getErrorStream(); - String resp = readStream(is); - conn.disconnect(); - if (code < 200 || code >= 300) { Log.e(TAG, "token HTTP " + code); return null; } - JSONObject json = new JSONObject(resp); - TokenResult result = new TokenResult(); - result.accessToken = json.optString("access_token", null); - result.refreshToken = json.optString("refresh_token", null); - result.accountId = json.optString("account_id", ""); - result.displayName = json.optString("displayName", ""); - long expiresIn = json.optLong("expires_in", 7200L); - String expiresAtStr = json.optString("expires_at", null); - long expiresAtMs; - if (expiresAtStr != null && !expiresAtStr.isEmpty()) { - try { expiresAtMs = parseIso8601(expiresAtStr); } catch (Exception e) { expiresAtMs = System.currentTimeMillis() + expiresIn * 1000L; } - } else expiresAtMs = System.currentTimeMillis() + expiresIn * 1000L; - result.expiresAt = expiresAtMs; - return result.accessToken != null ? result : null; - } catch (Exception e) { - Log.e(TAG, "token request failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static String getExchangeCode(String accessToken) { - try { - String resp = getRequest(EXCHANGE_URL, accessToken); - if (resp == null) { Log.i(TAG, "exchange code: no answer"); return null; } - return new JSONObject(resp).optString("code", null); - } catch (Exception e) { - Log.e(TAG, "exchange code failed: " + e.getClass().getSimpleName()); - return null; - } - } - - public static String getRequest(String urlStr, String accessToken) { - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(30000); - conn.setRequestProperty("Authorization", "Bearer " + accessToken); - conn.setRequestProperty("User-Agent", USER_AGENT); - int code = conn.getResponseCode(); - InputStream is = (code < 400) ? conn.getInputStream() : conn.getErrorStream(); - String resp = readStream(is); - if (code < 200 || code >= 300) { Log.e(TAG, "GET HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); return null; } - return resp; - } catch (Exception e) { - Log.e(TAG, "GET failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } finally { - if (conn != null) conn.disconnect(); - } - } - - public static byte[] getBytes(String urlStr, String accessToken) { - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setRequestMethod("GET"); - conn.setConnectTimeout(30000); - conn.setReadTimeout(120000); - if (accessToken != null) conn.setRequestProperty("Authorization", "Bearer " + accessToken); - conn.setRequestProperty("User-Agent", USER_AGENT); - int code = conn.getResponseCode(); - if (code < 200 || code >= 300) { Log.e(TAG, "bytes HTTP " + code + " from " + StoreLog.redactUrl(urlStr)); return null; } - return readAllBytes(conn.getInputStream()); - } catch (Exception e) { - Log.e(TAG, "bytes failed: " + StoreLog.redactUrl(urlStr) + " (" + e.getClass().getSimpleName() + ")"); - return null; - } finally { - if (conn != null) conn.disconnect(); - } - } - - static byte[] readAllBytes(InputStream in) throws IOException { - java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream(); - byte[] buf = new byte[131072]; - int n; - while ((n = in.read(buf)) != -1) out.write(buf, 0, n); - return out.toByteArray(); - } - - static String readStream(InputStream is) throws IOException { - if (is == null) return ""; - StringBuilder sb = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) sb.append(line); - } - return sb.toString(); - } - - private static long parseIso8601(String s) { - int year = Integer.parseInt(s.substring(0, 4)); - int month = Integer.parseInt(s.substring(5, 7)); - int day = Integer.parseInt(s.substring(8, 10)); - int hour = Integer.parseInt(s.substring(11, 13)); - int minute = Integer.parseInt(s.substring(14, 16)); - int second = Integer.parseInt(s.substring(17, 19)); - long jd = 367L * year - (7 * (year + (month + 9) / 12)) / 4 + (275 * month) / 9 + day + 1721013L; - long epochDays = jd - 2440588L; - return epochDays * 86400000L + hour * 3600000L + minute * 60000L + second * 1000L; - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt index 5603ba7424..f5e4ba0ac6 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt @@ -2,40 +2,37 @@ package com.droiddeck.launcher.stores.epic import android.content.Context import android.content.Intent -import android.util.Log -import android.webkit.WebView +import com.droiddeck.launcher.R import com.droiddeck.launcher.stores.CatalogItem import com.droiddeck.launcher.stores.EpicLaunchSupport import com.droiddeck.launcher.stores.InstalledStoreGame import com.droiddeck.launcher.stores.LoginFlows import com.droiddeck.launcher.stores.Store import com.droiddeck.launcher.stores.StoreBackend -import com.droiddeck.launcher.stores.StoreExe -import com.droiddeck.launcher.stores.StoreGameSidecar import com.droiddeck.launcher.stores.StoreInstallRoot -import com.droiddeck.launcher.stores.StoreInstalls import com.droiddeck.launcher.stores.StoreLoginActivity -import com.droiddeck.launcher.stores.StoreShelves import com.droiddeck.launcher.stores.StoresState import com.droiddeck.launcher.stores.download.DownloadEntry import com.droiddeck.launcher.stores.download.DownloadQueue -import com.droiddeck.launcher.stores.download.DownloadStage -import org.json.JSONObject -import java.io.File -import java.util.concurrent.atomic.AtomicBoolean +import com.droiddeck.launcher.stores.mergeItems object EpicBackend : StoreBackend, EpicLaunchSupport { - private const val TAG = "EpicBackend" override val store: Store = Store.EPIC - init { LoginFlows.epicFlow = { LoginFlow } } + init { + LoginFlows.epicFlow = { EpicLogin.create() } + } override fun signIn(context: Context) { - context.startActivity(Intent(context, StoreLoginActivity::class.java).putExtra(StoreLoginActivity.EXTRA_STORE, store.id).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + context.startActivity( + Intent(context, StoreLoginActivity::class.java) + .putExtra(StoreLoginActivity.EXTRA_STORE, store.id) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK), + ) } override fun signOut(context: Context) { - EpicPrefs.get(context).edit().remove("library_cache").remove("library_synced_at").apply() + runCatching { EpicLibrary.forget(context) } } override fun syncLibrary(context: Context, force: Boolean) { @@ -44,19 +41,20 @@ object EpicBackend : StoreBackend, EpicLaunchSupport { val cached = EpicLibrary.cached(app).map { EpicLibrary.toCatalogItem(app, it) } val quiet = cached.isNotEmpty() && !force StoresState.post { - if (cached.isNotEmpty()) StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], cached) - if (!quiet) StoresState.status[store] = app.getString(com.droiddeck.launcher.R.string.stores_sync_library) + if (cached.isNotEmpty()) StoresState.library[store] = mergeItems(StoresState.library[store], cached) + if (!quiet) StoresState.status[store] = app.getString(R.string.stores_sync_library) } val result = EpicLibrary.sync(app, force) { line -> if (!quiet) StoresState.post { StoresState.status[store] = line } } StoresState.post { StoresState.status.remove(store) when (result) { is EpicLibrary.SyncResult.Ok -> { - StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], result.games.map { EpicLibrary.toCatalogItem(app, it) }); StoresState.problems.remove(store) + StoresState.library[store] = mergeItems(StoresState.library[store], result.games.map { EpicLibrary.toCatalogItem(app, it) }) + StoresState.problems.remove(store) } is EpicLibrary.SyncResult.Failed -> StoresState.problems[store] = result.message EpicLibrary.SyncResult.NotLoggedIn -> StoresState.problems[store] = StoresState.notSignedInLine(store) - EpicLibrary.SyncResult.Throttled -> if (!StoresState.library.containsKey(store)) StoresState.library[store] = com.droiddeck.launcher.stores.mergeItems(StoresState.library[store], cached) + EpicLibrary.SyncResult.Throttled -> if (!StoresState.library.containsKey(store)) StoresState.library[store] = cached EpicLibrary.SyncResult.Busy -> {} } } @@ -65,126 +63,33 @@ object EpicBackend : StoreBackend, EpicLaunchSupport { override fun loadShelves(context: Context, force: Boolean) { val app = context.applicationContext - EpicStoreCatalog.cachedFeatured(app)?.let { StoresState.post { StoresState.shelves[store] = markOwned(it) } } + EpicShelves.cached(app)?.let { s -> StoresState.post { StoresState.shelves[store] = EpicShelves.markOwned(s, StoresState.library[store]) } } Thread({ - val shelves = EpicStoreCatalog.featured(app, force) - StoresState.post { if (shelves != null) StoresState.shelves[store] = markOwned(shelves) } + val shelves = runCatching { EpicShelves.featured(app, force) }.getOrNull() + if (shelves != null) StoresState.post { StoresState.shelves[store] = EpicShelves.markOwned(shelves, StoresState.library[store]) } }, "epic-shelves").start() } - private fun markOwned(s: StoreShelves): StoreShelves { - val owned = StoresState.library[store] ?: return s - val byCatalogItem = owned.associateBy { it.extra["catalogItemId"] ?: "" } - fun mark(l: List) = l.map { i -> - val match = i.extra["items"]?.split(',')?.firstNotNullOfOrNull { byCatalogItem[it] } - if (match != null) match.copy( - imageUrl = i.imageUrl ?: match.imageUrl, tallImageUrl = i.tallImageUrl ?: match.tallImageUrl, - developer = match.developer.ifBlank { i.developer }, publisher = match.publisher.ifBlank { i.publisher }, - releaseDate = match.releaseDate.ifBlank { i.releaseDate }, - ) else i - } - return StoreShelves(mark(s.whatsNew), mark(s.deals), mark(s.free), mark(s.trending)) - } - - override fun install(context: Context, item: CatalogItem, root: File) { + override fun install(context: Context, item: CatalogItem, root: java.io.File) { val app = context.applicationContext - val namespace = item.extra["namespace"] ?: "" - val catalogItemId = item.extra["catalogItemId"] ?: "" - if (namespace.isEmpty() || catalogItemId.isEmpty()) { StoresState.logLine("Epic: \"${item.title}\" is not in your library"); return } + if (item.extra["namespace"].isNullOrEmpty() || item.extra["catalogItemId"].isNullOrEmpty()) { + StoresState.logLine("Epic: \"${item.title}\" is not in your library") + return + } val folder = StoreInstallRoot.folderFor(app, Store.EPIC, item.id, item.title, root) - val entry = DownloadEntry(store, item.id, item.title, cover = item.imageUrl, bytesTotal = item.sizeBytes, location = StoreInstallRoot.labelFor(app, folder)) - DownloadQueue.enqueue(app, entry) { InstallJob(app, item, namespace, catalogItemId, folder) } + val entry = DownloadEntry( + Store.EPIC, item.id, item.title, cover = item.imageUrl, bytesTotal = item.sizeBytes, + location = StoreInstallRoot.labelFor(app, folder), + ) + DownloadQueue.enqueue(app, entry) { EpicInstaller(app, item, folder) } } override fun uninstall(context: Context, game: InstalledStoreGame) {} override fun exchangeCode(context: Context): String? { val token = EpicCredentialStore.getValidAccessToken(context) ?: return null - return EpicAuthClient.getExchangeCode(token) + return EpicApiClient.exchangeCode(token) } override fun signInExpired(context: Context): Boolean = EpicCredentialStore.expired(context) - - private class InstallJob(val app: Context, val item: CatalogItem, val namespace: String, val catalogItemId: String, val folder: File) : DownloadQueue.DownloadJob { - private val cancelled = AtomicBoolean(false) - private val scratch: File? = if (StoreInstallRoot.isRemovable(app, folder) && !hasChunks(File(folder, ".chunks"))) StoreInstallRoot.scratchDir(app, Store.EPIC, item.id) else null - - private fun hasChunks(dir: File): Boolean = dir.isDirectory && (dir.list()?.isNotEmpty() == true) - - override fun run(handle: DownloadQueue.JobHandle): String? { - handle.stage(DownloadStage.MANIFEST, app.getString(com.droiddeck.launcher.R.string.stores_sync_sign_in)) - val token = EpicCredentialStore.getValidAccessToken(app) ?: throw EpicDownloadManager.InstallException("Not signed in to Epic Games") - StoreInstalls.begin(folder, Store.EPIC, item.id, item.title, item.tallImageUrl ?: item.imageUrl, item.imageUrl) - if (scratch == null && hasChunks(File(folder, ".chunks"))) handle.log("epic: resuming \"${item.title}\" with the chunks already beside it") - val manifestJson = EpicApiClient.getManifestApiJson(token, namespace, catalogItemId, item.id) - ?: throw EpicDownloadManager.InstallException("The manifest could not be fetched") - val tags = EpicInstallTags.tagsForDevice() - var downloading = false - val result = EpicDownloadManager.install(app, manifestJson, folder.path, scratch?.path ?: "", tags, cancelled, object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) { - when { - message.startsWith("Writing") || message.startsWith("Complete") -> handle.stage(DownloadStage.INSTALL, message) - message.startsWith("Downloading chunks") -> { if (!downloading) { downloading = true; handle.stage(DownloadStage.DOWNLOAD, message) } else handle.progress(-1, -1, message) } - message.startsWith("Checking") -> handle.stage(DownloadStage.MANIFEST, message) - else -> handle.stage(DownloadStage.MANIFEST, message) - } - } - override fun onBytes(done: Long, total: Long, speedBps: Long) { handle.progress(done, total, null, speedBps) } - override fun onLog(line: String) { handle.log(line) } - override fun onSizes(downloadBytes: Long, diskBytes: Long) { - handle.diskSize(diskBytes) - EpicLibrary.rememberSize(app, item.id, diskBytes) - } - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - // Assembly overlaps chunk fetching; only the final install event advances the UI. - if (stage == "stream") return - handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL }) - handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install") - } - }) ?: return null - if (cancelled.get()) return null - handle.stage(DownloadStage.INSTALL, app.getString(com.droiddeck.launcher.R.string.stores_registering)) - EpicLibrary.rememberSize(app, item.id, result.bytes) - val deploymentId = runCatching { JSONObject(manifestJson).optString("deploymentId", "") }.getOrDefault("") - val exe = StoreExe.pick(folder, result.launchExe, item.title) - if (exe.isEmpty()) handle.log("epic: no exe found in ${folder.name}; pick one in Steam settings › Added games") - val args = EpicLaunchData.arguments(app, token, item.id, namespace, catalogItemId, deploymentId) - val sidecar = StoreGameSidecar( - Store.EPIC, item.id, item.title, exe = exe.ifEmpty { "game.exe" }, args = args, - installVersion = result.buildVersion, installedAt = System.currentTimeMillis(), - cover = item.tallImageUrl ?: item.imageUrl, hero = item.imageUrl, - extra = EpicLaunchData.extras(item.id, namespace, catalogItemId, deploymentId) + mapOf( - com.droiddeck.launcher.frontend.WinCompSources.EPIC_PREREQ_PATH to result.prereqPath, - com.droiddeck.launcher.frontend.WinCompSources.EPIC_PREREQ_NAME to result.prereqName, - ) + (if (result.prereqRead) mapOf(com.droiddeck.launcher.frontend.WinCompSources.LIST_READ to "1") else emptyMap()), - ) - StoreInstalls.complete(app, folder, sidecar) - return folder.path - } - - override fun cancel(deleteFiles: Boolean) { - cancelled.set(true) - if (deleteFiles) StoreInstalls.discard(app, folder, listOfNotNull(File(folder, ".chunks"), scratch, StoreInstallRoot.scratchDir(app, Store.EPIC, item.id))) - } - } - - private object LoginFlow : StoreLoginActivity.Flow { - private const val REDIRECT = "https://www.epicgames.com/id/api/redirect" - override val startUrl: String = "https://www.epicgames.com/id/login?redirectUrl=https%3A%2F%2Fwww.epicgames.com%2Fid%2Fapi%2Fredirect%3FclientId%3D${EpicAuthClient.CLIENT_ID}%26responseType%3Dcode" - override val userAgent: String get() = EpicAuthClient.USER_AGENT - override val readsPage: Boolean get() = true - override fun isRedirect(url: String): Boolean = url.startsWith(REDIRECT) - - override fun finish(activity: StoreLoginActivity, view: WebView, url: String): String? { - var text = url - if (text.startsWith("\"")) text = text.substring(1, text.length - 1) - text = text.replace("\\n", "").replace("\\r", "").replace("\\\"", "\"") - val code = Regex("\"authorizationCode\":\"([^\"]+)\"").find(text)?.groupValues?.get(1) - if (code.isNullOrEmpty()) { Log.w(TAG, "login: no authorization code on the redirect page"); return activity.getString(com.droiddeck.launcher.R.string.stores_login_error_verification) } - val result = EpicAuthClient.exchangeCode(code) ?: return activity.getString(com.droiddeck.launcher.R.string.stores_login_error_generic) - EpicCredentialStore.save(activity, result) - Log.i(TAG, "signed in") - return null - } - } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicCredentialStore.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicCredentialStore.kt index 58237eed96..38423bc6f8 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicCredentialStore.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicCredentialStore.kt @@ -7,48 +7,99 @@ import com.droiddeck.launcher.stores.StoreAccounts import org.json.JSONObject object EpicCredentialStore { - private const val TAG = "EpicAuth" + private const val TAG = "EpicCredentials" + private const val REFRESH_MARGIN_MS = 5 * 60 * 1000L + private val refreshLock = Any() - class Credentials(val accessToken: String, val refreshToken: String?, val accountId: String, val displayName: String, val expiresAt: Long) + data class Credentials( + val accessToken: String, + val refreshToken: String?, + val accountId: String, + val displayName: String, + val expiresAt: Long, + ) - @JvmStatic - fun save(context: Context, result: EpicAuthClient.TokenResult) { - val json = JSONObject().put("access_token", result.accessToken).put("account_id", result.accountId ?: "") - .put("display_name", result.displayName ?: "").put("expires_at", result.expiresAt) - if (!result.refreshToken.isNullOrEmpty()) json.put("refresh_token", result.refreshToken) - StoreAccounts.write(context, Store.EPIC, json) + class TokenResult( + val accessToken: String, + val refreshToken: String?, + val accountId: String?, + val displayName: String?, + val expiresAt: Long, + ) { + companion object { + fun from(json: JSONObject): TokenResult? { + if (json.has("error")) return null + val access = json.optString("access_token", "").ifEmpty { return null } + return TokenResult( + access, + json.optString("refresh_token", "").ifEmpty { null }, + json.optString("account_id", "").ifEmpty { null }, + json.optString("display_name", "").ifEmpty { null }, + json.optLong("expires_at", 0L), + ) + } + } + } + + internal fun fromJson(o: JSONObject): Credentials? { + val access = o.optString("access_token", "").ifEmpty { return null } + return Credentials( + accessToken = access, + refreshToken = o.optString("refresh_token", "").ifEmpty { null }, + accountId = o.optString("account_id", ""), + displayName = o.optString("display_name", ""), + expiresAt = o.optLong("expires_at", 0L), + ) + } + + internal fun toJson(c: Credentials): JSONObject = JSONObject().apply { + put("access_token", c.accessToken) + c.refreshToken?.takeIf { it.isNotEmpty() }?.let { put("refresh_token", it) } + put("account_id", c.accountId) + put("display_name", c.displayName) + put("expires_at", c.expiresAt) } @JvmStatic - fun load(context: Context): Credentials? { - val json = StoreAccounts.read(context, Store.EPIC) ?: return null - val token = json.optString("access_token", "").ifEmpty { return null } - return Credentials(token, json.optString("refresh_token", "").ifEmpty { null }, json.optString("account_id", ""), json.optString("display_name", ""), json.optLong("expires_at", 0L)) + fun load(context: Context): Credentials? = StoreAccounts.read(context.applicationContext, Store.EPIC)?.let(::fromJson) + + fun save(context: Context, result: TokenResult, merge: Boolean = true) { + val before = if (merge) load(context) else null + val next = Credentials( + accessToken = result.accessToken, + refreshToken = result.refreshToken ?: before?.refreshToken, + accountId = result.accountId ?: before?.accountId.orEmpty(), + displayName = result.displayName ?: before?.displayName.orEmpty(), + expiresAt = result.expiresAt, + ) + StoreAccounts.write(context.applicationContext, Store.EPIC, toJson(next)) } - @JvmStatic fun isLoggedIn(context: Context): Boolean = load(context) != null - @JvmStatic fun expired(context: Context): Boolean { - val creds = load(context) ?: return false - return creds.expiresAt in 1 until System.currentTimeMillis() + val c = load(context) ?: return false + return c.expiresAt > 0 && c.expiresAt < System.currentTimeMillis() } + private fun fresh(c: Credentials): Boolean = + c.expiresAt - System.currentTimeMillis() > REFRESH_MARGIN_MS || c.refreshToken.isNullOrEmpty() + + @JvmStatic fun getValidAccessToken(context: Context): String? { - val creds = load(context) ?: return null - if (creds.expiresAt - System.currentTimeMillis() < 5L * 60L * 1000L && creds.refreshToken != null) { - val result = EpicAuthClient.refreshToken(creds.refreshToken) - if (result != null) { - if (result.refreshToken.isNullOrEmpty()) result.refreshToken = creds.refreshToken - if (result.accountId.isNullOrEmpty()) result.accountId = creds.accountId - if (result.displayName.isNullOrEmpty()) result.displayName = creds.displayName - save(context, result) - Log.i(TAG, "token refreshed") - return result.accessToken + val current = load(context) ?: return null + if (fresh(current)) return current.accessToken + synchronized(refreshLock) { + val again = load(context) ?: return null + if (fresh(again)) return again.accessToken + val answer = EpicNative.call("auth.refresh", JSONObject().put("refreshToken", again.refreshToken)) + val result = answer?.let(TokenResult::from) + if (result == null) { + Log.w(TAG, "token refresh failed: ${answer?.optString("error").orEmpty().ifEmpty { "no answer" }}") + return again.accessToken } - Log.w(TAG, "token refresh failed; using the stored token") + save(context, result) + return result.accessToken } - return creds.accessToken } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java deleted file mode 100644 index 88307fd25e..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java +++ /dev/null @@ -1,1595 +0,0 @@ -package com.droiddeck.launcher.stores.epic; - -import android.content.Context; -import android.util.Log; - -import com.droiddeck.launcher.stores.StoreLog; -import com.droiddeck.launcher.stores.StoreInstallRoot; -import com.droiddeck.launcher.stores.StoreInstalls; -import com.droiddeck.launcher.stores.StoresNative; -import com.droiddeck.launcher.stores.download.StoreDownloadTier; - -import org.json.JSONArray; -import org.json.JSONObject; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.security.MessageDigest; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.zip.Inflater; - -public final class EpicDownloadManager { - - private static final String TAG = "EpicDownload"; - private static final String UA = EpicAuthClient.USER_AGENT; - /** Keep reusable chunk data bounded even when a manifest shares chunks across distant files. */ - static final long MAX_RETAINED_CHUNK_CACHE_BYTES = 64L * 1024 * 1024; - static final int PREFETCH_PARTS = 8; - static final String STAGING_DIRECTORY = ".droiddeck-epic-staging"; - private static final String STAGING_OWNER = "DroidDeck Epic staging v1\n"; - /** Most the adaptive engine caches of shared chunks; more only saves the occasional refetch. */ - static final long ADAPTIVE_CACHE_MAX_BYTES = 1024L * 1024 * 1024; - /** Its budget when free space cannot be read. */ - static final long ADAPTIVE_CACHE_DEFAULT_BYTES = 256L * 1024 * 1024; - /** - * Below this, internal scratch is too small to be worth a cache of its own: the shared chunks - * then go beside the game (still released at their last use) rather than failing the install. - */ - static final long ADAPTIVE_CACHE_MIN_SCRATCH_BYTES = MAX_RETAINED_CHUNK_CACHE_BYTES; - /** Chunks being decoded, written or released at any moment, beyond the budget itself. */ - static final long ADAPTIVE_TRANSIENT_BYTES = 64L * 1024 * 1024; - - static final String MODE_ADAPTIVE = "adaptive"; - static final String MODE_LOOKAHEAD = "lookahead"; - static final String MODE_BATCH = "batch"; - static final String MODE_JAVA = "java"; - - /** Which writer {@link #install} asks for and how wide the native fetch may go. */ - static final class NativeOptions { - final String mode; - final int maxWorkers, processWorkers; - final String tierId; - NativeOptions(String mode, int maxWorkers, int processWorkers, String tierId) { - this.mode = mode; this.maxWorkers = maxWorkers; this.processWorkers = processWorkers; this.tierId = tierId; - } - /** The default: the adaptive engine at the Fast tier's window. */ - static NativeOptions defaults() { return new NativeOptions(MODE_ADAPTIVE, 32, 2, "fast"); } - } - - public interface Callback { - void onProgress(String message, int pct); - default void onBytes(long done, long total, long speedBps) {} - default void onLog(String line) {} - default void onStage(String stage, long done, long total, int items, int itemsTotal) {} - default void onSizes(long downloadBytes, long diskBytes) {} - } - - private EpicDownloadManager() {} - - public static final class CdnUrl { - public final String baseUrl, cloudDir, authParams; - CdnUrl(String baseUrl, String cloudDir, String authParams) { this.baseUrl = baseUrl; this.cloudDir = cloudDir; this.authParams = authParams; } - } - - public static final class ChunkInfo { - public int[] guid = new int[4]; - public long hash; - public byte[] sha1; - public int groupNum; - public int windowSize; - public long fileSize; - public String guidStr() { return String.format("%08X%08X%08X%08X", guid[0], guid[1], guid[2], guid[3]); } - public String getPath(String chunkDir) { return chunkDir + "/" + String.format("%02d", groupNum) + "/" + String.format("%016X", hash) + "_" + guidStr() + ".chunk"; } - } - - public static final class ChunkPart { - public int[] guid = new int[4]; - public int offset, size; - public String guidStr() { return String.format("%08X%08X%08X%08X", guid[0], guid[1], guid[2], guid[3]); } - } - - public static final class FileInfo { - public String filename = ""; - public List parts = new ArrayList<>(); - public List installTags = new ArrayList<>(); - public byte[] sha1 = null; - public long fileSize() { long t = 0; for (ChunkPart p : parts) t += (p.size & 0xFFFFFFFFL); return t; } - } - - public static final class Manifest { - public String chunkDir = "ChunksV4"; - public List uniqueChunks = new ArrayList<>(); - public List files = new ArrayList<>(); - public String launchExe = ""; - public String launchCommand = ""; - public String buildVersion = ""; - public String prereqName = ""; - public String prereqPath = ""; - public boolean prereqRead = false; - public List cdnUrls = new ArrayList<>(); - /** The manifest as downloaded, for the native engines (which parse it themselves); null = Java only. */ - byte[] rawBytes = null; - } - - public static final class Result { - public final String launchExe; - public final String buildVersion; - public final long bytes; - public String prereqName = ""; - public String prereqPath = ""; - Result(String launchExe, String buildVersion, long bytes) { this.launchExe = launchExe; this.buildVersion = buildVersion; this.bytes = bytes; } - public boolean prereqRead = false; - Result withPrereq(Manifest m) { prereqName = m.prereqName; prereqPath = m.prereqPath; prereqRead = m.prereqRead; return this; } - } - - public static final class InstallException extends Exception { - InstallException(String message) { super(message); } - } - - public static Result install(Context ctx, String manifestApiJson, String installDirPath, String chunkCacheDirPath, List installTags, AtomicBoolean cancel, Callback cb) throws InstallException { - try { - if (cancel.get()) return null; - cb.onProgress("Reading CDN list…", 0); - List cdnUrls = parseCdnUrls(manifestApiJson); - if (cdnUrls.isEmpty()) throw new InstallException("No CDN in the manifest answer"); - for (CdnUrl c : cdnUrls) cb.onLog("epic: CDN " + c.baseUrl + " auth=" + (c.authParams.isEmpty() ? "no" : "yes")); - cb.onProgress("Downloading manifest…", 0); - byte[] manifestBytes = downloadManifest(manifestApiJson, cdnUrls); - if (manifestBytes == null) throw new InstallException("The manifest could not be downloaded"); - if (cancel.get()) return null; - cb.onProgress("Parsing manifest…", 0); - Manifest manifest = parseManifest(manifestBytes); - if (manifest == null) throw new InstallException("The manifest could not be parsed"); - manifest.cdnUrls = cdnUrls; - manifest.rawBytes = manifestBytes; - cb.onLog("epic: chunkDir=" + manifest.chunkDir + " chunks=" + manifest.uniqueChunks.size() + " files=" + manifest.files.size() + " exe=" + manifest.launchExe); - StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); - String mode = com.droiddeck.launcher.session.SessionPrefs.INSTANCE.storesEpicMode(ctx); - NativeOptions options = new NativeOptions(mode, Math.max(1, Math.min(128, tier.getNetworkWindow())), Math.max(2, tier.getProcessWorkers()), tier.getId()); - return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, options); - } catch (InstallException e) { - throw e; - } catch (Exception e) { - Log.w(TAG, "install failed", e); - throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "")); - } - } - - static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, - List installTags, AtomicBoolean cancel, Callback cb) throws InstallException { - return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, NativeOptions.defaults()); - } - - static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, - List installTags, AtomicBoolean cancel, Callback cb, - NativeOptions nativeOptions) throws InstallException { - Result result = installSelected(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, nativeOptions); - // A finished install leaves no staging; the owned namespace is recreated by the next run. - if (result != null) releaseOwnedStaging(new File(installDirPath)); - return result; - } - - private static Result installSelected(Manifest manifest, String installDirPath, String chunkCacheDirPath, - List installTags, AtomicBoolean cancel, Callback cb, - NativeOptions nativeOptions) throws InstallException { - try { - List cdnUrls = manifest.cdnUrls; - File installDir = new File(installDirPath); - installDir.mkdirs(); - final String cachePath = chunkCacheDirPath == null ? "" : chunkCacheDirPath; - final File chunkCacheDir = cachePath.isEmpty() ? new File(installDir, ".chunks") : new File(cachePath); - chunkCacheDir.mkdirs(); - File stagingDir = recoverOwnedStaging(installDir, manifest.files); - - List selected = resolveInstallFiles(manifest, installTags); - cb.onLog("epic: " + selected.size() + "/" + manifest.files.size() + " files for tags " + (installTags == null ? "(all)" : installTags.toString())); - long installBytes = 0; - for (FileInfo f : selected) installBytes += f.fileSize(); - final long planned = installBytes; - - cb.onProgress("Checking files…", 0); - List pending = new ArrayList<>(selected.size()); - int checked = 0, good = 0; - long checkedBytes = 0; - cb.onStage("check", 0, installBytes, 0, selected.size()); - for (FileInfo f : selected) { - if (cancel.get()) return null; - File out = new File(installDir, f.filename.replace("\\", "/")); - if (fileExistsWithCorrectHash(out, f.fileSize(), f.sha1)) good++; else pending.add(f); - checked++; - checkedBytes += f.fileSize(); - if ((checked & 15) == 0 || checked == selected.size()) cb.onStage("check", checkedBytes, installBytes, checked, selected.size()); - if ((checked & 63) == 0) cb.onProgress("Checking files… (" + checked + "/" + selected.size() + ")", 0); - } - cb.onLog("epic: delta " + good + " up to date, " + pending.size() + " to download"); - if (pending.isEmpty()) { - deleteDir(chunkCacheDir); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } - List needed = uniqueChunksForFiles(manifest, pending); - Map chunksByGuid = new LinkedHashMap<>(); - for (ChunkInfo c : needed) chunksByGuid.put(c.guidStr(), c); - if (!chunkCacheDir.isDirectory() && !chunkCacheDir.mkdirs()) throw new InstallException("Could not create Epic chunk cache"); - long pendingBytes = 0; - Map oldFileBytes = new LinkedHashMap<>(); - for (FileInfo f : pending) { - pendingBytes += f.fileSize(); - File output = new File(installDir, f.filename.replace("\\", "/")); - if (output.isFile()) oldFileBytes.put(f.filename, output.length()); - } - NativeOptions options = nativeOptions != null ? nativeOptions : NativeOptions.defaults(); - String mode = options.mode; - if (!MODE_LOOKAHEAD.equals(mode) && !MODE_BATCH.equals(mode) && !MODE_JAVA.equals(mode)) mode = MODE_ADAPTIVE; - boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded(); - if (!nativeAvailable && !MODE_JAVA.equals(mode)) { - cb.onLog("epic: native engine unavailable; mode=java-bounded"); - mode = MODE_JAVA; - } else if (MODE_JAVA.equals(mode)) { - cb.onLog("epic: mode=java-bounded chosen by storesEpicMode"); - } - if (MODE_ADAPTIVE.equals(mode)) { - NativeAttempt attempt = installAdaptive(manifest, pending, pendingBytes, oldFileBytes, planned, installDir, cachePath, chunkCacheDir, stagingDir, options, cancel, cb); - if (attempt.handled) return attempt.result; - // Its space plan did not fit: the lookahead engine sizes its window to what is free. - mode = MODE_LOOKAHEAD; - } else if (MODE_BATCH.equals(mode)) { - return installBatch(manifest, pending, needed, pendingBytes, planned, installDir, cachePath, chunkCacheDir, options, cancel, cb); - } - boolean lookahead = MODE_LOOKAHEAD.equals(mode); - Map cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed); - boolean cacheOnInstallVolume = cachePath.isEmpty(); - long peakGameAdditional = lookahead ? 0 : cacheOnInstallVolume - ? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true, oldFileBytes) - : estimateStagedFilePeak(pending, oldFileBytes); - long peakCacheAdditional = lookahead || cacheOnInstallVolume - ? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false); - long freeGame = StoreInstallRoot.availableBytes(installDir); - // The lookahead engine plans its own window against both free-space readings. - if (!lookahead && cacheOnInstallVolume) { - if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free"); - } else if (!lookahead) { - long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir); - if (StoreInstallRoot.insufficientSpace(peakCacheAdditional, freeCache)) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free"); - if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need " + fmt(peakGameAdditional) + ", only " + fmt(freeGame) + " free"); - } - if (cancel.get()) return null; - if (lookahead) { - JSONObject request = nativeRequest(manifest, pending, chunksByGuid, cachedChunkBytes, - oldFileBytes, installDir, chunkCacheDir, stagingDir, cacheOnInstallVolume); - request.put("free_game", StoreInstallRoot.availableBytes(installDir)); - request.put("free_cache", StoreInstallRoot.availableBytes(chunkCacheDir)); - final long nativeTotal = pendingBytes; - final long nativeStarted = System.nanoTime(); - cb.onSizes(nativeTotal, planned); - cb.onBytes(0, nativeTotal, 0); - EpicNative.Result outcome = EpicNative.installAdaptive(request.toString(), cancel, new EpicNative.InstallListener() { - @Override public void onPlan(int chunks, long bytes, String plan) { - cb.onLog("epic: mode=lookahead engine=lookahead-rust " + plan); - } - @Override public void onProgress(long done, long total, int files, int filesTotal) { - long elapsedMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000); - cb.onBytes(done, total, done * 1000 / elapsedMs); - cb.onProgress("Downloading chunks…", (int) (done * 80L / Math.max(1, total))); - cb.onStage("stream", done, total, files, filesTotal); - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytes) {} - @Override public boolean beforeSwap() { - try { - if (cancel.get()) return false; - StoreInstalls.markUpdating(installDir); - return !cancel.get(); - } catch (Exception e) { - cb.onLog("epic: cannot mark updating: " + e.getMessage()); - return false; - } - } - }); - if (outcome.cancelled || cancel.get()) return null; - if (!outcome.success) throw new InstallException("Native Epic install failed: " + outcome.error); - long lookaheadMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000); - cb.onLog("epic: summary mode=lookahead files=" + pending.size() + " bytes_written=" + nativeTotal + " elapsed_ms=" + lookaheadMs - + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", nativeTotal * 1000.0 / lookaheadMs / 1048576.0)); - cb.onStage("install", nativeTotal, nativeTotal, pending.size(), pending.size()); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } - cb.onLog("epic: cache " + (cacheOnInstallVolume ? "beside the game" : "scratch") + ", peak game additional=" + fmt(peakGameAdditional) + (cacheOnInstallVolume ? "" : ", peak cache additional=" + fmt(peakCacheAdditional)) + " for " + fmt(pendingBytes) + " of files"); - final long fTotalBytes = pendingBytes; - final int totalChunks = needed.size(); - cb.onSizes(fTotalBytes, planned); - cb.onBytes(0, fTotalBytes, 0); - cb.onLog("epic: " + fmt(fTotalBytes) + " to write using " + totalChunks + " unique chunks"); - cb.onLog("epic: engine=bounded-java (8-worker prefetch; native availability does not change cache semantics)"); - long completedBytes = 0, writtenBytes = 0; - long transferStarted = System.nanoTime(); - int completedCount = 0, doneFiles = 0, totalFiles = pending.size(); - Set reservedPaths = new LinkedHashSet<>(); - for (FileInfo file : manifest.files) reservedPaths.add(new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath()); - boolean updateRecorded = false; - try (ChunkPrefetch prefetch = new ChunkPrefetch(pending, chunksByGuid, cachedChunkBytes, - chunkCacheDir, manifest.chunkDir, cdnUrls, cancel)) { - for (FileInfo file : pending) { - if (cancel.get()) return null; - String relPath = file.filename.replace("\\", "/"); - File outFile = new File(installDir, relPath); - cb.onProgress("Downloading chunks: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), (int) (completedBytes * 80L / Math.max(1, fTotalBytes))); - MessageDigest fileSha = expectedDigest(file.sha1); - File partFile = createStagingFile(outFile, reservedPaths, stagingDir); - try { - try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) { - for (ChunkPart part : file.parts) { - if (cancel.get()) throw new IOException("cancelled"); - String guid = part.guidStr(); - File cachedChunk = prefetch.await(guid); - completedCount++; - cb.onProgress("Downloading chunks (part " + completedCount + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes))); - copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath); - completedBytes += part.size & 0xFFFFFFFFL; - long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); - cb.onBytes(completedBytes, fTotalBytes, completedBytes * 1000 / elapsedMs); - prefetch.consume(); - } - bos.flush(); - fos.getFD().sync(); - } - if (cancel.get()) return null; - if (partFile.length() != file.fileSize()) throw new InstallException("Size mismatch for " + relPath); - if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) { - throw new InstallException("SHA-1 mismatch for " + relPath); - } - // Once a swap can mix versions, the old installed record is no longer truthful. - if (!updateRecorded) { - StoreInstalls.markUpdating(installDir); - updateRecorded = true; - } - if (cancel.get()) return null; - Files.move(partFile.toPath(), outFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - if (cancel.get()) return null; - throw new InstallException("Could not assemble or atomically replace " + relPath + ": " + e.getMessage()); - } finally { - partFile.delete(); - } - doneFiles++; - writtenBytes += file.fileSize(); - cb.onStage("stream", writtenBytes, pendingBytes, doneFiles, totalFiles); - } - } - deleteDir(chunkCacheDir); - long boundedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); - cb.onLog("epic: summary mode=java-bounded files=" + doneFiles + " bytes_written=" + writtenBytes + " elapsed_ms=" + boundedMs + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", writtenBytes * 1000.0 / boundedMs / 1048576.0)); - cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } catch (InstallException e) { - throw e; - } catch (Exception e) { - Log.w(TAG, "install failed", e); - throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "")); - } - } - - /** What a native attempt did: {@code handled} = the install ended there ({@code result} null when cancelled). */ - private static final class NativeAttempt { - final boolean handled; - final Result result; - private NativeAttempt(boolean handled, Result result) { this.handled = handled; this.result = result; } - static final NativeAttempt FALLBACK = new NativeAttempt(false, null); - static NativeAttempt done(Result result) { return new NativeAttempt(true, result); } - } - - /** The pending files as indices into the manifest's file list, what the native runs take; null when one is not in it. */ - static int[] pendingIndices(Manifest manifest, List pending) { - java.util.IdentityHashMap index = new java.util.IdentityHashMap<>(); - for (int i = 0; i < manifest.files.size(); i++) index.put(manifest.files.get(i), i); - int[] pendingIdx = new int[pending.size()]; - for (int i = 0; i < pending.size(); i++) { - Integer k = index.get(pending.get(i)); - if (k == null) return null; - pendingIdx[i] = k; - } - return pendingIdx; - } - - private static String[] cdnPrefixes(List cdnUrls) { - String[] prefixes = new String[cdnUrls.size()]; - for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir; - return prefixes; - } - - /** - * Where the adaptive engine's shared-chunk cache goes and how big it may get, and what each - * volume must hold. A card install keeps the cache on internal scratch (each byte hits the - * card once); when internal storage cannot hold a useful cache, the cache goes beside the game - * instead of failing. {@code error} names the storage that is short, and by how much. - */ - static final class AdaptiveSpacePlan { - boolean cacheBesideGame; - long cacheBudget; - /** - * Most planned bytes of replaced files the engine may stage at once (each sits beside its - * old copy until its swap); -1 = nothing is replaced, no limit. - */ - long overlapBudget = -1; - long gameNeed; - long scratchNeed; - String error; - } - - /** The part of free space the shared cache may use, after {@code need}: half the headroom, capped. */ - static long adaptiveCacheBudget(long free, long need) { - if (free < 0) return ADAPTIVE_CACHE_DEFAULT_BYTES; - long headroom = free - need - ADAPTIVE_TRANSIENT_BYTES; - if (headroom <= 0) return 0; - return Math.min(ADAPTIVE_CACHE_MAX_BYTES, headroom / 2); - } - - /** - * The game volume's net growth at its worst point when files are swapped in order and each - * replaced file's old copy is released at its swap: the running Σ(new − old) after each file. - * What a staged file costs while it sits beside its old copy is the overlap budget's part. - */ - static long settledNetPeak(List files, Map oldFileBytes) { - long running = 0, peak = 0; - for (FileInfo file : files) { - Long old = oldFileBytes.get(file.filename); - running += file.fileSize() - (old == null ? 0 : old); - peak = Math.max(peak, running); - } - return peak; - } - - /** The largest pending file whose destination has data (the engine's smallest overlap budget). */ - static long largestReplacedFile(List files, Map oldFileBytes) { - long largest = 0; - for (FileInfo file : files) { - Long old = oldFileBytes.get(file.filename); - if (old != null && old > 0) largest = Math.max(largest, file.fileSize()); - } - return largest; - } - - /** - * The cache placement and budget as before; for an update, the replacement-overlap budget is - * what the game volume has left after the net growth and the cache, never below the largest - * replaced file (one at a time is the minimum that still progresses). The check is the - * lookahead planner's: the need must not exceed the free reading, no extra margin. - */ - static AdaptiveSpacePlan planAdaptiveSpace(long netPeak, long largestReplaced, long freeGame, boolean scratchRequested, long freeScratch) { - AdaptiveSpacePlan plan = new AdaptiveSpacePlan(); - long minimum = netPeak + largestReplaced; - long scratchBudget = scratchRequested ? adaptiveCacheBudget(freeScratch, 0) : 0; - plan.cacheBesideGame = !scratchRequested || (freeScratch >= 0 && scratchBudget < ADAPTIVE_CACHE_MIN_SCRATCH_BYTES); - long cacheOnGame; - if (plan.cacheBesideGame) { - plan.cacheBudget = adaptiveCacheBudget(freeGame, minimum); - cacheOnGame = plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; - plan.scratchNeed = 0; - } else { - plan.cacheBudget = scratchBudget; - cacheOnGame = 0; - plan.scratchNeed = plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; - } - long overlap = 0; - if (largestReplaced > 0) { - overlap = freeGame < 0 ? largestReplaced : Math.max(largestReplaced, freeGame - netPeak - cacheOnGame); - plan.overlapBudget = overlap; - } - plan.gameNeed = netPeak + overlap + cacheOnGame; - if (StoreInstallRoot.insufficientSpace(plan.gameNeed, freeGame)) { - plan.error = "Not enough free space on the game's storage: need " + fmt(plan.gameNeed) - + (plan.cacheBesideGame && scratchRequested ? " (download cache moved beside the game: internal storage is full)" : "") - + ", only " + fmt(freeGame) + " free (short by " + fmt(plan.gameNeed - freeGame) + ")"; - } else if (!plan.cacheBesideGame && StoreInstallRoot.insufficientSpace(plan.scratchNeed, freeScratch)) { - plan.error = "Not enough internal space for the download cache: need " + fmt(plan.scratchNeed) + ", only " + fmt(freeScratch) - + " free (short by " + fmt(plan.scratchNeed - freeScratch) + ")"; - } - return plan; - } - - /** - * The adaptive native install; the library is loaded. Hands back {@code FALLBACK} (nothing - * fetched or written) only when its space plan does not fit, so the caller can run the - * lookahead engine, which narrows its window to the free space. Any other failure is final. - */ - private static NativeAttempt installAdaptive(Manifest manifest, List pending, long pendingBytes, Map oldFileBytes, - long planned, File installDir, String cachePath, File chunkCacheDir, File stagingDir, - NativeOptions options, AtomicBoolean cancel, Callback cb) throws Exception { - int[] pendingIdx = pendingIndices(manifest, pending); - if (manifest.rawBytes == null || pendingIdx == null) { - throw new InstallException("Native Epic install failed: the manifest is not available to the native engine"); - } - long netPeak = settledNetPeak(pending, oldFileBytes); - long largestReplaced = largestReplacedFile(pending, oldFileBytes); - long freeGame = StoreInstallRoot.availableBytes(installDir); - boolean scratchRequested = !cachePath.isEmpty(); - long freeScratch = scratchRequested ? StoreInstallRoot.availableBytes(chunkCacheDir) : -1; - AdaptiveSpacePlan space = planAdaptiveSpace(netPeak, largestReplaced, freeGame, scratchRequested, freeScratch); - if (space.error != null) { - cb.onLog("epic: adaptive space plan does not fit (" + space.error + "); mode=lookahead"); - return NativeAttempt.FALLBACK; - } - String engineCache = space.cacheBesideGame ? "" : cachePath; - File engineCacheDir = engineCache.isEmpty() ? new File(installDir, ".chunks") : chunkCacheDir; - if (space.cacheBesideGame && scratchRequested) { - cb.onLog("epic: internal storage cannot hold the shared-chunk cache (" + fmt(freeScratch) + " free); cache beside the game"); - } - if (cancel.get()) return NativeAttempt.done(null); - cb.onLog("epic: mode=adaptive tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers - + " cache=" + (engineCache.isEmpty() ? "beside the game" : "scratch") + " budget=" + fmt(space.cacheBudget) - + " overlap_budget=" + (space.overlapBudget < 0 ? "none" : fmt(space.overlapBudget)) - + " game need=" + fmt(space.gameNeed) + " of " + fmt(freeGame) + (space.cacheBesideGame ? "" : ", scratch need=" + fmt(space.scratchNeed) + " of " + fmt(freeScratch))); - cb.onSizes(pendingBytes, planned); - cb.onBytes(0, pendingBytes, 0); - final int totalFiles = pending.size(); - final File fInstallDir = installDir; - final Object lock = new Object(); - final long started = System.nanoTime(); - final long[] state = new long[] {0, 0, System.currentTimeMillis(), 0, 0}; // bytes, files, speed ms, speed bytes, speed bps - EpicNative.Result res = EpicNative.stream(manifest.rawBytes, installDir.getPath(), engineCache, stagingDir.getCanonicalPath(), cdnPrefixes(manifest.cdnUrls), pendingIdx, pendingBytes, - space.cacheBudget, space.overlapBudget, "", options.maxWorkers, options.processWorkers, cancel, new EpicNative.StreamListener() { - @Override public void onStreamProgress(long bytesWritten, long bytesTotal, int filesDone, int filesTotal, long filesBytes) { - synchronized (lock) { - if (bytesWritten < state[0] && filesDone <= state[1]) return; // a late, older sample - state[0] = Math.max(state[0], bytesWritten); - long nowMs = System.currentTimeMillis(); - if (nowMs - state[2] >= 500) { - state[4] = (state[0] - state[3]) * 1000L / Math.max(1, nowMs - state[2]); - state[2] = nowMs; - state[3] = state[0]; - } - cb.onBytes(state[0], pendingBytes, state[4]); - cb.onProgress("Downloading chunks (" + filesDone + "/" + totalFiles + " files)" + speedSuffix(state[4]), - (int) (state[0] * 80L / Math.max(1, pendingBytes))); - if (filesDone > state[1]) { - state[1] = filesDone; - cb.onStage("stream", filesBytes, pendingBytes, filesDone, totalFiles); - } - } - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public boolean onBeforeReplace() { - // Once a swap can mix versions, the old installed record is no longer truthful. - try { - StoreInstalls.markUpdating(fInstallDir); - return true; - } catch (Exception e) { - cb.onLog("epic: could not mark the install as updating: " + e.getMessage()); - return false; - } - } - @Override public void onComplete(boolean success, String error, long bytes) {} - }); - if (!res.started) throw new InstallException("Native Epic install failed: " + res.error); - long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); - if (res.cancelled) return NativeAttempt.done(null); - if (!res.success) throw new InstallException("Native Epic install failed: " + res.error); - deleteDir(engineCacheDir); - deleteDir(chunkCacheDir); - cb.onLog("epic: summary mode=adaptive files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs - + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); - cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); - cb.onProgress("Complete", 100); - return NativeAttempt.done(new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest)); - } - - /** - * The older native batch install, kept for A/B benchmarks: every chunk fetched into the cache - * in parallel, then the files assembled from it. It needs the whole cache and the files at - * once (about twice the game), and it writes files in place rather than staged, so an - * existing install is marked updating before anything is fetched. - */ - private static Result installBatch(Manifest manifest, List pending, List needed, long pendingBytes, - long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options, - AtomicBoolean cancel, Callback cb) throws InstallException { - int[] pendingIdx = pendingIndices(manifest, pending); - if (manifest.rawBytes == null || pendingIdx == null) { - throw new InstallException("Native Epic install failed: the manifest is not available to the native engine"); - } - long cacheMissing = 0, totalBytes = 0; - for (ChunkInfo c : needed) { - if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += chunkWindowSize(c); - totalBytes += Math.max(c.fileSize, 1); - } - long freeGame = StoreInstallRoot.availableBytes(installDir); - if (cachePath.isEmpty()) { - long need = cacheMissing + pendingBytes; - if (StoreInstallRoot.insufficientSpace(need, freeGame)) throw new InstallException("Not enough free space for the batch Epic download: need " + fmt(need) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(freeGame) + " free (short by " + fmt(need - freeGame) + ")"); - } else { - long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir); - if (StoreInstallRoot.insufficientSpace(cacheMissing, freeCache)) throw new InstallException("Not enough internal space for the batch download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free (short by " + fmt(cacheMissing - freeCache) + ")"); - if (StoreInstallRoot.insufficientSpace(pendingBytes, freeGame)) throw new InstallException("Not enough free space on the game's storage: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free (short by " + fmt(pendingBytes - freeGame) + ")"); - } - if (cancel.get()) return null; - final long fTotal = totalBytes; - final int totalChunks = needed.size(); - final int totalFiles = pending.size(); - cb.onLog("epic: mode=batch tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers + (cachePath.isEmpty() ? "" : " cache=scratch") + " cache_missing=" + fmt(cacheMissing)); - cb.onSizes(pendingBytes, planned); - final AtomicLong lastMs = new AtomicLong(System.currentTimeMillis()), lastBytes = new AtomicLong(), speed = new AtomicLong(); - final long started = System.nanoTime(); - EpicNative.Result fetch = EpicNative.run(manifest.rawBytes, installDir.getPath(), cachePath, cdnPrefixes(manifest.cdnUrls), pendingIdx, totalChunks, fTotal, "", - options.maxWorkers, options.processWorkers, cancel, new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); } - @Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) { - sampleSpeed(bytesDone, lastMs, lastBytes, speed); - cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speed.get()), (int) (bytesDone * 80L / Math.max(1, fTotal))); - cb.onBytes(bytesDone, fTotal, speed.get()); - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytes) {} - }); - if (!fetch.started) throw new InstallException("Native Epic install failed: " + fetch.error); - if (fetch.cancelled) return null; - if (!fetch.success) throw new InstallException("Epic batch download failed: " + fetch.error); - // Assembly overwrites files in place: the old installed record stops being truthful now. - try { StoreInstalls.markUpdating(installDir); } catch (Exception e) { throw new InstallException("Could not mark the install as updating: " + e.getMessage()); } - EpicNative.Result asm = EpicNative.assemble(manifest.rawBytes, installDir.getPath(), cachePath, pendingIdx, cancel, new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {} - @Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) { - cb.onStage("stream", bytesDone, bytesTotal, done, Math.max(total, totalFiles)); - cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles)))); - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytes) {} - }); - if (asm.cancelled) return null; - if (!asm.started || !asm.success) throw new InstallException(asm.error == null || asm.error.isEmpty() ? "Epic batch assembly failed" : asm.error); - long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); - cb.onLog("epic: summary mode=batch files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs - + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); - deleteDir(chunkCacheDir); - cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } - - private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); - if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { - long prevB = lastSpeedBytes.getAndSet(done); - long dt = nowMs - prevMs; - if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); - } - } - - private static String speedSuffix(long bps) { - if (bps <= 0) return ""; - return " " + (bps >= 1048576 ? String.format(java.util.Locale.ROOT, "%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); - } - - static File recoverOwnedStaging(File installDir, List manifestFiles) throws IOException { - File staging = new File(installDir, STAGING_DIRECTORY); - Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY); - if (!staging.getCanonicalFile().toPath().equals(expected) || - (Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS) && - !Files.isDirectory(staging.toPath(), LinkOption.NOFOLLOW_LINKS))) { - throw new IOException("Unsafe Epic staging namespace; refusing to traverse it"); - } - String namespace = expected.toString(); - for (FileInfo file : manifestFiles) { - String path = new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath(); - if (path.equals(namespace) || path.startsWith(namespace + File.separator)) { - throw new IOException("Manifest conflicts with Epic staging namespace"); - } - } - File owner = new File(staging, "owner"); - if (Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS)) { - if (!Files.isRegularFile(owner.toPath(), LinkOption.NOFOLLOW_LINKS) || - !owner.getCanonicalFile().toPath().equals(expected.resolve("owner")) || - owner.length() != STAGING_OWNER.length() || - !STAGING_OWNER.equals(new String(Files.readAllBytes(owner.toPath()), StandardCharsets.UTF_8))) { - throw new IOException("Unowned Epic staging namespace; refusing to delete it"); - } - File[] files = staging.listFiles(); - if (files == null) throw new IOException("Could not inspect Epic staging"); - for (File file : files) { - if (file.getName().equals("owner")) continue; - if (!Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS) || - !file.getCanonicalFile().toPath().equals(expected.resolve(file.getName())) || - !file.getName().endsWith(".stage")) throw new IOException("Unexpected Epic staging entry"); - } - for (File file : files) { - if (file.getName().equals("owner")) continue; - if (!file.delete()) throw new IOException("Could not recover orphan Epic staging"); - } - } else { - if (!staging.mkdir()) throw new IOException("Could not create Epic staging namespace"); - try (FileOutputStream out = new FileOutputStream(owner)) { - out.write(STAGING_OWNER.getBytes(StandardCharsets.UTF_8)); - out.getFD().sync(); - } - } - return staging; - } - - /** - * Remove the owned staging namespace when it holds nothing but its own marker. Same no-follow - * and canonical-path checks as {@link #recoverOwnedStaging}; anything unexpected is left alone. - */ - static void releaseOwnedStaging(File installDir) { - try { - File staging = new File(installDir, STAGING_DIRECTORY); - if (!Files.isDirectory(staging.toPath(), LinkOption.NOFOLLOW_LINKS)) return; - Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY); - if (!staging.getCanonicalFile().toPath().equals(expected)) return; - String[] names = staging.list(); - if (names == null || names.length != 1 || !names[0].equals("owner")) return; - File owner = new File(staging, "owner"); - if (!Files.isRegularFile(owner.toPath(), LinkOption.NOFOLLOW_LINKS) - || owner.length() != STAGING_OWNER.length() - || !STAGING_OWNER.equals(new String(Files.readAllBytes(owner.toPath()), StandardCharsets.UTF_8))) return; - if (owner.delete()) staging.delete(); - } catch (IOException e) { - Log.w(TAG, "staging namespace kept: " + e.getMessage()); - } - } - - private static JSONObject nativeRequest(Manifest manifest, List files, - Map chunks, Map cached, Map old, - File install, File cache, File staging, boolean sameVolume) throws Exception { - JSONObject request = new JSONObject(); - request.put("schema_version", 1); - request.put("install_dir", install.getCanonicalPath()); - request.put("cache_dir", cache.getCanonicalPath()); - request.put("staging_dir", staging.getCanonicalPath()); - request.put("same_volume", sameVolume); - request.put("chunk_dir", manifest.chunkDir); - request.put("max_workers", 32); - JSONArray hosts = new JSONArray(); - for (CdnUrl cdn : manifest.cdnUrls) hosts.put(cdn.baseUrl + cdn.cloudDir); - request.put("hosts", hosts); - JSONArray chunkList = new JSONArray(); - for (ChunkInfo chunk : chunks.values()) { - JSONObject item = new JSONObject(); - item.put("guid", chunk.guidStr()); - item.put("path", chunk.getPath(manifest.chunkDir)); - item.put("size", chunkWindowSize(chunk)); - item.put("exact_size", chunk.windowSize > 0); - item.put("wire_size", Math.max(1, chunk.fileSize)); - item.put("sha1", hexDigest(chunk.sha1)); - item.put("cached", cached.containsKey(chunk.guidStr()) ? cached.get(chunk.guidStr()) : -1); - chunkList.put(item); - } - request.put("chunks", chunkList); - JSONArray fileList = new JSONArray(); - for (FileInfo file : files) { - JSONObject item = new JSONObject(); - item.put("path", file.filename.replace("\\", "/")); - item.put("sha1", hexDigest(file.sha1)); - item.put("old_size", old.containsKey(file.filename) ? old.get(file.filename) : 0); - JSONArray parts = new JSONArray(); - for (ChunkPart part : file.parts) { - parts.put(new JSONObject().put("guid", part.guidStr()).put("offset", part.offset & 0xFFFFFFFFL).put("size", part.size & 0xFFFFFFFFL)); - } - item.put("parts", parts); - fileList.put(item); - } - request.put("files", fileList); - return request; - } - - private static String hexDigest(byte[] digest) { - if (digest == null || digest.length != 20) return ""; - StringBuilder out = new StringBuilder(); - for (byte b : digest) out.append(String.format("%02x", b & 255)); - return out.toString(); - } - - private static File createStagingFile(File output, Set reservedPaths, File staging) throws IOException { - File parent = output.getParentFile(); - if (!parent.isDirectory() && !parent.mkdirs()) throw new IOException("Could not create output directory"); - // Owned staging is on the output volume, never a manifest's literal ".part". - while (true) { - File stage = File.createTempFile("file-", ".stage", staging); - if (!reservedPaths.contains(stage.getCanonicalPath())) return stage; - if (!stage.delete()) throw new IOException("Could not clear staging collision"); - } - } - - static long estimateStagedFilePeak(List files, Map oldFileBytes) { - long delta = 0, peak = 0; - for (FileInfo file : files) { - delta += file.fileSize(); - peak = Math.max(peak, delta); - delta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0; - } - return peak; - } - - /** - * Discard abandoned/orphaned cache entries and retain only verified chunks this install uses. - * A bounded run consumes each chunk at its last part reference, so stale chunks from an older - * all-at-once run must not silently occupy the space needed by the game files. - */ - static Map prepareBoundedCache(File cacheDir, List needed) throws InstallException { - Map byGuid = new LinkedHashMap<>(); - for (ChunkInfo chunk : needed) byGuid.put(chunk.guidStr(), chunk); - Map valid = new LinkedHashMap<>(); - File[] entries = cacheDir.listFiles(); - if (entries == null) throw new InstallException("Could not inspect Epic chunk cache"); - for (File entry : entries) { - ChunkInfo chunk = byGuid.get(entry.getName()); - if (chunk == null || !isValidCachedChunk(chunk, entry)) { - deleteDir(entry); - if (entry.exists()) throw new InstallException("Could not clear stale Epic cache entry " + entry.getName()); - continue; - } - valid.put(entry.getName(), entry.length()); - } - trimCacheToBound(cacheDir, valid); - return valid; - } - - /** Evict oldest reusable chunks until retained cache data is under the fixed disk ceiling. */ - private static void trimCacheToBound(File cacheDir, Map cacheSizes) throws InstallException { - for (String guid : evictOldestReusableChunks(cacheSizes).keySet()) { - File chunk = new File(cacheDir, guid); - if (chunk.exists() && !chunk.delete() && chunk.exists()) { - throw new InstallException("Could not release Epic cache chunk " + guid); - } - } - } - - /** - * The writer and space planner use the same oldest-first retained-cache policy. Entries leave - * this map as soon as their last reference is consumed; an oversized incoming chunk is - * therefore allowed to coexist with the retained cache until its current part is written. - */ - private static Map evictOldestReusableChunks(Map cacheSizes) { - long retained = 0; - for (long bytes : cacheSizes.values()) retained += bytes; - Map evicted = new LinkedHashMap<>(); - java.util.Iterator> iterator = cacheSizes.entrySet().iterator(); - while (retained > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) { - Map.Entry entry = iterator.next(); - retained -= entry.getValue(); - evicted.put(entry.getKey(), entry.getValue()); - iterator.remove(); - } - return evicted; - } - - private static boolean consumeChunkReference(Map refs, String guid) { - Integer count = refs.get(guid); - if (count == null) return false; - if (count <= 1) { - refs.remove(guid); - return true; - } - refs.put(guid, count - 1); - return false; - } - - /** - * Pure disk schedule shared by execution and preflight. The next eight part references pin - * at most eight GUIDs (including across file boundaries); other reusable data is oldest-first - * capped at 64 MiB. Incoming downloads reserve their entire decoded size before submission. - */ - static final class PrefetchSchedule { - final List parts = new ArrayList<>(); - final Map resident; - final Map refs; - final Map chunks; - int cursor; - - PrefetchSchedule(List files, Map chunks, Map cached) { - for (FileInfo file : files) for (ChunkPart part : file.parts) parts.add(part.guidStr()); - this.chunks = chunks; - resident = new LinkedHashMap<>(cached); - refs = chunkReferences(files); - } - - Set window() { - return new LinkedHashSet<>(parts.subList(cursor, Math.min(parts.size(), cursor + PREFETCH_PARTS))); - } - - List fill() { - List fetch = new ArrayList<>(); - for (String guid : window()) if (!resident.containsKey(guid)) { - ChunkInfo chunk = chunks.get(guid); - if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid); - resident.put(guid, chunkWindowSize(chunk)); - fetch.add(guid); - } - return fetch; - } - - List consume() { - List release = new ArrayList<>(); - String guid = parts.get(cursor++); - if (consumeChunkReference(refs, guid)) { - resident.remove(guid); - release.add(guid); - } - Set pinned = window(); - long reusable = 0; - for (Map.Entry entry : resident.entrySet()) { - if (!pinned.contains(entry.getKey())) reusable += entry.getValue(); - } - Iterator> it = resident.entrySet().iterator(); - while (reusable > MAX_RETAINED_CHUNK_CACHE_BYTES && it.hasNext()) { - Map.Entry entry = it.next(); - if (pinned.contains(entry.getKey())) continue; - reusable -= entry.getValue(); - release.add(entry.getKey()); - it.remove(); - } - return release; - } - - long bytes() { - long bytes = 0; - for (long size : resident.values()) bytes += size; - return bytes; - } - } - - private static final class ChunkPrefetch implements AutoCloseable { - final PrefetchSchedule schedule; - final File cacheDir; - final AtomicBoolean cancel; - final AtomicBoolean stopped = new AtomicBoolean(); - final ExecutorService workers = Executors.newFixedThreadPool(PREFETCH_PARTS); - final Map> jobs = new LinkedHashMap<>(); - final Set connections = ConcurrentHashMap.newKeySet(); - final String chunkDir; - final List cdns; - - ChunkPrefetch(List files, Map chunks, Map cached, - File cacheDir, String chunkDir, List cdns, AtomicBoolean cancel) { - schedule = new PrefetchSchedule(files, chunks, cached); - this.cacheDir = cacheDir; - this.chunkDir = chunkDir; - this.cdns = cdns; - this.cancel = cancel; - } - - File await(String guid) throws Exception { - for (String fetch : schedule.fill()) { - jobs.put(fetch, workers.submit(() -> downloadChunkStreaming(schedule.chunks.get(fetch), - chunkDir, cdns, new File(cacheDir, fetch), cancel, stopped, connections))); - } - Future job = jobs.get(guid); - if (job != null) while (true) { - if (cancel.get()) throw new IOException("cancelled"); - try { - if (!job.get(100, TimeUnit.MILLISECONDS)) { - if (cancel.get()) throw new IOException("cancelled"); - throw new InstallException("Failed to download Epic chunk " + guid); - } - break; - } catch (TimeoutException ignored) { } - } - return new File(cacheDir, guid); - } - - void consume() throws InstallException { - for (String guid : schedule.consume()) { - // Only the writer advances the schedule, after closing its chunk reader. - jobs.remove(guid); - File chunk = new File(cacheDir, guid); - if (chunk.exists() && !chunk.delete() && chunk.exists()) { - throw new InstallException("Could not release Epic cache chunk " + guid); - } - } - } - - @Override public void close() { - stopped.set(true); - for (HttpURLConnection connection : connections) connection.disconnect(); - workers.shutdownNow(); - boolean interrupted = false; - while (true) { - try { - if (workers.awaitTermination(100, TimeUnit.MILLISECONDS)) break; - for (HttpURLConnection connection : connections) connection.disconnect(); - } catch (InterruptedException e) { interrupted = true; } - } - if (interrupted) Thread.currentThread().interrupt(); - } - } - - static boolean isValidCachedChunk(ChunkInfo chunk, File file) { - if (!file.isFile()) return false; - if (chunk.windowSize > 0 && file.length() != chunk.windowSize) return false; - byte[] expected = chunk.sha1; - if (expected == null || expected.length != 20) return true; - boolean allZero = true; - for (byte b : expected) if (b != 0) { allZero = false; break; } - if (allZero) return true; - try (FileInputStream in = new FileInputStream(file)) { - MessageDigest sha = MessageDigest.getInstance("SHA-1"); - byte[] buffer = new byte[64 * 1024]; - int count; - while ((count = in.read(buffer)) != -1) sha.update(buffer, 0, count); - return MessageDigest.isEqual(sha.digest(), expected); - } catch (Exception e) { - return false; - } - } - - private static MessageDigest expectedDigest(byte[] expected) throws InstallException { - if (expected == null || expected.length != 20) return null; - for (byte b : expected) if (b != 0) { - try { return MessageDigest.getInstance("SHA-1"); } - catch (java.security.NoSuchAlgorithmException e) { throw new InstallException("SHA-1 verification unavailable"); } - } - return null; - } - - static Map chunkReferences(List files) { - Map refs = new LinkedHashMap<>(); - for (FileInfo file : files) for (ChunkPart part : file.parts) { - String guid = part.guidStr(); - refs.put(guid, refs.containsKey(guid) ? refs.get(guid) + 1 : 1); - } - return refs; - } - - /** - * Simulates reserved parallel prefetch → append part → last-reference eviction order. The returned - * additional-space peak is measured against the current free-space reading: existing verified - * chunks are already reflected in that reading and count as reclaimed only when actually freed. - * Old output bytes are reclaimed after the complete new file is staged and verified, never - * before its parts are written. The same schedule is used for scratch-cache game-volume checks. - */ - static long estimatePeakAdditionalBytes(List files, Map chunksByGuid, - Map cachedChunkBytes, boolean includeFileWrites) { - return estimatePeakAdditionalBytes(files, chunksByGuid, cachedChunkBytes, includeFileWrites, new LinkedHashMap<>()); - } - - static long estimatePeakAdditionalBytes(List files, Map chunksByGuid, - Map cachedChunkBytes, boolean includeFileWrites, - Map oldFileBytes) { - PrefetchSchedule schedule = new PrefetchSchedule(files, chunksByGuid, cachedChunkBytes); - long initialCache = schedule.bytes(); - long fileDelta = 0, peak = 0; - for (FileInfo file : files) { - for (ChunkPart part : file.parts) { - schedule.fill(); - peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache); - if (includeFileWrites) { - fileDelta += part.size & 0xFFFFFFFFL; - peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache); - } - schedule.consume(); - } - // The verified staging file coexists with the old output until its atomic swap. - if (includeFileWrites) fileDelta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0; - } - return Math.max(0, peak); - } - - private static void copyChunkPart(File chunkFile, ChunkPart part, BufferedOutputStream out, - MessageDigest fileSha, AtomicBoolean cancel, String relPath) throws IOException { - long offset = part.offset & 0xFFFFFFFFL; - long remaining = part.size & 0xFFFFFFFFL; - long length = chunkFile.length(); - if (offset > length || remaining > length - offset) { - throw new IOException("Chunk " + part.guidStr() + " is too short for " + relPath); - } - try (FileInputStream in = new FileInputStream(chunkFile)) { - in.getChannel().position(offset); - byte[] buffer = new byte[64 * 1024]; - while (remaining > 0) { - if (cancel.get()) throw new IOException("cancelled"); - int count = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); - if (count < 0) throw new IOException("Unexpected end of chunk " + part.guidStr()); - out.write(buffer, 0, count); - if (fileSha != null) fileSha.update(buffer, 0, count); - remaining -= count; - } - } - } - - - public static List parseCdnUrls(String json) { - List result = new ArrayList<>(); - try { - JSONArray manifests = new JSONObject(json).optJSONArray("manifests"); - if (manifests == null) return result; - for (int i = 0; i < manifests.length(); i++) { - JSONObject m = manifests.optJSONObject(i); - if (m == null) continue; - String uri = m.optString("uri", ""); - int buildsIdx = uri.indexOf("/Builds"); - if (buildsIdx < 0) continue; - String baseUrl = uri.substring(0, buildsIdx); - if (!baseUrl.startsWith("http") || baseUrl.contains("cloudflare.epicgamescdn.com")) continue; - String afterBase = uri.substring(buildsIdx); - int q = afterBase.indexOf('?'); - if (q >= 0) afterBase = afterBase.substring(0, q); - int lastSlash = afterBase.lastIndexOf('/'); - if (lastSlash < 0) continue; - String cloudDir = afterBase.substring(0, lastSlash); - StringBuilder auth = new StringBuilder(); - JSONArray params = m.optJSONArray("queryParams"); - if (params != null) for (int j = 0; j < params.length(); j++) { - JSONObject p = params.optJSONObject(j); - if (p == null) continue; - auth.append(auth.length() == 0 ? "?" : "&").append(p.optString("name")).append('=').append(p.optString("value")); - } - result.add(new CdnUrl(baseUrl, cloudDir, auth.toString())); - } - } catch (Exception e) { - Log.e(TAG, "CDN parse: " + e.getClass().getSimpleName()); - } - return result; - } - - public static byte[] downloadManifest(String json, List cdnUrls) { - try { - JSONArray manifests = new JSONObject(json).optJSONArray("manifests"); - if (manifests == null || manifests.length() == 0) return null; - String firstUri = manifests.getJSONObject(0).optString("uri", ""); - String uriPath = firstUri.contains("?") ? firstUri.substring(0, firstUri.indexOf('?')) : firstUri; - int lastSlash = uriPath.lastIndexOf('/'); - if (lastSlash < 0) return null; - String manifestFilename = uriPath.substring(lastSlash + 1); - for (CdnUrl cdn : cdnUrls) { - byte[] bytes = downloadBytes(cdn.baseUrl + cdn.cloudDir + "/" + manifestFilename + cdn.authParams); - if (bytes != null && bytes.length > 4) return bytes; - } - } catch (Exception e) { - Log.e(TAG, "manifest download: " + e.getClass().getSimpleName()); - } - return null; - } - - - public static Manifest parseManifest(byte[] bytes) { - try { - ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN); - int magic = buf.getInt(); - if (magic != 0x44BEC00C) return parseJsonManifest(bytes); - int headerSize = buf.getInt(); - int sizeUncompressed = buf.getInt(); - buf.getInt(); - buf.position(buf.position() + 20); - int storedAs = buf.get() & 0xFF; - int version = buf.getInt(); - String chunkDir = version >= 15 ? "ChunksV4" : version >= 6 ? "ChunksV3" : version >= 3 ? "ChunksV2" : "Chunks"; - buf.position(headerSize); - byte[] body = new byte[buf.remaining()]; - buf.get(body); - if ((storedAs & 1) != 0) { - Inflater inflater = new Inflater(); - inflater.setInput(body); - byte[] decomp = new byte[sizeUncompressed]; - int got = inflater.inflate(decomp); - inflater.end(); - if (got != sizeUncompressed) { Log.e(TAG, "manifest inflate size mismatch"); return null; } - body = decomp; - } - ByteBuffer b = ByteBuffer.wrap(body).order(ByteOrder.LITTLE_ENDIAN); - Manifest m = new Manifest(); - m.chunkDir = chunkDir; - int metaStart = b.position(); - int metaSize = b.getInt(); - try { - b.get(); - b.getInt(); - b.get(); - b.getInt(); - readFString(b); - m.buildVersion = readFString(b); - m.launchExe = readFString(b).replace('\\', '/'); - m.launchCommand = readFString(b); - int prereqIds = b.getInt(); - for (int i = 0; i < prereqIds && i < 64; i++) readFString(b); - m.prereqName = readFString(b); - m.prereqPath = readFString(b).replace('\\', '/'); - readFString(b); - m.prereqRead = true; - } catch (Exception e) { - Log.w(TAG, "manifest meta: " + e.getClass().getSimpleName()); - } - b.position(metaStart + metaSize); - - int cdlStart = b.position(); - int cdlSize = b.getInt(); - b.get(); - int chunkCount = b.getInt(); - List chunks = new ArrayList<>(chunkCount); - for (int i = 0; i < chunkCount; i++) chunks.add(new ChunkInfo()); - for (ChunkInfo c : chunks) { c.guid[0] = b.getInt(); c.guid[1] = b.getInt(); c.guid[2] = b.getInt(); c.guid[3] = b.getInt(); } - for (ChunkInfo c : chunks) c.hash = b.getLong(); - for (ChunkInfo c : chunks) { c.sha1 = new byte[20]; b.get(c.sha1); } - for (ChunkInfo c : chunks) c.groupNum = b.get() & 0xFF; - for (ChunkInfo c : chunks) c.windowSize = b.getInt(); - for (ChunkInfo c : chunks) c.fileSize = b.getLong(); - b.position(cdlStart + cdlSize); - - int fmlStart = b.position(); - int fmlSize = b.getInt(); - b.get(); - int fileCount = b.getInt(); - List files = new ArrayList<>(fileCount); - for (int i = 0; i < fileCount; i++) files.add(new FileInfo()); - for (FileInfo f : files) f.filename = readFString(b); - for (int i = 0; i < fileCount; i++) readFString(b); - for (FileInfo f : files) { f.sha1 = new byte[20]; b.get(f.sha1); } - b.position(b.position() + fileCount); - for (int i = 0; i < fileCount; i++) { - int tagCount = b.getInt(); - for (int j = 0; j < tagCount; j++) { String tag = readFString(b); if (tag != null && !tag.isEmpty()) files.get(i).installTags.add(tag); } - } - for (FileInfo f : files) { - int partCount = b.getInt(); - for (int j = 0; j < partCount; j++) { - int partStart = b.position(); - int partStructSize = b.getInt(); - ChunkPart part = new ChunkPart(); - part.guid[0] = b.getInt(); part.guid[1] = b.getInt(); part.guid[2] = b.getInt(); part.guid[3] = b.getInt(); - part.offset = b.getInt(); part.size = b.getInt(); - f.parts.add(part); - b.position(partStart + partStructSize); - } - } - b.position(fmlStart + fmlSize); - Map seen = new LinkedHashMap<>(chunkCount * 2); - for (ChunkInfo c : chunks) seen.put(c.guidStr(), c); - m.uniqueChunks = new ArrayList<>(seen.values()); - m.files = files; - return m; - } catch (Exception e) { - Log.e(TAG, "manifest parse: " + e.getClass().getSimpleName()); - return null; - } - } - - private static Manifest parseJsonManifest(byte[] bytes) { - try { - JSONObject root = new JSONObject(new String(bytes, StandardCharsets.UTF_8)); - int manifestVersion = 0; - try { manifestVersion = (int) blobToNum(root.optString("ManifestFileVersion", "0")); } catch (NumberFormatException ignored) {} - Manifest m = new Manifest(); - m.chunkDir = manifestVersion >= 15 ? "ChunksV4" : manifestVersion >= 6 ? "ChunksV3" : manifestVersion >= 3 ? "ChunksV2" : "ChunksV4"; - m.launchExe = root.optString("LaunchExeString", root.optString("LaunchExe", "")).replace('\\', '/'); - m.launchCommand = root.optString("LaunchCommand", ""); - m.buildVersion = root.optString("BuildVersionString", ""); - m.prereqName = root.optString("PrereqName", ""); - m.prereqPath = root.optString("PrereqPath", "").replace('\\', '/'); - m.prereqRead = true; - JSONObject chunkHashList = root.optJSONObject("ChunkHashList"); - JSONObject dataGroupList = root.optJSONObject("DataGroupList"); - JSONObject chunkFilesizeList = root.optJSONObject("ChunkFilesizeList"); - if (chunkHashList == null) return null; - Map chunkMap = new LinkedHashMap<>(); - Iterator keys = chunkHashList.keys(); - while (keys.hasNext()) { - String guidHex = keys.next(); - if (guidHex.length() < 32) continue; - ChunkInfo c = new ChunkInfo(); - for (int k = 0; k < 4; k++) c.guid[k] = (int) Long.parseLong(guidHex.substring(k * 8, k * 8 + 8), 16); - try { c.hash = blobToNum(chunkHashList.getString(guidHex)); } catch (NumberFormatException ignored) {} - if (dataGroupList != null) { try { c.groupNum = (int) blobToNum(dataGroupList.optString(guidHex, "0")); } catch (NumberFormatException ignored) {} } - if (chunkFilesizeList != null) { try { c.fileSize = blobToNum(chunkFilesizeList.optString(guidHex, "0")); } catch (NumberFormatException ignored) {} } - chunkMap.put(guidHex, c); - } - JSONArray fileList = root.optJSONArray("FileManifestList"); - if (fileList == null) return null; - List files = new ArrayList<>(fileList.length()); - for (int i = 0; i < fileList.length(); i++) { - JSONObject fo = fileList.getJSONObject(i); - FileInfo fi = new FileInfo(); - fi.filename = fo.optString("Filename", ""); - JSONArray tags = fo.optJSONArray("InstallTags"); - if (tags != null) for (int t = 0; t < tags.length(); t++) { String tag = tags.optString(t, ""); if (!tag.isEmpty()) fi.installTags.add(tag); } - JSONArray parts = fo.optJSONArray("FileChunkParts"); - if (parts != null) for (int j = 0; j < parts.length(); j++) { - JSONObject po = parts.getJSONObject(j); - ChunkPart part = new ChunkPart(); - String g = po.optString("Guid", ""); - if (g.length() >= 32) for (int k = 0; k < 4; k++) part.guid[k] = (int) Long.parseLong(g.substring(k * 8, k * 8 + 8), 16); - try { part.offset = (int) blobToNum(po.optString("Offset", "0")); } catch (NumberFormatException ignored) {} - try { part.size = (int) blobToNum(po.optString("Size", "0")); } catch (NumberFormatException ignored) {} - fi.parts.add(part); - } - files.add(fi); - } - m.uniqueChunks = new ArrayList<>(chunkMap.values()); - m.files = files; - return m; - } catch (Exception e) { - Log.e(TAG, "json manifest parse: " + e.getClass().getSimpleName()); - return null; - } - } - - static long blobToNum(String blob) { - long num = 0; - for (int i = 0, shift = 0; i < blob.length(); i += 3, shift += 8) { - String group = blob.substring(i, Math.min(i + 3, blob.length())); - for (int k = 0; k < group.length(); k++) { - char ch = group.charAt(k); - if (ch < '0' || ch > '9') throw new NumberFormatException("not a blob: " + blob); - } - if (shift < 64) num += Long.parseLong(group) << shift; - } - return num; - } - - public static String readFString(ByteBuffer buf) { - int length = buf.getInt(); - if (length == 0) return ""; - if (length < 0) { - int chars = (-length) - 1; - byte[] bytes = new byte[chars * 2]; - buf.get(bytes); - buf.getShort(); - return new String(bytes, StandardCharsets.UTF_16LE); - } - byte[] bytes = new byte[length - 1]; - buf.get(bytes); - buf.get(); - return new String(bytes, StandardCharsets.US_ASCII); - } - - - public static byte[] downloadBytes(String urlStr) { - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(urlStr).openConnection(); - conn.setConnectTimeout(30000); - conn.setReadTimeout(60000); - conn.setRequestProperty("User-Agent", UA); - if (conn.getResponseCode() != 200) { Log.w(TAG, "HTTP " + conn.getResponseCode() + " for " + StoreLog.redactUrl(urlStr)); return null; } - return EpicAuthClient.readAllBytes(conn.getInputStream()); - } catch (Exception e) { - Log.w(TAG, "download error [" + StoreLog.redactUrl(urlStr) + "]: " + e.getClass().getSimpleName()); - return null; - } finally { - if (conn != null) conn.disconnect(); - } - } - - private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List cdnUrls, - File outFile, AtomicBoolean cancel, AtomicBoolean stopped, - Set connections) { - String chunkPath = chunk.getPath(chunkDir); - File tmp = new File(outFile.getPath() + ".part"); - tmp.delete(); - for (CdnUrl cdn : cdnUrls) { - if (cancel.get() || stopped.get()) return false; - HttpURLConnection conn = null; - try { - conn = (HttpURLConnection) new URL(cdn.baseUrl + cdn.cloudDir + "/" + chunkPath).openConnection(); - connections.add(conn); - if (cancel.get() || stopped.get()) return false; - conn.setConnectTimeout(30000); - conn.setReadTimeout(60000); - conn.setRequestProperty("User-Agent", UA); - if (conn.getResponseCode() != 200) { conn.disconnect(); continue; } - MessageDigest sha = null; - if (chunk.sha1 != null && chunk.sha1.length == 20) { - boolean allZero = true; - for (byte x : chunk.sha1) if (x != 0) { allZero = false; break; } - if (!allZero) { try { sha = MessageDigest.getInstance("SHA-1"); } catch (Exception ignored) {} } - } - try (InputStream in = conn.getInputStream(); FileOutputStream fos = new FileOutputStream(tmp)) { - byte[] hdrBuf = new byte[41]; - readFully(in, hdrBuf); - ByteBuffer hdr = ByteBuffer.wrap(hdrBuf).order(ByteOrder.LITTLE_ENDIAN); - if (hdr.getInt() != 0xB1FE3AA2) { Log.w(TAG, "bad chunk magic"); continue; } - hdr.getInt(); - int headerSize = hdr.getInt(); - int compressedSize = hdr.getInt(); - hdr.position(hdr.position() + 24); - int storedAs = hdr.get() & 0xFF; - if (headerSize > 41) skipFully(in, headerSize - 41); - byte[] iobuf = new byte[131072]; - long outputBytes = 0; - if ((storedAs & 1) != 0) { - Inflater inflater = new Inflater(); - byte[] obuf = new byte[131072]; - int remaining = compressedSize; - try { - while (remaining > 0 && !inflater.finished()) { - if (cancel.get() || stopped.get()) break; - if (inflater.needsInput()) { - int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); - if (n <= 0) break; - remaining -= n; - inflater.setInput(iobuf, 0, n); - } - int out = inflater.inflate(obuf); - if (out > 0) { - outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); - fos.write(obuf, 0, out); - if (sha != null) sha.update(obuf, 0, out); - } - if (out == 0 && !inflater.needsInput() && !inflater.finished()) { - throw new IOException("invalid compressed Epic chunk"); - } - } - int out; - while ((out = inflater.inflate(obuf)) > 0) { - if (cancel.get() || stopped.get()) break; - outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); - fos.write(obuf, 0, out); - if (sha != null) sha.update(obuf, 0, out); - } - if (!cancel.get() && !stopped.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk"); - } finally { - inflater.end(); - } - } else { - int remaining = compressedSize; - while (remaining > 0) { - if (cancel.get() || stopped.get()) break; - int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); - if (n <= 0) break; - outputBytes = checkedChunkWriteSize(chunk, outputBytes, n); - fos.write(iobuf, 0, n); - if (sha != null) sha.update(iobuf, 0, n); - remaining -= n; - } - if (!cancel.get() && !stopped.get() && remaining != 0) throw new IOException("truncated Epic chunk"); - } - } - if (cancel.get() || stopped.get()) { tmp.delete(); conn.disconnect(); return false; } - if (chunk.windowSize > 0 && tmp.length() != chunk.windowSize) { tmp.delete(); conn.disconnect(); continue; } - if (sha != null && !MessageDigest.isEqual(sha.digest(), chunk.sha1)) { tmp.delete(); conn.disconnect(); continue; } - if (!tmp.renameTo(outFile)) { tmp.delete(); conn.disconnect(); continue; } - conn.disconnect(); - return true; - } catch (Exception e) { - Log.w(TAG, "chunk " + chunk.guidStr() + " failed: " + e.getMessage()); - tmp.delete(); - if (conn != null) conn.disconnect(); - } finally { - if (conn != null) { - connections.remove(conn); - conn.disconnect(); - } - tmp.delete(); - } - } - tmp.delete(); - Log.e(TAG, "every CDN failed for chunk " + chunk.guidStr()); - return false; - } - - private static long checkedChunkWriteSize(ChunkInfo chunk, long written, int count) throws IOException { - long next = written + count; - if (next > chunkWindowSize(chunk)) throw new IOException("chunk exceeds expected window size"); - return next; - } - - private static long chunkWindowSize(ChunkInfo chunk) { - // Legacy JSON manifests omit window sizes; their chunks use 1 MiB windows. - return chunk.windowSize > 0 ? chunk.windowSize : 1024L * 1024; - } - - private static void readFully(InputStream in, byte[] buf) throws IOException { - int off = 0; - while (off < buf.length) { int n = in.read(buf, off, buf.length - off); if (n < 0) throw new IOException("stream ended"); off += n; } - } - - private static void skipFully(InputStream in, int count) throws IOException { - byte[] skip = new byte[Math.min(count, 4096)]; - int remaining = count; - while (remaining > 0) { int n = in.read(skip, 0, Math.min(skip.length, remaining)); if (n < 0) throw new IOException("stream ended"); remaining -= n; } - } - - - public static List getRequiredInstallFiles(Manifest manifest) { - List required = new ArrayList<>(); - for (FileInfo f : manifest.files) if (f.installTags.isEmpty()) required.add(f); - return required.isEmpty() ? new ArrayList<>(manifest.files) : required; - } - - public static List resolveInstallFiles(Manifest manifest, List selectedTags) { - if (selectedTags == null) return new ArrayList<>(manifest.files); - if (selectedTags.isEmpty()) return getRequiredInstallFiles(manifest); - Set want = new LinkedHashSet<>(selectedTags); - List out = new ArrayList<>(); - for (FileInfo f : manifest.files) { - if (f.installTags.isEmpty()) { out.add(f); continue; } - for (String t : f.installTags) if (want.contains(t)) { out.add(f); break; } - } - return out.isEmpty() ? getRequiredInstallFiles(manifest) : out; - } - - public static List uniqueChunksForFiles(Manifest manifest, List files) { - Map byGuid = new LinkedHashMap<>(); - for (ChunkInfo c : manifest.uniqueChunks) byGuid.put(c.guidStr(), c); - Set seen = new LinkedHashSet<>(); - List chunks = new ArrayList<>(); - for (FileInfo f : files) for (ChunkPart p : f.parts) { String g = p.guidStr(); if (seen.add(g)) { ChunkInfo c = byGuid.get(g); if (c != null) chunks.add(c); } } - return chunks; - } - - public static boolean fileExistsWithCorrectHash(File outputFile, long expectedSize, byte[] expectedHash) { - if (outputFile == null || !outputFile.exists() || outputFile.length() != expectedSize) return false; - if (expectedHash == null || expectedHash.length != 20) return false; - boolean allZero = true; - for (byte x : expectedHash) if (x != 0) { allZero = false; break; } - if (allZero) return false; - try (FileInputStream fis = new FileInputStream(outputFile)) { - MessageDigest digest = MessageDigest.getInstance("SHA-1"); - byte[] buf = new byte[64 * 1024]; - int n; - while ((n = fis.read(buf)) != -1) digest.update(buf, 0, n); - return MessageDigest.isEqual(digest.digest(), expectedHash); - } catch (Exception e) { - return false; - } - } - - public static String[] fetchPrereq(String manifestApiJson) { - try { - List cdnUrls = parseCdnUrls(manifestApiJson); - if (cdnUrls.isEmpty()) return null; - byte[] bytes = downloadManifest(manifestApiJson, cdnUrls); - if (bytes == null) return null; - Manifest m = parseManifest(bytes); - if (m == null || !m.prereqRead) { Log.w(TAG, "manifest prerequisite not read"); return null; } - Log.i(TAG, "manifest " + m.buildVersion + ": " + m.files.size() + " files, prerequisite " + (m.prereqName.isEmpty() && m.prereqPath.isEmpty() ? "none" : m.prereqName + " (" + m.prereqPath + ")")); - return new String[] { m.prereqName, m.prereqPath }; - } catch (Exception e) { - return null; - } - } - - public static long fetchInstallSizeBytes(String manifestApiJson, List installTags) { - try { - List cdnUrls = parseCdnUrls(manifestApiJson); - if (cdnUrls.isEmpty()) return -1; - byte[] bytes = downloadManifest(manifestApiJson, cdnUrls); - if (bytes == null) return -1; - Manifest m = parseManifest(bytes); - if (m == null) return -1; - long total = 0; - for (FileInfo f : resolveInstallFiles(m, installTags)) total += Math.max(f.fileSize(), 0); - return total > 0 ? total : -1; - } catch (Exception e) { - return -1; - } - } - - - static void deleteDir(File dir) { - if (!dir.exists()) return; - File[] files = dir.listFiles(); - if (files != null) for (File f : files) { if (f.isDirectory()) deleteDir(f); else f.delete(); } - dir.delete(); - } - - private static String fmt(long bytes) { - if (bytes >= 1073741824L) return String.format("%.1f GB", bytes / 1073741824.0); - if (bytes >= 1048576L) return String.format("%.1f MB", bytes / 1048576.0); - return String.format("%.0f KB", bytes / 1024.0); - } -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.kt new file mode 100644 index 0000000000..4595c1938c --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.kt @@ -0,0 +1,76 @@ +package com.droiddeck.launcher.stores.epic + +import org.json.JSONArray +import org.json.JSONObject + +class EpicCdn(val base: String, val dir: String, val auth: String) + +class EpicManifest internal constructor( + internal val bytes: ByteArray, + val files: List, + val buildVersion: String, + val launchExe: String, + val prereqName: String, + val prereqPath: String, + val prereqRead: Boolean, +) { + class FileEntry(val path: String, private val size: Long, val tags: List) { + fun fileSize(): Long = size + } +} + +object EpicDownloadManager { + private fun info(json: String): JSONObject? = runCatching { JSONObject(json) }.getOrNull() + + @JvmStatic + fun parseCdnUrls(json: String): List { + val info = info(json) ?: return emptyList() + val list = EpicNative.call("build.cdns", JSONObject().put("info", info))?.optJSONArray("cdns") ?: return emptyList() + return (0 until list.length()).mapNotNull { list.optJSONObject(it) }.map { + EpicCdn(it.optString("base"), it.optString("dir"), it.optString("auth")) + } + } + + @JvmStatic + fun downloadManifest(json: String, cdns: List): ByteArray? { + val info = info(json) ?: return null + if (cdns.isEmpty()) return null + val list = JSONArray() + cdns.forEach { list.put(JSONObject().put("base", it.base).put("dir", it.dir).put("auth", it.auth)) } + return EpicNative.fetch("manifest.download", JSONObject().put("info", info).put("cdns", list)) + } + + @JvmStatic + fun parseManifest(bytes: ByteArray): EpicManifest? { + val summary = EpicNative.inspect(bytes, JSONObject().put("op", "summary")) ?: return null + if (summary.has("error")) return null + val list = summary.optJSONArray("files") ?: return null + val files = (0 until list.length()).mapNotNull { list.optJSONObject(it) }.map { f -> + val tags = f.optJSONArray("tags") + EpicManifest.FileEntry(f.optString("path"), f.optLong("size", 0L), List(tags?.length() ?: 0) { tags!!.optString(it) }) + } + return EpicManifest( + bytes, files, + buildVersion = summary.optString("buildVersion"), + launchExe = summary.optString("launchExe"), + prereqName = summary.optString("prereqName"), + prereqPath = summary.optString("prereqPath"), + prereqRead = summary.optBoolean("prereqRead", false), + ) + } + + @JvmStatic + fun resolveInstallFiles(manifest: EpicManifest, tags: List): List { + val answer = EpicNative.inspect(manifest.bytes, JSONObject().put("op", "select").put("tags", JSONArray(tags))) ?: return emptyList() + val indices = answer.optJSONArray("indices") ?: return emptyList() + return (0 until indices.length()).mapNotNull { manifest.files.getOrNull(indices.optInt(it, -1)) } + } + + @JvmStatic + fun fetchPrereq(json: String): Array? { + val info = info(json) ?: return null + val answer = EpicNative.call("build.prereq", JSONObject().put("info", info)) ?: return null + if (answer.has("error") || !answer.optBoolean("read", false)) return null + return arrayOf(answer.optString("name"), answer.optString("path")) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicGame.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicGame.java deleted file mode 100644 index 43ba8069b3..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicGame.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.droiddeck.launcher.stores.epic; - -public class EpicGame { - public String appName = ""; - public String namespace = ""; - public String catalogItemId = ""; - public String title = ""; - public String developer = ""; - public String description = ""; - public String artCover = ""; - public String artSquare = ""; - public String version = ""; - public long installSize = 0L; - public boolean canRunOffline = true; - public boolean isDLC = false; - public String baseGameCatalogItemId = ""; - public String releaseDate = ""; - public String publisher = ""; -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstallTags.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstallTags.kt index 9ad136a849..8d1018c540 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstallTags.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstallTags.kt @@ -1,37 +1,24 @@ package com.droiddeck.launcher.stores.epic +import org.json.JSONObject import java.util.Locale object EpicInstallTags { - private val LANGUAGE_TO_TAGS: Map> = mapOf( - "ar" to listOf("Arabic", "ar"), "bg" to listOf("Bulgarian", "bg-BG", "bg"), - "zh-Hans" to listOf("Chinese", "ChineseSimplified", "zh-Hans", "zh_Hans", "zh"), "zh-Hant" to listOf("ChineseTraditional", "zh-Hant", "zh_Hant"), - "cs" to listOf("Czech", "cs-CZ", "cs"), "da" to listOf("Danish", "da-DK", "da"), "nl" to listOf("Dutch", "nl-NL", "nl"), - "en" to listOf("English", "en-US", "en"), "fi" to listOf("Finnish", "fi-FI", "fi"), "fr" to listOf("French", "fr-FR", "fr"), - "de" to listOf("German", "de-DE", "de"), "el" to listOf("Greek", "el-GR", "el"), "hu" to listOf("Hungarian", "hu-HU", "hu"), - "it" to listOf("Italian", "it-IT", "it"), "ja" to listOf("Japanese", "ja-JP", "ja"), "ko" to listOf("Korean", "ko-KR", "ko"), - "no" to listOf("Norwegian", "nb-NO", "no"), "pl" to listOf("Polish", "pl-PL", "pl"), "pt" to listOf("Portuguese", "pt-PT", "pt"), - "pt-BR" to listOf("PortugueseBrazilian", "Brazilian", "pt-BR", "br"), "ro" to listOf("Romanian", "ro-RO", "ro"), - "ru" to listOf("Russian", "ru-RU", "ru"), "es" to listOf("Spanish", "es-ES", "es"), "es-419" to listOf("SpanishLatinAmerica", "Latam", "es-MX", "es_mx"), - "sv" to listOf("Swedish", "sv-SE", "sv"), "th" to listOf("Thai", "th-TH", "th"), "tr" to listOf("Turkish", "tr-TR", "tr"), - "uk" to listOf("Ukrainian", "uk-UA", "uk"), "vi" to listOf("Vietnamese", "vi-VN", "vi"), - ) + private class Answer(val locale: Locale, val tags: List, val launchLocale: String) - private fun key(locale: Locale): String? { - val l = locale.language.lowercase(Locale.ROOT) - val r = locale.country.uppercase(Locale.ROOT) - return when (l) { - "zh" -> if (r == "TW" || r == "HK" || r == "MO") "zh-Hant" else "zh-Hans" - "pt" -> if (r == "BR") "pt-BR" else "pt" - "es" -> if (r == "ES" || r.isEmpty()) "es" else "es-419" - "nb", "nn" -> "no" - else -> if (LANGUAGE_TO_TAGS.containsKey(l)) l else null - } + @Volatile private var last: Answer? = null + + private fun current(): Answer? { + val locale = Locale.getDefault() + last?.takeIf { it.locale == locale }?.let { return it } + val answer = EpicNative.call("locale", JSONObject().put("language", locale.language).put("region", locale.country)) ?: return null + val list = answer.optJSONArray("tags") + val tags = List(list?.length() ?: 0) { list!!.optString(it) }.filter { it.isNotEmpty() } + return Answer(locale, tags, answer.optString("launchLocale", "en").ifEmpty { "en" }).also { last = it } } @JvmStatic - fun tagsForDevice(): List = LANGUAGE_TO_TAGS[key(Locale.getDefault()) ?: "en"] ?: LANGUAGE_TO_TAGS.getValue("en") + fun tagsForDevice(): List = current()?.tags ?: emptyList() - @JvmStatic - fun localeCodeForDevice(): String = key(Locale.getDefault())?.substringBefore('-') ?: "en" + fun launchLocale(): String = current()?.launchLocale ?: "en" } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstaller.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstaller.kt new file mode 100644 index 0000000000..ee6a2181d0 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicInstaller.kt @@ -0,0 +1,144 @@ +package com.droiddeck.launcher.stores.epic + +import android.content.Context +import com.droiddeck.launcher.R +import com.droiddeck.launcher.frontend.WinCompSources +import com.droiddeck.launcher.stores.CatalogItem +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreExe +import com.droiddeck.launcher.stores.StoreGameSidecar +import com.droiddeck.launcher.stores.StoreInstallRoot +import com.droiddeck.launcher.stores.StoreInstalls +import com.droiddeck.launcher.stores.download.DownloadQueue +import com.droiddeck.launcher.stores.download.DownloadStage +import com.droiddeck.launcher.stores.download.StoreDownloadTier +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong + +internal class EpicInstaller( + private val app: Context, + private val item: CatalogItem, + private val folder: File, +) : DownloadQueue.DownloadJob { + private val cancelled = AtomicBoolean(false) + private val namespace = item.extra["namespace"].orEmpty() + private val catalogItemId = item.extra["catalogItemId"].orEmpty() + + private fun scratch(): File = StoreInstallRoot.scratchDir(app, Store.EPIC, item.id) + + override fun run(handle: DownloadQueue.JobHandle): String? { + handle.stage(DownloadStage.MANIFEST, app.getString(R.string.stores_sync_sign_in)) + val token = EpicCredentialStore.getValidAccessToken(app) ?: throw IllegalStateException("Not signed in to Epic Games") + val cover = item.tallImageUrl ?: item.imageUrl + StoreInstalls.begin(folder, Store.EPIC, item.id, item.title, cover, item.imageUrl) + if (cancelled.get()) return null + + handle.stage(DownloadStage.MANIFEST, "Reading the build list…") + val info = EpicApiClient.getManifestApiJson(token, namespace, catalogItemId, item.id) + ?: throw IllegalStateException("The manifest could not be fetched") + if (cancelled.get()) return null + + val tier = StoreDownloadTier.current(app) + val request = JSONObject() + .put("info", JSONObject(info)) + .put("gameDir", folder.absolutePath) + .put("scratchDir", if (StoreInstallRoot.isRemovable(app, folder)) scratch().absolutePath else "") + .put("storageLabel", StoreInstallRoot.labelFor(app, folder)) + .put("tags", JSONArray(EpicInstallTags.tagsForDevice())) + .put("maxWorkers", tier.networkWindow.coerceIn(1, 128)) + .put("processWorkers", tier.processWorkers) + .put("caBundle", "") + + val shown = AtomicLong(-1) + val installing = AtomicBoolean(false) + val prefs = EpicPrefs.get(app) + val listener = object : EpicNative.InstallListener { + override fun onStatus(text: String) { handle.stage(DownloadStage.MANIFEST, text) } + + override fun onCheck(doneBytes: Long, totalBytes: Long, files: Int, filesTotal: Int) { + handle.stage(DownloadStage.MANIFEST, "Checking files…") + handle.stageProgress(doneBytes, totalBytes, files, filesTotal) + } + + override fun onSizes(pendingBytes: Long, diskBytes: Long) { + handle.diskSize(diskBytes) + if (diskBytes > 0) prefs.edit().putLong("size_${item.id}", diskBytes).apply() + } + + override fun onProgress(written: Long, total: Long, files: Int, filesTotal: Int) { + if (installing.get()) return + while (true) { + val before = shown.get() + if (written < before) return + if (shown.compareAndSet(before, written)) break + } + handle.progress(written, total, "Downloading chunks ($files/$filesTotal files)") + } + + override fun onInstall(done: Long, total: Long, files: Int, filesTotal: Int) { + installing.set(true) + handle.stage(DownloadStage.INSTALL) + handle.stageProgress(done, total, files, filesTotal, bytes = true) + } + + override fun onLog(line: String) { handle.log("epic: $line") } + + override fun beforeReplace(): Boolean = try { + StoreInstalls.markUpdating(folder) + true + } catch (e: Exception) { + handle.log("epic: the install record could not be switched to updating (${e.javaClass.simpleName})") + false + } + + override fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) = Unit + } + + val outcome = EpicNative.install(request, listener, cancelled) + if (outcome.cancelled || cancelled.get() && !outcome.success) return null + if (!outcome.success) throw IllegalStateException(outcome.error.ifEmpty { "Native Epic install failed" }) + val result = outcome.result ?: JSONObject() + + handle.stage(DownloadStage.INSTALL, app.getString(R.string.stores_registering)) + val picked = StoreExe.pick(folder, result.optString("launchExe"), item.title) + if (picked.isEmpty()) handle.log("epic: no exe found in ${folder.name}; pick one in Steam settings › Added games") + val creds = EpicCredentialStore.load(app) + val additional = EpicApiClient.getCustomAttribute(token, namespace, catalogItemId, "AdditionalCommandLine").orEmpty() + val deployment = result.optString("deploymentId") + val argsAnswer = EpicNative.call( + "launch.args", + JSONObject().put("displayName", creds?.displayName.orEmpty()).put("accountId", creds?.accountId.orEmpty()) + .put("namespace", namespace).put("locale", EpicInstallTags.launchLocale()).put("appName", item.id) + .put("deploymentId", deployment).put("additional", additional), + ) + val argList = argsAnswer?.optJSONArray("args") + val args = List(argList?.length() ?: 0) { argList!!.optString(it) }.filter { it.isNotEmpty() } + + val extra = LinkedHashMap() + extra["appName"] = item.id + extra["namespace"] = namespace + extra["catalogItemId"] = catalogItemId + if (deployment.isNotEmpty()) extra["deploymentId"] = deployment + extra[WinCompSources.EPIC_PREREQ_PATH] = result.optString("prereqPath") + extra[WinCompSources.EPIC_PREREQ_NAME] = result.optString("prereqName") + if (result.optBoolean("prereqRead", false)) extra[WinCompSources.LIST_READ] = "1" + + val sidecar = StoreGameSidecar( + Store.EPIC, item.id, item.title, exe = picked.ifEmpty { "game.exe" }, args = args, + installVersion = result.optString("buildVersion"), installedAt = System.currentTimeMillis(), + cover = cover, hero = item.imageUrl, extra = extra, + ) + StoreInstalls.complete(app, folder, sidecar, onStep = { done, steps -> handle.stageProgress(done.toLong(), steps.toLong()) }) + val disk = result.optLong("diskBytes", 0L) + if (disk > 0) prefs.edit().putLong("size_${item.id}", disk).apply() + return folder.path + } + + override fun cancel(deleteFiles: Boolean) { + cancelled.set(true) + if (deleteFiles) StoreInstalls.discard(app, folder, listOf(File(folder, ".chunks"), scratch())) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLaunchData.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLaunchData.kt deleted file mode 100644 index 1452ef447f..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLaunchData.kt +++ /dev/null @@ -1,41 +0,0 @@ -package com.droiddeck.launcher.stores.epic - -import android.content.Context -import android.util.Log -import com.droiddeck.launcher.stores.StoreGameSidecar - -object EpicLaunchData { - private const val TAG = "EpicLaunch" - - fun arguments(context: Context, token: String?, appName: String, namespace: String, catalogItemId: String, deploymentId: String): List { - val creds = EpicCredentialStore.load(context) - val displayName = creds?.displayName?.takeIf { it.isNotEmpty() } ?: "EpicUser" - val accountId = creds?.accountId?.takeIf { it.isNotEmpty() } ?: "0" - val args = ArrayList() - args += "-EpicPortal" - args += "-epicusername=\"${sanitize(displayName).replace("\"", "")}\"" - args += "-epicuserid=${sanitize(accountId)}" - args += "-epicsandboxid=${sanitize(namespace)}" - args += "-epiclocale=${EpicInstallTags.localeCodeForDevice()}" - args += "-epicapp=${sanitize(appName)}" - args += "-epicenv=Prod" - if (deploymentId.isNotEmpty()) args += "-epicdeploymentid=${sanitize(deploymentId)}" - if (token != null && catalogItemId.isNotEmpty()) { - val extra = EpicApiClient.getAdditionalCommandLine(token, namespace, catalogItemId) - if (!extra.isNullOrBlank()) { - val clean = extra.filter { it >= ' ' && it != '\u007f' }.trim() - if (clean.isNotEmpty()) { args += clean; Log.i(TAG, "$appName: extra command line present") } - } - } - return args - } - - fun extras(appName: String, namespace: String, catalogItemId: String, deploymentId: String): Map = buildMap { - put("appName", appName); put("namespace", namespace); put("catalogItemId", catalogItemId) - if (deploymentId.isNotEmpty()) put("deploymentId", deploymentId) - } - - fun wantsExchangeCode(sidecar: StoreGameSidecar): Boolean = sidecar.store == com.droiddeck.launcher.stores.Store.EPIC - - private fun sanitize(s: String): String = s.replace("\n", "").replace("\r", "").replace("\u0000", "") -} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLibrary.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLibrary.kt index f84fa1dec9..be665881b8 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLibrary.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLibrary.kt @@ -1,116 +1,180 @@ package com.droiddeck.launcher.stores.epic import android.content.Context -import android.content.SharedPreferences -import android.util.Log import com.droiddeck.launcher.R import com.droiddeck.launcher.stores.CatalogItem import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.cleanStoreText import org.json.JSONArray import org.json.JSONObject +import java.net.URLEncoder import java.util.concurrent.atomic.AtomicBoolean +data class EpicGame( + val appName: String, + val namespace: String, + val catalogItemId: String, + val title: String, + val artCover: String = "", + val artSquare: String = "", + val developer: String = "", + val description: String = "", + val version: String = "", + val installSize: Long = 0L, + val canRunOffline: Boolean = true, + val releaseDate: String = "", + val publisher: String = "", + val isDlc: Boolean = false, +) + object EpicLibrary { - private const val TAG = "EpicLibrary" - private const val CACHE_KEY = "library_cache" - private const val LAST_SYNC_KEY = "library_synced_at" - private const val THROTTLE_MS = 6L * 60L * 60L * 1000L + const val KEY_CACHE = "library_cache" + const val KEY_SYNCED_AT = "library_synced_at" + private const val THROTTLE_MS = 6 * 60 * 60 * 1000L + private const val MAX_SIZE = 1L shl 40 - private val syncing = AtomicBoolean(false) - - sealed interface SyncResult { - class Ok(val games: List) : SyncResult - class Failed(val message: String) : SyncResult - object NotLoggedIn : SyncResult - object Busy : SyncResult - object Throttled : SyncResult + sealed class SyncResult { + class Ok(val games: List) : SyncResult() + class Failed(val message: String) : SyncResult() + object NotLoggedIn : SyncResult() + object Throttled : SyncResult() + object Busy : SyncResult() } - fun cached(context: Context): List { - val json = EpicPrefs.get(context).getString(CACHE_KEY, null) ?: return emptyList() - return runCatching { - val arr = JSONArray(json) - val list = ArrayList(arr.length()) - for (i in 0 until arr.length()) { - val j = arr.optJSONObject(i) ?: continue - val g = EpicGame() - g.appName = j.optString("appName", ""); g.namespace = j.optString("namespace", ""); g.catalogItemId = j.optString("catalogItemId", "") - g.title = j.optString("title", ""); g.artCover = j.optString("artCover", ""); g.artSquare = j.optString("artSquare", "") - g.developer = j.optString("developer", ""); g.description = j.optString("description", ""); g.version = j.optString("version", "") - g.installSize = j.optLong("installSize", 0L).let { if (it > 1_099_511_627_776L) 0L else it } - g.canRunOffline = j.optBoolean("canRunOffline", true) - g.releaseDate = j.optString("releaseDate", "") - g.publisher = j.optString("publisher", "") - if (g.appName.isNotEmpty()) list.add(g) - } - list.distinctBy { it.appName }.sortedBy { it.title.lowercase() } - }.getOrDefault(emptyList()) + private val running = AtomicBoolean(false) + + fun toCacheJson(games: List): String = JSONArray().apply { + for (g in games) put( + JSONObject() + .put("appName", g.appName).put("namespace", g.namespace).put("catalogItemId", g.catalogItemId) + .put("title", g.title).put("artCover", g.artCover).put("artSquare", g.artSquare) + .put("developer", g.developer).put("description", g.description).put("version", g.version) + .put("installSize", g.installSize).put("canRunOffline", g.canRunOffline) + .put("releaseDate", g.releaseDate).put("publisher", g.publisher), + ) + }.toString() + + fun fromCacheJson(text: String?): List { + if (text.isNullOrBlank()) return emptyList() + val array = runCatching { JSONArray(text) }.getOrNull() ?: return emptyList() + val games = (0 until array.length()).mapNotNull { array.optJSONObject(it) }.mapNotNull { o -> + val appName = o.optString("appName", "").ifEmpty { return@mapNotNull null } + EpicGame( + appName = appName, + namespace = o.optString("namespace", ""), + catalogItemId = o.optString("catalogItemId", ""), + title = o.optString("title", "").ifEmpty { appName }, + artCover = o.optString("artCover", ""), + artSquare = o.optString("artSquare", ""), + developer = o.optString("developer", ""), + description = o.optString("description", ""), + version = o.optString("version", ""), + installSize = o.optLong("installSize", 0L).takeIf { it in 0..MAX_SIZE } ?: 0L, + canRunOffline = o.optBoolean("canRunOffline", true), + releaseDate = o.optString("releaseDate", ""), + publisher = o.optString("publisher", ""), + ) + } + return tidy(games) } - fun rememberSize(context: Context, appName: String, bytes: Long) { - if (bytes <= 0) return - val p = EpicPrefs.get(context) - p.edit().putLong("size_$appName", bytes).apply() + private fun tidy(games: List): List = + games.distinctBy { it.appName }.sortedBy { it.title.lowercase() } + + fun cached(context: Context): List = fromCacheJson(EpicPrefs.get(context).getString(KEY_CACHE, null)) + + fun forget(context: Context) { + EpicPrefs.get(context).edit().remove(KEY_CACHE).remove(KEY_SYNCED_AT).apply() } - fun sync(context: Context, force: Boolean = false, onStatus: (String) -> Unit = {}): SyncResult { - if (!syncing.compareAndSet(false, true)) return SyncResult.Busy + fun toCatalogItem(context: Context, game: EpicGame): CatalogItem { + val prefs = EpicPrefs.get(context) + val wide = game.artSquare.ifEmpty { null } + val tall = game.artCover.ifEmpty { null } + val title = game.title.ifEmpty { game.appName } + return CatalogItem( + store = Store.EPIC, + id = game.appName, + title = title, + imageUrl = wide ?: tall, + tallImageUrl = tall ?: wide, + developer = game.developer, + publisher = game.publisher, + releaseDate = game.releaseDate.ifEmpty { prefs.getString("release_${game.appName}", "").orEmpty() }, + description = cleanStoreText(game.description), + owned = true, + sizeBytes = prefs.getLong("size_${game.appName}", 0L).takeIf { it > 0 } ?: game.installSize, + storeUrl = "https://store.epicgames.com/en-US/browse?q=" + URLEncoder.encode(title, "UTF-8"), + extra = mapOf("namespace" to game.namespace, "catalogItemId" to game.catalogItemId, "version" to game.version), + ) + } + + internal fun gameFrom(record: JSONObject, details: JSONObject?, before: EpicGame?): EpicGame { + val appName = record.optString("appName") + val d = details ?: JSONObject() + fun pick(key: String, fallback: String) = d.optString(key, "").ifEmpty { fallback } + return EpicGame( + appName = appName, + namespace = record.optString("namespace"), + catalogItemId = record.optString("catalogItemId"), + title = pick("title", before?.title?.takeIf { it != appName } ?: appName), + artCover = pick("tall", before?.artCover.orEmpty()), + artSquare = pick("wide", before?.artSquare.orEmpty()), + developer = pick("developer", before?.developer.orEmpty()), + description = pick("description", before?.description.orEmpty()), + version = before?.version.orEmpty(), + installSize = before?.installSize ?: 0L, + canRunOffline = if (details != null) d.optBoolean("canRunOffline", true) else before?.canRunOffline ?: true, + releaseDate = pick("releaseDate", before?.releaseDate.orEmpty()), + publisher = pick("publisher", before?.publisher.orEmpty()), + isDlc = d.optBoolean("dlc", false), + ) + } + + internal fun visible(games: List): List { + val base = games.filterNot { it.isDlc } + return tidy(if (base.isEmpty()) games else base) + } + + fun sync(context: Context, force: Boolean, status: (String) -> Unit): SyncResult { + if (!running.compareAndSet(false, true)) return SyncResult.Busy try { val app = context.applicationContext - val p = EpicPrefs.get(app) - val cachedList = cached(app) - val lastSync = p.getLong(LAST_SYNC_KEY, 0L) - if (!force && cachedList.isNotEmpty() && System.currentTimeMillis() - lastSync < THROTTLE_MS) return SyncResult.Throttled - onStatus(app.getString(R.string.stores_sync_sign_in)) + val prefs = EpicPrefs.get(app) + val cache = cached(app) + status(app.getString(R.string.stores_sync_sign_in)) + if (!EpicCredentialStore.isLoggedIn(app)) return SyncResult.NotLoggedIn + val syncedAt = prefs.getLong(KEY_SYNCED_AT, 0L) + if (!force && cache.isNotEmpty() && System.currentTimeMillis() - syncedAt in 0 until THROTTLE_MS) return SyncResult.Throttled val token = EpicCredentialStore.getValidAccessToken(app) ?: return SyncResult.NotLoggedIn - onStatus(app.getString(R.string.stores_sync_game_list)) - val raw = EpicApiClient.getLibraryItems(token) - if (raw.isNullOrEmpty()) return if (cachedList.isEmpty()) SyncResult.Failed(app.getString(R.string.stores_sync_no_games, Store.EPIC.label)) else SyncResult.Ok(cachedList) - val total = raw.size - var done = 0 - for (game in raw) { - EpicApiClient.enrichFromCatalog(token, game) - done++ - if (done % 5 == 0) onStatus(app.getString(R.string.stores_sync_details, done, total)) + + status(app.getString(R.string.stores_sync_game_list)) + val answer = EpicNative.call("library.owned", JSONObject().put("accessToken", token)) + ?: return SyncResult.Failed(app.getString(R.string.stores_sync_failed)) + if (answer.optBoolean("signIn", false)) return SyncResult.NotLoggedIn + if (answer.has("error")) return SyncResult.Failed(answer.optString("error").ifEmpty { app.getString(R.string.stores_sync_failed) }) + val records = answer.optJSONArray("records") ?: JSONArray() + if (records.length() == 0) { + return if (cache.isNotEmpty()) SyncResult.Ok(cache) + else SyncResult.Failed(app.getString(R.string.stores_sync_no_games, Store.EPIC.label)) } - val main = raw.filter { !it.isDLC } - val display = (if (main.isEmpty()) raw else main).distinctBy { it.appName }.sortedWith { a, b -> a.title.compareTo(b.title, ignoreCase = true) } - for (fresh in display) { - val old = cachedList.firstOrNull { it.appName == fresh.appName } ?: continue - fresh.version = old.version - fresh.installSize = old.installSize + + val before = cache.associateBy { it.appName } + val total = records.length() + val games = ArrayList(total) + for (i in 0 until total) { + val record = records.optJSONObject(i) ?: continue + if (i % 5 == 0) status(app.getString(R.string.stores_sync_details, i, total)) + val details = EpicApiClient.catalogDetails(token, record.optString("namespace"), record.optString("catalogItemId")) + games += gameFrom(record, details, before[record.optString("appName")]) } - saveCache(p, display) - p.edit().putLong(LAST_SYNC_KEY, System.currentTimeMillis()).apply() - Log.i(TAG, "sync: raw=${raw.size} shown=${display.size}") - return SyncResult.Ok(display) + val shown = visible(games) + prefs.edit().putString(KEY_CACHE, toCacheJson(shown)).putLong(KEY_SYNCED_AT, System.currentTimeMillis()).apply() + return SyncResult.Ok(shown) } catch (e: Exception) { - Log.w(TAG, "sync failed: ${e.message}") return SyncResult.Failed(e.message ?: context.getString(R.string.stores_sync_failed)) } finally { - syncing.set(false) + running.set(false) } } - - private fun saveCache(prefs: SharedPreferences, games: List) { - runCatching { - val arr = JSONArray() - for (g in games) arr.put(JSONObject().put("appName", g.appName).put("namespace", g.namespace).put("catalogItemId", g.catalogItemId) - .put("title", g.title).put("artCover", g.artCover).put("artSquare", g.artSquare).put("developer", g.developer) - .put("description", g.description).put("version", g.version).put("installSize", g.installSize).put("canRunOffline", g.canRunOffline) - .put("releaseDate", g.releaseDate).put("publisher", g.publisher)) - prefs.edit().putString(CACHE_KEY, arr.toString()).apply() - } - } - - fun toCatalogItem(context: Context, g: EpicGame): CatalogItem = CatalogItem( - store = Store.EPIC, id = g.appName, title = g.title.ifBlank { g.appName }, - imageUrl = g.artSquare.ifBlank { null } ?: g.artCover.ifBlank { null }, - tallImageUrl = g.artCover.ifBlank { null } ?: g.artSquare.ifBlank { null }, - developer = g.developer, publisher = g.publisher, description = com.droiddeck.launcher.stores.cleanStoreText(g.description), owned = true, - releaseDate = g.releaseDate.ifEmpty { EpicPrefs.get(context).getString("release_${g.appName}", "").orEmpty() }, - sizeBytes = EpicPrefs.get(context).getLong("size_${g.appName}", g.installSize), - storeUrl = "https://store.epicgames.com/en-US/browse?q=${java.net.URLEncoder.encode(g.title, "UTF-8")}", - extra = mapOf("namespace" to g.namespace, "catalogItemId" to g.catalogItemId, "version" to (g.version ?: "")), - ) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLogin.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLogin.kt new file mode 100644 index 0000000000..ff804a85e3 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicLogin.kt @@ -0,0 +1,49 @@ +package com.droiddeck.launcher.stores.epic + +import android.util.Log +import android.webkit.WebView +import com.droiddeck.launcher.R +import com.droiddeck.launcher.stores.StoreLoginActivity +import org.json.JSONObject + +internal class EpicLogin private constructor( + override val startUrl: String, + override val userAgent: String, + private val redirectPrefix: String, +) : StoreLoginActivity.Flow { + override val readsPage: Boolean get() = true + + override fun isRedirect(url: String): Boolean = url.startsWith(redirectPrefix) + + override fun finish(activity: StoreLoginActivity, view: WebView, url: String): String? { + val code = EpicNative.call("auth.codeFromPage", JSONObject().put("text", url))?.optString("code", "").orEmpty() + if (code.isEmpty()) { + Log.w(TAG, "the redirect page carried no authorization code") + return activity.getString(R.string.stores_login_error_verification) + } + val answer = EpicNative.call("auth.signIn", JSONObject().put("code", code)) + val tokens = answer?.let(EpicCredentialStore.TokenResult::from) + if (tokens == null) { + Log.w(TAG, "code exchange failed: ${answer?.optString("error").orEmpty().ifEmpty { "no answer" }}") + return activity.getString(R.string.stores_login_error_generic) + } + return try { + EpicCredentialStore.save(activity, tokens, merge = false) + null + } catch (e: Exception) { + Log.w(TAG, "saving the sign-in failed: ${e.javaClass.simpleName}") + activity.getString(R.string.stores_login_error_generic) + } + } + + companion object { + private const val TAG = "EpicLogin" + + fun create(): StoreLoginActivity.Flow? { + val config = EpicNative.call("auth.config") ?: return null + val start = config.optString("startUrl").ifEmpty { return null } + val prefix = config.optString("redirectPrefix").ifEmpty { return null } + return EpicLogin(start, config.optString("webViewUserAgent"), prefix) + } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt index 41b6637c29..2cf02a3bb8 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt @@ -1,189 +1,130 @@ package com.droiddeck.launcher.stores.epic import android.util.Log +import com.droiddeck.launcher.BuildConfig import com.droiddeck.launcher.stores.StoresNative +import org.json.JSONObject import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicReference object EpicNative { private const val TAG = "EpicNative" + private const val CANCEL_POLL_MS = 250L - interface Listener { - fun onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String) - fun onProgress(bytesDone: Long, bytesTotal: Long, done: Int, total: Int) - fun onLog(line: String) - fun onComplete(success: Boolean, error: String, bytes: Long) - } + @Volatile private var configured = false - interface InstallListener : Listener { - /** Synchronous approval before the first verified atomic replacement. False aborts. */ - fun beforeSwap(): Boolean - } - - /** One native budgeted fetch/assembly run. A loaded-engine failure is never a Java retry. */ - @JvmStatic - fun installAdaptive(request: String, cancel: AtomicBoolean, listener: InstallListener): Result = - drive("nativeInstallAdaptive", cancel, listener, strict = true) { inner -> - nativeInstallAdaptive(request, inner) - } - - /** Callbacks from a [stream] run; every method is called on a native thread. */ - interface StreamListener { - /** File bytes written so far (staging included), and the files verified and swapped in so far. */ - fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) - /** Engine log line (already in logcat under `EpicNative`). */ - fun onLog(line: String) - /** Once, before the first file replaces its destination; false (or a throw) fails the run with nothing replaced. */ - fun onBeforeReplace(): Boolean - /** Terminal; `bytes` = file bytes written. */ - fun onComplete(success: Boolean, error: String, bytes: Long) - } - - class Result( - @JvmField val started: Boolean, @JvmField val success: Boolean, @JvmField val cancelled: Boolean, - @JvmField val error: String, @JvmField val bytes: Long, @JvmField val done: Int, @JvmField val total: Int, - ) - - private class Completion(val success: Boolean, val error: String, val bytes: Long) - - @JvmStatic - fun run( - manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedChunks: Int, expectedBytes: Long, - caBundlePath: String, maxWorkers: Int, processWorkers: Int, cancel: AtomicBoolean?, listener: Listener, - ): Result = drive("nativeStart", cancel, listener) { inner -> - nativeStart(manifest, installDir, chunkCacheDir, cdnPrefixes, pendingFileIdx, expectedChunks, expectedBytes, caBundlePath, maxWorkers, processWorkers, inner) - } - - @JvmStatic - fun assemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, cancel: AtomicBoolean?, listener: Listener): Result = - drive("nativeAssemble", cancel, listener) { inner -> nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, inner) } - - /** - * The adaptive install of the files [pendingFileIdx] (indices into the manifest's file list), - * with at most [cacheBudget] bytes of shared chunks cached and at most [overlapBudget] planned - * bytes of replaced files staged at once (`-1` = no limit), staged in [stagingDir] (the - * manager's owned staging directory). `started == false` means nothing was fetched or written. - */ - @JvmStatic - fun stream( - manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, - cacheBudget: Long, overlapBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, cancel: AtomicBoolean?, listener: StreamListener, - ): Result { - if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) - val latch = CountDownLatch(1) - val completion = AtomicReference(null) - val doneRef = AtomicReference(0) - val totalRef = AtomicReference(pendingFileIdx.size) - val inner = object : StreamListener { - override fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) { - doneRef.set(filesDone); totalRef.set(filesTotal) - listener.onStreamProgress(bytesWritten, bytesTotal, filesDone, filesTotal, filesBytes) - } - override fun onLog(line: String) = listener.onLog(line) - override fun onBeforeReplace(): Boolean = runCatching { listener.onBeforeReplace() }.getOrDefault(false) - override fun onComplete(success: Boolean, error: String, bytes: Long) { - completion.set(Completion(success, error, bytes)) - latch.countDown() - runCatching { listener.onComplete(success, error, bytes) } + fun ready(): Boolean { + if (!StoresNative.ensureLoaded()) return false + if (!configured) synchronized(this) { + if (!configured) { + runCatching { nativeConfigure(BuildConfig.VERSION_NAME) } + .onFailure { Log.w(TAG, "configure: ${it.javaClass.simpleName}") } + configured = true } } - val handle: Long = try { - nativeStream(manifest, installDir, chunkCacheDir, stagingDir, cdnPrefixes, pendingFileIdx, expectedBytes, cacheBudget, overlapBudget, caBundlePath, maxWorkers, processWorkers, inner) + return true + } + + fun call(op: String, request: JSONObject = JSONObject()): JSONObject? { + if (!ready()) return null + val text = try { + nativeCall(op, request.toString()) } catch (t: Throwable) { - Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}") - return Result(false, false, false, "nativeStream: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0) + Log.w(TAG, "$op: ${t.javaClass.simpleName}") + null + } ?: return null + return runCatching { JSONObject(text) }.getOrNull() + } + + fun fetch(op: String, request: JSONObject): ByteArray? { + if (!ready()) return null + return try { + nativeFetch(op, request.toString()) + } catch (t: Throwable) { + Log.w(TAG, "$op: ${t.javaClass.simpleName}") + null } - if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) - var cancelSent = false + } + + fun send(op: String, request: JSONObject, body: ByteArray): JSONObject? { + if (!ready()) return null + val text = try { + nativeSend(op, request.toString(), body) + } catch (t: Throwable) { + Log.w(TAG, "$op: ${t.javaClass.simpleName}") + null + } ?: return null + return runCatching { JSONObject(text) }.getOrNull() + } + + fun inspect(manifest: ByteArray, request: JSONObject): JSONObject? { + if (!ready()) return null + val text = try { + nativeInspect(manifest, request.toString()) + } catch (t: Throwable) { + Log.w(TAG, "inspect: ${t.javaClass.simpleName}") + null + } ?: return null + return runCatching { JSONObject(text) }.getOrNull() + } + + interface InstallListener { + fun onStatus(text: String) + fun onCheck(doneBytes: Long, totalBytes: Long, files: Int, filesTotal: Int) + fun onSizes(pendingBytes: Long, diskBytes: Long) + fun onProgress(written: Long, total: Long, files: Int, filesTotal: Int) + fun onInstall(done: Long, total: Long, files: Int, filesTotal: Int) + fun onLog(line: String) + fun beforeReplace(): Boolean + fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) + } + + class InstallOutcome(val success: Boolean, val cancelled: Boolean, val error: String, val result: JSONObject?) + + fun install(request: JSONObject, events: InstallListener, cancelled: AtomicBoolean): InstallOutcome { + if (!ready()) return InstallOutcome(false, false, "Native Epic install failed: the store engine is not available", null) + val done = CountDownLatch(1) + val holder = arrayOfNulls(1) + val bridge = object : InstallListener by events { + override fun onComplete(success: Boolean, cancelled: Boolean, error: String, result: String) { + holder[0] = InstallOutcome(success, cancelled, error, result.takeIf { it.isNotEmpty() }?.let { runCatching { JSONObject(it) }.getOrNull() }) + done.countDown() + } + } + val handle = try { + nativeInstall(request.toString(), bridge) + } catch (t: Throwable) { + return InstallOutcome(false, false, "Native Epic install failed: ${t.javaClass.simpleName}", null) + } + if (handle == 0L) return holder[0] ?: InstallOutcome(false, false, "Native Epic install failed: the install could not start", null) try { - while (!latch.await(250, TimeUnit.MILLISECONDS)) { - if (!cancelSent && cancel != null && cancel.get()) { + var forwarded = false + while (true) { + val finished = try { + done.await(CANCEL_POLL_MS, TimeUnit.MILLISECONDS) + } catch (e: InterruptedException) { + cancelled.set(true) + false + } + if (finished) break + if (!forwarded && cancelled.get()) { nativeCancel(handle) - cancelSent = true + forwarded = true } } } finally { - if (completion.get() == null) nativeCancel(handle) nativeRelease(handle) } - val c = completion.get() - val cancelled = cancelSent || (c != null && !c.success && c.error == "cancelled") - val success = !cancelled && c != null && c.success - val error = when { cancelled -> "cancelled"; c == null -> "no completion"; else -> c.error } - return Result(true, success, cancelled, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get()) + return holder[0] ?: InstallOutcome(false, false, "Native Epic install failed: no result", null) } - private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, strict: Boolean = false, start: (Listener) -> Long): Result { - if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) - val latch = CountDownLatch(1) - val completion = AtomicReference(null) - val doneRef = AtomicReference(0) - val totalRef = AtomicReference(0) - val inner = object : InstallListener { - override fun beforeSwap(): Boolean = (listener as? InstallListener)?.beforeSwap() ?: false - override fun onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String) { totalRef.set(chunksTotal); listener.onPlan(chunksTotal, bytesTotal, chunkDir) } - override fun onProgress(bytesDone: Long, bytesTotal: Long, done: Int, total: Int) { doneRef.set(done); totalRef.set(total); listener.onProgress(bytesDone, bytesTotal, done, total) } - override fun onLog(line: String) = listener.onLog(line) - override fun onComplete(success: Boolean, error: String, bytes: Long) { - completion.set(Completion(success, error, bytes)) - latch.countDown() - runCatching { listener.onComplete(success, error, bytes) } - } - } - val handle: Long = try { - start(inner) - } catch (t: Throwable) { - Log.w(TAG, "${if (strict) "engine start failed (no fallback)" else "engine unavailable"}: ${t.javaClass.simpleName}: ${t.message}") - return Result(strict, false, false, "$what: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0) - } - if (handle == 0L) return Result(strict, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) - var cancelSent = false - try { - while (!latch.await(250, TimeUnit.MILLISECONDS)) { - if (cancel != null && cancel.get()) { - nativeCancel(handle) - cancelSent = true - if (strict) { - while (!latch.await(250, TimeUnit.MILLISECONDS)) { } - } else { - latch.await(5, TimeUnit.SECONDS) - } - break - } - } - } finally { - if (completion.get() == null) nativeCancel(handle) - nativeRelease(handle) - } - val c = completion.get() - val success = !cancelSent && c != null && c.success - val error = when { cancelSent -> "cancelled"; c == null -> "no completion"; else -> c.error } - return Result(true, success, cancelSent, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get()) - } - - @JvmStatic - private external fun nativeInstallAdaptive(request: String, listener: Listener): Long - - @JvmStatic - private external fun nativeStart( - manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedChunks: Int, expectedBytes: Long, - caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: Listener, - ): Long - - @JvmStatic - private external fun nativeAssemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, listener: Listener): Long - - @JvmStatic - private external fun nativeStream( - manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, - cacheBudget: Long, overlapBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: StreamListener, - ): Long - - @JvmStatic - private external fun nativeCancel(handle: Long) - - @JvmStatic - private external fun nativeRelease(handle: Long) + @JvmStatic private external fun nativeConfigure(appVersion: String) + @JvmStatic private external fun nativeCall(op: String, request: String): String? + @JvmStatic private external fun nativeFetch(op: String, request: String): ByteArray? + @JvmStatic private external fun nativeSend(op: String, request: String, body: ByteArray): String? + @JvmStatic private external fun nativeInspect(manifest: ByteArray, request: String): String? + @JvmStatic private external fun nativeInstall(request: String, listener: InstallListener): Long + @JvmStatic private external fun nativeCancel(handle: Long) + @JvmStatic private external fun nativeRelease(handle: Long) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicPrefs.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicPrefs.kt index 1510dbef9e..e1556aaa58 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicPrefs.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicPrefs.kt @@ -4,6 +4,8 @@ import android.content.Context import android.content.SharedPreferences object EpicPrefs { + const val FILE = "stores.epic" + @JvmStatic - fun get(context: Context): SharedPreferences = context.applicationContext.getSharedPreferences("stores.epic", Context.MODE_PRIVATE) + fun get(context: Context): SharedPreferences = context.applicationContext.getSharedPreferences(FILE, Context.MODE_PRIVATE) } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveSync.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveSync.kt new file mode 100644 index 0000000000..b23dfbc132 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveSync.kt @@ -0,0 +1,45 @@ +package com.droiddeck.launcher.stores.epic + +import com.droiddeck.launcher.stores.CloudFile +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException + +object EpicSaveSync { + private fun base(token: String, accountId: String, appName: String) = + JSONObject().put("accessToken", token).put("accountId", accountId).put("appName", appName) + + fun list(token: String, accountId: String, appName: String): List { + val answer = EpicNative.call("cloud.list", base(token, accountId, appName)) ?: throw IOException("Epic cloud is unavailable") + if (answer.has("error")) throw IOException(answer.optString("error")) + val files = answer.optJSONArray("files") ?: return emptyList() + return (0 until files.length()).mapNotNull { files.optJSONObject(it) }.map { f -> + CloudFile( + f.optString("name"), + f.optLong("modified", -1L), + md5 = f.optString("hash").ifEmpty { null }, + readLink = f.optString("readLink").ifEmpty { null }, + ) + } + } + + fun read(file: CloudFile): Pair { + val link = file.readLink ?: throw IOException("no download link for ${file.name}") + val bytes = EpicNative.fetch("cloud.read", JSONObject().put("url", link)) ?: throw IOException("Epic cloud download failed") + return bytes to file.modifiedMs + } + + fun write(token: String, accountId: String, appName: String, files: Map): Map { + if (files.isEmpty()) return emptyMap() + val answer = EpicNative.call("cloud.writeLinks", base(token, accountId, appName).put("names", JSONArray(files.keys.toList()))) + ?: throw IOException("Epic cloud is unavailable") + if (answer.has("error")) throw IOException(answer.optString("error")) + val links = answer.optJSONObject("links") ?: JSONObject() + return files.mapValues { (name, data) -> + val link = links.optString(name, "").ifEmpty { throw IOException("no upload link for $name") } + val sent = EpicNative.send("cloud.write", JSONObject().put("url", link), data) ?: throw IOException("Epic cloud upload failed") + if (sent.has("error")) throw IOException(sent.optString("error")) + -1L + } + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplate.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplate.kt new file mode 100644 index 0000000000..41af522d82 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplate.kt @@ -0,0 +1,31 @@ +package com.droiddeck.launcher.stores.epic + +object EpicSaveTemplate { + enum class Root { INSTALL_DIR, USER_PROFILE, DOCUMENTS, SAVED_GAMES, LOCAL_APPDATA, ROAMING_APPDATA } + + class Parsed(val root: Root, val rest: String) + + private val ROOTS = mapOf( + "installdir" to Root.INSTALL_DIR, + "userprofile" to Root.USER_PROFILE, + "userdir" to Root.DOCUMENTS, + "usersavedgames" to Root.SAVED_GAMES, + "appdata" to Root.LOCAL_APPDATA, + "localappdata" to Root.LOCAL_APPDATA, + "roamingappdata" to Root.ROAMING_APPDATA, + ) + + private val LEAD = Regex("^\\{([A-Za-z]+)\\}") + private val EPIC_ID = Regex("\\{EpicId\\}", RegexOption.IGNORE_CASE) + private val APP_NAME = Regex("\\{AppName\\}", RegexOption.IGNORE_CASE) + + fun parse(template: String, accountId: String, appName: String): Parsed? { + val norm = template.replace('\\', '/').trim() + val match = LEAD.find(norm) ?: return null + val root = ROOTS[match.groupValues[1].lowercase()] ?: return null + val rest = norm.substring(match.value.length) + .replace(EPIC_ID) { accountId } + .replace(APP_NAME) { appName } + return Parsed(root, rest) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicShelves.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicShelves.kt new file mode 100644 index 0000000000..71e538d674 --- /dev/null +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicShelves.kt @@ -0,0 +1,116 @@ +package com.droiddeck.launcher.stores.epic + +import android.content.Context +import com.droiddeck.launcher.stores.CatalogItem +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreShelves +import com.droiddeck.launcher.stores.cleanStoreText +import com.droiddeck.launcher.stores.matureByTags +import org.json.JSONArray +import org.json.JSONObject +import java.util.Locale + +object EpicShelves { + const val KEY_FEATURED = "store_featured" + const val KEY_FEATURED_AT = "store_featured_at" + private const val FRESH_MS = 3 * 60 * 60 * 1000L + private val SHELF_KEYS = listOf("new", "deals", "free", "trending") + + @Volatile private var memory: StoreShelves? = null + @Volatile private var memoryAt = 0L + + private fun fresh(at: Long) = System.currentTimeMillis() - at in 0 until FRESH_MS + + fun itemFrom(o: JSONObject): CatalogItem? { + val id = o.optString("id", "").ifEmpty { return null } + val title = o.optString("title", "").ifEmpty { return null } + val extra = LinkedHashMap() + o.optJSONObject("extra")?.let { e -> e.keys().forEach { k -> extra[k] = e.optString(k) } } + val mature = if (o.has("mature")) o.optBoolean("mature") else { + val tags = o.optJSONArray("allTags") + matureByTags(List(tags?.length() ?: 0) { tags!!.optString(it) }) + } + return CatalogItem( + store = Store.EPIC, id = id, title = title, + imageUrl = o.optString("image", "").ifEmpty { null }, + tallImageUrl = o.optString("tall", "").ifEmpty { null }, + tags = o.optString("tags", ""), + isFree = o.optBoolean("free", false), + hasPrice = o.optBoolean("hasPrice", false), + finalPrice = o.optString("final", ""), + originalPrice = o.optString("orig", ""), + discountPercent = o.optInt("disc", 0), + storeUrl = o.optString("url", ""), + developer = o.optString("dev", ""), + publisher = o.optString("pub", ""), + releaseDate = o.optString("rel", ""), + description = cleanStoreText(o.optString("desc", "")), + extra = extra, + mature = mature, + ) + } + + fun itemJson(i: CatalogItem): JSONObject = JSONObject() + .put("id", i.id).put("title", i.title).put("image", i.imageUrl ?: "").put("tall", i.tallImageUrl ?: "") + .put("tags", i.tags).put("free", i.isFree).put("hasPrice", i.hasPrice).put("final", i.finalPrice) + .put("orig", i.originalPrice).put("disc", i.discountPercent).put("url", i.storeUrl).put("dev", i.developer) + .put("pub", i.publisher).put("rel", i.releaseDate).put("desc", i.description) + .put("extra", JSONObject(i.extra as Map<*, *>)).put("mature", i.mature) + + fun fromJson(text: String?): StoreShelves? { + if (text.isNullOrBlank()) return null + val o = runCatching { JSONObject(text) }.getOrNull() ?: return null + fun shelf(key: String): List { + val a = o.optJSONArray(key) ?: return emptyList() + return (0 until a.length()).mapNotNull { a.optJSONObject(it) }.mapNotNull(::itemFrom).distinctBy { it.id } + } + val s = StoreShelves(shelf("new"), shelf("deals"), shelf("free"), shelf("trending")) + return if (s.isEmpty) null else s + } + + fun toJson(s: StoreShelves): String = JSONObject().apply { + listOf(s.whatsNew, s.deals, s.free, s.trending).zip(SHELF_KEYS).forEach { (items, key) -> + put(key, JSONArray().apply { items.forEach { put(itemJson(it)) } }) + } + }.toString() + + fun cached(context: Context): StoreShelves? = + memory ?: fromJson(EpicPrefs.get(context).getString(KEY_FEATURED, null))?.also { memory = it } + + fun featured(context: Context, force: Boolean): StoreShelves? { + val prefs = EpicPrefs.get(context) + if (!force) { + memory?.takeIf { fresh(memoryAt) }?.let { return it } + if (fresh(prefs.getLong(KEY_FEATURED_AT, 0L))) fromJson(prefs.getString(KEY_FEATURED, null))?.let { + memory = it; memoryAt = prefs.getLong(KEY_FEATURED_AT, 0L); return it + } + } + val country = Locale.getDefault().country.takeIf { it.length == 2 && it.all(Char::isLetter) }?.uppercase() ?: "US" + val answer = EpicNative.call("shop.shelves", JSONObject().put("country", country)) + val shelves = fromJson(answer?.toString()) + if (shelves == null) return fromJson(prefs.getString(KEY_FEATURED, null)) + val now = System.currentTimeMillis() + prefs.edit().putString(KEY_FEATURED, toJson(shelves)).putLong(KEY_FEATURED_AT, now).apply() + memory = shelves + memoryAt = now + return shelves + } + + fun markOwned(s: StoreShelves, library: List?): StoreShelves { + if (library.isNullOrEmpty()) return s + val byCatalogId = HashMap() + for (item in library) item.extra["catalogItemId"]?.takeIf { it.isNotEmpty() }?.let { byCatalogId[it] = item } + fun mark(list: List) = list.map { shelf -> + val owned = shelf.extra["items"].orEmpty().split(',').map { it.trim() }.firstNotNullOfOrNull { byCatalogId[it] } + ?: return@map shelf + owned.copy( + imageUrl = shelf.imageUrl ?: owned.imageUrl, + tallImageUrl = shelf.tallImageUrl ?: owned.tallImageUrl, + developer = owned.developer.ifBlank { shelf.developer }, + publisher = owned.publisher.ifBlank { shelf.publisher }, + releaseDate = owned.releaseDate.ifBlank { shelf.releaseDate }, + ) + } + return StoreShelves(mark(s.whatsNew), mark(s.deals), mark(s.free), mark(s.trending)) + } +} diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicStoreCatalog.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicStoreCatalog.kt deleted file mode 100644 index b7b6ac3643..0000000000 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicStoreCatalog.kt +++ /dev/null @@ -1,218 +0,0 @@ -package com.droiddeck.launcher.stores.epic - -import android.content.Context -import android.util.Log -import com.droiddeck.launcher.stores.CatalogItem -import com.droiddeck.launcher.stores.Store -import com.droiddeck.launcher.stores.StoreNet -import com.droiddeck.launcher.stores.StoreShelves -import org.json.JSONArray -import org.json.JSONObject -import java.text.SimpleDateFormat -import java.util.Date -import java.util.Locale -import java.util.TimeZone -import java.util.concurrent.ConcurrentHashMap - -object EpicStoreCatalog { - private const val TAG = "EpicStore" - private const val GRAPHQL = "https://store.epicgames.com/graphql" - private const val PROMOS = "https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions" - private const val FEATURED_TTL_MS = 3 * 60 * 60 * 1000L - private const val SEARCH_TTL_MS = 5 * 60 * 1000L - private const val KEY_FEATURED = "store_featured" - - private const val ELEMENT_FIELDS = - "title id namespace description effectiveDate releaseDate offerType productSlug urlSlug " + - "items{id namespace} keyImages{type url} seller{name} tags{name} customAttributes{key value} " + - "catalogNs{mappings(pageType:\"productHome\"){pageSlug pageType}} " + - "price(country:\$country){totalPrice{discountPrice originalPrice discount currencyCode " + - "fmtPrice(locale:\$locale){originalPrice discountPrice}}}" - - private class Cached(val value: T, val at: Long) - private var featuredCache: Cached? = null - private val searchCache = ConcurrentHashMap>>() - - private fun country(): String = Locale.getDefault().country.takeIf { it.length == 2 }?.uppercase(Locale.ROOT) ?: "US" - - private fun nowIso(): String { - val f = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) - f.timeZone = TimeZone.getTimeZone("UTC") - return f.format(Date()) - } - - private fun searchStore(count: Int, keywords: String? = null, sortBy: String = "relevancy", sortDir: String = "DESC", onSale: Boolean? = null, freeGame: Boolean? = null, releasedOnly: Boolean = false): List { - val args = StringBuilder("category:\"games/edition/base\",count:\$count,country:\$country,locale:\$locale,sortBy:\$sortBy,sortDir:\$sortDir,allowCountries:\$country") - if (keywords != null) args.append(",keywords:\$keywords") - if (onSale != null) args.append(",onSale:$onSale") - if (freeGame != null) args.append(",freeGame:$freeGame") - if (releasedOnly) args.append(",releaseDate:\"[,${nowIso()}]\"") - val query = "query q(\$count:Int,\$country:String!,\$locale:String,\$sortBy:String,\$sortDir:String" + - (if (keywords != null) ",\$keywords:String" else "") + "){Catalog{searchStore($args){elements{$ELEMENT_FIELDS} paging{total}}}}" - val vars = JSONObject().put("count", count).put("country", country()).put("locale", "en-US").put("sortBy", sortBy).put("sortDir", sortDir) - if (keywords != null) vars.put("keywords", keywords) - val resp = StoreNet.postJson(GRAPHQL, JSONObject().put("query", query).put("variables", vars).toString()) ?: return emptyList() - val elements = runCatching { JSONObject(resp).optJSONObject("data")?.optJSONObject("Catalog")?.optJSONObject("searchStore")?.optJSONArray("elements") } - .onFailure { Log.w(TAG, "searchStore parse failed: ${it.message}") }.getOrNull() ?: return emptyList() - return parseElements(elements) - } - - private fun parseElements(arr: JSONArray): List { - val out = ArrayList(arr.length()) - for (i in 0 until arr.length()) arr.optJSONObject(i)?.let { e -> parseElement(e)?.let { out.add(it) } } - return out.distinctBy { it.id } - } - - fun parseElement(e: JSONObject): CatalogItem? { - val id = e.optString("id", "") - val title = e.optString("title", "") - if (id.isEmpty() || title.isEmpty()) return null - val ns = e.optString("namespace", "") - var wide: String? = null; var tall: String? = null; var thumb: String? = null - e.optJSONArray("keyImages")?.let { imgs -> - for (k in 0 until imgs.length()) { - val img = imgs.optJSONObject(k) ?: continue - val url = img.optString("url", "") - if (url.isBlank()) continue - when (img.optString("type", "")) { - "DieselGameBox" -> wide = url - "OfferImageWide", "DieselStoreFrontWide" -> if (wide == null) wide = url - "DieselGameBoxTall" -> tall = url - "OfferImageTall", "DieselStoreFrontTall" -> if (tall == null) tall = url - "Thumbnail" -> if (thumb == null) thumb = url - } - } - } - val price = e.optJSONObject("price")?.optJSONObject("totalPrice") - val original = price?.optLong("originalPrice", -1L) ?: -1L - val discounted = price?.optLong("discountPrice", -1L) ?: -1L - val fmt = price?.optJSONObject("fmtPrice") - val fmtOriginal = fmt?.optString("originalPrice", "").orEmpty() - val fmtDiscount = fmt?.optString("discountPrice", "").orEmpty() - val hasPrice = original >= 0 && discounted >= 0 - val isFree = hasPrice && discounted == 0L - val discountPct = if (hasPrice && original > 0 && discounted < original) ((original - discounted) * 100 / original).toInt() else 0 - val finalPrice = when { isFree -> "Free"; fmtDiscount.isNotBlank() && fmtDiscount != "0" -> fmtDiscount; else -> fmtOriginal } - val tagLine = buildList { - e.optJSONArray("tags")?.let { tags -> for (t in 0 until tags.length()) { val n = tags.optJSONObject(t)?.optString("name").orEmpty(); if (n.isNotBlank() && n != "Windows" && size < 3) add(n) } } - }.joinToString(", ") - val allTags = buildList { e.optJSONArray("tags")?.let { t -> for (k in 0 until t.length()) t.optJSONObject(k)?.optString("name")?.let { add(it) } } } - val itemIds = buildList { - e.optJSONArray("items")?.let { items -> for (j in 0 until items.length()) items.optJSONObject(j)?.optString("id")?.takeIf { it.isNotBlank() }?.let { add(it) } } - } - val slug = pageSlugOf(e) - val storeUrl = if (slug.isNotBlank()) "https://store.epicgames.com/en-US/p/$slug" else "https://store.epicgames.com/en-US/browse?q=${java.net.URLEncoder.encode(title, "UTF-8")}" - return CatalogItem( - store = Store.EPIC, id = id, title = title, imageUrl = wide ?: thumb, tallImageUrl = tall ?: thumb, tags = tagLine, - isFree = isFree, hasPrice = hasPrice, finalPrice = finalPrice, originalPrice = if (discountPct > 0) fmtOriginal else "", discountPercent = discountPct, - storeUrl = storeUrl, developer = attribute(e, "developerName"), - publisher = attribute(e, "publisherName").ifEmpty { e.optJSONObject("seller")?.optString("name", "").orEmpty() }, - releaseDate = e.optString("releaseDate", e.optString("effectiveDate", "")), description = com.droiddeck.launcher.stores.cleanStoreText(e.optString("description", "")), - extra = buildMap { put("namespace", ns); if (itemIds.isNotEmpty()) put("items", itemIds.joinToString(",")); if (slug.isNotBlank()) put("slug", slug) }, - mature = com.droiddeck.launcher.stores.matureByTags(allTags), - ) - } - - private fun attribute(e: JSONObject, key: String): String { - val attrs = e.optJSONArray("customAttributes") ?: return "" - return (0 until attrs.length()).mapNotNull { attrs.optJSONObject(it) }.firstOrNull { it.optString("key") == key } - ?.optString("value", "")?.trim().orEmpty() - } - - private fun pageSlugOf(e: JSONObject): String { - e.optJSONObject("catalogNs")?.optJSONArray("mappings")?.let { m -> - for (i in 0 until m.length()) { val map = m.optJSONObject(i) ?: continue; if (map.optString("pageType") == "productHome") { val s = map.optString("pageSlug", ""); if (s.isNotBlank()) return s } } - } - var slug = e.optString("productSlug", "") - if (slug == "null") slug = "" - return slug.removeSuffix("/home") - } - - fun featured(context: Context, force: Boolean = false): StoreShelves? { - val now = System.currentTimeMillis() - featuredCache?.let { if (!force && now - it.at < FEATURED_TTL_MS) return it.value } - if (!force && now - EpicPrefs.get(context).getLong("store_featured_at", 0L) < FEATURED_TTL_MS) cachedFeatured(context)?.let { return it } - val freeNow = fetchPromos() - val onSale = searchStore(24, sortBy = "currentPrice", sortDir = "ASC", onSale = true).filter { it.discountPercent > 0 } - val newest = searchStore(24, sortBy = "releaseDate", sortDir = "DESC", releasedOnly = true) - val freeToPlay = searchStore(24, sortBy = "relevancy", sortDir = "DESC", freeGame = true) - val result = StoreShelves(whatsNew = newest, deals = onSale, free = (freeNow + freeToPlay).distinctBy { it.id }, trending = searchStore(24, sortBy = "relevancy", sortDir = "DESC", releasedOnly = true)) - if (!result.isEmpty) { - featuredCache = Cached(result, now) - persist(context, result) - Log.i(TAG, "shelves: freeNow=${freeNow.size} sale=${onSale.size} new=${newest.size} f2p=${freeToPlay.size}") - return result - } - Log.w(TAG, "every store feed came back empty; using the on-disk mirror") - return loadPersisted(context) - } - - fun cachedFeatured(context: Context): StoreShelves? = featuredCache?.value ?: loadPersisted(context) - - fun search(query: String): List { - val term = query.trim() - if (term.length < 2) return emptyList() - val key = term.lowercase(Locale.ROOT) - val now = System.currentTimeMillis() - searchCache[key]?.let { if (now - it.at < SEARCH_TTL_MS) return it.value } - val results = searchStore(30, keywords = term) - searchCache[key] = Cached(results, now) - return results - } - - private fun fetchPromos(): List { - val body = StoreNet.get("$PROMOS?locale=en-US&country=${country()}&allowCountries=${country()}") ?: return emptyList() - return runCatching { - val elements = JSONObject(body).optJSONObject("data")?.optJSONObject("Catalog")?.optJSONObject("searchStore")?.optJSONArray("elements") ?: JSONArray() - val out = ArrayList() - for (i in 0 until elements.length()) { - val el = elements.optJSONObject(i) ?: continue - val current = el.optJSONObject("promotions")?.optJSONArray("promotionalOffers") - if (current == null || current.length() == 0) continue - val inner = current.optJSONObject(0)?.optJSONArray("promotionalOffers") ?: continue - if (inner.length() == 0) continue - val discount = inner.optJSONObject(0)?.optJSONObject("discountSetting") - if (discount == null || discount.optInt("discountPercentage", -1) != 0) continue - val end = inner.optJSONObject(0)?.optString("endDate", "").orEmpty() - val base = parseElement(el) ?: continue - out.add(base.copy(isFree = true, hasPrice = true, finalPrice = "Free", originalPrice = base.originalPrice.ifBlank { - el.optJSONObject("price")?.optJSONObject("totalPrice")?.optJSONObject("fmtPrice")?.optString("originalPrice", "").orEmpty() - }, discountPercent = 100, tags = if (end.length >= 10) "Free until ${end.take(10)}" else base.tags)) - } - out.distinctBy { it.id } - }.onFailure { Log.w(TAG, "promos parse failed: ${it.message}") }.getOrDefault(emptyList()) - } - - private fun itemToJson(i: CatalogItem): JSONObject = JSONObject().apply { - put("id", i.id); put("title", i.title); put("image", i.imageUrl ?: ""); put("tall", i.tallImageUrl ?: ""); put("tags", i.tags) - put("free", i.isFree); put("hasPrice", i.hasPrice); put("final", i.finalPrice); put("orig", i.originalPrice); put("disc", i.discountPercent) - put("url", i.storeUrl); put("dev", i.developer); put("pub", i.publisher); put("rel", i.releaseDate); put("desc", i.description); put("extra", JSONObject(i.extra as Map<*, *>)) - if (i.mature) put("mature", true) - } - - private fun itemFromJson(o: JSONObject): CatalogItem { - val extra = HashMap() - o.optJSONObject("extra")?.let { e -> e.keys().forEach { k -> extra[k] = e.optString(k) } } - return CatalogItem( - store = Store.EPIC, id = o.optString("id"), title = o.optString("title"), imageUrl = o.optString("image").ifBlank { null }, tallImageUrl = o.optString("tall").ifBlank { null }, - tags = o.optString("tags"), isFree = o.optBoolean("free"), hasPrice = o.optBoolean("hasPrice"), finalPrice = o.optString("final"), originalPrice = o.optString("orig"), - discountPercent = o.optInt("disc"), storeUrl = o.optString("url"), developer = o.optString("dev"), publisher = o.optString("pub"), releaseDate = o.optString("rel"), description = o.optString("desc"), extra = extra, - mature = o.optBoolean("mature"), - ) - } - - private fun listToJson(l: List) = JSONArray().apply { l.forEach { put(itemToJson(it)) } } - private fun listFromJson(a: JSONArray?): List = if (a == null) emptyList() else List(a.length()) { a.optJSONObject(it) }.filterNotNull().map(::itemFromJson) - - private fun persist(context: Context, f: StoreShelves) { - runCatching { - EpicPrefs.get(context).edit().putString(KEY_FEATURED, JSONObject().put("new", listToJson(f.whatsNew)).put("deals", listToJson(f.deals)).put("free", listToJson(f.free)).put("trending", listToJson(f.trending)).toString()).putLong("store_featured_at", System.currentTimeMillis()).apply() - } - } - - private fun loadPersisted(context: Context): StoreShelves? = runCatching { - val s = EpicPrefs.get(context).getString(KEY_FEATURED, null) ?: return null - val o = JSONObject(s) - StoreShelves(listFromJson(o.optJSONArray("new")), listFromJson(o.optJSONArray("deals")), listFromJson(o.optJSONArray("free")), listFromJson(o.optJSONArray("trending"))).takeIf { !it.isEmpty } - }.getOrNull() -} diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt index 41e46975cc..f6bc614b9d 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt @@ -58,7 +58,8 @@ import com.droiddeck.launcher.stores.formatBytes @Composable -internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: FrontEndActions, onBack: () -> Unit) { +internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: FrontEndActions, lead: androidx.compose.ui.focus.FocusRequester, onBack: () -> Unit) { + val leadOn = Modifier.focusRequester(lead) val ctx = LocalContext.current val colors = MaterialTheme.colorScheme val pal = LocalPalette.current @@ -117,19 +118,19 @@ internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: Fro ActionChip(stringResource(R.string.stores_in_steam), ok = true) val gone = StoresState.uninstalling["${store.id}:$id"] if (gone != null) SecondaryButton(stringResource(R.string.stores_uninstalling), enabled = false, progress = gone) {} - else UninstallButton(title, installed, enabled = download == null) { + else UninstallButton(title, installed, enabled = download == null, modifier = leadOn) { StoresState.uninstall(ctx, installed) { StoresState.refresh(ctx); a.onLibraryChanged() } } } download != null -> { ActionChip(downloadLabel(download), ok = false) - if (download.state != DownloadState.PAUSED) SecondaryButton(stringResource(R.string.stores_dl_pause)) { com.droiddeck.launcher.stores.download.DownloadQueue.pause(download.key) } - else PrimaryButton(stringResource(R.string.stores_dl_resume), main = true) { com.droiddeck.launcher.stores.download.DownloadQueue.resume(ctx, download.key) } + if (download.state != DownloadState.PAUSED) SecondaryButton(stringResource(R.string.stores_dl_pause), modifier = leadOn) { com.droiddeck.launcher.stores.download.DownloadQueue.pause(download.key) } + else PrimaryButton(stringResource(R.string.stores_dl_resume), main = true, modifier = leadOn) { com.droiddeck.launcher.stores.download.DownloadQueue.resume(ctx, download.key) } ConfirmButton(stringResource(R.string.stores_dl_cancel), stringResource(R.string.stores_dl_cancel_confirm)) { com.droiddeck.launcher.stores.download.DownloadQueue.cancel(ctx, download.key) } } failedDownload != null -> { ActionChip(stringResource(R.string.stores_dl_failed), ok = false) - PrimaryButton(stringResource(R.string.stores_dl_resume), main = true) { + PrimaryButton(stringResource(R.string.stores_dl_resume), main = true, modifier = leadOn) { if (item != null) StoresState.requestInstall(ctx, item) else com.droiddeck.launcher.stores.download.DownloadQueue.retry(ctx, failedDownload.key) } @@ -138,14 +139,15 @@ internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: Fro } } item?.owned == true && StoresState.isUnfinished(item) -> { - InstallButton(stringResource(R.string.stores_resume_install)) { StoresState.requestInstall(ctx, item) } + InstallButton(stringResource(R.string.stores_resume_install), leadOn) { StoresState.requestInstall(ctx, item) } ConfirmButton(stringResource(R.string.stores_dl_clear), stringResource(R.string.stores_dl_cancel_confirm), compact = true) { StoresState.clearUnfinished(ctx, item) } } item?.owned == true -> InstallButton( if (item.sizeBytes > 0) stringResource(R.string.stores_install_size, formatBytes(item.sizeBytes)) else stringResource(R.string.stores_install), + leadOn, ) { StoresState.requestInstall(ctx, item) } - item != null && item.isFree -> PrimaryButton(stringResource(R.string.stores_get_free), main = true) { openStoreUrl(ctx, item) } - item != null -> PrimaryButton(if (item.hasPrice && item.finalPrice.isNotBlank()) stringResource(R.string.stores_buy_price, item.finalPrice) else stringResource(R.string.stores_view_on, store.shortLabel), main = true) { openStoreUrl(ctx, item) } + item != null && item.isFree -> PrimaryButton(stringResource(R.string.stores_get_free), main = true, modifier = leadOn) { openStoreUrl(ctx, item) } + item != null -> PrimaryButton(if (item.hasPrice && item.finalPrice.isNotBlank()) stringResource(R.string.stores_buy_price, item.finalPrice) else stringResource(R.string.stores_view_on, store.shortLabel), main = true, modifier = leadOn) { openStoreUrl(ctx, item) } } BusyChip(s) } @@ -221,7 +223,7 @@ private fun StoreAbout(text: String, height: Dp, modifier: Modifier = Modifier) } @Composable -private fun UninstallButton(title: String, game: com.droiddeck.launcher.stores.InstalledStoreGame, enabled: Boolean, onUninstall: () -> Unit) { +private fun UninstallButton(title: String, game: com.droiddeck.launcher.stores.InstalledStoreGame, enabled: Boolean, modifier: Modifier = Modifier, onUninstall: () -> Unit) { val colors = MaterialTheme.colorScheme val placed = remember { arrayOfNulls(1) } val heading = stringResource(R.string.stores_uninstall_title, title) @@ -230,7 +232,7 @@ private fun UninstallButton(title: String, game: com.droiddeck.launcher.stores.I val keep = stringResource(R.string.stores_notification_keep) val remove = stringResource(R.string.stores_uninstall) Box(Modifier.onGloballyPositioned { placed[0] = it }) { - SecondaryButton(remove, enabled = enabled) { + SecondaryButton(remove, enabled = enabled, modifier = modifier) { val at = placed[0]?.takeIf { it.isAttached }?.boundsInRoot() ?: return@SecondaryButton var confirmed = false StoresMotion.ask(StepAsk(anchor = at, side = StepSide.Right, accent = colors.error, items = 3, pillCorner = 12.dp, maxWidth = 340.dp, @@ -249,10 +251,10 @@ private fun UninstallButton(title: String, game: com.droiddeck.launcher.stores.I } @Composable -private fun InstallButton(label: String, onInstall: () -> Unit) { +private fun InstallButton(label: String, modifier: Modifier = Modifier, onInstall: () -> Unit) { val placed = remember { arrayOfNulls(1) } Box(Modifier.onGloballyPositioned { placed[0] = it }) { - PrimaryButton(label, main = true) { + PrimaryButton(label, main = true, modifier = modifier) { StoresMotion.markInstall(placed[0]?.takeIf { it.isAttached }?.boundsInRoot(), label) onInstall() } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt index c0471d5605..e2c64e56ca 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt @@ -171,6 +171,8 @@ internal fun StoresPage(s: FrontEndState, a: FrontEndActions, modifier: Modifier val inputMode = androidx.compose.ui.platform.LocalInputModeManager.current var focusMove by remember { mutableStateOf?>(null) } var lastOpened by remember { mutableStateOf(null) } + val heroFocus = remember { androidx.compose.ui.focus.FocusRequester() } + var detailFocused by remember { mutableStateOf(false) } fun move(kind: String) { focusMove = kind to (focusMove?.second ?: 0) + 1 } fun open(key: String) { lastOpened = key; openGame = key; move("detail") } fun closeGame() { openGame = null; move("card") } @@ -238,8 +240,6 @@ internal fun StoresPage(s: FrontEndState, a: FrontEndActions, modifier: Modifier fun target(): Pair? { val kind = focusMove?.first return when { - openGame != null -> if (ff.primaryAttached > 0) ff.primary to FrontFocus.PRIMARY - else "back:${store?.label}".let { id -> ff.items[id]?.takeIf { attached(id) }?.let { it to id } } kind == "card" -> lastOpened?.let { "card:$it" }?.let { id -> ff.items[id]?.takeIf { attached(id) }?.let { it to id } } ?: ff.firstTile.takeIf { ff.firstTileAttached > 0 }?.let { it to null } kind == "first" -> ff.firstTile.takeIf { ff.firstTileAttached > 0 }?.let { it to null } @@ -250,6 +250,16 @@ internal fun StoresPage(s: FrontEndState, a: FrontEndActions, modifier: Modifier } } repeat(60) { if (!HeldKeys.confirm) return@repeat; androidx.compose.runtime.withFrameNanos { } } + if (openGame != null) { + val back = "back:${ctx.getString(R.string.common_back)}" + repeat(45) { n -> + androidx.compose.runtime.withFrameNanos { } + if (detailFocused) return@LaunchedEffect + val req = if (n < 20) heroFocus else ff.items[back]?.takeIf { attached(back) } ?: heroFocus + runCatching { req.requestFocus() } + } + return@LaunchedEffect + } repeat(45) { androidx.compose.runtime.withFrameNanos { } val (req, id) = target() ?: return@repeat @@ -374,7 +384,12 @@ internal fun StoresPage(s: FrontEndState, a: FrontEndActions, modifier: Modifier Modifier.fillMaxSize().pointerInput(Unit) { detectTapGestures { } } .verticalScroll(detailScroll).padding(top = 4.dp, bottom = 16.dp), ) { - if (detailStore != null) Column(Modifier.fillMaxWidth().focusGroup()) { StoreGameDetail(detailStore, key, s, a, onBack = { closeGame() }) } + if (detailStore != null) Column( + Modifier.fillMaxWidth() + .onFocusChanged { detailFocused = it.hasFocus } + .focusProperties { exit = { if (coveringNow.value) androidx.compose.ui.focus.FocusRequester.Cancel else androidx.compose.ui.focus.FocusRequester.Default } } + .focusGroup(), + ) { StoreGameDetail(detailStore, key, s, a, heroFocus, onBack = { closeGame() }) } } } val from = detailFrom diff --git a/app/src/main/rust/stores/Cargo.lock b/app/src/main/rust/stores/Cargo.lock index d46174b207..90055ee4c7 100644 --- a/app/src/main/rust/stores/Cargo.lock +++ b/app/src/main/rust/stores/Cargo.lock @@ -51,6 +51,12 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.11.1" @@ -121,6 +127,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + [[package]] name = "crc32fast" version = "1.5.0" @@ -169,6 +190,7 @@ dependencies = [ "futures-util", "jni", "libc", + "lzma-rs", "reqwest", "serde_json", "sha1", @@ -592,6 +614,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + [[package]] name = "memchr" version = "2.8.1" diff --git a/app/src/main/rust/stores/Cargo.toml b/app/src/main/rust/stores/Cargo.toml index f9e9ce49f0..d21d5132be 100644 --- a/app/src/main/rust/stores/Cargo.toml +++ b/app/src/main/rust/stores/Cargo.toml @@ -12,6 +12,7 @@ crate-type = ["cdylib"] [dependencies] jni = "0.21" libc = "0.2" +lzma-rs = { version = "0.3", default-features = false } flate2 = { version = "1", default-features = false, features = ["rust_backend"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip"] } tokio = { version = "1", default-features = false, features = ["rt", "net", "time", "sync"] } diff --git a/app/src/main/rust/stores/JNI.md b/app/src/main/rust/stores/JNI.md index 9cd314dfd3..b50981385d 100644 --- a/app/src/main/rust/stores/JNI.md +++ b/app/src/main/rust/stores/JNI.md @@ -2,9 +2,12 @@ The Kotlin side of the store download engines mirrors this file exactly. Every symbol the library exports is listed here with the class it binds to, the JNI signature, when it is called and on -which thread. The engines came from Bannerlator's `bl-steam-client` crate; the method names and -signatures are identical to Bannerlator's `BlGogDownload` / `BlEpicDownload` / `BlAmazonDownload`, -only the package and class names changed. +which thread. The GOG engine and the shared fetch core came from Bannerlator's +`bl-steam-client` crate (method names and signatures match Bannerlator's `BlGogDownload`, only the +package and class names changed). The Epic module (`src/epic/`) and the Amazon module +(`src/amazon/`) are DroidDeck's own, each written from a functional spec (see +`docs/development/epic-cleanroom.md` and `docs/development/amazon-cleanroom.md`); their JNI +surfaces below are new. Load it once with `System.loadLibrary("droiddeckstores")`. The library is built by Gradle (`buildRustStores`, see `app/build.gradle`) and is absent from a `-PskipRust=true` build, so every @@ -112,250 +115,172 @@ On-disk protocol (what a resumed run, by either engine, relies on): a file is as ## `com.droiddeck.launcher.stores.epic.EpicNative` -The manager's default Epic install is the **stream** run (`nativeStream`, mode `adaptive`, -below): the chunks are fetched in parallel and written straight into staged files as they arrive; -on device it measured about 83 MB/s on a 71 GB install. The **lookahead** install -(`nativeInstallAdaptive`, mode `lookahead`) is its fallback when the stream engine's space plan -does not fit, and is selectable directly, as is the legacy batch pair (`nativeStart` + -`nativeAssemble`, mode `batch`), through the hidden `storesEpicMode` preference. +Everything Epic runs in the Rust `epic` module: the web-service calls (sign-in, refresh, exchange +codes, library, catalog, build information, shop shelves, achievements, cloud saves), manifest +parsing and the install engine. Tokens are passed in per call and never stored or logged by the +library; the Kotlin facade keeps them in the shared Keystore-sealed account store. All requests send +one User-Agent, `DroidDeck/` (`epic/net.rs::USER_AGENT_PRODUCT`; the version comes from +`nativeConfigure`). -### Lookahead install (snapshot schema v1) - -`nativeInstallAdaptive(request: String, listener: EpicNative.Listener): Long` -exports `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive`. -Its exact Kotlin descriptor is -`(Ljava/lang/String;Lcom/droiddeck/launcher/stores/epic/EpicNative$Listener;)J`. -The listener must also implement `EpicNative.InstallListener.beforeSwap(): Boolean` (`()Z`). -Rust invokes it synchronously after staging/hash/sync and before the first atomic replacement; -false or a Java exception aborts without replacing that output. Java marks the sidecar updating -there. Later verified swaps preserve mixed-update/resume semantics. - -The UTF-8 JSON request contains: - -- `schema_version: 1`, absolute `install_dir`, `cache_dir`, owned `staging_dir`, - `same_volume`, signed `free_game`/`free_cache` (`-1` unknown), `max_workers`, CDN `hosts`. -- `chunks`: distinct `guid`, relative CDN `path`, decoded `size`, `exact_size`, `wire_size`, - SHA-1 hex `sha1` (empty/all-zero means absent), `cached` verified length (`-1` absent). -- `files`: pending manifest-order relative `path`, `sha1`, current `old_size`, ordered `parts` - containing `guid`, unsigned `offset` and `size`. - -Parse, owned-namespace validation and adaptive storage planning happen synchronously. Errors -return zero after `onComplete(false, error, 0)` and are **terminal**, not fallback signals. -`onPlan` reports the selected lookahead/retention/peaks; `onProgress` counts staged file bytes -and verified swapped file counts, not fetched wire bytes. Fetch and staged assembly overlap -in one native run, using the same pure reservation/release policy as the peak planner. -Cancellation/release joins native workers; no worker survives the terminal callback/handle -release. Only `StoresNative.ensureLoaded() == false` selects bounded parallel Java; the same rule -holds for `nativeStream`. - -### Legacy batch APIs (mode `batch`, for A/B runs) - -Two explicit calls provide the legacy batch interface: - -1. **Fetch** (`nativeStart`): fills the chunk cache with verified, decompressed chunks for the - files the manager says are pending. The Kotlin manager still fetches and parses the manifest - API JSON, downloads the manifest, selects files (install tags), runs the delta/verify pass and - does every post-install step. The engine re-parses the same manifest bytes (ChunksV4 binary - or legacy JSON), rebuilds the chunk plan for the pending files, skips chunks already in the - cache, and writes verified chunks with a `.part` + rename protocol, so the on-disk state is the - same whichever side fetched. -2. **Assemble** (`nativeAssemble`): writes the pending files from the cache, one sequential write - per file, SHA-1 checked as the bytes go out when the manifest hashes the file. This is the - same loop the Kotlin manager runs today (`for part in file.parts: write(chunk[offset..+size])`, - same "Missing chunk X for Y" wording); the Kotlin loop remains a valid fallback. - -**The chunk cache directory** is the `chunkCacheDir` parameter of both calls (always pass the -same value to both for one install): - -- `""` — today's behaviour: the cache is `/.chunks`. The fetch leaves it in place; - `nativeAssemble` removes it whole at the end of a successful run (as the Kotlin loop's - `deleteDir(.chunks)` does), and never deletes a chunk early. -- any other absolute path — a **scratch cache**, meant for internal storage when the game - installs to an SD card (each byte then hits the card once instead of twice; the card's write - speed stops starving the fetchers). The fetch fills it exactly like `.chunks`. The assembler - reference-counts chunk GUIDs across the files it was given and **deletes each cached chunk as - soon as the last file that uses it has been written and verified**, then removes the directory - at the end of a successful run. A cancelled or failed run (fetch or assembly) leaves the cache - exactly as it stands, for resume: the next run's delta pass skips finished files and the next - fetch skips chunks still cached. The directory is created by the fetch if it does not exist; - give each install its own (e.g. `/epic/`), since a successful assembly - removes it. - -| Method | JNI signature | Symbol | +| Method | JNI signature | Symbol suffix | |---|---|---| -| `nativeStart(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;[Ljava/lang/String;[IIJLjava/lang/String;IILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart` | -| `nativeAssemble(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;[ILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble` | -| `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCancel` | -| `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeRelease` | +| `nativeConfigure(appVersion: String)` | `(Ljava/lang/String;)V` | `nativeConfigure` | +| `nativeCall(op: String, request: String): String?` | `(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;` | `nativeCall` | +| `nativeFetch(op: String, request: String): ByteArray?` | `(Ljava/lang/String;Ljava/lang/String;)[B` | `nativeFetch` | +| `nativeSend(op: String, request: String, body: ByteArray): String?` | `(Ljava/lang/String;Ljava/lang/String;[B)Ljava/lang/String;` | `nativeSend` | +| `nativeInspect(manifest: ByteArray, request: String): String?` | `([BLjava/lang/String;)Ljava/lang/String;` | `nativeInspect` | +| `nativeInstall(request: String, listener: InstallListener): Long` | `(Ljava/lang/String;Lcom/droiddeck/launcher/stores/epic/EpicNative$InstallListener;)J` | `nativeInstall` | +| `nativeCancel(handle: Long)` | `(J)V` | `nativeCancel` | +| `nativeRelease(handle: Long)` | `(J)V` | `nativeRelease` | -`nativeCancel` / `nativeRelease` take the handle of either run. There is no `nativeProbe`; -Bannerlator probes by catching the `Throwable` from `loadLibrary` / the first `nativeStart`. +Full symbols are `Java_com_droiddeck_launcher_stores_epic_EpicNative_`. Every call is +synchronous on the calling thread (network calls block: call them off the main thread) except +`nativeInstall`, which returns at once. A Rust panic is caught at the boundary (`null` / 0 result). -```kotlin -@JvmStatic private external fun nativeStart( - manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, - pendingFileIdx: IntArray, expectedChunks: Int, expectedBytes: Long, caBundlePath: String, - maxWorkers: Int, processWorkers: Int, listener: Listener, -): Long -@JvmStatic private external fun nativeAssemble( - manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, listener: Listener, -): Long -``` +### `nativeCall` operations -### `nativeStart` parameters, in order +`request` and the answer are JSON objects. A failure answers `{"error": "…"}`. -| # | Parameter | Meaning | +| op | request | answer | |---|---|---| -| 1 | `manifest: ByteArray` | the raw manifest bytes the manager downloaded (binary ChunksV4 or JSON) | -| 2 | `installDir: String` | absolute install directory | -| 3 | `chunkCacheDir: String` | `""` = `/.chunks`; else the scratch cache directory (see above) | -| 4 | `cdnPrefixes: Array` | the CDN base URLs from the manifest API answer (`.../ChunksV4//` style); one or many, distinct hosts widen the window | -| 5 | `pendingFileIdx: IntArray` | indices into the manifest's file list of the files still to fetch (the manager's delta/verify result) | -| 6 | `expectedChunks: Int` | the manager's own needed-chunk count for a cross-check; `-1` = no cross-check | -| 7 | `expectedBytes: Long` | the manager's `Σ max(fileSize, 1)` for a cross-check; `-1` = no cross-check | -| 8 | `caBundlePath: String` | see conventions | -| 9 | `maxWorkers: Int` | window ceiling (Bannerlator: 8) | -| 10 | `processWorkers: Int` | inflate/verify/write threads | -| 11 | `listener` | see below | +| `auth.config` | – | `startUrl`, `redirectPrefix`, `webViewUserAgent` for the login WebView | +| `auth.codeFromPage` | `text` (the redirect page body as the WebView returns it) | `code`, or `{}` | +| `auth.signIn` | `code` | `access_token`, `refresh_token`?, `account_id`?, `display_name`?, `expires_at` (epoch ms) | +| `auth.refresh` | `refreshToken` | same as `auth.signIn` (absent fields: keep the stored ones) | +| `auth.exchange` | `accessToken` | `code` (single-use launch exchange code) | +| `library.owned` | `accessToken` | `records[]` of `appName`, `namespace`, `catalogItemId` (all pages; filtered to installable Windows games). A refused token answers `{"error":…, "signIn": true}`; any failed page fails the whole call | +| `catalog.details` | `accessToken`, `namespace`, `catalogItemId` | `details`: `title`, `developer`, `description`, `publisher`, `releaseDate`, `tall`, `wide`, `canRunOffline`, `dlc`, `mainGameId` | +| `catalog.attribute` | `accessToken`, `namespace`, `catalogItemId`, `name` | `value` (`""` = absent), or `unknown: true` when the service could not be asked | +| `build.info` | `accessToken`, `namespace`, `catalogItemId`, `appName` | `info`: the opaque build description (`manifest`, `buildVersion`, `deploymentId`, `cdns[]` of `base`/`dir`/`auth`) | +| `build.cdns` | `info` | `cdns[]` | +| `build.prereq` | `info` | `name`, `path`, `read` (downloads and parses the manifest) | +| `shop.shelves` | `country` | `new`, `deals`, `free`, `trending`: arrays in the `store_featured` item shape plus `allTags` | +| `achievements` | `accessToken`, `namespace`, `accountId`, `locale` | `list[]` of `id`, `name`, `description`, `hidden`, `unlocked`, `unlockedAt` (s), `icon`, `iconLocked` | +| `cloud.list` | `accessToken`, `accountId`, `appName` | `files[]` of `name`, `modified` (ms, -1 unknown), `hash`, `readLink` | +| `cloud.writeLinks` | `accessToken`, `accountId`, `appName`, `names[]` | `links`: name → write link | +| `launch.args` | `displayName`, `accountId`, `namespace`, `locale`, `appName`, `deploymentId`, `additional` | `args[]` | +| `locale` | `language`, `region` | `key`, `tags[]`, `launchLocale` | -Planning (manifest parse + plan + cross-check) runs synchronously on the calling thread, so a -plan failure returns `0` BEFORE any fetch; the manager then runs its own fallback loop. On `0`, -`onComplete(false, reason, 0)` has already been called synchronously (except for a null listener). -On success, `onPlan` has been called synchronously before `nativeStart` returns, then the fetch -runs on a new thread. The engine's `plan` log line names the cache: -`plan chunk_dir=ChunksV4 cache_dir= scratch= files_pending=… chunks=… bytes=… …`. +`nativeFetch` ops: `manifest.download` (`info`, optional `cdns`) → the manifest bytes; +`cloud.read` (`url`) → the file. `null` = failed. `nativeSend` op: `cloud.write` (`url`) with the +body → `{"ok": true}` or an error. `nativeInspect` parses manifest bytes: `{"op":"summary"}` → +`buildVersion`, `launchExe`, `prereqName`, `prereqPath`, `prereqRead`, `files[]` (`path`, `size`, +`tags`); `{"op":"select","tags":[…]}` → `indices[]` of the files an install with those tags gets. -### `nativeAssemble` parameters, in order +### `nativeInstall` -| # | Parameter | Meaning | +`request` (JSON): `info` (from `build.info`), absolute `gameDir`, `scratchDir` (`""` or the internal +scratch cache for a card install), `storageLabel` (for space messages), `tags[]` (device install +tags), `maxWorkers` (network window, 1–128), `processWorkers` (decode/write threads), `caBundle` +(`""` = the roots rustls ships with). + +An unreadable request returns `0` after a synchronous `onComplete(false, false, error, "")`. +Otherwise the run happens on a thread named `epic-install` (background priority) and +`onComplete` fires exactly once, last. `nativeCancel` is idempotent and may be called at any time +before `nativeRelease`; call `nativeRelease` exactly once per non-zero handle, after `onComplete`. +The Kotlin wrapper (`EpicNative.install`) polls its cancel flag every 250 ms, forwards it, and waits +for `onComplete` without a time limit: the run removes its unfinished stage files before it +completes. + +| `InstallListener` method | JNI signature | When | |---|---|---| -| 1 | `manifest: ByteArray` | the same manifest bytes as the fetch | -| 2 | `installDir: String` | absolute install directory (files are written at `installDir/`) | -| 3 | `chunkCacheDir: String` | the same value the fetch was given (`""` or the scratch directory) | -| 4 | `fileIdx: IntArray` | indices into the manifest's file list of the files to write — the fetch's `pendingFileIdx`, in one call | -| 5 | `listener` | see below | +| `onStatus(text)` | `(Ljava/lang/String;)V` | manifest-stage status lines | +| `onCheck(doneBytes, totalBytes, files, filesTotal)` | `(JJII)V` | the delta pass, every 16 files and at its end | +| `onSizes(pendingBytes, diskBytes)` | `(JJ)V` | once pending work is known | +| `onProgress(written, total, files, filesTotal)` | `(JJII)V` | pending-file bytes written; at most every 50 ms plus per finished file; may arrive out of order from several threads | +| `onInstall(done, total, files, filesTotal)` | `(JJII)V` | every file is written and verified | +| `onLog(line)` | `(Ljava/lang/String;)V` | `manifest …`, `plan …`, the fetch core's lines, `summary …`; also in logcat under `EpicNative` | +| `beforeReplace(): Boolean` | `()Z` | once, before the first file replaces its destination; `false` or a throw fails the run with nothing replaced | +| `onComplete(success, cancelled, error, result)` | `(ZZLjava/lang/String;Ljava/lang/String;)V` | terminal. `result` (on success) is JSON: `buildVersion`, `launchExe`, `prereqName`, `prereqPath`, `prereqRead`, `deploymentId`, `diskBytes`, `bytesWritten`, `filesWritten`; `error` is `"cancelled"` on cancel | -Parse + plan run on the calling thread: an unparsable manifest, an index out of range or an -empty `installDir` returns `0` after a synchronous `onComplete(false, reason, 0)` and nothing is -written. Otherwise the assembly runs on a new thread. Each file is created fresh and written in -part order; on any error (missing chunk, short chunk, I/O error, SHA-1 mismatch) the file being -written is deleted, the run fails with that error, no further files are touched and the cache is -left as it is. A manifest SHA-1 of all zeros (and a JSON manifest, which has none) means no hash -check — the Kotlin delta pass treats those the same way. The run's log lines: -`assemble cache_dir=… scratch=… files=… bytes=… chunks_referenced=…`, then `cache removed …` / -`cache not removed …: ` (not fatal) and `assembled files=… bytes=… chunks_removed_early=…`. - -### Listener - -Suggested: `com.droiddeck.launcher.stores.epic.EpicNative.Listener` (nested interface, as in -Bannerlator). One interface serves both runs. - -| Listener method | JNI signature | When | -|---|---|---| -| `onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String)` | `(IJLjava/lang/String;)V` | fetch only: once, synchronously inside `nativeStart`, before any fetch: the plan the engine derived (`chunkDir` is the manifest's, e.g. `ChunksV4`, not the cache directory) | -| `onProgress(bytesDone: Long, bytesTotal: Long, done: Int, total: Int)` | `(JJII)V` | fetch: per accounted chunk (cached-skip or fetched), `done`/`total` = chunks; assembly: after each file written, `bytesDone` = bytes written so far, `done`/`total` = files. Native threads. | -| `onLog(line: String)` | `(Ljava/lang/String;)V` | engine log line, already in logcat under `EpicNative`; any native thread (also synchronously from `nativeStart`/`nativeAssemble` for the start line) | -| `onComplete(success: Boolean, error: String, bytes: Long)` | `(ZLjava/lang/String;J)V` | terminal, exactly once (worker thread; or synchronously when the call returns `0`). Fetch: `bytes` = credited bytes (cached-skip + fetched). Assembly: `bytes` = bytes written. `error` is `"cancelled"` on cancel. | - -Bannerlator's `BlEpicDownload.run(...)` wraps `nativeStart` in a blocking call that polls an -`AtomicBoolean` cancel flag every 250 ms, calls `nativeCancel` on cancel, waits up to 5 s for -`onComplete`, then `nativeRelease` in `finally`. The same wrapper shape fits `nativeAssemble` -(cancel is honoured between files; a multi-GB file finishes before the run stops). - -### `nativeStream` — the adaptive install - -| Method | JNI signature | Symbol | -|---|---|---| -| `nativeStream(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;[IJJJLjava/lang/String;IILcom/droiddeck/launcher/stores/epic/EpicNative$StreamListener;)J` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream` | - -```kotlin -@JvmStatic private external fun nativeStream( - manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, - cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, cacheBudget: Long, - overlapBudget: Long, caBundlePath: String, - maxWorkers: Int, processWorkers: Int, listener: StreamListener, -): Long -``` - -Parameters as `nativeStart`, plus `stagingDir` (the manager's owned -`/.droiddeck-epic-staging`, which must exist; `""` = stage beside each file), and -except `expectedBytes` (the manager's `Σ fileSize` of the pending -files, `-1` = no cross-check; no chunk count) and `cacheBudget` (the most bytes of shared chunks -the cache may hold; the manager sizes it from free space, `0` = never cache, refetch instead) and -`overlapBudget` (the most planned bytes of *replaced* files - destination already holds data - -staged at once, since each sits beside its old copy until its swap; one is always allowed so the -run progresses; `-1` = no limit). Files are admitted in order and a chunk is dispatched only once -the file of its first use is admitted; fresh files are never held back. -`chunkCacheDir` is `""` (`/.chunks`) or a scratch directory; the manager falls back to -`""` for a card install when internal storage cannot hold the budget. - -What the run does (`epic/stream.rs`): a chunk one file part uses is decompressed, verified and -written at the part's offset into a unique `stream-*.stage` file in `stagingDir`; a -chunk several parts use is also kept in the cache, decompressed, as `` (a later copy after -an eviction as `.`) and deleted after its last covered part. The plan evicts by furthest -next use when the budget would be exceeded and fetches the evicted chunk again later. Each file, -once complete: synced, size and SHA-1 checked, `onBeforeReplace()` once for the whole run, then -renamed over its destination. Cancel or failure removes every unfinished stage file before -`onComplete`, leaves the destinations and the cache; success removes the cache directory. - -Planning runs on the calling thread and also prepares the cache (keeps verified chunks this -install needs, deletes everything else, as the Java writer does). A refusal returns `0` after a -synchronous `onComplete(false, reason, 0)`; nothing was fetched or written, and the manager fails -the install with that reason (it does not switch engines once the library is loaded). - -| `StreamListener` method | JNI signature | When | -|---|---|---| -| `onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long)` | `(JJIIJ)V` | after written parts (at most every 50 ms) and after every finished file; `filesBytes` = sizes of the files swapped in | -| `onLog(line: String)` | `(Ljava/lang/String;)V` | engine lines: `stream-plan …`, the fetch core's lines, `stream-summary mode=adaptive …` (peak cache, peak staging, refetches, direct vs cached bytes) | -| `onBeforeReplace(): Boolean` | `()Z` | once, before the first file replaces its destination; `false` or a throw fails the run with nothing replaced | -| `onComplete(success: Boolean, error: String, bytes: Long)` | `(ZLjava/lang/String;J)V` | terminal, exactly once; `bytes` = file bytes written; `error` = `"cancelled"` on cancel | - -Unlike the batch runs, the Kotlin wrapper (`EpicNative.stream`) waits for `onComplete` after a -cancel without a time limit: the stage-file cleanup must be finished before the manager returns. +On-disk protocol: game files are staged in `/.droiddeck-epic-staging/*.stage` (owner +marker `owner` = `DroidDeck Epic staging v1\n`) and renamed over their destination after size and +SHA-1 checks; reusable chunks live, decompressed, as `` (extra copies `.`, writes via +`.part`) in `/.chunks` or the scratch directory. ## `com.droiddeck.launcher.stores.amazon.AmazonNative` -The file-fetch pool of an Amazon Games install: one streamed HTTP GET per manifest file, SHA-256 -checked, written as `.tmp` and renamed. The manager keeps manifest/auth/markers/post-install -and hands the engine a JSON plan. +Everything Amazon Games runs in the Rust `amazon` module (`src/amazon/`, written from a functional +spec, see `docs/development/amazon-cleanroom.md`): device sign-in values and registration, token +refresh, deregistration, the entitlement library, the download lookup, manifest framing, +decompression and parsing, the install engine, and the launch details (exe, arguments, FuelPump +environment). Tokens are passed in per call and never stored or logged by the library; the Kotlin +facade keeps them in the shared Keystore-sealed account store. -| Method | JNI signature | Symbol | +User-Agents (all in `amazon/net.rs`): the sign-in WebView and `api.amazon.com` get a desktop +Firefox string (`BROWSER_USER_AGENT`); the distribution service (`gaming.amazon.com`) gets the +Windows games launcher's `User-Agent: AGSLauncher/1.0.0` plus its non-standard +`UserAgent: com.amazon.agslauncher.win/3.0.9202.1` header; the CDN (manifest and files) gets +`DroidDeck/` (`USER_AGENT_PRODUCT`; the version comes from `nativeConfigure`). + +| Method | JNI signature | Symbol suffix | |---|---|---| -| `nativeStart(...)`: Long | `(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;IILjava/lang/Object;)J` (last parameter is the listener) | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart` | -| `nativeCancel(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel` | -| `nativeRelease(handle: Long)` | `(J)V` | `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease` | +| `nativeConfigure(appVersion: String)` | `(Ljava/lang/String;)V` | `nativeConfigure` | +| `nativeCall(op: String, request: String): String?` | `(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;` | `nativeCall` | +| `nativeFetch(op: String, request: String): ByteArray?` | `(Ljava/lang/String;Ljava/lang/String;)[B` | `nativeFetch` | +| `nativeInspect(manifest: ByteArray, request: String): String?` | `([BLjava/lang/String;)Ljava/lang/String;` | `nativeInspect` | +| `nativeInstall(request: String, listener: InstallListener): Long` | `(Ljava/lang/String;Lcom/droiddeck/launcher/stores/amazon/AmazonNative$InstallListener;)J` | `nativeInstall` | +| `nativeCancel(handle: Long)` | `(J)V` | `nativeCancel` | +| `nativeRelease(handle: Long)` | `(J)V` | `nativeRelease` | -`nativeStart` parameters, in order: +Full symbols are `Java_com_droiddeck_launcher_stores_amazon_AmazonNative_`. Every call is +synchronous on the calling thread (network calls block: call them off the main thread) except +`nativeInstall`, which returns at once. A Rust panic is caught at the boundary (`null` / 0 result). +The Amazon module writes nothing to logcat itself; install lines reach the app through `onLog`. -| # | Parameter | Meaning | +### `nativeCall` operations + +`request` and the answer are JSON objects. A failure answers `{"error": "…"}`. + +| op | request | answer | |---|---|---| -| 1 | `planJson: String` | a JSON array, one object per manifest file: `{"relPath": "...", "url": "https://...", "size": , "sha256hex": "..."}` (`sha256hex` may be `""` = no hash check). Every manifest file goes in: the engine applies the size-based resume-skip itself. | -| 2 | `installDir: String` | absolute install directory | -| 3 | `caBundlePath: String` | see conventions | -| 4 | `maxWorkers: Int` | window size; `<= 0` = the engine default (8, Amazon's own `MAX_PARALLEL`) | -| 5 | `processWorkers: Int` | sync verify+write threads (`<= 0` → 1) | -| 6 | `listener` | see below | +| `auth.start` | – | `startUrl` (the sign-in page with a fresh PKCE challenge), `userAgent` (for the WebView), `redirectMarker` (`openid.oa2.authorization_code=`), and the attempt's `serial`, `clientId`, `verifier` (kept by the flow object until `auth.register`) | +| `auth.register` | `url` (the return URL) or `code`, plus `serial`, `clientId`, `verifier` | `access_token`, `refresh_token`, `expires_at` (epoch ms), `device_serial`, `client_id`, `display_name`? — the credential record's own key names; `{"error":…, "noCode": true}` when the URL carries no code | +| `auth.refresh` | `refreshToken` | `access_token`, `expires_at` (the refresh token does not change) | +| `auth.deregister` | `accessToken` | `{}` (the answer is not interpreted) | +| `library.entitlements` | `accessToken`, `deviceSerial` | `total` (entitlements before filtering) and `games[]` of `productId`, `entitlementId`, `title`, `sku`, `art`, `hero`, `developer`, `publisher`, `dlc`, `parentProductId` — all pages, de-duplicated by product id, add-ons with a parent hidden (unless that hides everything), sorted by title. A refused token (HTTP 401/403, or 400 `NotAuthorizedException`) answers `{"error":…, "signIn": true}`; any failed page fails the whole call | +| `download.lookup` | `accessToken`, `entitlementId` | `downloadUrl`, `versionId` | +| `download.size` | `accessToken`, `entitlementId` | `bytes` (0 = unknown) | +| `launch.spec` | `gameDir`, `title` | `exe` (relative, `/`, `""` when the folder has no `.exe`), `args[]`, `source` (`fuel` / `score`), `warning` (or null) | +| `launch.env` | `entitlementId`, `sku`, `displayName` | `env[]` of `[name, value]` pairs, in order | -Returns `0` when the listener is null (no callback), or when `planJson` / `installDir` is empty -(`onComplete(false, "empty plan or install dir", 0)` fires synchronously first), or when the -listener `GlobalRef` cannot be taken (no callback). The engine keeps the `JavaVM` it takes from -the first `nativeStart`. +`nativeFetch` op: `cdn.get` (`url`, optional `accessToken` sent as `x-amzn-token`) → the body, or +`null`. `nativeInspect` parses a manifest object: `totalBytes`, `files`, or an error. -Suggested listener: `com.droiddeck.launcher.stores.amazon.AmazonNative.Listener` (an interface). +### `nativeInstall` -| Listener method | JNI signature | When | +`request` (JSON): `accessToken`, `entitlementId`, absolute `gameDir`, `maxWorkers` (request window, +1–128), `processWorkers` (verify/write workers; the engine uses at least half the CPU cores), +`caBundle` (`""` = the roots rustls ships with). + +An unusable request returns `0` after a synchronous `onComplete(false, false, error, "")`. +Otherwise the run happens on a thread named `amazon-install` (background priority) and +`onComplete` fires exactly once, last. `nativeCancel` is idempotent and may be called at any time +before `nativeRelease`; call `nativeRelease` exactly once per non-zero handle, after `onComplete`. +The Kotlin wrapper (`AmazonNative.install`) polls its cancel flag every 250 ms, forwards it once, +and waits for `onComplete` without a time limit: the run removes its `.tmp` files before it +completes. + +| `InstallListener` method | JNI signature | When | |---|---|---| -| `onProgress(bytesDone: Long, bytesTotal: Long, filesDone: Long, filesTotal: Long)` | `(JJJJ)V` | once up front with the resume-skipped credit, then after each committed file (and as pieces stream in); `bytesDone` includes skipped files. Native threads. | -| `onLog(line: String)` | `(Ljava/lang/String;)V` | one diagnostic line (`engine=rust …`, `fetch-window …`, `summary …`). The Amazon engine does NOT write to logcat itself: the Kotlin side is the one place that logs these, so each line appears once. | -| `onComplete(success: Boolean, error: String, bytesWritten: Long)` | `(ZLjava/lang/String;J)V` | exactly once, when the run succeeds, fails or is cancelled (`error = "cancelled"`). | +| `onStatus(text)` | `(Ljava/lang/String;)V` | manifest-stage status lines | +| `onCheck(doneBytes, totalBytes, files, filesTotal)` | `(JJII)V` | the check of files already on disk, at most every 200 ms and at its end | +| `onSizes(pendingBytes, totalBytes)` | `(JJ)V` | once the pending work is known | +| `onProgress(done, total, files, filesTotal)` | `(JJII)V` | bytes of the whole install that are complete (skipped files included); at most every 50 ms plus per placed file; never moves backwards on the native side, but may arrive out of order from several threads | +| `onLog(line)` | `(Ljava/lang/String;)V` | `manifest …`, `resume …`, `engine …`, the fetch core's lines, `summary …` (URL query strings removed) | +| `beforeReplace(): Boolean` | `()Z` | once, before the first file is put in place; `false` or a throw fails the run with nothing replaced | +| `onComplete(success, cancelled, error, result)` | `(ZZLjava/lang/String;Ljava/lang/String;)V` | terminal. `result` (on success) is JSON: `versionId`, `totalBytes`, `fileCount`, `filesWritten`, `bytesWritten`; `error` is `"cancelled"` on cancel, else a user-facing message | -Bannerlator's `BlAmazonDownload.runBlocking(...)` polls a cancel probe every 100 ms, calls -`nativeCancel` once, and `nativeRelease` in `finally` after `onComplete`; on thread interrupt it -cancels and waits up to 30 s for the run to wind down before releasing. +On-disk protocol: each file is written to `.tmp` beside its destination and renamed over it +after the size and SHA-256 checks; leftover `.tmp` siblings of manifest files are removed when a +run starts. A file whose size and SHA-256 already match is skipped. Empty files whose hash is +absent or that of empty input are created without a request. `.amazon_download_in_progress` is +created when a run starts and removed when it succeeds, which also creates +`.amazon_download_complete`; nothing reads either marker. ## Log tags -Lines go to logcat under `GogNative` and `EpicNative` (the Amazon engine logs only through -`onLog`). Bannerlator used `BL_GOG_DL` / `BL_EPIC_DL` / `BL_AMAZON_DL`; the line grammar +Lines go to logcat under `GogNative` and `EpicNative` (the Amazon module logs only through +`onLog`). Bannerlator used `BL_GOG_DL` for the GOG engine; the fetch core's line grammar (`fetch-start`, `fetch-window`, `throughput`, `fetch-end`, `summary`) is unchanged, so its tuning notes still read against a DroidDeck log. diff --git a/app/src/main/rust/stores/src/amazon.rs b/app/src/main/rust/stores/src/amazon.rs deleted file mode 100644 index 1f24533b2d..0000000000 --- a/app/src/main/rust/stores/src/amazon.rs +++ /dev/null @@ -1,888 +0,0 @@ - -pub mod jni; - -use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; -use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use std::fs::{self, File}; -use std::io::Write; -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -pub const MAX_PARALLEL: usize = 8; -pub const DOWNLOAD_USER_AGENT: &str = "nile/0.1 Amazon"; -pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); -pub const TMP_SUFFIX: &str = ".tmp"; -pub const LABEL: &str = "amazon"; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PlanEntry { - pub rel_path: String, - pub url: String, - pub size: u64, - pub sha256: Vec, -} - -pub fn parse_plan(json: &str) -> Result, String> { - let value: serde_json::Value = - serde_json::from_str(json).map_err(|err| format!("plan json: {err}"))?; - let Some(array) = value.as_array() else { - return Err("plan json: expected an array".to_string()); - }; - let mut entries = Vec::with_capacity(array.len()); - for (index, item) in array.iter().enumerate() { - let rel_path = item - .get("relPath") - .and_then(|v| v.as_str()) - .map(unix_path) - .unwrap_or_default(); - if rel_path.is_empty() { - return Err(format!("plan json: entry {index} has no relPath")); - } - let url = item - .get("url") - .and_then(|v| v.as_str()) - .unwrap_or_default() - .to_string(); - if url.is_empty() { - return Err(format!("plan json: entry {index} ({rel_path}) has no url")); - } - let size = item - .get("size") - .and_then(|v| v.as_u64().or_else(|| v.as_i64().map(|s| s.max(0) as u64))) - .unwrap_or(0); - let sha_hex = item - .get("sha256hex") - .and_then(|v| v.as_str()) - .unwrap_or_default(); - let sha256 = if sha_hex.is_empty() { - Vec::new() - } else { - hex_decode(sha_hex) - .ok_or_else(|| format!("plan json: entry {index} ({rel_path}) bad sha256hex"))? - }; - entries.push(PlanEntry { - rel_path, - url, - size, - sha256, - }); - } - Ok(entries) -} - -pub fn unix_path(path: &str) -> String { - path.replace('\\', "/") -} - -pub fn hex_decode(hex: &str) -> Option> { - let bytes = hex.as_bytes(); - if bytes.len() % 2 != 0 { - return None; - } - let nibble = |b: u8| -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } - }; - let mut out = Vec::with_capacity(bytes.len() / 2); - for pair in bytes.chunks(2) { - out.push((nibble(pair[0])? << 4) | nibble(pair[1])?); - } - Some(out) -} - -pub fn dest_path(install_dir: &str, rel_path: &str) -> PathBuf { - let base = install_dir.trim_end_matches('/'); - PathBuf::from(format!("{base}/{rel_path}")) -} - -pub fn tmp_path(install_dir: &str, rel_path: &str) -> PathBuf { - dest_path(install_dir, &format!("{rel_path}{TMP_SUFFIX}")) -} - -pub fn is_present_and_complete(install_dir: &str, entry: &PlanEntry) -> bool { - fs::metadata(dest_path(install_dir, &entry.rel_path)) - .map(|meta| meta.len() == entry.size) - .unwrap_or(false) -} - -pub fn host_key(url: &str) -> String { - let (scheme, rest) = match url.find("://") { - Some(idx) => (&url[..idx], &url[idx + 3..]), - None => ("https", url), - }; - let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); - format!("{scheme}://{}", &rest[..end]) -} - -pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize { - let hosts = distinct_hosts.max(1); - let per_host = max_workers.max(1).div_ceil(hosts); - per_host.max(crate::fetch_tuning::PER_HOST_CAP) -} - -#[derive(Debug, Default)] -pub struct Plan { - pub items: Vec, - pub entries: Vec, - pub total_bytes: u64, - pub skipped_bytes: u64, - pub skipped_files: u64, - pub hosts: Vec, -} - -pub fn build_plan(entries: Vec, install_dir: &str) -> Plan { - let mut plan = Plan { - total_bytes: entries.iter().map(|e| e.size).sum(), - ..Plan::default() - }; - for (index, entry) in entries.iter().enumerate() { - if is_present_and_complete(install_dir, entry) { - plan.skipped_bytes = plan.skipped_bytes.saturating_add(entry.size); - plan.skipped_files += 1; - continue; - } - let host = host_key(&entry.url); - if !plan.hosts.contains(&host) { - plan.hosts.push(host); - } - plan.items.push(FetchItem { - id: index as u64, - urls: vec![entry.url.clone()], - reserve: entry.size, - range: None, - }); - } - if plan.hosts.is_empty() { - plan.hosts.push("https://amazon".to_string()); - } - plan.entries = entries; - plan -} - -struct OpenTmp { - file: File, - hasher: Sha256, - written: u64, -} - -pub struct AmazonSink { - install_dir: String, - entries: Vec, - cancel: Arc, - files_done: AtomicU64, - open: Mutex>>>, -} - -impl AmazonSink { - pub fn new(install_dir: &str, entries: Vec, cancel: Arc) -> Self { - Self { - install_dir: install_dir.to_string(), - entries, - cancel, - files_done: AtomicU64::new(0), - open: Mutex::new(HashMap::new()), - } - } - - pub fn files_done(&self) -> u64 { - self.files_done.load(Ordering::Relaxed) - } - - fn entry(&self, item: &FetchItem) -> Result<&PlanEntry, SinkError> { - self.entries - .get(item.id as usize) - .ok_or_else(|| SinkError::Fatal(format!("plan index {} out of range", item.id))) - } - - fn ensure_parent(&self, entry: &PlanEntry) { - if let Some(parent) = dest_path(&self.install_dir, &entry.rel_path).parent() { - let _ = fs::create_dir_all(parent); - } - } - - fn rename_into_place(&self, entry: &PlanEntry) -> Result<(), SinkError> { - let dest = dest_path(&self.install_dir, &entry.rel_path); - let tmp = tmp_path(&self.install_dir, &entry.rel_path); - if dest.exists() { - let _ = fs::remove_file(&dest); - } - if let Err(err) = fs::rename(&tmp, &dest) { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Fatal(format!( - "Failed to rename tmp → {} ({err})", - dest.display() - ))); - } - self.files_done.fetch_add(1, Ordering::Relaxed); - Ok(()) - } - - fn verify(&self, entry: &PlanEntry, digest: &[u8]) -> Result<(), SinkError> { - if !entry.sha256.is_empty() && digest != entry.sha256.as_slice() { - let _ = fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path)); - return Err(SinkError::Retry(format!( - "SHA-256 mismatch for: {}", - entry.rel_path - ))); - } - Ok(()) - } - - pub fn commit(&self, entry: &PlanEntry, body: &[u8]) -> Result { - if !entry.sha256.is_empty() { - let digest = Sha256::digest(body); - if digest.as_slice() != entry.sha256.as_slice() { - return Err(SinkError::Retry(format!( - "SHA-256 mismatch for: {}", - entry.rel_path - ))); - } - } - self.ensure_parent(entry); - let tmp = tmp_path(&self.install_dir, &entry.rel_path); - if let Err(err) = fs::write(&tmp, body) { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Retry(format!("write {}: {err}", tmp.display()))); - } - self.rename_into_place(entry)?; - Ok(body.len() as u64) - } - - pub fn cleanup_partials(&self) -> usize { - let leftovers: Vec<(u64, Arc>)> = match self.open.lock() { - Ok(mut open) => open.drain().collect(), - Err(_) => return 0, - }; - let mut removed = 0; - for (id, open) in leftovers { - drop(open); - if let Some(entry) = self.entries.get(id as usize) { - if fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path)).is_ok() { - removed += 1; - } - } - } - removed - } -} - -impl FetchSink for AmazonSink { - fn process(&self, item: &FetchItem, body: Vec) -> Result { - if self.cancel.load(Ordering::Relaxed) { - return Err(SinkError::Fatal("cancelled".to_string())); - } - let entry = self.entry(item)?; - self.commit(entry, &body) - } - - fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { - if self.cancel.load(Ordering::Relaxed) { - return Err(SinkError::Fatal("cancelled".to_string())); - } - let entry = self.entry(item)?; - let tmp = tmp_path(&self.install_dir, &entry.rel_path); - let open = if offset == 0 { - self.ensure_parent(entry); - let file = File::create(&tmp).map_err(|err| { - let _ = fs::remove_file(&tmp); - SinkError::Retry(format!("create {}: {err}", tmp.display())) - })?; - let open = Arc::new(Mutex::new(OpenTmp { - file, - hasher: Sha256::new(), - written: 0, - })); - match self.open.lock() { - Ok(mut map) => { - map.insert(item.id, Arc::clone(&open)); - } - Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), - } - open - } else { - let found = match self.open.lock() { - Ok(map) => map.get(&item.id).cloned(), - Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), - }; - match found { - Some(open) => open, - None => { - return Err(SinkError::Retry(format!( - "piece at offset {offset} without a stream start for: {}", - entry.rel_path - ))) - } - } - }; - let Ok(mut state) = open.lock() else { - return Err(SinkError::Fatal("sink item poisoned".to_string())); - }; - if state.written != offset { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Retry(format!( - "piece out of order ({} != {offset}) for: {}", - state.written, entry.rel_path - ))); - } - if let Err(err) = state.file.write_all(data) { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Retry(format!("write {}: {err}", tmp.display()))); - } - state.hasher.update(data); - state.written = state.written.saturating_add(data.len() as u64); - Ok(()) - } - - fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result { - let entry = self.entry(item)?; - let tmp = tmp_path(&self.install_dir, &entry.rel_path); - let taken = match self.open.lock() { - Ok(mut map) => map.remove(&item.id), - Err(_) => return Err(SinkError::Fatal("sink map poisoned".to_string())), - }; - let (written, digest) = match taken { - Some(open) => { - let Ok(mut state) = open.lock() else { - return Err(SinkError::Fatal("sink item poisoned".to_string())); - }; - if let Err(err) = state.file.flush() { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Retry(format!("flush {}: {err}", tmp.display()))); - } - let written = state.written; - let digest = std::mem::replace(&mut state.hasher, Sha256::new()).finalize(); - (written, digest) - } - None => { - if total_len != 0 { - return Err(SinkError::Retry(format!( - "finish without a stream for: {}", - entry.rel_path - ))); - } - self.ensure_parent(entry); - if let Err(err) = File::create(&tmp) { - return Err(SinkError::Retry(format!("create {}: {err}", tmp.display()))); - } - (0, Sha256::new().finalize()) - } - }; - if written != total_len { - let _ = fs::remove_file(&tmp); - return Err(SinkError::Retry(format!( - "stream length {written} != {total_len} for: {}", - entry.rel_path - ))); - } - self.verify(entry, digest.as_slice())?; - self.rename_into_place(entry)?; - Ok(0) - } -} - -#[derive(Clone, Debug, Default)] -pub struct RunResult { - pub success: bool, - pub cancelled: bool, - pub error: String, - pub bytes_written: u64, - pub files_done: u64, - pub files_total: u64, -} - -struct SpeedMeter { - started: Instant, - last_at: Instant, - last_bytes: u64, - peak_bps: f64, -} - -impl SpeedMeter { - fn new() -> Self { - let now = Instant::now(); - Self { - started: now, - last_at: now, - last_bytes: 0, - peak_bps: 0.0, - } - } - - fn observe(&mut self, bytes: u64) { - let now = Instant::now(); - let dt = now.duration_since(self.last_at).as_secs_f64(); - if dt >= 0.5 { - let bps = bytes.saturating_sub(self.last_bytes) as f64 / dt; - if bps > self.peak_bps { - self.peak_bps = bps; - } - self.last_at = now; - self.last_bytes = bytes; - } - } - - fn summary(&self, bytes: u64) -> (f64, f64, f64) { - let elapsed = self.started.elapsed().as_secs_f64().max(0.001); - let avg = bytes as f64 / elapsed; - (elapsed, avg * 8.0 / 1_000_000.0, self.peak_bps * 8.0 / 1_000_000.0) - } -} - -pub fn run_download( - plan_json: &str, - install_dir: &str, - ca_bundle_path: &str, - max_workers: usize, - process_workers: usize, - cancel: Arc, - progress: &(dyn Fn(u64, u64, u64, u64) + Sync), - log: &(dyn Fn(&str) + Sync), -) -> RunResult { - let max_workers = if max_workers == 0 { MAX_PARALLEL } else { max_workers }; - let process_workers = process_workers.max(1); - let entries = match parse_plan(plan_json) { - Ok(entries) => entries, - Err(err) => { - log(&format!("engine=rust plan error: {err}")); - return RunResult { - error: err, - ..RunResult::default() - }; - } - }; - let plan = build_plan(entries, install_dir); - let files_total = plan.entries.len() as u64; - let fetch_bytes: u64 = plan.items.iter().map(|item| item.reserve).sum(); - let per_host_cap = per_host_cap_for(max_workers, plan.hosts.len()); - log(&format!( - "engine=rust mode=stream plan={} files skip={} ({} bytes) fetch={} ({} bytes) hosts={} workers={} per_host_cap={} process={} dir={} {}", - files_total, - plan.skipped_files, - plan.skipped_bytes, - plan.items.len(), - fetch_bytes, - plan.hosts.len(), - max_workers, - per_host_cap, - process_workers, - install_dir, - crate::priority::log_field() - )); - progress(plan.skipped_bytes, plan.total_bytes, plan.skipped_files, files_total); - if plan.items.is_empty() { - log("summary bytes=0 elapsed=0.000 avg_mbps=0.0 peak_mbps=0.0 files=all-present"); - return RunResult { - success: true, - files_done: plan.skipped_files, - files_total, - ..RunResult::default() - }; - } - if cancel.load(Ordering::Relaxed) { - return RunResult { - cancelled: true, - error: "cancelled".to_string(), - files_done: plan.skipped_files, - files_total, - ..RunResult::default() - }; - } - - let opts = FetchOptions { - max_workers, - per_host_cap, - timeout: REQUEST_TIMEOUT, - headers: vec![("User-Agent".to_string(), DOWNLOAD_USER_AGENT.to_string())], - ca_bundle_path: ca_bundle_path.to_string(), - process_workers, - label: LABEL.to_string(), - stream: true, - ..FetchOptions::default() - }; - let sink = AmazonSink::new(install_dir, plan.entries.clone(), Arc::clone(&cancel)); - let meter = Mutex::new(SpeedMeter::new()); - let skipped_bytes = plan.skipped_bytes; - let skipped_files = plan.skipped_files; - let total_bytes = plan.total_bytes; - let progress_cb = |credited: u64, items_ok: u64| { - if let Ok(mut meter) = meter.lock() { - meter.observe(credited); - } - progress( - skipped_bytes.saturating_add(credited), - total_bytes, - skipped_files.saturating_add(items_ok), - files_total, - ); - }; - let outcome = run_fetch( - plan.items, - &plan.hosts, - &opts, - &sink, - cancel.as_ref(), - &progress_cb, - log, - ); - let leftovers = sink.cleanup_partials(); - let (elapsed, avg_mbps, peak_mbps) = meter - .lock() - .map(|meter| meter.summary(outcome.bytes_credited)) - .unwrap_or((0.0, 0.0, 0.0)); - let files_done = skipped_files.saturating_add(sink.files_done()); - log(&format!( - "summary bytes={} elapsed={:.3} avg_mbps={:.1} peak_mbps={:.1} files={}/{} items_ok={} tmp_removed={} cancelled={} error={}", - outcome.bytes_credited, - elapsed, - avg_mbps, - peak_mbps, - files_done, - files_total, - outcome.items_ok, - leftovers, - outcome.cancelled, - outcome.error.as_deref().unwrap_or("") - )); - let cancelled = outcome.cancelled || cancel.load(Ordering::Relaxed); - let error = if cancelled { - outcome.error.unwrap_or_else(|| "cancelled".to_string()) - } else { - outcome.error.unwrap_or_default() - }; - RunResult { - success: !cancelled && error.is_empty(), - cancelled, - error, - bytes_written: outcome.bytes_credited, - files_done, - files_total, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::AtomicUsize; - - static COUNTER: AtomicUsize = AtomicUsize::new(0); - - fn scratch_dir() -> String { - let n = COUNTER.fetch_add(1, Ordering::Relaxed); - let dir = std::env::temp_dir().join(format!( - "amazon-test-{}-{}", - std::process::id(), - n - )); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - dir.to_string_lossy().into_owned() - } - - fn sha_hex(data: &[u8]) -> String { - crate::gog::to_hex(Sha256::digest(data).as_slice()) - } - - fn item_for(entry: &PlanEntry, id: u64) -> FetchItem { - FetchItem { - id, - urls: vec![entry.url.clone()], - reserve: entry.size, - range: None, - } - } - - const PLAN: &str = r#"[ - {"relPath":"Binaries\\Win64\\Game.exe","url":"https://d1.cdn.example/base/files/aa?Sig=1","size":4,"sha256hex":"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"}, - {"relPath":"data/level1.pak","url":"https://d1.cdn.example/base/files/bb?Sig=1","size":2,"sha256hex":""}, - {"relPath":"readme.txt","url":"https://d2.cdn.example/base/files/cc?Sig=1","size":0,"sha256hex":""} - ]"#; - - #[test] - fn parses_plan_and_normalises_paths() { - let entries = parse_plan(PLAN).unwrap(); - assert_eq!(entries.len(), 3); - assert_eq!(entries[0].rel_path, "Binaries/Win64/Game.exe"); - assert_eq!(entries[0].size, 4); - assert_eq!(entries[0].sha256.len(), 32); - assert!(entries[1].sha256.is_empty()); - assert_eq!(entries[2].size, 0); - } - - #[test] - fn rejects_bad_plan() { - assert!(parse_plan("{}").is_err()); - assert!(parse_plan(r#"[{"url":"x","size":1}]"#).is_err()); - assert!(parse_plan(r#"[{"relPath":"a","size":1}]"#).is_err()); - assert!(parse_plan(r#"[{"relPath":"a","url":"u","size":1,"sha256hex":"zz"}]"#).is_err()); - } - - #[test] - fn hex_roundtrip() { - assert_eq!(hex_decode("00ff10"), Some(vec![0x00, 0xff, 0x10])); - assert_eq!(hex_decode("00FF"), Some(vec![0x00, 0xff])); - assert_eq!(hex_decode("0"), None); - assert_eq!(hex_decode("0g"), None); - } - - #[test] - fn per_host_cap_gives_single_host_the_whole_ceiling() { - assert_eq!(per_host_cap_for(32, 1), 32); - assert_eq!(per_host_cap_for(32, 3), 11); - assert_eq!(per_host_cap_for(8, 1), 8); - assert_eq!(per_host_cap_for(4, 1), 6); - assert_eq!(per_host_cap_for(0, 0), 6); - } - - #[test] - fn host_key_strips_path_and_query() { - assert_eq!( - host_key("https://d1.cdn.example/base/files/aa?Sig=1"), - "https://d1.cdn.example" - ); - assert_eq!(host_key("http://h:8080/x"), "http://h:8080"); - assert_eq!(host_key("nohost"), "https://nohost"); - } - - #[test] - fn dest_path_never_escapes_install_dir() { - assert_eq!( - dest_path("/inst/", "a/b.txt"), - PathBuf::from("/inst/a/b.txt") - ); - assert_eq!(dest_path("/inst", "/abs.txt"), PathBuf::from("/inst//abs.txt")); - assert_eq!(tmp_path("/inst", "a.bin"), PathBuf::from("/inst/a.bin.tmp")); - } - - #[test] - fn build_plan_skips_only_size_matched_files() { - let dir = scratch_dir(); - let entries = parse_plan(PLAN).unwrap(); - fs::create_dir_all(format!("{dir}/data")).unwrap(); - fs::write(format!("{dir}/data/level1.pak"), b"zz").unwrap(); - fs::create_dir_all(format!("{dir}/Binaries/Win64")).unwrap(); - fs::write(format!("{dir}/Binaries/Win64/Game.exe"), b"tes").unwrap(); - let plan = build_plan(entries, &dir); - assert_eq!(plan.total_bytes, 6); - assert_eq!(plan.skipped_files, 1); - assert_eq!(plan.skipped_bytes, 2); - assert_eq!(plan.items.len(), 2); - assert_eq!(plan.items[0].id, 0); - assert_eq!(plan.items[0].reserve, 4); - assert!(plan.items[0].range.is_none()); - assert_eq!(plan.items[1].id, 2); - assert_eq!( - plan.hosts, - vec!["https://d1.cdn.example".to_string(), "https://d2.cdn.example".to_string()] - ); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn zero_size_missing_file_is_fetched_and_present_zero_is_skipped() { - let dir = scratch_dir(); - let entry = PlanEntry { - rel_path: "empty.txt".into(), - url: "https://h/x".into(), - size: 0, - sha256: Vec::new(), - }; - assert!(!is_present_and_complete(&dir, &entry)); - fs::write(format!("{dir}/empty.txt"), b"").unwrap(); - assert!(is_present_and_complete(&dir, &entry)); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn whole_body_sink_verifies_writes_and_renames() { - let dir = scratch_dir(); - let body = b"test".to_vec(); - let entry = PlanEntry { - rel_path: "sub/Game.exe".into(), - url: "https://h/x".into(), - size: 4, - sha256: hex_decode(&sha_hex(&body)).unwrap(), - }; - let cancel = Arc::new(AtomicBool::new(false)); - let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::clone(&cancel)); - let item = item_for(&entry, 0); - match sink.process(&item, b"nope".to_vec()) { - Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")), - Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"), - Ok(_) => panic!("expected Retry, got Ok"), - } - assert!(!dest_path(&dir, "sub/Game.exe").exists()); - assert!(!tmp_path(&dir, "sub/Game.exe").exists()); - assert_eq!(sink.process(&item, body.clone()).unwrap(), 4); - assert_eq!(fs::read(dest_path(&dir, "sub/Game.exe")).unwrap(), body); - assert!(!tmp_path(&dir, "sub/Game.exe").exists()); - assert_eq!(sink.files_done(), 1); - assert_eq!(sink.process(&item, body.clone()).unwrap(), 4); - cancel.store(true, Ordering::Relaxed); - assert!(matches!( - sink.process(&item, body), - Err(SinkError::Fatal(_)) - )); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn stream_sink_appends_verifies_and_renames() { - let dir = scratch_dir(); - let body = b"hello world".to_vec(); - let entry = PlanEntry { - rel_path: "sub/big.bin".into(), - url: "https://h/x".into(), - size: body.len() as u64, - sha256: hex_decode(&sha_hex(&body)).unwrap(), - }; - let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); - let item = item_for(&entry, 0); - sink.on_chunk(&item, 0, b"hello").unwrap(); - assert!(tmp_path(&dir, "sub/big.bin").exists()); - sink.on_chunk(&item, 5, b" wor").unwrap(); - sink.on_chunk(&item, 9, b"ld").unwrap(); - assert_eq!(sink.on_finish(&item, 11).unwrap(), 0); - assert_eq!(fs::read(dest_path(&dir, "sub/big.bin")).unwrap(), body); - assert!(!tmp_path(&dir, "sub/big.bin").exists()); - assert_eq!(sink.files_done(), 1); - assert_eq!(sink.cleanup_partials(), 0); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn stream_sink_restart_truncates_and_mismatch_retries() { - let dir = scratch_dir(); - let body = b"abcdef".to_vec(); - let entry = PlanEntry { - rel_path: "f.bin".into(), - url: "https://h/x".into(), - size: 6, - sha256: hex_decode(&sha_hex(&body)).unwrap(), - }; - let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); - let item = item_for(&entry, 0); - sink.on_chunk(&item, 0, b"zzzz").unwrap(); - sink.on_chunk(&item, 0, b"abc").unwrap(); - sink.on_chunk(&item, 3, b"xyz").unwrap(); - match sink.on_finish(&item, 6) { - Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")), - Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"), - Ok(_) => panic!("expected Retry, got Ok"), - } - assert!(!tmp_path(&dir, "f.bin").exists()); - assert!(!dest_path(&dir, "f.bin").exists()); - sink.on_chunk(&item, 0, b"abc").unwrap(); - assert!(matches!( - sink.on_chunk(&item, 5, b"f"), - Err(SinkError::Retry(_)) - )); - sink.on_chunk(&item, 0, b"abc").unwrap(); - assert!(matches!(sink.on_finish(&item, 6), Err(SinkError::Retry(_)))); - assert!(matches!( - sink.on_chunk(&item, 3, b"def"), - Err(SinkError::Retry(_)) - )); - sink.on_chunk(&item, 0, b"abc").unwrap(); - sink.on_chunk(&item, 3, b"def").unwrap(); - assert_eq!(sink.on_finish(&item, 6).unwrap(), 0); - assert_eq!(fs::read(dest_path(&dir, "f.bin")).unwrap(), body); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn stream_sink_cleanup_removes_unfinished_tmp_and_zero_len_finishes() { - let dir = scratch_dir(); - let big = PlanEntry { - rel_path: "cut.bin".into(), - url: "https://h/x".into(), - size: 10, - sha256: Vec::new(), - }; - let empty = PlanEntry { - rel_path: "empty.bin".into(), - url: "https://h/y".into(), - size: 0, - sha256: Vec::new(), - }; - let cancel = Arc::new(AtomicBool::new(false)); - let sink = AmazonSink::new(&dir, vec![big.clone(), empty.clone()], Arc::clone(&cancel)); - sink.on_chunk(&item_for(&big, 0), 0, b"12345").unwrap(); - assert!(tmp_path(&dir, "cut.bin").exists()); - assert_eq!(sink.on_finish(&item_for(&empty, 1), 0).unwrap(), 0); - assert_eq!(fs::read(dest_path(&dir, "empty.bin")).unwrap(), b""); - cancel.store(true, Ordering::Relaxed); - assert!(matches!( - sink.on_chunk(&item_for(&big, 0), 5, b"6"), - Err(SinkError::Fatal(_)) - )); - assert_eq!(sink.cleanup_partials(), 1); - assert!(!tmp_path(&dir, "cut.bin").exists()); - assert!(!dest_path(&dir, "cut.bin").exists()); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn stream_sink_skips_hash_when_manifest_has_none() { - let dir = scratch_dir(); - let entry = PlanEntry { - rel_path: "a.bin".into(), - url: "https://h/x".into(), - size: 3, - sha256: Vec::new(), - }; - let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false))); - let item = item_for(&entry, 0); - sink.on_chunk(&item, 0, b"abc").unwrap(); - assert_eq!(sink.on_finish(&item, 3).unwrap(), 0); - assert_eq!(fs::read(dest_path(&dir, "a.bin")).unwrap(), b"abc"); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn run_download_all_present_short_circuits() { - let dir = scratch_dir(); - fs::write(format!("{dir}/a.bin"), b"abc").unwrap(); - let plan = r#"[{"relPath":"a.bin","url":"https://h/x","size":3,"sha256hex":""}]"#; - let seen = Mutex::new(Vec::new()); - let logs = Mutex::new(Vec::new()); - let result = run_download( - plan, - &dir, - "", - 0, - 1, - Arc::new(AtomicBool::new(false)), - &|done, total, fd, ft| seen.lock().unwrap().push((done, total, fd, ft)), - &|line| logs.lock().unwrap().push(line.to_string()), - ); - assert!(result.success); - assert_eq!(result.files_done, 1); - assert_eq!(result.files_total, 1); - assert_eq!(seen.lock().unwrap().as_slice(), &[(3, 3, 1, 1)]); - assert!(logs.lock().unwrap()[0].starts_with("engine=rust mode=stream")); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn run_download_reports_plan_error() { - let result = run_download( - "not json", - "/nonexistent", - "", - 8, - 2, - Arc::new(AtomicBool::new(false)), - &|_, _, _, _| {}, - &|_| {}, - ); - assert!(!result.success); - assert!(result.error.contains("plan json")); - } -} diff --git a/app/src/main/rust/stores/src/amazon/auth.rs b/app/src/main/rust/stores/src/amazon/auth.rs new file mode 100644 index 0000000000..8f0e87f171 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/auth.rs @@ -0,0 +1,292 @@ +use std::io::Read; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::net::{self, Call}; + +pub const DEVICE_TYPE: &str = "A2UMVHOX7UP4V7"; +pub const APP_NAME: &str = "AGSLauncher for Windows"; +pub const APP_VERSION: &str = "1.0.0"; +pub const OS_VERSION: &str = "10.0.19044.0"; +pub const DEVICE_MODEL: &str = "Windows"; +pub const CLIENT_DOMAIN: &str = "DeviceLegacy"; +pub const REGISTRATION_DOMAIN: &str = "Device"; +pub const ASSOCIATION_HANDLE: &str = "amzn_sonic_games_launcher"; +pub const MARKETPLACE_US: &str = "ATVPDKIKX0DER"; + +pub const SIGN_IN_PAGE: &str = "https://www.amazon.com/ap/signin"; +pub const RETURN_TO: &str = "https://www.amazon.com"; +pub const CODE_PARAMETER: &str = "openid.oa2.authorization_code"; +pub const REDIRECT_MARKER: &str = "openid.oa2.authorization_code="; + +const REGISTER_URL: &str = "https://api.amazon.com/auth/register"; +const TOKEN_URL: &str = "https://api.amazon.com/auth/token"; +const DEREGISTER_URL: &str = "https://api.amazon.com/auth/deregister"; +const DEFAULT_EXPIRES_IN: i64 = 3600; + +fn random_bytes(n: usize) -> Result, String> { + let mut out = vec![0u8; n]; + std::fs::File::open("/dev/urandom") + .and_then(|mut f| f.read_exact(&mut out)) + .map_err(|e| format!("no randomness: {e}"))?; + Ok(out) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Attempt { + pub serial: String, + pub client_id: String, + pub verifier: String, +} + +pub fn serial_from(random: &[u8; 16]) -> String { + let mut b = *random; + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + net::hex_upper(&b) +} + +pub fn client_id_for(serial: &str) -> String { + net::hex_lower(format!("{serial}#{DEVICE_TYPE}").as_bytes()) +} + +pub fn challenge_for(verifier: &str) -> String { + net::base64url(&Sha256::digest(verifier.as_bytes())) +} + +impl Attempt { + pub fn new() -> Result { + let random = random_bytes(48)?; + let uuid: [u8; 16] = random[..16].try_into().expect("16 bytes"); + let serial = serial_from(&uuid); + Ok(Attempt { client_id: client_id_for(&serial), serial, verifier: net::base64url(&random[16..48]) }) + } + + pub fn start_url(&self) -> String { + let client = format!("device:{}", self.client_id); + let challenge = challenge_for(&self.verifier); + let params: [(&str, &str); 17] = [ + ("openid.ns", "http://specs.openid.net/auth/2.0"), + ("openid.claimed_id", "http://specs.openid.net/auth/2.0/identifier_select"), + ("openid.identity", "http://specs.openid.net/auth/2.0/identifier_select"), + ("openid.mode", "checkid_setup"), + ("openid.oa2.scope", "device_auth_access"), + ("openid.ns.oa2", "http://www.amazon.com/ap/ext/oauth/2"), + ("openid.oa2.response_type", "code"), + ("openid.oa2.code_challenge_method", "S256"), + ("openid.oa2.client_id", client.as_str()), + ("language", "en_US"), + ("marketPlaceId", MARKETPLACE_US), + ("openid.return_to", RETURN_TO), + ("openid.pape.max_auth_age", "0"), + ("openid.ns.pape", "http://specs.openid.net/extensions/pape/1.0"), + ("openid.assoc_handle", ASSOCIATION_HANDLE), + ("pageId", ASSOCIATION_HANDLE), + ("openid.oa2.code_challenge", challenge.as_str()), + ]; + let query: Vec = params.iter().map(|(k, v)| format!("{k}={}", net::percent_encode(v))).collect(); + format!("{SIGN_IN_PAGE}?{}", query.join("&")) + } +} + +pub fn code_from_url(url: &str) -> Option { + let query = &url[url.find('?')? + 1..]; + let query = query.split('#').next().unwrap_or(""); + query + .split('&') + .filter_map(|pair| pair.split_once('=')) + .find(|(k, _)| *k == CODE_PARAMETER) + .map(|(_, v)| net::percent_decode(v)) + .filter(|v| !v.is_empty()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Registered { + pub access_token: String, + pub refresh_token: String, + pub expires_at_ms: i64, + pub device_serial: String, + pub client_id: String, + pub display_name: String, +} + +impl Registered { + pub fn to_json(&self) -> Value { + let mut out = json!({ + "access_token": self.access_token, + "refresh_token": self.refresh_token, + "device_serial": self.device_serial, + "client_id": self.client_id, + "expires_at": self.expires_at_ms, + }); + if !self.display_name.is_empty() { + out["display_name"] = json!(self.display_name); + } + out + } +} + +fn expires_in(v: Option<&Value>) -> i64 { + let secs = v.and_then(|v| v.as_i64().or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))); + secs.filter(|s| *s > 0).unwrap_or(DEFAULT_EXPIRES_IN) +} + +pub fn registration_body(code: &str, attempt: &Attempt) -> Value { + json!({ + "auth_data": { + "authorization_code": code, + "client_domain": CLIENT_DOMAIN, + "client_id": attempt.client_id, + "code_algorithm": "SHA-256", + "code_verifier": attempt.verifier, + "use_global_authentication": false, + }, + "registration_data": { + "app_name": APP_NAME, + "app_version": APP_VERSION, + "device_model": DEVICE_MODEL, + "device_name": null, + "device_serial": attempt.serial, + "device_type": DEVICE_TYPE, + "domain": REGISTRATION_DOMAIN, + "os_version": OS_VERSION, + }, + "requested_extensions": ["customer_info", "device_info"], + "requested_token_type": ["bearer", "mac_dms"], + "user_context_map": {}, + }) +} + +pub fn read_registration(answer: &Value, attempt: &Attempt, now_ms: i64) -> Option { + let success = answer.get("response")?.get("success")?; + let bearer = success.get("tokens")?.get("bearer")?; + let access_token = net::str_of(bearer, "access_token"); + let refresh_token = net::str_of(bearer, "refresh_token"); + if access_token.is_empty() || refresh_token.is_empty() { + return None; + } + let serial = net::str_at(success, &["extensions", "device_info", "device_serial_number"]); + Some(Registered { + access_token: access_token.to_string(), + refresh_token: refresh_token.to_string(), + expires_at_ms: now_ms + expires_in(bearer.get("expires_in")) * 1000, + device_serial: if serial.is_empty() { attempt.serial.clone() } else { serial.to_string() }, + client_id: attempt.client_id.clone(), + display_name: net::str_at(success, &["extensions", "customer_info", "name"]).trim().to_string(), + }) +} + +fn identity_call(url: &str, body: &Value) -> Call { + Call::post_json(url, net::BROWSER_USER_AGENT.to_string(), body) +} + +pub fn register(code: &str, attempt: &Attempt) -> Result { + let reply = net::send(identity_call(REGISTER_URL, ®istration_body(code, attempt)))?; + if !reply.ok() { + return Err(format!("registration answered HTTP {}", reply.status)); + } + let answer = reply.json().ok_or("registration answer is not JSON")?; + read_registration(&answer, attempt, net::now_ms()).ok_or_else(|| "registration answer has no tokens".into()) +} + +pub fn refresh(refresh_token: &str) -> Result<(String, i64), String> { + let body = json!({ + "source_token": refresh_token, + "source_token_type": "refresh_token", + "requested_token_type": "access_token", + "app_name": APP_NAME, + "app_version": APP_VERSION, + }); + let call = identity_call(TOKEN_URL, &body).header("x-amzn-identity-auth-domain", "api.amazon.com"); + let reply = net::send(call)?; + if !reply.ok() { + return Err(format!("token refresh answered HTTP {}", reply.status)); + } + let answer = reply.json().ok_or("token answer is not JSON")?; + let access = net::str_of(&answer, "access_token"); + if access.is_empty() { + return Err("token answer has no access token".into()); + } + Ok((access.to_string(), net::now_ms() + expires_in(answer.get("expires_in")) * 1000)) +} + +pub fn deregister(access_token: &str) -> Result<(), String> { + let body = json!({ "requested_extensions": ["device_info", "customer_info"] }); + let call = identity_call(DEREGISTER_URL, &body).header("Authorization", &format!("Bearer {access_token}")); + net::send(call).map(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn attempt() -> Attempt { + let serial = serial_from(&[0xab; 16]); + Attempt { client_id: client_id_for(&serial), serial, verifier: "dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk".into() } + } + + #[test] + fn device_values() { + let a = attempt(); + assert_eq!(a.serial, "ABABABABABAB4BABABABABABABABABAB"); + assert_eq!(a.serial.len(), 32); + assert_eq!(a.client_id.len(), 94); + assert_eq!(a.client_id, net::hex_lower(format!("{}#A2UMVHOX7UP4V7", a.serial).as_bytes())); + assert!(a.client_id.starts_with("4142")); + assert_eq!(challenge_for("dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk"), "E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM"); + let fresh = Attempt::new().unwrap(); + assert_eq!(fresh.verifier.len(), 43); + assert!(fresh.serial.chars().all(|c| c.is_ascii_hexdigit() && !c.is_ascii_lowercase())); + assert_eq!(&fresh.serial[12..13], "4"); + assert_ne!(fresh, Attempt::new().unwrap()); + } + + #[test] + fn sign_in_url() { + let a = attempt(); + let url = a.start_url(); + assert!(url.starts_with("https://www.amazon.com/ap/signin?openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&")); + assert!(url.contains(&format!("&openid.oa2.client_id=device%3A{}&", a.client_id))); + assert!(url.contains("&marketPlaceId=ATVPDKIKX0DER&")); + assert!(url.contains("&openid.assoc_handle=amzn_sonic_games_launcher&pageId=amzn_sonic_games_launcher&")); + assert!(url.ends_with("&openid.oa2.code_challenge=E9Melhoa2OwvFrEMTJguCHaoeK1t8URWbuGJSstw-cM")); + assert_eq!(url.matches('&').count(), 16); + } + + #[test] + fn codes_from_return_urls() { + assert_eq!( + code_from_url("https://www.amazon.com/?openid.assoc_handle=x&openid.oa2.authorization_code=ANabc%2Bd&openid.mode=id_res").as_deref(), + Some("ANabc+d") + ); + assert_eq!(code_from_url("https://www.amazon.com/?openid.oa2.authorization_code=Z#frag").as_deref(), Some("Z")); + assert_eq!(code_from_url("https://www.amazon.com/?openid.oa2.authorization_code=&x=1"), None); + assert_eq!(code_from_url("https://www.amazon.com/?x.openid.oa2.authorization_code=1"), None); + assert_eq!(code_from_url("https://www.amazon.com/"), None); + } + + #[test] + fn registration_round() { + let a = attempt(); + let body = registration_body("CODE", &a); + assert_eq!(body["auth_data"]["client_id"], a.client_id.as_str()); + assert_eq!(body["auth_data"]["use_global_authentication"], false); + assert!(body["registration_data"]["device_name"].is_null()); + assert_eq!(body["registration_data"]["device_serial"], a.serial.as_str()); + let answer = json!({"response":{"success":{ + "tokens":{"bearer":{"access_token":"Atna|x","refresh_token":"Atnr|y","expires_in":"3600"},"mac_dms":{}}, + "extensions":{"customer_info":{"name":"Sam"},"device_info":{"device_serial_number":"SERVER"}}}}}); + let r = read_registration(&answer, &a, 1000).unwrap(); + assert_eq!(r.expires_at_ms, 3_601_000); + assert_eq!(r.device_serial, "SERVER"); + assert_eq!(r.client_id, a.client_id); + assert_eq!(r.to_json()["display_name"], "Sam"); + let bare = json!({"response":{"success":{"tokens":{"bearer":{"access_token":"A","refresh_token":"R"}}}}}); + let r = read_registration(&bare, &a, 0).unwrap(); + assert_eq!(r.device_serial, a.serial); + assert_eq!(r.expires_at_ms, 3_600_000); + assert!(r.to_json().get("display_name").is_none()); + assert!(read_registration(&json!({"response":{"error":{}}}), &a, 0).is_none()); + } +} diff --git a/app/src/main/rust/stores/src/amazon/cdn.rs b/app/src/main/rust/stores/src/amazon/cdn.rs new file mode 100644 index 0000000000..de3b43c242 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/cdn.rs @@ -0,0 +1,58 @@ +use super::net::{self, Call}; + +pub const MANIFEST_SEGMENT: &str = "manifest.proto"; + +pub fn append_path(base: &str, segment: &str) -> String { + match base.find('?') { + Some(q) => format!("{}/{segment}{}", &base[..q], &base[q..]), + None => format!("{base}/{segment}"), + } +} + +pub fn file_url(base: &str, cdn_name: &str) -> String { + append_path(base, &format!("files/{cdn_name}")) +} + +pub fn origin(url: &str) -> String { + let after_scheme = url.find("://").map(|i| i + 3).unwrap_or(0); + let end = url[after_scheme..].find(['/', '?', '#']).map(|i| after_scheme + i).unwrap_or(url.len()); + url[..end].to_string() +} + +pub fn get(url: &str, access_token: Option<&str>) -> Result, String> { + let mut call = Call::get(url, net::user_agent()).read_timeout(net::BYTES_READ_TIMEOUT); + if let Some(token) = access_token.filter(|t| !t.is_empty()) { + call = call.header("x-amzn-token", token); + } + let reply = net::send(call)?; + if !reply.ok() { + return Err(format!("the CDN answered HTTP {}", reply.status)); + } + Ok(reply.body) +} + +pub fn fetch_manifest(base: &str, access_token: &str) -> Result, String> { + get(&append_path(base, MANIFEST_SEGMENT), Some(access_token)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn segments_go_before_the_query() { + assert_eq!( + append_path("https://cdn.example/v1/abc?Expires=1&Signature=x%2By", "manifest.proto"), + "https://cdn.example/v1/abc/manifest.proto?Expires=1&Signature=x%2By" + ); + assert_eq!(append_path("https://cdn.example/v1/abc", "manifest.proto"), "https://cdn.example/v1/abc/manifest.proto"); + assert_eq!(append_path("https://h/p?a=1?b=2", "s"), "https://h/p/s?a=1?b=2"); + assert_eq!(file_url("https://h/base?q", "00ff"), "https://h/base/files/00ff?q"); + } + + #[test] + fn origins() { + assert_eq!(origin("https://cdn.example:8443/x?y"), "https://cdn.example:8443"); + assert_eq!(origin("https://cdn.example?y"), "https://cdn.example"); + } +} diff --git a/app/src/main/rust/stores/src/amazon/codec.rs b/app/src/main/rust/stores/src/amazon/codec.rs new file mode 100644 index 0000000000..36da49d10b --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/codec.rs @@ -0,0 +1,95 @@ +use std::io::{self, Write}; + +const XZ_MAGIC_PREFIX: [u8; 2] = [0xfd, 0x37]; + +pub const MAX_MESSAGE_BYTES: usize = 256 * 1024 * 1024; + +pub fn body(object: &[u8]) -> Result<&[u8], String> { + if object.len() < 4 { + return Err("the manifest is too short".into()); + } + let header_len = u32::from_be_bytes([object[0], object[1], object[2], object[3]]); + if header_len > i32::MAX as u32 || 4 + header_len as u64 > object.len() as u64 { + return Err(format!("the manifest has an invalid header size ({header_len})")); + } + Ok(&object[4 + header_len as usize..]) +} + +struct Capped(Vec); + +impl Write for Capped { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.0.len() + buf.len() > MAX_MESSAGE_BYTES { + return Err(io::Error::other("the manifest decompresses to more than the allowed size")); + } + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub fn decompress(body: &[u8]) -> Result, String> { + if body.len() < 2 { + return Ok(body.to_vec()); + } + let mut input = io::BufReader::new(body); + let mut out = Capped(Vec::new()); + if body[..2] == XZ_MAGIC_PREFIX { + lzma_rs::xz_decompress(&mut input, &mut out).map_err(|e| format!("xz body: {e:?}"))?; + } else { + lzma_rs::lzma_decompress(&mut input, &mut out).map_err(|e| format!("lzma body: {e:?}"))?; + } + Ok(out.0) +} + +pub fn message(object: &[u8]) -> Result, String> { + decompress(body(object)?) +} + +#[cfg(test)] +pub mod testing { + pub fn frame(message: &[u8], header: &[u8], xz: bool) -> Vec { + let mut body = Vec::new(); + let mut input = std::io::BufReader::new(message); + if xz { + lzma_rs::xz_compress(&mut input, &mut body).unwrap(); + } else { + lzma_rs::lzma_compress(&mut input, &mut body).unwrap(); + } + let mut out = (header.len() as u32).to_be_bytes().to_vec(); + out.extend_from_slice(header); + out.extend_from_slice(&body); + out + } +} + +#[cfg(test)] +mod tests { + use super::testing::frame; + use super::*; + + #[test] + fn both_containers_round_trip() { + let message: Vec = (0..5000u32).flat_map(|i| (i % 251).to_le_bytes()).collect(); + let xz = frame(&message, b"\x08\x01", true); + assert_eq!(&xz[6..8], &XZ_MAGIC_PREFIX); + assert_eq!(super::message(&xz).unwrap(), message); + let alone = frame(&message, b"", false); + assert_eq!(super::message(&alone).unwrap(), message); + } + + #[test] + fn framing_is_validated() { + assert!(body(&[0, 0, 0]).is_err()); + assert!(body(&[0, 0, 0, 5, 1, 2]).is_err()); + assert!(body(&[0x80, 0, 0, 0, 1]).is_err()); + assert_eq!(body(&[0, 0, 0, 1, 9, 7, 8]).unwrap(), &[7, 8]); + assert_eq!(body(&[0, 0, 0, 0]).unwrap(), &[] as &[u8]); + assert_eq!(decompress(&[5]).unwrap(), vec![5]); + assert!(decompress(&[0xfd, 0x37, 1, 2, 3]).is_err()); + assert!(decompress(&[0xff, 0, 0, 0, 1]).is_err(), "invalid LZMA properties byte"); + } +} diff --git a/app/src/main/rust/stores/src/amazon/distribution.rs b/app/src/main/rust/stores/src/amazon/distribution.rs new file mode 100644 index 0000000000..208e94d9a4 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/distribution.rs @@ -0,0 +1,92 @@ +use serde_json::{json, Value}; + +use super::net::{self, Call, Reply}; + +pub const ENTITLEMENTS_URL: &str = "https://gaming.amazon.com/api/distribution/entitlements"; +pub const PUBLIC_URL: &str = "https://gaming.amazon.com/api/distribution/v2/public"; + +pub const ENTITLEMENTS_TARGET: &str = + "com.amazon.animusdistributionservice.entitlement.AnimusEntitlementsService.GetEntitlements"; +pub const GAME_DOWNLOAD_TARGET: &str = + "com.amazon.animusdistributionservice.external.AnimusDistributionService.GetGameDownload"; + +#[derive(Debug, PartialEq, Eq)] +pub enum Failure { + SignIn, + Other(String), +} + +impl Failure { + pub fn message(&self) -> String { + match self { + Failure::SignIn => "Amazon refused the session".into(), + Failure::Other(m) => m.clone(), + } + } +} + +pub fn refused(reply: &Reply) -> bool { + matches!(reply.status, 401 | 403) + || (reply.status == 400 && String::from_utf8_lossy(&reply.body).contains("NotAuthorizedException")) +} + +pub fn call(url: &str, target: &str, access_token: &str, body: &Value) -> Result { + let call = Call::post_json(url, net::LAUNCHER_USER_AGENT.to_string(), body) + .header("X-Amz-Target", target) + .header("x-amzn-token", access_token) + .header(net::LAUNCHER_PACKAGE.0, net::LAUNCHER_PACKAGE.1) + .header("Content-Encoding", "amz-1.0"); + let reply = net::send(call).map_err(Failure::Other)?; + if !reply.ok() { + if refused(&reply) { + return Err(Failure::SignIn); + } + return Err(Failure::Other(format!("Amazon answered HTTP {} ({})", reply.status, reply.snippet()))); + } + reply.json().ok_or_else(|| Failure::Other("Amazon's answer could not be read".into())) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Download { + pub base_url: String, + pub version_id: String, +} + +pub fn read_download(answer: &Value) -> Option { + let base_url = net::str_of(answer, "downloadUrl"); + if base_url.is_empty() { + return None; + } + Some(Download { base_url: base_url.to_string(), version_id: net::str_of(answer, "versionId").to_string() }) +} + +pub fn game_download(access_token: &str, entitlement_id: &str) -> Result { + let body = json!({ "entitlementId": entitlement_id, "Operation": "GetGameDownload" }); + let answer = call(PUBLIC_URL, GAME_DOWNLOAD_TARGET, access_token, &body)?; + read_download(&answer).ok_or_else(|| Failure::Other("the answer has no download URL".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn download_answers() { + let d = read_download(&json!({"downloadUrl":"https://cdn/x?sig=1","versionId":"v7"})).unwrap(); + assert_eq!(d.base_url, "https://cdn/x?sig=1"); + assert_eq!(d.version_id, "v7"); + assert_eq!(read_download(&json!({"downloadUrl":"","versionId":"v"})), None); + assert_eq!(read_download(&json!({})), None); + assert_eq!(read_download(&json!({"downloadUrl":"https://cdn/x"})).unwrap().version_id, ""); + } + + #[test] + fn refusals() { + let r = |status, body: &str| Reply { status, body: body.as_bytes().to_vec() }; + assert!(refused(&r(401, ""))); + assert!(refused(&r(403, ""))); + assert!(refused(&r(400, r#"{"__type":"com.amazon.coral.service#NotAuthorizedException"}"#))); + assert!(!refused(&r(400, r#"{"__type":"com.amazon.coral.validate#ValidationException"}"#))); + assert!(!refused(&r(500, ""))); + } +} diff --git a/app/src/main/rust/stores/src/amazon/ffi.rs b/app/src/main/rust/stores/src/amazon/ffi.rs new file mode 100644 index 0000000000..8c0ca0cd06 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/ffi.rs @@ -0,0 +1,248 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use jni::objects::{GlobalRef, JByteArray, JClass, JObject, JString, JValue}; +use jni::sys::{jlong, jstring, jbyteArray, JNI_FALSE, JNI_TRUE}; +use jni::{JNIEnv, JavaVM}; + +use super::install::{self, Events, Stop}; +use super::{net, service}; + +fn read_string(env: &mut JNIEnv, value: &JString) -> String { + if value.is_null() { + return String::new(); + } + env.get_string(value).map(|s| s.to_string_lossy().into_owned()).unwrap_or_default() +} + +fn new_string(env: &mut JNIEnv, text: &str) -> jstring { + env.new_string(text).map(|s| s.into_raw()).unwrap_or(std::ptr::null_mut()) +} + +fn guarded(fallback: T, f: impl FnOnce() -> T) -> T { + catch_unwind(AssertUnwindSafe(f)).unwrap_or(fallback) +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeConfigure( + mut env: JNIEnv, + _class: JClass, + app_version: JString, +) { + let version = read_string(&mut env, &app_version); + net::set_app_version(&version); +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCall( + mut env: JNIEnv, + _class: JClass, + op: JString, + request: JString, +) -> jstring { + let op = read_string(&mut env, &op); + let request = read_string(&mut env, &request); + match guarded(None, || Some(service::call(&op, &request))) { + Some(text) => new_string(&mut env, &text), + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeFetch( + mut env: JNIEnv, + _class: JClass, + op: JString, + request: JString, +) -> jbyteArray { + let op = read_string(&mut env, &op); + let request = read_string(&mut env, &request); + match guarded(None, || service::fetch(&op, &request)) { + Some(bytes) => env.byte_array_from_slice(&bytes).map(|a| a.into_raw()).unwrap_or(std::ptr::null_mut()), + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeInspect( + mut env: JNIEnv, + _class: JClass, + manifest: JByteArray, + request: JString, +) -> jstring { + let request = read_string(&mut env, &request); + let bytes = if manifest.is_null() { Vec::new() } else { env.convert_byte_array(&manifest).unwrap_or_default() }; + match guarded(None, || Some(service::inspect(&bytes, &request))) { + Some(text) => new_string(&mut env, &text), + None => std::ptr::null_mut(), + } +} + +struct Listener { + vm: JavaVM, + target: GlobalRef, +} + +impl Listener { + fn with_env(&self, fallback: R, f: impl FnOnce(&mut JNIEnv) -> R) -> R { + let Ok(mut env) = self.vm.attach_current_thread_as_daemon() else { + return fallback; + }; + let out = f(&mut env); + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } + out + } + + fn text(&self, method: &str, text: &str) { + self.with_env((), |env| { + let Ok(s) = env.new_string(text) else { return }; + let s = JObject::from(s); + let _ = env.call_method(self.target.as_obj(), method, "(Ljava/lang/String;)V", &[JValue::Object(&s)]); + let _ = env.delete_local_ref(s); + }); + } + + fn counts(&self, method: &str, a: u64, b: u64, c: u32, d: u32) { + self.with_env((), |env| { + let _ = env.call_method( + self.target.as_obj(), + method, + "(JJII)V", + &[ + JValue::Long(a.min(i64::MAX as u64) as i64), + JValue::Long(b.min(i64::MAX as u64) as i64), + JValue::Int(c.min(i32::MAX as u32) as i32), + JValue::Int(d.min(i32::MAX as u32) as i32), + ], + ); + }); + } + + fn complete(&self, success: bool, cancelled: bool, error: &str, result: &str) { + self.with_env((), |env| { + let (Ok(e), Ok(r)) = (env.new_string(error), env.new_string(result)) else { return }; + let (e, r) = (JObject::from(e), JObject::from(r)); + let _ = env.call_method( + self.target.as_obj(), + "onComplete", + "(ZZLjava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), + JValue::Bool(if cancelled { JNI_TRUE } else { JNI_FALSE }), + JValue::Object(&e), + JValue::Object(&r), + ], + ); + let _ = env.delete_local_ref(e); + let _ = env.delete_local_ref(r); + }); + } +} + +impl Events for Listener { + fn log(&self, line: &str) { + self.text("onLog", &net::scrub_urls(line)); + } + + fn status(&self, text: &str) { + self.text("onStatus", text); + } + + fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32) { + self.counts("onCheck", done_bytes, total_bytes, files, files_total); + } + + fn sizes(&self, pending_bytes: u64, total_bytes: u64) { + self.with_env((), |env| { + let _ = env.call_method( + self.target.as_obj(), + "onSizes", + "(JJ)V", + &[ + JValue::Long(pending_bytes.min(i64::MAX as u64) as i64), + JValue::Long(total_bytes.min(i64::MAX as u64) as i64), + ], + ); + }); + } + + fn progress(&self, done: u64, total: u64, files: u32, files_total: u32) { + self.counts("onProgress", done, total, files, files_total); + } + + fn before_replace(&self) -> bool { + self.with_env(false, |env| match env.call_method(self.target.as_obj(), "beforeReplace", "()Z", &[]) { + Ok(v) => v.z().unwrap_or(false), + Err(_) => false, + }) + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeInstall( + mut env: JNIEnv, + _class: JClass, + request: JString, + listener: JObject, +) -> jlong { + if listener.is_null() { + return 0; + } + let Ok(vm) = env.get_java_vm() else { return 0 }; + let Ok(target) = env.new_global_ref(&listener) else { return 0 }; + let listener = Listener { vm, target }; + let text = read_string(&mut env, &request); + let parsed = serde_json::from_str::(&text) + .map_err(|_| "The install request could not be read".to_string()) + .and_then(|v| install::Request::from_json(&v)); + let request = match parsed { + Ok(r) => r, + Err(e) => { + listener.complete(false, false, &e, ""); + return 0; + } + }; + let cancel = Arc::new(AtomicBool::new(false)); + let worker_cancel = Arc::clone(&cancel); + let spawned = std::thread::Builder::new().name("amazon-install".into()).spawn(move || { + crate::priority::background(); + let result = catch_unwind(AssertUnwindSafe(|| install::run(&request, &worker_cancel, &listener))) + .unwrap_or_else(|_| Err(Stop::Failed("The Amazon install stopped on an internal error".into()))); + match result { + Ok(done) => listener.complete(true, false, "", &done.to_json().to_string()), + Err(Stop::Cancelled) => listener.complete(false, true, "cancelled", ""), + Err(Stop::Failed(e)) => listener.complete(false, false, &e, ""), + } + }); + if spawned.is_err() { + return 0; + } + Box::into_raw(Box::new(cancel)) as jlong +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + let ptr = handle as *const Arc; + if let Some(cancel) = unsafe { ptr.as_ref() } { + cancel.store(true, Ordering::SeqCst); + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + let ptr = handle as *mut Arc; + if !ptr.is_null() { + drop(unsafe { Box::from_raw(ptr) }); + } +} diff --git a/app/src/main/rust/stores/src/amazon/install/check.rs b/app/src/main/rust/stores/src/amazon/install/check.rs new file mode 100644 index 0000000000..e53a082bf4 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/install/check.rs @@ -0,0 +1,165 @@ +use std::fs; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use sha2::{Digest, Sha256}; + +use crate::amazon::manifest::{GameFile, TEMP_SUFFIX}; + +#[derive(Debug, PartialEq, Eq)] +pub enum State { + Complete, + Pending, + Cancelled, +} + +pub fn sha256_of(path: &Path, cancel: &AtomicBool) -> Result, ()> { + let Ok(mut f) = fs::File::open(path) else { return Ok(None) }; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; 1024 * 1024]; + loop { + if cancel.load(Ordering::Relaxed) { + return Err(()); + } + match f.read(&mut buf) { + Ok(0) => return Ok(Some(hasher.finalize().into())), + Ok(n) => hasher.update(&buf[..n]), + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(_) => return Ok(None), + } + } +} + +pub fn state(dest: &Path, file: &GameFile, cancel: &AtomicBool) -> State { + match fs::symlink_metadata(dest) { + Ok(m) if m.is_file() && m.len() == file.size => {} + _ => return State::Pending, + } + let Some(expected) = file.sha256() else { return State::Complete }; + match sha256_of(dest, cancel) { + Err(()) => State::Cancelled, + Ok(Some(got)) if got == expected => State::Complete, + Ok(_) => State::Pending, + } +} + +pub fn temp_path(dest: &Path) -> PathBuf { + let mut name = dest.as_os_str().to_owned(); + name.push(TEMP_SUFFIX); + PathBuf::from(name) +} + +pub fn remove_stray_temps(game_dir: &Path, files: &[GameFile]) -> usize { + files + .iter() + .map(|f| temp_path(&game_dir.join(&f.path))) + .filter(|t| fs::symlink_metadata(t).map(|m| m.is_file() || m.file_type().is_symlink()).unwrap_or(false)) + .filter(|t| fs::remove_file(t).is_ok()) + .count() +} + +pub fn free_bytes(path: &Path) -> i64 { + let mut probe = Some(path); + while let Some(p) = probe { + if p.exists() { + return available(p); + } + probe = p.parent(); + } + -1 +} + +#[cfg(unix)] +fn available(path: &Path) -> i64 { + use std::os::unix::ffi::OsStrExt; + let Ok(c) = std::ffi::CString::new(path.as_os_str().as_bytes()) else { return -1 }; + let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + return -1; + } + (st.f_bavail as u64).saturating_mul(st.f_frsize as u64).min(i64::MAX as u64) as i64 +} + +#[cfg(not(unix))] +fn available(_path: &Path) -> i64 { + -1 +} + +pub fn human(bytes: u64) -> String { + const KIB: f64 = 1024.0; + let b = bytes as f64; + if b >= KIB * KIB * KIB { + format!("{:.1} GB", b / (KIB * KIB * KIB)) + } else if b >= KIB * KIB { + format!("{:.1} MB", b / (KIB * KIB)) + } else { + format!("{:.1} KB", b / KIB) + } +} + +pub fn space_problem(needed: u64, free: i64) -> Option { + if needed > 0 && free >= 0 && (free as u64) < needed { + Some(format!("Not enough free space: need {}, only {} free", human(needed), human(free as u64))) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(path: &str, size: u64, digest: Vec) -> GameFile { + GameFile { path: path.into(), size, algorithm: 0, digest } + } + + #[test] + fn complete_needs_size_and_hash() { + let dir = std::env::temp_dir().join(format!("dd-amazon-check-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let p = dir.join("a.bin"); + fs::write(&p, b"hello").unwrap(); + let good = Sha256::digest(b"hello").to_vec(); + let cancel = AtomicBool::new(false); + assert_eq!(state(&p, &file("a.bin", 5, good.clone()), &cancel), State::Complete); + assert_eq!(state(&p, &file("a.bin", 5, Sha256::digest(b"HELLO").to_vec()), &cancel), State::Pending); + assert_eq!(state(&p, &file("a.bin", 6, good.clone()), &cancel), State::Pending); + let other = GameFile { path: "a.bin".into(), size: 5, algorithm: 3, digest: vec![1, 2] }; + assert_eq!(state(&p, &other, &cancel), State::Complete, "no SHA-256: size decides"); + assert_eq!(state(&dir, &file("x", 0, vec![]), &cancel), State::Pending, "a directory is not a file"); + assert_eq!(state(&dir.join("missing"), &file("m", 0, vec![]), &cancel), State::Pending); + cancel.store(true, Ordering::Relaxed); + assert_eq!(state(&p, &file("a.bin", 5, good), &cancel), State::Cancelled); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn stray_temps_are_removed_only_for_manifest_files() { + let dir = std::env::temp_dir().join(format!("dd-amazon-temps-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("sub")).unwrap(); + fs::write(dir.join("sub/a.bin.tmp"), b"x").unwrap(); + fs::write(dir.join("save.tmp"), b"keep").unwrap(); + let files = vec![file("sub/a.bin", 1, vec![0; 32]), file("b.bin", 1, vec![0; 32])]; + assert_eq!(temp_path(&dir.join("sub/a.bin")), dir.join("sub/a.bin.tmp")); + assert_eq!(remove_stray_temps(&dir, &files), 1); + assert!(!dir.join("sub/a.bin.tmp").exists()); + assert!(dir.join("save.tmp").exists()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn space_messages() { + assert_eq!(space_problem(0, 0), None); + assert_eq!(space_problem(10, -1), None); + assert_eq!(space_problem(10, 10), None); + assert_eq!( + space_problem(3 * 1024 * 1024 * 1024, 512 * 1024 * 1024).unwrap(), + "Not enough free space: need 3.0 GB, only 512.0 MB free" + ); + assert_eq!(space_problem(2048, 0).unwrap(), "Not enough free space: need 2.0 KB, only 0.0 KB free"); + assert!(free_bytes(&std::env::temp_dir()) != 0); + } +} diff --git a/app/src/main/rust/stores/src/amazon/install/mod.rs b/app/src/main/rust/stores/src/amazon/install/mod.rs new file mode 100644 index 0000000000..f9e85122cf --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/install/mod.rs @@ -0,0 +1,293 @@ +pub mod check; +pub mod writer; + +use std::fs; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::distribution::{self, Download}; +use super::manifest::{self, GameFile}; +use super::{cdn, net}; + +pub const IN_PROGRESS_MARKER: &str = ".amazon_download_in_progress"; +pub const COMPLETE_MARKER: &str = ".amazon_download_complete"; + +pub const NOT_SIGNED_IN: &str = "Not signed in to Amazon Games"; +pub const NO_DOWNLOAD: &str = "Amazon gave no download for this game"; +pub const NO_MANIFEST: &str = "The manifest could not be downloaded"; +pub const FILE_FAILED: &str = "A file failed to download"; +pub const REPLACE_REFUSED: &str = "The install record could not be switched to updating; nothing was replaced"; + +const CHECK_REPORT_EVERY: Duration = Duration::from_millis(200); + +pub trait Events: Sync { + fn log(&self, line: &str); + fn status(&self, text: &str); + fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32); + fn sizes(&self, pending_bytes: u64, total_bytes: u64); + fn progress(&self, done: u64, total: u64, files: u32, files_total: u32); + fn before_replace(&self) -> bool; +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Stop { + Cancelled, + Failed(String), +} + +pub struct ReplaceGate(Mutex>); + +impl ReplaceGate { + pub fn new() -> Self { + ReplaceGate(Mutex::new(None)) + } + + pub fn allow(&self, events: &dyn Events) -> bool { + let mut answer = self.0.lock().expect("gate lock"); + if answer.is_none() { + *answer = Some(events.before_replace()); + } + *answer == Some(true) + } +} + +impl Default for ReplaceGate { + fn default() -> Self { + Self::new() + } +} + +pub struct Request { + pub access_token: String, + pub entitlement_id: String, + pub game_dir: PathBuf, + pub max_workers: usize, + pub process_workers: usize, + pub ca_bundle: String, +} + +impl Request { + pub fn from_json(v: &Value) -> Result { + let game_dir = net::str_of(v, "gameDir"); + if game_dir.is_empty() || !game_dir.starts_with('/') { + return Err("No install folder was given".into()); + } + let access_token = net::str_of(v, "accessToken"); + if access_token.is_empty() { + return Err(NOT_SIGNED_IN.into()); + } + let entitlement_id = net::str_of(v, "entitlementId"); + if entitlement_id.is_empty() { + return Err(NO_DOWNLOAD.into()); + } + Ok(Request { + access_token: access_token.to_string(), + entitlement_id: entitlement_id.to_string(), + game_dir: PathBuf::from(game_dir), + max_workers: v.get("maxWorkers").and_then(|n| n.as_u64()).unwrap_or(8) as usize, + process_workers: v.get("processWorkers").and_then(|n| n.as_u64()).unwrap_or(2) as usize, + ca_bundle: net::str_of(v, "caBundle").to_string(), + }) + } +} + +#[derive(Debug, Default)] +pub struct Finished { + pub version_id: String, + pub total_bytes: u64, + pub file_count: usize, + pub files_written: usize, + pub bytes_written: u64, +} + +impl Finished { + pub fn to_json(&self) -> Value { + json!({ + "versionId": self.version_id, + "totalBytes": self.total_bytes, + "fileCount": self.file_count, + "filesWritten": self.files_written, + "bytesWritten": self.bytes_written, + }) + } +} + +fn cancelled(cancel: &AtomicBool) -> Result<(), Stop> { + if cancel.load(Ordering::SeqCst) { + Err(Stop::Cancelled) + } else { + Ok(()) + } +} + +pub fn load_manifest(download: &Download, access_token: &str, log: &dyn Fn(&str)) -> Result { + let bytes = cdn::fetch_manifest(&download.base_url, access_token).map_err(|e| { + log(&format!("manifest download: {e}")); + NO_MANIFEST.to_string() + })?; + manifest::parse(&bytes).map_err(|e| { + log(&format!("manifest parse: {e}")); + if e.starts_with("The manifest") { + e + } else { + format!("The manifest could not be read: {e}") + } + }) +} + +pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result { + ev.status("Asking Amazon for the download…"); + fs::create_dir_all(&req.game_dir).map_err(|e| Stop::Failed(format!("Could not create the install folder: {e}")))?; + let _ = fs::File::create(req.game_dir.join(IN_PROGRESS_MARKER)); + let download = distribution::game_download(&req.access_token, &req.entitlement_id).map_err(|f| { + ev.log(&format!("download lookup: {}", f.message())); + Stop::Failed(NO_DOWNLOAD.into()) + })?; + cancelled(cancel)?; + run_download(req, &download, cancel, ev) +} + +fn empty_sha256() -> [u8; 32] { + Sha256::digest(b"").into() +} + +pub fn run_download(req: &Request, download: &Download, cancel: &AtomicBool, ev: &dyn Events) -> Result { + let failed = Stop::Failed; + let started = Instant::now(); + fs::create_dir_all(&req.game_dir).map_err(|e| failed(format!("Could not create the install folder: {e}")))?; + let _ = fs::File::create(req.game_dir.join(IN_PROGRESS_MARKER)); + + ev.status("Downloading manifest…"); + let log = |line: &str| ev.log(line); + let manifest = load_manifest(download, &req.access_token, &log).map_err(failed)?; + cancelled(cancel)?; + for w in &manifest.warnings { + ev.log(w); + } + let total = manifest.total_size(); + let files_total = manifest.files.len() as u32; + ev.log(&format!( + "manifest version={} files={} entries={} total_bytes={total}", + if download.version_id.is_empty() { "?" } else { download.version_id.as_str() }, + manifest.files.len(), + manifest.entries, + )); + + let strays = check::remove_stray_temps(&req.game_dir, &manifest.files); + if strays > 0 { + ev.log(&format!("removed {strays} leftover temporary files")); + } + + ev.status("Checking files…"); + let mut pending: Vec<&GameFile> = Vec::new(); + let mut checked = 0u64; + let mut last_report = Instant::now(); + for (n, f) in manifest.files.iter().enumerate() { + match check::state(&req.game_dir.join(&f.path), f, cancel) { + check::State::Cancelled => return Err(Stop::Cancelled), + check::State::Complete => {} + check::State::Pending => pending.push(f), + } + checked += f.size; + if last_report.elapsed() >= CHECK_REPORT_EVERY || n + 1 == manifest.files.len() { + ev.checking(checked, total, (n + 1) as u32, files_total); + last_report = Instant::now(); + } + cancelled(cancel)?; + } + let pending_bytes: u64 = pending.iter().map(|f| f.size).sum(); + let skipped_files = manifest.files.len() - pending.len(); + ev.log(&format!("resume skipped_files={skipped_files} skipped_bytes={} pending_files={} pending_bytes={pending_bytes}", total - pending_bytes, pending.len())); + ev.sizes(pending_bytes, total); + + let free = check::free_bytes(&req.game_dir); + if let Some(message) = check::space_problem(pending_bytes, free) { + ev.log(&format!("space pending={pending_bytes} free={free}")); + return Err(failed(message)); + } + + let gate = ReplaceGate::new(); + let empty = empty_sha256(); + let (local, remote): (Vec<&GameFile>, Vec<&GameFile>) = pending + .into_iter() + .partition(|f| f.size == 0 && (f.digest.is_empty() || f.sha256() == Some(empty))); + for f in &local { + let dest = req.game_dir.join(&f.path); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).map_err(|e| failed(format!("Could not create the folder for {}: {e}", f.path)))?; + } + let tmp = check::temp_path(&dest); + fs::File::create(&tmp).map_err(|e| failed(format!("Could not write {}: {e}", f.path)))?; + if !gate.allow(ev) { + let _ = fs::remove_file(&tmp); + return Err(failed(REPLACE_REFUSED.into())); + } + if let Err(e) = fs::rename(&tmp, &dest) { + let _ = fs::remove_file(&tmp); + return Err(failed(format!("Could not put {} in place: {e}", f.path))); + } + } + cancelled(cancel)?; + + let jobs: Vec = remote + .iter() + .map(|f| writer::Job { + path: f.path.clone(), + dest: req.game_dir.join(&f.path), + size: f.size, + sha256: f.sha256(), + url: cdn::file_url(&download.base_url, &f.cdn_name()), + }) + .collect(); + let settings = writer::Settings { + max_workers: req.max_workers, + process_workers: req.process_workers, + ca_bundle: req.ca_bundle.clone(), + user_agent: net::user_agent(), + hosts: vec![cdn::origin(&download.base_url)], + }; + let base = writer::Baseline { + done_bytes: total - pending_bytes, + done_files: (skipped_files + local.len()) as u32, + total_bytes: total, + total_files: files_total, + }; + let result = writer::run(&jobs, &settings, base, &gate, cancel, ev); + let elapsed = started.elapsed().as_secs_f64().max(0.001); + let bytes_written = result.as_ref().map(|s| s.bytes_written).unwrap_or(0); + ev.log(&format!( + "summary files={}/{} bytes_written={bytes_written} elapsed_s={elapsed:.1} avg_mbps={:.1} cancelled={} error={}", + if result.is_ok() { manifest.files.len() } else { skipped_files + local.len() }, + manifest.files.len(), + bytes_written as f64 / 1_048_576.0 / elapsed, + matches!(result, Err(Stop::Cancelled)), + match &result { + Err(Stop::Failed(m)) => m.as_str(), + _ => "none", + }, + )); + let stats = result?; + + let _ = fs::remove_file(req.game_dir.join(IN_PROGRESS_MARKER)); + let _ = fs::File::create(req.game_dir.join(COMPLETE_MARKER)); + Ok(Finished { + version_id: download.version_id.clone(), + total_bytes: total, + file_count: manifest.files.len(), + files_written: stats.files_written + local.len(), + bytes_written, + }) +} + +pub fn size_of(access_token: &str, entitlement_id: &str) -> u64 { + let Ok(download) = distribution::game_download(access_token, entitlement_id) else { return 0 }; + load_manifest(&download, access_token, &|_: &str| {}).map(|m| m.total_size()).unwrap_or(0) +} + +#[cfg(test)] +mod tests; diff --git a/app/src/main/rust/stores/src/amazon/install/tests.rs b/app/src/main/rust/stores/src/amazon/install/tests.rs new file mode 100644 index 0000000000..0667186444 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/install/tests.rs @@ -0,0 +1,313 @@ +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use sha2::{Digest, Sha256}; + +use super::*; +use crate::amazon::codec::testing::frame; +use crate::amazon::manifest::testing::{message, Entry}; + +#[derive(Default)] +struct Seen { + paths: Vec, + queries_ok: bool, + manifest_token: Option, + file_tokens: usize, + agents: Vec, +} + +struct Server { + base: String, + seen: Arc>, + files: Arc>>>, +} + +fn serve(files: HashMap>) -> Server { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let files = Arc::new(Mutex::new(files)); + let seen = Arc::new(Mutex::new(Seen { queries_ok: true, ..Default::default() })); + let (f2, s2) = (Arc::clone(&files), Arc::clone(&seen)); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let (files, seen) = (Arc::clone(&f2), Arc::clone(&s2)); + std::thread::spawn(move || answer(stream, &files, &seen)); + } + }); + Server { base: format!("http://{addr}/v/abc?sig=s%2Bt&e=1"), seen, files } +} + +fn answer(mut stream: TcpStream, files: &Mutex>>, seen: &Mutex) { + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + if reader.read_line(&mut first).is_err() { + return; + } + let mut token = None; + let mut agent = String::new(); + loop { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) if line == "\r\n" => break, + Ok(_) => { + let lower = line.to_ascii_lowercase(); + if let Some(v) = lower.strip_prefix("x-amzn-token:") { + token = Some(v.trim().to_string()); + } + if lower.starts_with("user-agent:") { + agent = line["user-agent:".len()..].trim().to_string(); + } + } + } + } + let target = first.split_whitespace().nth(1).unwrap_or("/").to_string(); + let (path, query) = target.split_once('?').unwrap_or((target.as_str(), "")); + { + let mut s = seen.lock().unwrap(); + s.paths.push(path.to_string()); + s.queries_ok &= query == "sig=s%2Bt&e=1"; + s.agents.push(agent); + if path.ends_with("/manifest.proto") { + s.manifest_token = token; + } else if token.is_some() { + s.file_tokens += 1; + } + } + let body = files.lock().unwrap().get(path).cloned(); + let (status, body) = match body { + Some(b) => ("200 OK", b), + None => ("404 Not Found", Vec::new()), + }; + let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + let _ = stream.flush(); +} + +#[derive(Default)] +struct Recorder { + logs: Mutex>, + gate_calls: AtomicUsize, + refuse: bool, + last: Mutex<(u64, u64, u32, u32)>, + backwards: AtomicBool, + sizes: Mutex<(u64, u64)>, +} + +impl Events for Recorder { + fn log(&self, line: &str) { + self.logs.lock().unwrap().push(line.to_string()); + } + fn status(&self, _text: &str) {} + fn checking(&self, _d: u64, _t: u64, _f: u32, _ft: u32) {} + fn sizes(&self, pending: u64, total: u64) { + *self.sizes.lock().unwrap() = (pending, total); + } + fn progress(&self, done: u64, total: u64, files: u32, files_total: u32) { + let mut last = self.last.lock().unwrap(); + if done < last.0 { + self.backwards.store(true, Ordering::SeqCst); + } + *last = (done, total, files, files_total); + } + fn before_replace(&self) -> bool { + self.gate_calls.fetch_add(1, Ordering::SeqCst); + !self.refuse + } +} + +fn pattern(seed: u8, len: usize) -> Vec { + (0..len).map(|i| seed.wrapping_mul(29).wrapping_add((i * 13 % 251) as u8)).collect() +} + +struct Fixture { + server: Server, + expected: Vec<(&'static str, Vec)>, +} + +fn fixture_with(paths: &[(&'static str, Vec)]) -> Fixture { + let digests: Vec> = paths.iter().map(|(_, b)| Sha256::digest(b).to_vec()).collect(); + let mut packages = vec![Vec::new(), Vec::new()]; + for (i, (path, bytes)) in paths.iter().enumerate() { + let digest: &[u8] = if bytes.is_empty() { &[] } else { &digests[i] }; + packages[i % 2].push(Entry { path, size: bytes.len() as u64, algorithm: Some(0), digest }); + } + let manifest = frame(&message(&packages), b"\x08\x01\x12\x00", true); + let mut files = HashMap::new(); + files.insert("/v/abc/manifest.proto".to_string(), manifest); + for (i, (_, bytes)) in paths.iter().enumerate() { + files.insert(format!("/v/abc/files/{}", net::hex_lower(&digests[i])), bytes.clone()); + } + Fixture { server: serve(files), expected: paths.iter().map(|(p, b)| (*p, b.clone())).collect() } +} + +fn fixture() -> Fixture { + fixture_with(&[ + ("Game.exe", pattern(1, 300_000)), + ("Data\\pak0.pak", pattern(2, 70_000)), + ("Data\\Sub\\readme.txt", b"hello".to_vec()), + ("Data\\empty.cfg", Vec::new()), + ("fuel.json", br#"{"Main":{"Command":"Game.exe","Args":["-w"]}}"#.to_vec()), + ]) +} + +fn request(game: &Path) -> Request { + Request { + access_token: "token-1".into(), + entitlement_id: "ent".into(), + game_dir: game.to_path_buf(), + max_workers: 4, + process_workers: 2, + ca_bundle: String::new(), + } +} + +fn download(fx: &Fixture) -> Download { + Download { base_url: fx.server.base.clone(), version_id: "ver-9".into() } +} + +fn temp_game(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("dd-amazon-install-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + dir +} + +fn requests(fx: &Fixture) -> usize { + fx.server.seen.lock().unwrap().paths.len() +} + +fn no_temps(dir: &Path) -> bool { + fn walk(dir: &Path) -> bool { + fs::read_dir(dir).unwrap().flatten().all(|e| { + let p = e.path(); + if p.is_dir() { + walk(&p) + } else { + !e.file_name().to_string_lossy().ends_with(".tmp") + } + }) + } + walk(dir) +} + +#[test] +fn fresh_install_then_rerun_then_repair() { + let fx = fixture(); + let game = temp_game("e2e"); + let cancel = AtomicBool::new(false); + + let ev = Recorder::default(); + let done = run_download(&request(&game), &download(&fx), &cancel, &ev).unwrap(); + for (path, bytes) in &fx.expected { + assert_eq!(&fs::read(game.join(path.replace('\\', "/"))).unwrap(), bytes, "{path}"); + } + let total: u64 = fx.expected.iter().map(|(_, b)| b.len() as u64).sum(); + assert_eq!(done.version_id, "ver-9"); + assert_eq!(done.total_bytes, total); + assert_eq!(done.file_count, 5); + assert_eq!(done.files_written, 5); + assert_eq!(ev.gate_calls.load(Ordering::SeqCst), 1); + assert_eq!(*ev.last.lock().unwrap(), (total, total, 5, 5)); + assert!(!ev.backwards.load(Ordering::SeqCst)); + assert!(game.join(COMPLETE_MARKER).exists()); + assert!(!game.join(IN_PROGRESS_MARKER).exists()); + assert!(no_temps(&game)); + { + let seen = fx.server.seen.lock().unwrap(); + assert_eq!(seen.paths.len(), 5, "{:?}", seen.paths); + assert!(seen.queries_ok, "the signed query is kept on every URL"); + assert_eq!(seen.manifest_token.as_deref(), Some("token-1")); + assert_eq!(seen.file_tokens, 0, "file requests carry no token"); + assert!(seen.agents.iter().all(|a| a.starts_with("droiddeck/") || a.starts_with("DroidDeck/")), "{:?}", seen.agents); + } + + let before = requests(&fx); + let again = Recorder::default(); + run_download(&request(&game), &download(&fx), &cancel, &again).unwrap(); + assert_eq!(requests(&fx) - before, 1); + assert_eq!(again.sizes.lock().unwrap().0, 0); + assert_eq!(again.gate_calls.load(Ordering::SeqCst), 0); + + fs::write(game.join("Data/pak0.pak"), vec![0u8; 70_000]).unwrap(); + fs::write(game.join("Game.exe.tmp"), b"junk").unwrap(); + fs::write(game.join("notes.txt"), b"not in the manifest").unwrap(); + let before = requests(&fx); + let repair = Recorder::default(); + run_download(&request(&game), &download(&fx), &cancel, &repair).unwrap(); + assert_eq!(requests(&fx) - before, 2, "manifest + the damaged file"); + assert_eq!(fs::read(game.join("Data/pak0.pak")).unwrap(), fx.expected[1].1); + assert_eq!(*repair.sizes.lock().unwrap(), (70_000, total)); + assert_eq!(repair.gate_calls.load(Ordering::SeqCst), 1); + assert!(!game.join("Game.exe.tmp").exists()); + assert!(game.join("notes.txt").exists(), "files outside the manifest are left alone"); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn a_corrupt_file_fails_the_run_and_leaves_no_temporary_files() { + let fx = fixture(); + let digest = net::hex_lower(&Sha256::digest(&fx.expected[1].1)); + fx.server.files.lock().unwrap().insert(format!("/v/abc/files/{digest}"), vec![7u8; 70_000]); + let game = temp_game("corrupt"); + let cancel = AtomicBool::new(false); + let ev = Recorder::default(); + let err = run_download(&request(&game), &download(&fx), &cancel, &ev).unwrap_err(); + assert_eq!(err, Stop::Failed(FILE_FAILED.into())); + assert!(!game.join("Data/pak0.pak").exists()); + assert!(no_temps(&game)); + assert!(game.join(IN_PROGRESS_MARKER).exists()); + assert!(ev.logs.lock().unwrap().iter().any(|l| l.contains("SHA-256 mismatch"))); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn a_refused_switch_to_updating_replaces_nothing() { + let fx = fixture(); + let game = temp_game("refuse"); + fs::create_dir_all(&game).unwrap(); + fs::write(game.join("Game.exe"), b"old").unwrap(); + let cancel = AtomicBool::new(false); + let ev = Recorder { refuse: true, ..Default::default() }; + let err = run_download(&request(&game), &download(&fx), &cancel, &ev).unwrap_err(); + assert_eq!(err, Stop::Failed(REPLACE_REFUSED.into())); + assert_eq!(fs::read(game.join("Game.exe")).unwrap(), b"old"); + assert!(no_temps(&game)); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn cancel_and_bad_manifests() { + let fx = fixture(); + let game = temp_game("cancel"); + let cancel = AtomicBool::new(true); + assert_eq!(run_download(&request(&game), &download(&fx), &cancel, &Recorder::default()).unwrap_err(), Stop::Cancelled); + + let bad = fixture_with(&[("..\\escape.dll", b"x".to_vec())]); + let cancel = AtomicBool::new(false); + let err = run_download(&request(&game), &download(&bad), &cancel, &Recorder::default()).unwrap_err(); + match err { + Stop::Failed(m) => assert!(m.contains("..\\escape.dll"), "{m}"), + other => panic!("{other:?}"), + } + assert!(!game.parent().unwrap().join("escape.dll").exists()); + + let missing = Download { base_url: format!("{}x", fx.server.base.split('?').next().unwrap()), version_id: String::new() }; + let err = run_download(&request(&game), &missing, &cancel, &Recorder::default()).unwrap_err(); + assert_eq!(err, Stop::Failed(NO_MANIFEST.into())); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn requests_from_json() { + let ok = Request::from_json(&json!({"accessToken":"t","entitlementId":"e","gameDir":"/g","maxWorkers":16})).unwrap(); + assert_eq!((ok.max_workers, ok.process_workers), (16, 2)); + assert!(Request::from_json(&json!({"accessToken":"t","entitlementId":"e","gameDir":"rel"})).is_err()); + assert_eq!(Request::from_json(&json!({"entitlementId":"e","gameDir":"/g"})).err().unwrap(), NOT_SIGNED_IN); + assert!(Request::from_json(&json!({"accessToken":"t","gameDir":"/g"})).is_err()); +} diff --git a/app/src/main/rust/stores/src/amazon/install/writer.rs b/app/src/main/rust/stores/src/amazon/install/writer.rs new file mode 100644 index 0000000000..09aa5e0a6f --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/install/writer.rs @@ -0,0 +1,303 @@ +use std::fs::{self, File}; +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use sha2::{Digest, Sha256}; + +use super::check::temp_path; +use super::{Events, ReplaceGate, Stop}; +use crate::fetch_core::{distinct_host_count, run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; + +const PROGRESS_EVERY_MS: u64 = 50; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(120); + +pub struct Job { + pub path: String, + pub dest: PathBuf, + pub size: u64, + pub sha256: Option<[u8; 32]>, + pub url: String, +} + +pub struct Settings { + pub max_workers: usize, + pub process_workers: usize, + pub ca_bundle: String, + pub user_agent: String, + pub hosts: Vec, +} + +#[derive(Clone, Copy, Debug, Default)] +pub struct Baseline { + pub done_bytes: u64, + pub done_files: u32, + pub total_bytes: u64, + pub total_files: u32, +} + +#[derive(Debug, Default)] +pub struct Stats { + pub bytes_written: u64, + pub files_written: usize, +} + +struct Active { + file: File, + tmp: PathBuf, + hasher: Sha256, + written: u64, +} + +struct Placer<'a> { + jobs: &'a [Job], + slots: Vec>>, + placed: Vec, + events: &'a dyn Events, + gate: &'a ReplaceGate, + stop: &'a AtomicBool, + failure: Mutex>, + base: Baseline, + placed_bytes: AtomicU64, + in_flight: AtomicU64, + shown: AtomicU64, + bytes_written: AtomicU64, + files_placed: AtomicUsize, + last_emit: AtomicU64, + started: Instant, +} + +fn discard(active: Active) { + drop(active.file); + let _ = fs::remove_file(&active.tmp); +} + +impl Placer<'_> { + fn fail(&self, message: String) -> String { + let mut slot = self.failure.lock().expect("failure lock"); + if slot.is_none() { + *slot = Some(message.clone()); + } + drop(slot); + self.stop.store(true, Ordering::SeqCst); + message + } + + fn emit(&self, force: bool) { + let now = self.started.elapsed().as_millis() as u64; + let last = self.last_emit.load(Ordering::Relaxed); + if !force && now < last + PROGRESS_EVERY_MS { + return; + } + if !force && self.last_emit.compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed).is_err() { + return; + } + self.last_emit.store(now, Ordering::Relaxed); + let value = self.base.done_bytes + + self.placed_bytes.load(Ordering::Relaxed) + + self.in_flight.load(Ordering::Relaxed); + let shown = self.shown.fetch_max(value, Ordering::Relaxed).max(value); + self.events.progress( + shown.min(self.base.total_bytes), + self.base.total_bytes, + self.base.done_files + self.files_placed.load(Ordering::Relaxed) as u32, + self.base.total_files, + ); + } + + fn start(&self, job: &Job) -> Result { + if let Some(parent) = job.dest.parent() { + fs::create_dir_all(parent).map_err(|e| format!("Could not create the folder for {}: {e}", job.path))?; + } + let tmp = temp_path(&job.dest); + let file = File::create(&tmp).map_err(|e| format!("Could not write {}: {e}", job.path))?; + Ok(Active { file, tmp, hasher: Sha256::new(), written: 0 }) + } + + fn piece(&self, i: usize, offset: u64, data: &[u8]) -> Result<(), SinkError> { + let job = &self.jobs[i]; + let mut slot = self.slots[i].lock().expect("slot lock"); + if offset == 0 { + if let Some(old) = slot.take() { + self.in_flight.fetch_sub(old.written, Ordering::Relaxed); + discard(old); + } + *slot = Some(self.start(job).map_err(|e| SinkError::Fatal(self.fail(e)))?); + } + let Some(active) = slot.as_mut() else { + return Err(SinkError::Retry(format!("{}: data arrived before the start of the file", job.path))); + }; + if offset != active.written { + return Err(SinkError::Retry(format!("{}: a piece arrived out of order", job.path))); + } + if active.written + data.len() as u64 > job.size { + return Err(SinkError::Retry(format!("{}: the CDN sent more than the manifest size", job.path))); + } + active + .file + .write_all(data) + .map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?; + active.hasher.update(data); + active.written += data.len() as u64; + self.in_flight.fetch_add(data.len() as u64, Ordering::Relaxed); + self.bytes_written.fetch_add(data.len() as u64, Ordering::Relaxed); + drop(slot); + self.emit(false); + Ok(()) + } + + fn finish(&self, i: usize) -> Result { + let job = &self.jobs[i]; + let taken = self.slots[i].lock().expect("slot lock").take(); + let active = match taken { + Some(a) => a, + None => self.start(job).map_err(|e| SinkError::Fatal(self.fail(e)))?, + }; + self.in_flight.fetch_sub(active.written, Ordering::Relaxed); + if active.written != job.size { + let got = active.written; + discard(active); + return Err(SinkError::Retry(format!("{}: {got} bytes arrived, the manifest says {}", job.path, job.size))); + } + let Active { file, tmp, hasher, .. } = active; + if let Some(expected) = job.sha256 { + let got: [u8; 32] = hasher.finalize().into(); + if got != expected { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(SinkError::Retry(format!("{}: SHA-256 mismatch", job.path))); + } + } + if let Err(e) = file.sync_all() { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path)))); + } + drop(file); + if self.stop.load(Ordering::SeqCst) { + let _ = fs::remove_file(&tmp); + return Ok(0); + } + if !self.gate.allow(self.events) { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Fatal(self.fail(super::REPLACE_REFUSED.into()))); + } + if let Err(e) = fs::rename(&tmp, &job.dest) { + let _ = fs::remove_file(&tmp); + return Err(SinkError::Fatal(self.fail(format!("Could not put {} in place: {e}", job.path)))); + } + self.placed[i].store(true, Ordering::SeqCst); + self.placed_bytes.fetch_add(job.size, Ordering::Relaxed); + self.files_placed.fetch_add(1, Ordering::Relaxed); + self.emit(true); + Ok(0) + } +} + +struct Sink<'p, 'a>(&'p Placer<'a>); + +impl FetchSink for Sink<'_, '_> { + fn process(&self, item: &FetchItem, _body: Vec) -> Result { + Err(SinkError::Fatal(format!("item {} arrived whole; streaming was expected", item.id))) + } + + fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> { + self.0.piece(item.id as usize, offset, data) + } + + fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result { + self.0.finish(item.id as usize) + } +} + +pub fn worker_counts(max_workers: usize, process_workers: usize) -> (usize, usize) { + let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(2); + (max_workers.clamp(1, 128), process_workers.max(cores / 2).max(1)) +} + +pub fn run(jobs: &[Job], settings: &Settings, base: Baseline, gate: &ReplaceGate, stop: &AtomicBool, events: &dyn Events) -> Result { + let placer = Placer { + jobs, + slots: jobs.iter().map(|_| Mutex::new(None)).collect(), + placed: jobs.iter().map(|_| AtomicBool::new(false)).collect(), + events, + gate, + stop, + failure: Mutex::new(None), + base, + placed_bytes: AtomicU64::new(0), + in_flight: AtomicU64::new(0), + shown: AtomicU64::new(0), + bytes_written: AtomicU64::new(0), + files_placed: AtomicUsize::new(0), + last_emit: AtomicU64::new(0), + started: Instant::now(), + }; + let (window, process) = worker_counts(settings.max_workers, settings.process_workers); + let distinct = distinct_host_count(&settings.hosts).max(1); + let options = FetchOptions { + max_workers: window, + per_host_cap: 6usize.max(window.div_ceil(distinct)), + timeout: REQUEST_TIMEOUT, + headers: vec![("User-Agent".to_string(), settings.user_agent.clone())], + ca_bundle_path: settings.ca_bundle.clone(), + process_workers: process, + label: "amazon".to_string(), + stream: true, + }; + let items: Vec = jobs + .iter() + .enumerate() + .map(|(i, j)| FetchItem { id: i as u64, urls: vec![j.url.clone()], reserve: j.size, range: None }) + .collect(); + events.log(&format!( + "engine files={} window={window} process_workers={process} per_host_cap={} hosts={}", + jobs.len(), + options.per_host_cap, + settings.hosts.join(","), + )); + placer.emit(true); + let log = |line: &str| events.log(line); + let outcome = if items.is_empty() { + None + } else { + Some(run_fetch(items, &settings.hosts, &options, &Sink(&placer), stop, &|_: u64, _: u64| {}, &log)) + }; + + for slot in &placer.slots { + if let Some(active) = slot.lock().expect("slot lock").take() { + discard(active); + } + } + let placed_all = placer.placed.iter().all(|p| p.load(Ordering::SeqCst)); + if !placed_all { + for (job, placed) in jobs.iter().zip(&placer.placed) { + if !placed.load(Ordering::SeqCst) { + let _ = fs::remove_file(temp_path(&job.dest)); + } + } + } + placer.emit(true); + + let failure = placer.failure.lock().expect("failure lock").take(); + let stats = Stats { + bytes_written: placer.bytes_written.load(Ordering::Relaxed), + files_written: placer.files_placed.load(Ordering::Relaxed), + }; + if let Some(message) = failure { + return Err(Stop::Failed(message)); + } + if placed_all { + return Ok(stats); + } + if let Some(error) = outcome.as_ref().and_then(|o| o.error.clone()) { + events.log(&format!("download failed: {error}")); + return Err(Stop::Failed(super::FILE_FAILED.into())); + } + if stop.load(Ordering::SeqCst) { + return Err(Stop::Cancelled); + } + Err(Stop::Failed("The download ended before every file was written".into())) +} diff --git a/app/src/main/rust/stores/src/amazon/jni.rs b/app/src/main/rust/stores/src/amazon/jni.rs deleted file mode 100644 index 32ce4c62c0..0000000000 --- a/app/src/main/rust/stores/src/amazon/jni.rs +++ /dev/null @@ -1,207 +0,0 @@ - -use crate::amazon::{run_download, MAX_PARALLEL}; -use jni::objects::{GlobalRef, JClass, JObject, JString, JValue}; -use jni::sys::{jint, jlong, JNI_FALSE, JNI_TRUE}; -use jni::{JNIEnv, JavaVM}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, OnceLock}; -use std::thread; - -pub const LOG_TAG: &str = "AmazonNative"; - -static JVM: OnceLock = OnceLock::new(); - -fn clear_pending_exception(env: &mut JNIEnv) { - if env.exception_check().unwrap_or(false) { - let _ = env.exception_clear(); - } -} - -fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { - env.get_string(value) - .ok() - .map(|s| s.to_string_lossy().into_owned()) -} - -struct AmazonRun { - cancel: Arc, -} - -fn to_handle(run: Box) -> jlong { - Box::into_raw(run) as jlong -} - -unsafe fn from_handle<'a>(handle: jlong) -> Option<&'a AmazonRun> { - if handle == 0 { - None - } else { - Some(&*(handle as *const AmazonRun)) - } -} - -fn call_log(env: &mut JNIEnv, listener: &JObject, line: &str) { - let Ok(text) = env.new_string(line) else { - clear_pending_exception(env); - return; - }; - let text_obj = JObject::from(text); - let _ = env.call_method( - listener, - "onLog", - "(Ljava/lang/String;)V", - &[JValue::Object(&text_obj)], - ); - clear_pending_exception(env); -} - -fn call_progress( - env: &mut JNIEnv, - listener: &JObject, - bytes_done: u64, - bytes_total: u64, - files_done: u64, - files_total: u64, -) { - let _ = env.call_method( - listener, - "onProgress", - "(JJJJ)V", - &[ - JValue::Long(bytes_done as jlong), - JValue::Long(bytes_total as jlong), - JValue::Long(files_done as jlong), - JValue::Long(files_total as jlong), - ], - ); - clear_pending_exception(env); -} - -fn call_complete( - env: &mut JNIEnv, - listener: &JObject, - success: bool, - error: &str, - bytes_written: u64, -) { - let Ok(error) = env.new_string(error) else { - clear_pending_exception(env); - return; - }; - let error_obj = JObject::from(error); - let _ = env.call_method( - listener, - "onComplete", - "(ZLjava/lang/String;J)V", - &[ - JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), - JValue::Object(&error_obj), - JValue::Long(bytes_written as jlong), - ], - ); - clear_pending_exception(env); -} - -fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)) { - let Some(vm) = JVM.get() else { - return; - }; - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - f(&mut env, listener.as_obj()); -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart( - mut env: JNIEnv, - _class: JClass, - plan_json: JString, - install_dir: JString, - ca_bundle_path: JString, - max_workers: jint, - process_workers: jint, - listener: JObject, -) -> jlong { - if listener.is_null() { - return 0; - } - if JVM.get().is_none() { - if let Ok(vm) = env.get_java_vm() { - let _ = JVM.set(vm); - } - } - let plan_json = jstring_to_string(&mut env, &plan_json).unwrap_or_default(); - let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); - let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); - if plan_json.is_empty() || install_dir.is_empty() { - call_complete(&mut env, &listener, false, "empty plan or install dir", 0); - return 0; - } - let Ok(listener) = env.new_global_ref(&listener) else { - clear_pending_exception(&mut env); - return 0; - }; - let max_workers = if max_workers <= 0 { - MAX_PARALLEL - } else { - max_workers as usize - }; - let process_workers = process_workers.max(1) as usize; - let cancel = Arc::new(AtomicBool::new(false)); - let run = Box::new(AmazonRun { - cancel: Arc::clone(&cancel), - }); - - thread::spawn(move || { - crate::priority::background(); - let log_listener = listener.clone(); - let log = move |line: &str| { - with_attached_env(&log_listener, |env, obj| call_log(env, obj, line)); - }; - let progress_listener = listener.clone(); - let progress = move |bytes_done: u64, bytes_total: u64, files_done: u64, files_total: u64| { - with_attached_env(&progress_listener, |env, obj| { - call_progress(env, obj, bytes_done, bytes_total, files_done, files_total) - }); - }; - let result = run_download( - &plan_json, - &install_dir, - &ca_bundle_path, - max_workers, - process_workers, - cancel, - &progress, - &log, - ); - with_attached_env(&listener, |env, obj| { - call_complete(env, obj, result.success, &result.error, result.bytes_written) - }); - }); - - to_handle(run) -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel( - _env: JNIEnv, - _class: JClass, - handle: jlong, -) { - if let Some(run) = unsafe { from_handle(handle) } { - run.cancel.store(true, Ordering::Relaxed); - } -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease( - _env: JNIEnv, - _class: JClass, - handle: jlong, -) { - if handle != 0 { - unsafe { - drop(Box::from_raw(handle as *mut AmazonRun)); - } - } -} diff --git a/app/src/main/rust/stores/src/amazon/launch.rs b/app/src/main/rust/stores/src/amazon/launch.rs new file mode 100644 index 0000000000..248dfc93c5 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/launch.rs @@ -0,0 +1,306 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +pub const FUEL_FILE: &str = "fuel.json"; +const MAX_DEPTH: usize = 16; +const STUB_BYTES: u64 = 1_000_000; +const STUB_WORDS: &[&str] = &["crash", "handler", "viewer", "compiler", "tool", "setup", "unins", "eac", "launcher", "steam"]; +const SHIPPING_SUFFIXES: &[&str] = &["-win32.exe", "-win64.exe", "-win32-shipping.exe", "-win64-shipping.exe"]; +const UNREAL_DIRS: &[&str] = &["/binaries/win32/", "/binaries/win64/"]; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FuelMain { + pub command: String, + pub args: Vec, +} + +pub fn parse_fuel(text: &str) -> Option { + let v: Value = serde_json::from_str(text.trim_start_matches('\u{feff}')).ok()?; + let main = v.get("Main").filter(|m| m.is_object())?; + let command = main.get("Command").and_then(|c| c.as_str()).unwrap_or("").trim().to_string(); + let args = main + .get("Args") + .and_then(|a| a.as_array()) + .map(|a| a.iter().filter_map(|x| x.as_str()).filter(|s| !s.is_empty()).map(str::to_string).collect()) + .unwrap_or_default(); + Some(FuelMain { command, args }) +} + +fn relative_command(command: &str) -> Option { + let mut path = command.replace('\\', "/"); + while let Some(rest) = path.strip_prefix("./") { + path = rest.to_string(); + } + let path = path.trim_start_matches('/').to_string(); + let ok = !path.is_empty() && !path.contains(':') && path.split('/').all(|s| !s.is_empty() && s != "." && s != ".."); + ok.then_some(path) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Candidate { + pub path: String, + pub size: u64, +} + +fn file_name(path: &str) -> &str { + path.rsplit('/').next().unwrap_or(path) +} + +fn letter_digits_name(name_lower: &str) -> bool { + let Some(stem) = name_lower.strip_suffix(".exe") else { return false }; + let b = stem.as_bytes(); + (2..=4).contains(&b.len()) && b[0].is_ascii_lowercase() && b[1..].iter().all(u8::is_ascii_digit) +} + +pub fn likely_stub(c: &Candidate) -> bool { + let name = file_name(&c.path).to_lowercase(); + letter_digits_name(&name) || c.size < STUB_BYTES || STUB_WORDS.iter().any(|w| name.contains(w)) +} + +fn letters(text: &str) -> String { + text.to_lowercase().chars().filter(|c| c.is_ascii_lowercase()).collect() +} + +pub fn score(c: &Candidate, title: &str) -> i64 { + let rel = format!("/{}", c.path.to_lowercase()); + let name = file_name(&rel).to_string(); + let mut s = 50; + if SHIPPING_SUFFIXES.iter().any(|x| rel.ends_with(x)) { + s += 300; + } + if UNREAL_DIRS.iter().any(|x| rel.contains(x)) { + s += 250; + } + let title_lower = title.trim().to_lowercase(); + let title_letters = letters(title); + let stem_letters = letters(name.strip_suffix(".exe").unwrap_or(&name)); + let title_in_path = !title_lower.is_empty() && rel.contains(&title_lower); + let same_start = title_letters.len() >= 5 && stem_letters.len() >= 5 && title_letters[..5] == stem_letters[..5]; + if title_in_path || same_start { + s += 100; + } + if STUB_WORDS.iter().any(|w| rel.contains(w)) { + s -= 150; + } + if letter_digits_name(&name) { + s -= 200; + } + s +} + +pub fn choose(candidates: &[Candidate], title: &str) -> Option { + let real: Vec<&Candidate> = candidates.iter().filter(|c| !likely_stub(c)).collect(); + let pool: Vec<&Candidate> = if real.is_empty() { candidates.iter().collect() } else { real }; + pool.into_iter() + .max_by(|a, b| { + score(a, title) + .cmp(&score(b, title)) + .then(a.size.cmp(&b.size)) + .then_with(|| b.path.cmp(&a.path)) + }) + .cloned() +} + +fn collect(dir: &Path, rel: &str, depth: usize, out: &mut Vec) { + if depth > MAX_DEPTH { + return; + } + let Ok(entries) = fs::read_dir(dir) else { return }; + let mut entries: Vec<_> = entries.flatten().collect(); + entries.sort_by_key(|e| e.file_name()); + for entry in entries { + let Ok(meta) = entry.metadata() else { continue }; + let name = entry.file_name().to_string_lossy().into_owned(); + let child = if rel.is_empty() { name.clone() } else { format!("{rel}/{name}") }; + if meta.file_type().is_symlink() { + continue; + } else if meta.is_dir() { + collect(&entry.path(), &child, depth + 1, out); + } else if meta.is_file() && name.to_lowercase().ends_with(".exe") { + out.push(Candidate { path: child, size: meta.len() }); + } + } +} + +pub fn executables(game_dir: &Path) -> Vec { + let mut out = Vec::new(); + collect(game_dir, "", 0, &mut out); + out +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LaunchSpec { + pub exe: String, + pub args: Vec, + pub source: &'static str, + pub warning: Option, +} + +fn fuel_path(game_dir: &Path) -> Option { + let exact = game_dir.join(FUEL_FILE); + if exact.is_file() { + return Some(exact); + } + fs::read_dir(game_dir) + .ok()? + .flatten() + .find(|e| e.file_name().to_string_lossy().eq_ignore_ascii_case(FUEL_FILE) && e.path().is_file()) + .map(|e| e.path()) +} + +pub fn launch_spec(game_dir: &Path, title: &str) -> LaunchSpec { + let mut spec = LaunchSpec::default(); + if let Some(path) = fuel_path(game_dir) { + match fs::read_to_string(&path).ok().as_deref().and_then(parse_fuel) { + Some(main) => { + spec.args = main.args; + if let Some(rel) = relative_command(&main.command) { + let target = game_dir.join(&rel); + let regular = fs::symlink_metadata(&target).map(|m| m.is_file()).unwrap_or(false); + if regular { + spec.exe = rel; + spec.source = "fuel"; + return spec; + } + } + } + None => spec.warning = Some("fuel.json could not be read; choosing the exe by name".into()), + } + } + if let Some(best) = choose(&executables(game_dir), title) { + spec.exe = best.path; + spec.source = "score"; + } + spec +} + +pub fn fuel_environment(entitlement_id: &str, sku: &str, display_name: &str) -> Vec<(String, String)> { + let name = display_name.trim(); + let player = if name.is_empty() || name == "Amazon Games" { "Player" } else { name }; + vec![ + ("FUEL_DIR".into(), r"C:\ProgramData\Amazon Games Services\Legacy".into()), + ("AMAZON_GAMES_SDK_PATH".into(), r"C:\ProgramData\Amazon Games Services\AmazonGamesSDK".into()), + ("AMAZON_GAMES_FUEL_ENTITLEMENT_ID".into(), entitlement_id.into()), + ("AMAZON_GAMES_FUEL_PRODUCT_SKU".into(), sku.into()), + ("AMAZON_GAMES_FUEL_DISPLAY_NAME".into(), player.into()), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + const BIG: u64 = 50_000_000; + + fn c(path: &str, size: u64) -> Candidate { + Candidate { path: path.into(), size } + } + + #[test] + fn fuel_json_main_block() { + let f = parse_fuel(r#"{"Main":{"Command":" Bin\\Game.exe ","Args":["-a","",3,"-b"]},"PostInstall":[]}"#).unwrap(); + assert_eq!(f.command, "Bin\\Game.exe"); + assert_eq!(f.args, vec!["-a", "-b"]); + assert_eq!(parse_fuel(r#"{"Main":{}}"#).unwrap(), FuelMain::default()); + assert!(parse_fuel(r#"{"Main":"x"}"#).is_none()); + assert!(parse_fuel("not json").is_none()); + assert!(parse_fuel("\u{feff}{\"Main\":{\"Command\":\"a.exe\"}}").is_some()); + assert_eq!(relative_command("\\Bin\\Game.exe").as_deref(), Some("Bin/Game.exe")); + assert_eq!(relative_command("./Game.exe").as_deref(), Some("Game.exe")); + assert_eq!(relative_command("..\\Other\\x.exe"), None); + assert_eq!(relative_command("C:\\x.exe"), None); + assert_eq!(relative_command(""), None); + } + + #[test] + fn stubs() { + assert!(likely_stub(&c("a1.exe", BIG))); + assert!(likely_stub(&c("sub/X123.exe", BIG))); + assert!(!likely_stub(&c("ab1.exe", BIG))); + assert!(!likely_stub(&c("a1234.exe", BIG))); + assert!(likely_stub(&c("Game.exe", 999_999))); + assert!(likely_stub(&c("CrashReporter.exe", BIG))); + assert!(likely_stub(&c("UnityCrashHandler64.exe", BIG))); + assert!(!likely_stub(&c("Game.exe", BIG))); + } + + #[test] + fn unreal_shipping_binary_wins() { + let list = [ + c("Game.exe", 300_000_000), + c("MyGame/Binaries/Win64/MyGame-Win64-Shipping.exe", 80_000_000), + c("Engine/Extras/Redist/en-us/UE4PrereqSetup_x64.exe", 40_000_000), + ]; + assert_eq!(choose(&list, "My Game").unwrap().path, "MyGame/Binaries/Win64/MyGame-Win64-Shipping.exe"); + } + + #[test] + fn scoring_uses_relative_paths() { + let a = c("Dread Templar.exe", 10_000_000); + let b = c("bin/engine.exe", 20_000_000); + assert_eq!(score(&a, "Dread Templar"), 150); + assert_eq!(score(&b, "Dread Templar"), 50); + assert_eq!(choose(&[a.clone(), b.clone()], "Dread Templar").unwrap().path, "Dread Templar.exe"); + assert_eq!(score(&c("DreadT.exe", BIG), "Dread Templar"), 150); + assert_eq!(score(&c("Binaries/Win64/x.exe", BIG), ""), 300); + assert_eq!(score(&c("tools/a12.exe", BIG), "z"), 50 - 150 - 200); + } + + #[test] + fn ties_go_to_the_larger_file_and_stubs_are_a_last_resort() { + let list = [c("one.exe", 5_000_000), c("two.exe", 9_000_000)]; + assert_eq!(choose(&list, "zzz").unwrap().path, "two.exe"); + let stubs = [c("setup.exe", 5_000_000), c("small.exe", 10)]; + assert_eq!(choose(&stubs, "zzz").unwrap().path, "small.exe"); + assert_eq!(choose(&[], "zzz"), None); + } + + #[test] + fn spec_from_a_folder() { + let dir = std::env::temp_dir().join(format!("dd-amazon-launch-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("Bin")).unwrap(); + fs::write(dir.join("Bin/Game.exe"), vec![0u8; 1_100_000]).unwrap(); + fs::write(dir.join("Other.exe"), vec![0u8; 2_000_000]).unwrap(); + fs::write(dir.join("fuel.json"), r#"{"Main":{"Command":"Bin\\Game.exe","Args":["-x"]}}"#).unwrap(); + let s = launch_spec(&dir, "Something"); + assert_eq!((s.exe.as_str(), s.source), ("Bin/Game.exe", "fuel")); + assert_eq!(s.args, vec!["-x"]); + fs::write(dir.join("fuel.json"), r#"{"Main":{"Command":"Missing.exe","Args":["-y"]}}"#).unwrap(); + let s = launch_spec(&dir, "Something"); + assert_eq!((s.exe.as_str(), s.source), ("Other.exe", "score")); + assert_eq!(s.args, vec!["-y"]); + fs::write(dir.join("fuel.json"), "{").unwrap(); + let s = launch_spec(&dir, "Something"); + assert!(s.args.is_empty() && s.warning.is_some()); + assert_eq!(s.exe, "Other.exe"); + let empty = std::env::temp_dir().join(format!("dd-amazon-launch-empty-{}", std::process::id())); + fs::create_dir_all(&empty).unwrap(); + assert_eq!(launch_spec(&empty, "x").exe, ""); + let _ = fs::remove_dir_all(&dir); + let _ = fs::remove_dir_all(&empty); + } + + #[test] + fn environment() { + let env = fuel_environment("ent", "sku", "Amazon Games"); + let names: Vec<&str> = env.iter().map(|(k, _)| k.as_str()).collect(); + assert_eq!( + names, + vec![ + "FUEL_DIR", + "AMAZON_GAMES_SDK_PATH", + "AMAZON_GAMES_FUEL_ENTITLEMENT_ID", + "AMAZON_GAMES_FUEL_PRODUCT_SKU", + "AMAZON_GAMES_FUEL_DISPLAY_NAME" + ] + ); + assert_eq!(env[0].1, "C:\\ProgramData\\Amazon Games Services\\Legacy"); + assert_eq!(env[4].1, "Player"); + assert_eq!(fuel_environment("e", "", " ")[4].1, "Player"); + assert_eq!(fuel_environment("e", "", "Sam")[4].1, "Sam"); + assert_eq!(fuel_environment("e", "", "Sam")[2].1, "e"); + } +} diff --git a/app/src/main/rust/stores/src/amazon/library.rs b/app/src/main/rust/stores/src/amazon/library.rs new file mode 100644 index 0000000000..ec9ba55f86 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/library.rs @@ -0,0 +1,228 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use super::distribution::{self, Failure}; +use super::net::{self, str_at, str_of}; + +pub const CLIENT_ID: &str = "Sonic"; +pub const KEY_ID: &str = "d5dc8b8b-86c8-4fc4-ae93-18c0def5314d"; +pub const PAGE_SIZE: u32 = 50; +const MAX_PAGES: usize = 1000; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Game { + pub product_id: String, + pub entitlement_id: String, + pub title: String, + pub sku: String, + pub product_type: String, + pub parent_product_id: String, + pub art: String, + pub hero: String, + pub developer: String, + pub publisher: String, +} + +impl Game { + pub fn is_dlc(&self) -> bool { + (!self.product_type.is_empty() && !self.product_type.eq_ignore_ascii_case("GAME")) + || !self.parent_product_id.is_empty() + } + + pub fn to_json(&self) -> Value { + json!({ + "productId": self.product_id, + "entitlementId": self.entitlement_id, + "title": self.title, + "sku": self.sku, + "art": self.art, + "hero": self.hero, + "developer": self.developer, + "publisher": self.publisher, + "dlc": self.is_dlc(), + "parentProductId": self.parent_product_id, + }) + } +} + +pub fn hardware_hash(device_serial: &str) -> String { + net::hex_upper(&Sha256::digest(device_serial.as_bytes())) +} + +fn first<'v>(product: &'v Value, paths: &[&[&str]]) -> &'v str { + paths.iter().map(|p| str_at(product, p)).find(|s| !s.is_empty()).unwrap_or("") +} + +pub fn read_entitlement(e: &Value) -> Option { + let product = e.get("product").filter(|p| p.is_object())?; + let product_id = str_of(product, "id"); + if product_id.is_empty() { + return None; + } + let title = str_of(product, "title"); + Some(Game { + product_id: product_id.to_string(), + entitlement_id: str_of(e, "id").to_string(), + title: if title.is_empty() { "Unknown".to_string() } else { title.to_string() }, + sku: str_of(product, "sku").to_string(), + product_type: first(product, &[&["productType"], &["type"], &["productDetail", "details", "productType"]]).to_string(), + parent_product_id: first( + product, + &[&["parentProductId"], &["baseProductId"], &["parentId"], &["productDetail", "details", "parentProductId"]], + ) + .to_string(), + art: first(product, &[&["productDetail", "iconUrl"], &["productDetail", "details", "pgCrownImageUrl"]]).to_string(), + hero: first( + product, + &[ + &["productDetail", "details", "backgroundUrl2"], + &["productDetail", "details", "backgroundUrl1"], + &["productDetail", "details", "pgCrownImageUrl"], + ], + ) + .to_string(), + developer: str_at(product, &["productDetail", "details", "developer"]).to_string(), + publisher: str_at(product, &["productDetail", "details", "publisher"]).to_string(), + }) +} + +pub fn read_page(body: &Value) -> (Vec, Option) { + let games = body + .get("entitlements") + .and_then(|e| e.as_array()) + .map(|a| a.iter().filter_map(read_entitlement).collect()) + .unwrap_or_default(); + let next = Some(str_of(body, "nextToken")).filter(|t| !t.is_empty()).map(str::to_string); + (games, next) +} + +pub fn dedupe(games: Vec) -> Vec { + let mut out: Vec = Vec::with_capacity(games.len()); + let mut index: HashMap = HashMap::new(); + for g in games { + match index.get(&g.product_id) { + Some(&i) => out[i] = g, + None => { + index.insert(g.product_id.clone(), out.len()); + out.push(g); + } + } + } + out +} + +pub fn visible(games: Vec) -> Vec { + let kept: Vec = games.iter().filter(|g| !(g.is_dlc() && !g.parent_product_id.is_empty())).cloned().collect(); + let mut out = if kept.is_empty() { games } else { kept }; + out.sort_by_key(|g| g.title.to_lowercase()); + out +} + +pub fn page_body(device_serial: &str, next: Option<&str>) -> Value { + json!({ + "Operation": "GetEntitlements", + "clientId": CLIENT_ID, + "syncPoint": null, + "nextToken": next, + "maxResults": PAGE_SIZE, + "productIdFilter": null, + "keyId": KEY_ID, + "hardwareHash": hardware_hash(device_serial), + }) +} + +pub fn list(access_token: &str, device_serial: &str) -> Result, Failure> { + let mut all = Vec::new(); + let mut next: Option = None; + for _ in 0..MAX_PAGES { + let body = page_body(device_serial, next.as_deref()); + let answer = distribution::call( + distribution::ENTITLEMENTS_URL, + distribution::ENTITLEMENTS_TARGET, + access_token, + &body, + )?; + let (games, cursor) = read_page(&answer); + all.extend(games); + match cursor { + Some(c) if Some(&c) != next.as_ref() => next = Some(c), + Some(_) => return Err(Failure::Other("Amazon repeated a library page cursor".into())), + None => return Ok(dedupe(all)), + } + } + Err(Failure::Other("the Amazon library has too many pages".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn game(id: &str, title: &str, kind: &str, parent: &str) -> Game { + Game { + product_id: id.into(), + title: title.into(), + product_type: kind.into(), + parent_product_id: parent.into(), + ..Default::default() + } + } + + #[test] + fn hardware_hash_is_upper_hex_sha256() { + assert_eq!(hardware_hash("abc"), "BA7816BF8F01CFEA414140DE5DAE2223B00361A396177A9CB410FF61F20015AD"); + } + + #[test] + fn entitlement_fields_and_fallbacks() { + let e = json!({"id":"ent-1","product":{"id":"amzn1.adg.product.x","title":"Game","sku":"SKU1", + "productDetail":{"iconUrl":"","details":{"pgCrownImageUrl":"crown","backgroundUrl1":"bg1", + "developer":"Dev","publisher":"Pub","productType":"GAME"}}}}); + let g = read_entitlement(&e).unwrap(); + assert_eq!(g.entitlement_id, "ent-1"); + assert_eq!(g.art, "crown"); + assert_eq!(g.hero, "bg1"); + assert_eq!(g.product_type, "GAME"); + assert!(!g.is_dlc()); + let untitled = read_entitlement(&json!({"product":{"id":"p","type":"DLC","baseProductId":"q"}})).unwrap(); + assert_eq!(untitled.title, "Unknown"); + assert_eq!(untitled.entitlement_id, ""); + assert_eq!(untitled.parent_product_id, "q"); + assert!(untitled.is_dlc()); + assert!(read_entitlement(&json!({"id":"x"})).is_none()); + assert!(read_entitlement(&json!({"id":"x","product":{"id":""}})).is_none()); + } + + #[test] + fn pages_and_cursors() { + let (games, next) = read_page(&json!({"entitlements":[{"product":{"id":"a"}},{"product":{}}],"nextToken":"n2"})); + assert_eq!(games.len(), 1); + assert_eq!(next.as_deref(), Some("n2")); + assert_eq!(read_page(&json!({"nextToken":null})).1, None); + assert_eq!(read_page(&json!({"nextToken":""})).1, None); + assert!(read_page(&json!({})).0.is_empty()); + let first = page_body("SERIAL", None); + assert!(first["nextToken"].is_null()); + assert_eq!(first["maxResults"], 50); + assert_eq!(page_body("SERIAL", Some("n2"))["nextToken"], "n2"); + } + + #[test] + fn duplicates_and_visibility() { + let games = dedupe(vec![game("a", "b-old", "", ""), game("c", "zeta", "", ""), game("a", "Beta", "", "")]); + assert_eq!(games.len(), 2); + assert_eq!(games[0].title, "Beta"); + let shown = visible(vec![ + game("1", "zed", "GAME", ""), + game("2", "alpha", "", ""), + game("3", "addon", "DLC", "1"), + game("4", "soundtrack", "DLC", ""), + game("5", "Bravo", "game", ""), + ]); + let titles: Vec<&str> = shown.iter().map(|g| g.title.as_str()).collect(); + assert_eq!(titles, vec!["alpha", "Bravo", "soundtrack", "zed"]); + let only_addons = visible(vec![game("3", "addon", "DLC", "1")]); + assert_eq!(only_addons.len(), 1); + } +} diff --git a/app/src/main/rust/stores/src/amazon/manifest.rs b/app/src/main/rust/stores/src/amazon/manifest.rs new file mode 100644 index 0000000000..8bffbbead1 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/manifest.rs @@ -0,0 +1,304 @@ +use std::collections::HashMap; + +use super::codec; +use super::net::hex_lower; +use super::proto::{Fields, Value}; + +pub const SHA256_ALGORITHM: u64 = 0; + +pub const RESERVED_ROOT_NAMES: &[&str] = &[ + ".droiddeck-store.json", + ".droiddeck-store.json.tmp", + ".droiddeck-launch.bat", + ".amazon_download_in_progress", + ".amazon_download_complete", +]; + +pub const TEMP_SUFFIX: &str = ".tmp"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GameFile { + pub path: String, + pub size: u64, + pub algorithm: u64, + pub digest: Vec, +} + +impl GameFile { + pub fn sha256(&self) -> Option<[u8; 32]> { + if self.algorithm == SHA256_ALGORITHM { + self.digest.as_slice().try_into().ok() + } else { + None + } + } + + pub fn cdn_name(&self) -> String { + hex_lower(&self.digest) + } +} + +#[derive(Debug, Default)] +pub struct Manifest { + pub files: Vec, + pub entries: usize, + pub warnings: Vec, +} + +impl Manifest { + pub fn total_size(&self) -> u64 { + self.files.iter().map(|f| f.size).sum() + } +} + +pub fn check_path(raw: &str) -> Result { + let path = raw.replace('\\', "/"); + let refuse = |why: &str| Err(format!("The manifest has an unsafe path ({why}): {raw}")); + if path.starts_with('/') { + return refuse("absolute"); + } + if path.contains(':') { + return refuse("drive letter or stream name"); + } + if path.contains('\0') { + return refuse("NUL character"); + } + for segment in path.split('/') { + match segment { + "" => return refuse("empty segment"), + "." | ".." => return refuse("relative segment"), + _ => {} + } + } + if !path.contains('/') && RESERVED_ROOT_NAMES.iter().any(|r| r.eq_ignore_ascii_case(&path)) { + return refuse("DroidDeck's own file"); + } + Ok(path) +} + +fn parse_hash(data: &[u8]) -> Result<(u64, Vec), String> { + let mut algorithm = SHA256_ALGORITHM; + let mut digest = Vec::new(); + let mut fields = Fields::new(data); + while let Some((n, v)) = fields.next_field()? { + match (n, v) { + (1, Value::Varint(a)) => algorithm = a, + (2, Value::Bytes(b)) => digest = b.to_vec(), + (1 | 2, _) => return Err(format!("hash field {n} has the wrong type")), + _ => {} + } + } + Ok((algorithm, digest)) +} + +fn parse_file(data: &[u8]) -> Result, String> { + let mut path = String::new(); + let mut size = 0u64; + let mut hash = (SHA256_ALGORITHM, Vec::new()); + let mut fields = Fields::new(data); + while let Some((n, v)) = fields.next_field()? { + match (n, v) { + (1, Value::Bytes(b)) => { + path = String::from_utf8(b.to_vec()).map_err(|_| "a file path is not UTF-8".to_string())? + } + (3, Value::Varint(s)) => size = s, + (5, Value::Bytes(b)) => hash = parse_hash(b)?, + (1 | 3 | 5, _) => return Err(format!("file field {n} has the wrong type")), + _ => {} + } + } + if path.is_empty() { + return Ok(None); + } + let path = check_path(&path)?; + let (algorithm, digest) = hash; + if size > 0 && digest.is_empty() { + return Err(format!("The manifest gives no hash for {path}")); + } + if algorithm == SHA256_ALGORITHM && !digest.is_empty() && digest.len() != 32 { + return Err(format!("The manifest has a {}-byte SHA-256 for {path}", digest.len())); + } + Ok(Some(GameFile { path, size, algorithm, digest })) +} + +pub fn parse_message(data: &[u8]) -> Result { + let mut out = Manifest::default(); + let mut index: HashMap = HashMap::new(); + let mut top = Fields::new(data); + while let Some((n, v)) = top.next_field()? { + let package = match (n, v) { + (1, Value::Bytes(b)) => b, + (1, _) => return Err("package field has the wrong type".into()), + _ => continue, + }; + let mut fields = Fields::new(package); + while let Some((n, v)) = fields.next_field()? { + let entry = match (n, v) { + (2, Value::Bytes(b)) => b, + (2, _) => return Err("file field has the wrong type".into()), + _ => continue, + }; + let Some(file) = parse_file(entry)? else { continue }; + out.entries += 1; + match index.get(&file.path) { + Some(&i) => { + out.warnings.push(format!("duplicate manifest path {}; the later entry wins", file.path)); + out.files[i] = file; + } + None => { + index.insert(file.path.clone(), out.files.len()); + out.files.push(file); + } + } + } + } + for f in &out.files { + if let Some(base) = f.path.strip_suffix(TEMP_SUFFIX) { + if index.contains_key(base) { + return Err(format!("The manifest has a file that clashes with a temporary file: {}", f.path)); + } + } + } + Ok(out) +} + +pub fn parse(object: &[u8]) -> Result { + parse_message(&codec::message(object)?) +} + +#[cfg(test)] +pub mod testing { + use crate::amazon::proto::write; + + pub struct Entry<'a> { + pub path: &'a str, + pub size: u64, + pub algorithm: Option, + pub digest: &'a [u8], + } + + pub fn file_message(e: &Entry<'_>) -> Vec { + let mut hash = Vec::new(); + if let Some(a) = e.algorithm { + write::uint(&mut hash, 1, a); + } + write::bytes(&mut hash, 2, e.digest); + let mut f = Vec::new(); + write::bytes(&mut f, 1, e.path.as_bytes()); + write::uint(&mut f, 2, 1_700_000_000); + write::uint(&mut f, 3, e.size); + write::bytes(&mut f, 5, &hash); + f + } + + pub fn message(packages: &[Vec>]) -> Vec { + let mut m = Vec::new(); + write::uint(&mut m, 7, 3); + for files in packages { + let mut p = Vec::new(); + write::bytes(&mut p, 1, b"package-name"); + for e in files { + write::bytes(&mut p, 2, &file_message(e)); + } + write::bytes(&mut m, 1, &p); + } + m + } +} + +#[cfg(test)] +mod tests { + use super::testing::{message, Entry}; + use super::*; + use crate::amazon::codec::testing::frame; + use sha2::{Digest, Sha256}; + + #[test] + fn parses_files_across_packages() { + let d1: Vec = Sha256::digest(b"one").to_vec(); + let d2: Vec = Sha256::digest(b"two").to_vec(); + let msg = message(&[ + vec![ + Entry { path: "Binaries\\Win64\\Game.exe", size: 3, algorithm: Some(0), digest: &d1 }, + Entry { path: "", size: 9, algorithm: None, digest: &d1 }, + ], + vec![Entry { path: "fuel.json", size: 3, algorithm: None, digest: &d2 }], + ]); + for xz in [true, false] { + let m = parse(&frame(&msg, b"\x0a\x00", xz)).unwrap(); + assert_eq!(m.files.len(), 2); + assert_eq!(m.entries, 2); + assert_eq!(m.files[0].path, "Binaries/Win64/Game.exe"); + assert_eq!(m.files[0].sha256().unwrap().to_vec(), d1); + assert_eq!(m.files[0].cdn_name(), hex_lower(&d1)); + assert_eq!(m.files[1].algorithm, 0, "absent algorithm means SHA-256"); + assert_eq!(m.total_size(), 6); + } + } + + #[test] + fn other_algorithms_are_not_verified_but_still_named() { + let msg = message(&[vec![Entry { path: "a", size: 1, algorithm: Some(7), digest: &[0xab, 0xcd] }]]); + let m = parse_message(&msg).unwrap(); + assert_eq!(m.files[0].sha256(), None); + assert_eq!(m.files[0].cdn_name(), "abcd"); + } + + #[test] + fn duplicate_paths_keep_the_later_entry() { + let a = [1u8; 32]; + let b = [2u8; 32]; + let msg = message(&[vec![ + Entry { path: "x\\y.bin", size: 1, algorithm: Some(0), digest: &a }, + Entry { path: "x/y.bin", size: 2, algorithm: Some(0), digest: &b }, + ]]); + let m = parse_message(&msg).unwrap(); + assert_eq!(m.files.len(), 1); + assert_eq!(m.entries, 2); + assert_eq!(m.files[0].size, 2); + assert_eq!(m.warnings.len(), 1); + } + + #[test] + fn unsafe_paths_are_refused() { + for bad in [ + "\\abs.exe", + "/abs.exe", + "C:\\Windows\\x.dll", + "a\\..\\..\\b", + "..", + "a\\\\b", + "a\\", + "a\\.\\b", + ".droiddeck-store.json", + ".DroidDeck-Launch.BAT", + ".amazon_download_complete", + ] { + assert!(check_path(bad).is_err(), "{bad}"); + let msg = message(&[vec![Entry { path: bad, size: 1, algorithm: Some(0), digest: &[0; 32] }]]); + let err = parse_message(&msg).unwrap_err(); + assert!(err.contains(bad), "{err}"); + } + assert_eq!(check_path("sub\\.droiddeck-store.json").unwrap(), "sub/.droiddeck-store.json"); + assert_eq!(check_path("..hidden\\x..y").unwrap(), "..hidden/x..y"); + } + + #[test] + fn malformed_entries_are_errors() { + let no_hash = message(&[vec![Entry { path: "a", size: 5, algorithm: Some(0), digest: &[] }]]); + assert!(parse_message(&no_hash).is_err()); + let empty_ok = message(&[vec![Entry { path: "e", size: 0, algorithm: Some(0), digest: &[] }]]); + assert_eq!(parse_message(&empty_ok).unwrap().files[0].sha256(), None); + let short = message(&[vec![Entry { path: "a", size: 5, algorithm: Some(0), digest: &[1, 2] }]]); + assert!(parse_message(&short).is_err()); + let clash = message(&[vec![ + Entry { path: "a.bin", size: 1, algorithm: Some(0), digest: &[1; 32] }, + Entry { path: "a.bin.tmp", size: 1, algorithm: Some(0), digest: &[2; 32] }, + ]]); + assert!(parse_message(&clash).is_err()); + let mut truncated = message(&[vec![Entry { path: "a", size: 1, algorithm: Some(0), digest: &[1; 32] }]]); + truncated.truncate(truncated.len() - 3); + assert!(parse_message(&truncated).is_err()); + assert!(parse_message(&[0x08, 0x01]).is_err()); + } +} diff --git a/app/src/main/rust/stores/src/amazon/mod.rs b/app/src/main/rust/stores/src/amazon/mod.rs new file mode 100644 index 0000000000..dd2f1f7db8 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/mod.rs @@ -0,0 +1,12 @@ +pub mod auth; +pub mod cdn; +pub mod codec; +pub mod distribution; +pub mod ffi; +pub mod install; +pub mod launch; +pub mod library; +pub mod manifest; +pub mod net; +pub mod proto; +pub mod service; diff --git a/app/src/main/rust/stores/src/amazon/net.rs b/app/src/main/rust/stores/src/amazon/net.rs new file mode 100644 index 0000000000..2fc319ce28 --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/net.rs @@ -0,0 +1,295 @@ +use std::sync::OnceLock; +use std::time::Duration; + +pub const USER_AGENT_PRODUCT: &str = "DroidDeck"; + +pub const BROWSER_USER_AGENT: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0"; + +pub const LAUNCHER_USER_AGENT: &str = "AGSLauncher/1.0.0"; + +pub const LAUNCHER_PACKAGE: (&str, &str) = ("UserAgent", "com.amazon.agslauncher.win/3.0.9202.1"); + +static APP_VERSION: OnceLock = OnceLock::new(); + +pub fn set_app_version(version: &str) { + let clean: String = version.trim().chars().filter(|c| !c.is_control()).collect(); + if !clean.is_empty() { + let _ = APP_VERSION.set(clean); + } +} + +pub fn user_agent() -> String { + let version = APP_VERSION.get().map(String::as_str).unwrap_or(crate::stores_jni::VERSION); + format!("{USER_AGENT_PRODUCT}/{version}") +} + +pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +pub const JSON_READ_TIMEOUT: Duration = Duration::from_secs(30); +pub const BYTES_READ_TIMEOUT: Duration = Duration::from_secs(120); +const WHOLE_REQUEST_CAP: Duration = Duration::from_secs(600); +const MAX_BODY: usize = 256 * 1024 * 1024; + +pub struct Call { + pub post: bool, + pub url: String, + pub user_agent: String, + pub headers: Vec<(String, String)>, + pub json_body: Option, + pub read_timeout: Duration, +} + +impl Call { + pub fn get(url: &str, user_agent: String) -> Call { + Call { + post: false, + url: url.to_string(), + user_agent, + headers: Vec::new(), + json_body: None, + read_timeout: JSON_READ_TIMEOUT, + } + } + + pub fn post_json(url: &str, user_agent: String, body: &serde_json::Value) -> Call { + Call { post: true, json_body: Some(body.to_string()), ..Call::get(url, user_agent) } + } + + pub fn header(mut self, name: &str, value: &str) -> Call { + self.headers.push((name.to_string(), value.to_string())); + self + } + + pub fn read_timeout(mut self, timeout: Duration) -> Call { + self.read_timeout = timeout; + self + } +} + +pub struct Reply { + pub status: u16, + pub body: Vec, +} + +impl Reply { + pub fn ok(&self) -> bool { + (200..300).contains(&self.status) + } + + pub fn json(&self) -> Option { + serde_json::from_slice(&self.body).ok() + } + + pub fn snippet(&self) -> String { + let text = String::from_utf8_lossy(&self.body[..self.body.len().min(200)]).into_owned(); + text.chars().map(|c| if c.is_control() { ' ' } else { c }).collect() + } +} + +pub fn send(call: Call) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("runtime: {e}"))?; + runtime.block_on(send_async(call)) +} + +async fn send_async(call: Call) -> Result { + use reqwest::header::{ACCEPT, CONTENT_TYPE}; + let client = reqwest::Client::builder() + .user_agent(call.user_agent.as_str()) + .connect_timeout(CONNECT_TIMEOUT) + .read_timeout(call.read_timeout) + .timeout(WHOLE_REQUEST_CAP) + .build() + .map_err(|e| format!("http client: {}", e.without_url()))?; + let mut builder = if call.post { client.post(call.url.as_str()) } else { client.get(call.url.as_str()) }; + for (name, value) in &call.headers { + builder = builder.header(name.as_str(), value.as_str()); + } + if let Some(body) = call.json_body { + builder = builder.header(CONTENT_TYPE, "application/json").header(ACCEPT, "application/json").body(body); + } + let mut response = builder.send().await.map_err(describe)?; + let status = response.status().as_u16(); + let mut body = Vec::new(); + while let Some(piece) = response.chunk().await.map_err(describe)? { + if body.len() + piece.len() > MAX_BODY { + return Err("answer is too large".into()); + } + body.extend_from_slice(&piece); + } + Ok(Reply { status, body }) +} + +fn describe(err: reqwest::Error) -> String { + let kind = if err.is_timeout() { + "timed out" + } else if err.is_connect() { + "could not connect" + } else if err.is_body() || err.is_decode() { + "answer could not be read" + } else { + "request failed" + }; + format!("{kind}: {}", err.without_url()) +} + +const HEX_LOWER: &[u8; 16] = b"0123456789abcdef"; +const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF"; + +pub fn hex_lower(bytes: &[u8]) -> String { + bytes.iter().flat_map(|b| [HEX_LOWER[(b >> 4) as usize] as char, HEX_LOWER[(b & 15) as usize] as char]).collect() +} + +pub fn hex_upper(bytes: &[u8]) -> String { + bytes.iter().flat_map(|b| [HEX_UPPER[(b >> 4) as usize] as char, HEX_UPPER[(b & 15) as usize] as char]).collect() +} + +pub fn percent_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for b in text.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') { + out.push(b as char); + } else { + out.push('%'); + out.push(HEX_UPPER[(b >> 4) as usize] as char); + out.push(HEX_UPPER[(b & 15) as usize] as char); + } + } + out +} + +pub fn percent_decode(text: &str) -> String { + fn nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } + } + let bytes = text.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' && i + 2 < bytes.len() { + if let (Some(h), Some(l)) = (nibble(bytes[i + 1]), nibble(bytes[i + 2])) { + out.push(h << 4 | l); + i += 3; + continue; + } + } + out.push(bytes[i]); + i += 1; + } + String::from_utf8_lossy(&out).into_owned() +} + +pub fn base64url(data: &[u8]) -> String { + const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for group in data.chunks(3) { + let n = (group[0] as u32) << 16 | (*group.get(1).unwrap_or(&0) as u32) << 8 | *group.get(2).unwrap_or(&0) as u32; + out.push(TABLE[(n >> 18) as usize & 63] as char); + out.push(TABLE[(n >> 12) as usize & 63] as char); + if group.len() > 1 { + out.push(TABLE[(n >> 6) as usize & 63] as char); + } + if group.len() > 2 { + out.push(TABLE[n as usize & 63] as char); + } + } + out +} + +pub fn scrub_urls(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut rest = line; + loop { + let at = [rest.find("https://"), rest.find("http://")].into_iter().flatten().min(); + let Some(at) = at else { + out.push_str(rest); + return out; + }; + out.push_str(&rest[..at]); + let tail = &rest[at..]; + let end = tail.find(|c: char| c.is_whitespace() || matches!(c, ')' | '"' | '\'' | '>' | ',')).unwrap_or(tail.len()); + let url = &tail[..end]; + let cut = url.find(['?', '#']).unwrap_or(url.len()); + out.push_str(&url[..cut]); + rest = &tail[end..]; + } +} + +pub fn str_of<'v>(value: &'v serde_json::Value, key: &str) -> &'v str { + value.get(key).and_then(|v| v.as_str()).unwrap_or("") +} + +pub fn str_at<'v>(value: &'v serde_json::Value, path: &[&str]) -> &'v str { + let mut v = value; + for key in path { + match v.get(key) { + Some(next) => v = next, + None => return "", + } + } + v.as_str().unwrap_or("") +} + +pub fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hex_both_cases() { + assert_eq!(hex_lower(&[0x00, 0xab, 0x7f]), "00ab7f"); + assert_eq!(hex_upper(&[0x00, 0xab, 0x7f]), "00AB7F"); + } + + #[test] + fn percent_round_trip() { + assert_eq!(percent_encode("device:ab/c d"), "device%3Aab%2Fc%20d"); + assert_eq!(percent_decode("a%3Ab%2fc+d"), "a:b/c+d"); + assert_eq!(percent_decode("bad%zzend%4"), "bad%zzend%4"); + assert_eq!(percent_decode("%41"), "A"); + } + + #[test] + fn base64url_vectors() { + assert_eq!(base64url(b""), ""); + assert_eq!(base64url(b"f"), "Zg"); + assert_eq!(base64url(b"fo"), "Zm8"); + assert_eq!(base64url(b"foo"), "Zm9v"); + assert_eq!(base64url(&[0xfb, 0xff]), "-_8"); + } + + #[test] + fn urls_lose_their_query_in_logs() { + assert_eq!( + scrub_urls("GET https://cdn.example/x/files/ab?sig=SECRET&e=1 failed (https://a.b/c#frag)"), + "GET https://cdn.example/x/files/ab failed (https://a.b/c)" + ); + assert_eq!(scrub_urls("no url here"), "no url here"); + } + + #[test] + fn nested_strings() { + let v = serde_json::json!({"a":{"b":{"c":"x"}},"n":1}); + assert_eq!(str_at(&v, &["a", "b", "c"]), "x"); + assert_eq!(str_at(&v, &["a", "z"]), ""); + assert_eq!(str_of(&v, "n"), ""); + } + + #[test] + fn product_user_agent() { + assert!(user_agent().starts_with("DroidDeck/")); + } +} diff --git a/app/src/main/rust/stores/src/amazon/proto.rs b/app/src/main/rust/stores/src/amazon/proto.rs new file mode 100644 index 0000000000..60712d9e0a --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/proto.rs @@ -0,0 +1,155 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Value<'a> { + Varint(u64), + Fixed64(u64), + Bytes(&'a [u8]), + Fixed32(u32), +} + +pub struct Fields<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Fields<'a> { + pub fn new(data: &'a [u8]) -> Self { + Fields { data, pos: 0 } + } + + fn varint(&mut self) -> Result { + let mut value = 0u64; + for i in 0..10 { + let Some(&b) = self.data.get(self.pos) else { + return Err("a number runs past the end of the message".into()); + }; + self.pos += 1; + if i == 9 && b > 1 { + return Err("a number is longer than 64 bits".into()); + } + value |= ((b & 0x7f) as u64) << (7 * i); + if b & 0x80 == 0 { + return Ok(value); + } + } + Err("a number is longer than 10 bytes".into()) + } + + fn take(&mut self, n: u64) -> Result<&'a [u8], String> { + let left = (self.data.len() - self.pos) as u64; + if n > left { + return Err(format!("a field of {n} bytes runs past the end of the message ({left} left)")); + } + let start = self.pos; + self.pos += n as usize; + Ok(&self.data[start..self.pos]) + } + + pub fn next_field(&mut self) -> Result)>, String> { + if self.pos >= self.data.len() { + return Ok(None); + } + let key = self.varint()?; + let field = key >> 3; + if field == 0 || field > u32::MAX as u64 { + return Err(format!("invalid field number {field}")); + } + let value = match key & 7 { + 0 => Value::Varint(self.varint()?), + 1 => { + let b = self.take(8)?; + Value::Fixed64(u64::from_le_bytes(b.try_into().expect("8 bytes"))) + } + 2 => { + let n = self.varint()?; + Value::Bytes(self.take(n)?) + } + 5 => { + let b = self.take(4)?; + Value::Fixed32(u32::from_le_bytes(b.try_into().expect("4 bytes"))) + } + other => return Err(format!("unsupported wire type {other} for field {field}")), + }; + Ok(Some((field as u32, value))) + } +} + +#[cfg(test)] +pub mod write { + pub fn varint(out: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } + } + + pub fn key(out: &mut Vec, field: u32, wire: u8) { + varint(out, (field as u64) << 3 | wire as u64); + } + + pub fn uint(out: &mut Vec, field: u32, v: u64) { + key(out, field, 0); + varint(out, v); + } + + pub fn bytes(out: &mut Vec, field: u32, data: &[u8]) { + key(out, field, 2); + varint(out, data.len() as u64); + out.extend_from_slice(data); + } +} + +#[cfg(test)] +mod tests { + use super::write; + use super::*; + + fn all(data: &[u8]) -> Result)>, String> { + let mut f = Fields::new(data); + let mut out = Vec::new(); + while let Some(x) = f.next_field()? { + out.push(x); + } + Ok(out) + } + + #[test] + fn reads_every_supported_wire_type() { + let mut m = Vec::new(); + write::uint(&mut m, 1, 300); + write::key(&mut m, 2, 1); + m.extend_from_slice(&7u64.to_le_bytes()); + write::bytes(&mut m, 3, b"hi"); + write::key(&mut m, 4, 5); + m.extend_from_slice(&9u32.to_le_bytes()); + write::uint(&mut m, 5, u64::MAX); + let got = all(&m).unwrap(); + assert_eq!( + got, + vec![ + (1, Value::Varint(300)), + (2, Value::Fixed64(7)), + (3, Value::Bytes(b"hi")), + (4, Value::Fixed32(9)), + (5, Value::Varint(u64::MAX)), + ] + ); + } + + #[test] + fn malformed_messages_are_errors() { + assert!(all(&[0x0a, 0x05, b'a']).is_err()); + assert!(all(&[0x08, 0xff, 0xff]).is_err()); + assert!(all(&[0x08, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01]).is_err()); + assert!(all(&[0x08, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02]).is_err()); + assert!(all(&[0x0b]).is_err()); + assert!(all(&[0x0f]).is_err()); + assert!(all(&[0x00, 0x01]).is_err()); + assert!(all(&[0x0d, 1, 2]).is_err()); + assert_eq!(all(&[]).unwrap(), vec![]); + } +} diff --git a/app/src/main/rust/stores/src/amazon/service.rs b/app/src/main/rust/stores/src/amazon/service.rs new file mode 100644 index 0000000000..b009e0aece --- /dev/null +++ b/app/src/main/rust/stores/src/amazon/service.rs @@ -0,0 +1,150 @@ +use std::path::Path; + +use serde_json::{json, Value}; + +use super::auth::{self, Attempt}; +use super::distribution::{self, Failure}; +use super::net::str_of; +use super::{cdn, install, launch, library, manifest}; + +fn error(message: impl Into) -> Value { + json!({ "error": message.into() }) +} + +pub fn call(op: &str, request: &str) -> String { + let req: Value = serde_json::from_str(request).unwrap_or(Value::Null); + dispatch(op, &req).to_string() +} + +fn dispatch(op: &str, req: &Value) -> Value { + let token = str_of(req, "accessToken"); + match op { + "auth.start" => match Attempt::new() { + Ok(a) => json!({ + "startUrl": a.start_url(), + "userAgent": super::net::BROWSER_USER_AGENT, + "redirectMarker": auth::REDIRECT_MARKER, + "serial": a.serial, + "clientId": a.client_id, + "verifier": a.verifier, + }), + Err(e) => error(e), + }, + "auth.register" => { + let code = match str_of(req, "code") { + "" => auth::code_from_url(str_of(req, "url")), + c => Some(c.to_string()), + }; + let Some(code) = code else { + return json!({ "error": "the sign-in page returned no code", "noCode": true }); + }; + let attempt = Attempt { + serial: str_of(req, "serial").to_string(), + client_id: str_of(req, "clientId").to_string(), + verifier: str_of(req, "verifier").to_string(), + }; + if attempt.serial.is_empty() || attempt.client_id.is_empty() || attempt.verifier.is_empty() { + return error("the sign-in attempt is incomplete"); + } + match auth::register(&code, &attempt) { + Ok(r) => r.to_json(), + Err(e) => error(e), + } + } + "auth.refresh" => match auth::refresh(str_of(req, "refreshToken")) { + Ok((access, expires_at)) => json!({ "access_token": access, "expires_at": expires_at }), + Err(e) => error(e), + }, + "auth.deregister" => match auth::deregister(token) { + Ok(()) => json!({}), + Err(e) => error(e), + }, + "library.entitlements" => match library::list(token, str_of(req, "deviceSerial")) { + Ok(all) => { + let total = all.len(); + let games = library::visible(all); + json!({ "total": total, "games": games.iter().map(library::Game::to_json).collect::>() }) + } + Err(Failure::SignIn) => json!({ "error": "Amazon refused the session", "signIn": true }), + Err(Failure::Other(e)) => error(e), + }, + "download.lookup" => match distribution::game_download(token, str_of(req, "entitlementId")) { + Ok(d) => json!({ "downloadUrl": d.base_url, "versionId": d.version_id }), + Err(f) => error(f.message()), + }, + "download.size" => json!({ "bytes": install::size_of(token, str_of(req, "entitlementId")) }), + "launch.spec" => { + let dir = str_of(req, "gameDir"); + if dir.is_empty() { + return error("no install folder"); + } + let spec = launch::launch_spec(Path::new(dir), str_of(req, "title")); + json!({ "exe": spec.exe, "args": spec.args, "source": spec.source, "warning": spec.warning }) + } + "launch.env" => { + let env = launch::fuel_environment(str_of(req, "entitlementId"), str_of(req, "sku"), str_of(req, "displayName")); + json!({ "env": env.iter().map(|(k, v)| json!([k, v])).collect::>() }) + } + _ => error(format!("unknown operation {op}")), + } +} + +pub fn fetch(op: &str, request: &str) -> Option> { + let req: Value = serde_json::from_str(request).ok()?; + match op { + "cdn.get" => { + let url = str_of(&req, "url"); + if url.is_empty() { + return None; + } + cdn::get(url, Some(str_of(&req, "accessToken"))).ok() + } + _ => None, + } +} + +pub fn inspect(bytes: &[u8], _request: &str) -> String { + match manifest::parse(bytes) { + Ok(m) => json!({ "totalBytes": m.total_size(), "files": m.files.len() }).to_string(), + Err(e) => error(e).to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::amazon::codec::testing::frame; + use crate::amazon::manifest::testing::{message, Entry}; + + #[test] + fn local_operations() { + let v: Value = serde_json::from_str(&call("auth.start", "{}")).unwrap(); + assert!(v["startUrl"].as_str().unwrap().starts_with("https://www.amazon.com/ap/signin?")); + assert_eq!(v["redirectMarker"], "openid.oa2.authorization_code="); + assert!(v["userAgent"].as_str().unwrap().contains("Firefox")); + assert_eq!(v["serial"].as_str().unwrap().len(), 32); + let v: Value = serde_json::from_str(&call("auth.register", r#"{"url":"https://www.amazon.com/?a=1"}"#)).unwrap(); + assert_eq!(v["noCode"], true); + let v: Value = serde_json::from_str(&call("auth.register", r#"{"code":"c"}"#)).unwrap(); + assert!(v["error"].is_string() && v.get("noCode").is_none()); + let v: Value = serde_json::from_str(&call("launch.env", r#"{"entitlementId":"e","sku":"s","displayName":""}"#)).unwrap(); + assert_eq!(v["env"][2], json!(["AMAZON_GAMES_FUEL_ENTITLEMENT_ID", "e"])); + assert_eq!(v["env"][4][1], "Player"); + let v: Value = serde_json::from_str(&call("nope", "{}")).unwrap(); + assert!(v["error"].is_string()); + assert!(fetch("cdn.get", "{}").is_none()); + } + + #[test] + fn inspect_sizes() { + let msg = message(&[vec![ + Entry { path: "a", size: 10, algorithm: Some(0), digest: &[1; 32] }, + Entry { path: "b\\c", size: 5, algorithm: Some(0), digest: &[2; 32] }, + ]]); + let v: Value = serde_json::from_str(&inspect(&frame(&msg, b"", true), "{}")).unwrap(); + assert_eq!(v["totalBytes"], 15); + assert_eq!(v["files"], 2); + let v: Value = serde_json::from_str(&inspect(b"xx", "{}")).unwrap(); + assert!(v["error"].is_string()); + } +} diff --git a/app/src/main/rust/stores/src/epic.rs b/app/src/main/rust/stores/src/epic.rs deleted file mode 100644 index 9e131391b2..0000000000 --- a/app/src/main/rust/stores/src/epic.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Derived from GameNative's Epic manager (itself derived from Legendary); both are GPL-3.0 and this crate carries the same licence (see Cargo.toml). - -pub mod assemble; -pub mod adaptive; -pub mod chunk; -pub mod driver; -pub mod jni; -pub mod manifest; -pub mod plan; -pub mod stream; - -pub const USER_AGENT: &str = - "UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; - -pub const CHUNK_CACHE_DIR: &str = ".chunks"; - -pub const JAVA_POOL_THREADS: usize = 8; - -pub const LOG_TAG: &str = "EpicNative"; diff --git a/app/src/main/rust/stores/src/epic/account.rs b/app/src/main/rust/stores/src/epic/account.rs new file mode 100644 index 0000000000..81fe89d812 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/account.rs @@ -0,0 +1,167 @@ +use serde_json::{json, Value}; + +use super::net::{self, Body, Request}; + +pub const CLIENT_ID: &str = "34a02cf8f4414e29b15921876da36f9a"; +pub const CLIENT_SECRET: &str = "daafbccc737745039dffe53d94fc76cf"; +const ACCOUNT_HOST: &str = "https://account-public-service-prod03.ol.epicgames.com"; + +pub const REDIRECT_PREFIX: &str = "https://www.epicgames.com/id/api/redirect"; + +pub const WEBVIEW_USER_AGENT: &str = + "UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; + +const DEFAULT_EXPIRES_IN: i64 = 7200; + +pub fn start_url() -> String { + let redirect = format!("{REDIRECT_PREFIX}?clientId={CLIENT_ID}&responseType=code"); + format!("https://www.epicgames.com/id/login?redirectUrl={}", net::percent_encode(&redirect)) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tokens { + pub access_token: String, + pub refresh_token: Option, + pub account_id: Option, + pub display_name: Option, + pub expires_at_ms: i64, +} + +impl Tokens { + pub fn to_json(&self) -> Value { + let mut out = json!({ "access_token": self.access_token, "expires_at": self.expires_at_ms }); + if let Some(v) = &self.refresh_token { + out["refresh_token"] = json!(v); + } + if let Some(v) = &self.account_id { + out["account_id"] = json!(v); + } + if let Some(v) = &self.display_name { + out["display_name"] = json!(v); + } + out + } +} + +fn non_empty(v: &Value, key: &str) -> Option { + Some(net::str_of(v, key).to_string()).filter(|s| !s.is_empty()) +} + +pub fn parse_token_reply(body: &Value, now_ms: i64) -> Option { + let access_token = non_empty(body, "access_token")?; + let from_date = body + .get("expires_at") + .and_then(|v| v.as_str()) + .and_then(super::timefmt::utc_prefix_seconds) + .map(|s| s * 1000); + let expires_at_ms = from_date.unwrap_or_else(|| { + let secs = body.get("expires_in").and_then(|v| v.as_i64()).unwrap_or(DEFAULT_EXPIRES_IN); + now_ms + secs * 1000 + }); + Some(Tokens { + access_token, + refresh_token: non_empty(body, "refresh_token"), + account_id: non_empty(body, "account_id"), + display_name: non_empty(body, "displayName"), + expires_at_ms, + }) +} + +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +fn token_request(grant: &[(&str, &str)]) -> Result { + let url = format!("{ACCOUNT_HOST}/account/api/oauth/token"); + let mut form: Vec<(String, String)> = grant.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + form.push(("token_type".into(), "eg1".into())); + let reply = net::send(Request::post(&url, Body::Form(form)).basic(CLIENT_ID, CLIENT_SECRET))?; + if !reply.ok() { + return Err(format!("token request answered HTTP {}", reply.status)); + } + let body = reply.json().ok_or("token answer is not JSON")?; + parse_token_reply(&body, now_ms()).ok_or_else(|| "token answer has no access token".into()) +} + +pub fn sign_in_with_code(code: &str) -> Result { + token_request(&[("grant_type", "authorization_code"), ("code", code)]) +} + +pub fn refresh(refresh_token: &str) -> Result { + token_request(&[("grant_type", "refresh_token"), ("refresh_token", refresh_token)]) +} + +pub fn exchange_code(access_token: &str) -> Result { + let url = format!("{ACCOUNT_HOST}/account/api/oauth/exchange"); + let reply = net::send(Request::get(&url).bearer(access_token))?; + if !reply.ok() { + return Err(format!("exchange answered HTTP {}", reply.status)); + } + let body = reply.json().ok_or("exchange answer is not JSON")?; + non_empty(&body, "code").ok_or_else(|| "exchange answer has no code".into()) +} + +pub fn code_from_page_text(raw: &str) -> Option { + let mut text = raw.trim(); + if text.len() >= 2 && text.starts_with('"') && text.ends_with('"') { + text = &text[1..text.len() - 1]; + } + let text = text.replace("\\n", "").replace("\\r", "").replace("\\\"", "\""); + let key = "\"authorizationCode\""; + let mut from = 0; + while let Some(at) = text[from..].find(key) { + let rest = text[from + at + key.len()..].trim_start(); + from += at + key.len(); + let Some(rest) = rest.strip_prefix(':') else { continue }; + let Some(rest) = rest.trim_start().strip_prefix('"') else { continue }; + let Some(end) = rest.find('"') else { continue }; + let value = &rest[..end]; + if !value.is_empty() { + return Some(value.to_string()); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_url_wraps_the_redirect() { + assert_eq!( + start_url(), + "https://www.epicgames.com/id/login?redirectUrl=https%3A%2F%2Fwww.epicgames.com%2Fid%2Fapi%2Fredirect%3FclientId%3D34a02cf8f4414e29b15921876da36f9a%26responseType%3Dcode" + ); + } + + #[test] + fn code_from_the_redirect_page() { + let page = r#""{\"warning\":\"x\",\"redirectUrl\":\"https://localhost/launcher/authorized?code=abc\",\"authorizationCode\":\"0123abcd\",\"exchangeCode\":null,\"sid\":null}\n""#; + assert_eq!(code_from_page_text(page).as_deref(), Some("0123abcd")); + assert_eq!(code_from_page_text(r#"{"authorizationCode": "zz"}"#).as_deref(), Some("zz")); + assert_eq!(code_from_page_text(r#"{"authorizationCode":null}"#), None); + assert_eq!(code_from_page_text(r#"{"authorizationCode":""}"#), None); + assert_eq!(code_from_page_text("sign in"), None); + } + + #[test] + fn token_answers() { + let t = parse_token_reply( + &json!({"access_token":"A","refresh_token":"R","account_id":"acc","displayName":"Me","expires_at":"2026-10-13T12:34:56.789Z","expires_in":7200}), + 0, + ) + .unwrap(); + assert_eq!(t.expires_at_ms, 1_791_894_896_000); + assert_eq!(t.display_name.as_deref(), Some("Me")); + let fallback = parse_token_reply(&json!({"access_token":"A","expires_in":60}), 1000).unwrap(); + assert_eq!(fallback.expires_at_ms, 61_000); + assert_eq!(fallback.refresh_token, None); + let default = parse_token_reply(&json!({"access_token":"A","expires_at":"soon"}), 0).unwrap(); + assert_eq!(default.expires_at_ms, 7_200_000); + assert!(parse_token_reply(&json!({"refresh_token":"R"}), 0).is_none()); + } +} diff --git a/app/src/main/rust/stores/src/epic/achievements.rs b/app/src/main/rust/stores/src/epic/achievements.rs new file mode 100644 index 0000000000..e934a0a5a4 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/achievements.rs @@ -0,0 +1,127 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; + +use super::net::{self, Body, Request}; + +const GRAPHQL_URL: &str = "https://launcher.store.epicgames.com/graphql"; + +const DEFINITIONS_QUERY: &str = "query Achievement($sandboxId: String!, $locale: String!) { \ +Achievement { productAchievementsRecordBySandbox(sandboxId: $sandboxId, locale: $locale) { \ +achievements { achievement { name hidden unlockedDisplayName lockedDisplayName \ +unlockedDescription lockedDescription unlockedIconLink lockedIconLink XP } } } } }"; + +const PROGRESS_QUERY: &str = "query PlayerAchievement($epicAccountId: String!, $sandboxId: String!) { \ +PlayerAchievement { playerAchievementGameRecordsBySandbox(epicAccountId: $epicAccountId, sandboxId: $sandboxId) { \ +records { playerAchievements { playerAchievement { achievementName unlocked unlockDate XP } } } } } }"; + +fn ask(access_token: &str, query: &str, variables: Value) -> Option { + let body = json!({ "query": query, "variables": variables }).to_string(); + let reply = net::send(Request::post(GRAPHQL_URL, Body::Json(body)) + .agent(net::EPIC_LAUNCHER_USER_AGENT) + .bearer(access_token)).ok()?; + if !reply.ok() { + return None; + } + reply.json() +} + +pub fn unlocks_of(answer: &Value) -> HashMap { + let mut out = HashMap::new(); + let records = answer + .pointer("/data/PlayerAchievement/playerAchievementGameRecordsBySandbox/records") + .and_then(|r| r.as_array()); + for record in records.into_iter().flatten() { + let list = record.get("playerAchievements").and_then(|p| p.as_array()); + for entry in list.into_iter().flatten() { + let Some(a) = entry.get("playerAchievement") else { continue }; + if a.get("unlocked").and_then(|u| u.as_bool()) != Some(true) { + continue; + } + let name = net::str_of(a, "achievementName"); + if !name.is_empty() { + let at = super::timefmt::iso_seconds(net::str_of(a, "unlockDate")).unwrap_or(0); + out.insert(name.to_string(), at); + } + } + } + out +} + +pub fn merge(definitions: &Value, unlocks: &HashMap) -> Vec { + let list = definitions + .pointer("/data/Achievement/productAchievementsRecordBySandbox/achievements") + .and_then(|a| a.as_array()); + let mut out = Vec::new(); + for entry in list.into_iter().flatten() { + let Some(a) = entry.get("achievement") else { continue }; + let id = net::str_of(a, "name"); + if id.is_empty() { + continue; + } + let unlocked_at = unlocks.get(id).copied(); + let unlocked = unlocked_at.is_some(); + let pick = |when_unlocked: &str, when_locked: &str| { + let first = net::str_of(a, if unlocked { when_unlocked } else { when_locked }); + if !first.is_empty() { first.to_string() } else { net::str_of(a, when_unlocked).to_string() } + }; + let name = Some(pick("unlockedDisplayName", "lockedDisplayName")).filter(|n| !n.is_empty()).unwrap_or_else(|| id.to_string()); + out.push(json!({ + "id": id, + "name": name, + "description": pick("unlockedDescription", "lockedDescription"), + "hidden": a.get("hidden").and_then(|h| h.as_bool()).unwrap_or(false), + "unlocked": unlocked, + "unlockedAt": unlocked_at.unwrap_or(0), + "icon": net::str_of(a, "unlockedIconLink"), + "iconLocked": net::str_of(a, "lockedIconLink"), + })); + } + out +} + +pub fn fetch(access_token: &str, namespace: &str, account_id: &str, locale: &str) -> Option> { + let definitions = ask(access_token, DEFINITIONS_QUERY, json!({ "sandboxId": namespace, "locale": locale }))?; + let unlocks = if account_id.is_empty() { + HashMap::new() + } else { + let progress = ask( + access_token, + PROGRESS_QUERY, + json!({ "epicAccountId": account_id, "sandboxId": namespace }), + )?; + unlocks_of(&progress) + }; + Some(merge(&definitions, &unlocks)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn definitions_and_unlocks_merge() { + let defs = json!({"data": {"Achievement": {"productAchievementsRecordBySandbox": {"achievements": [ + {"achievement": {"name": "a1", "hidden": false, "unlockedDisplayName": "Won", "lockedDisplayName": "???", + "unlockedDescription": "You won", "lockedDescription": "", "unlockedIconLink": "u", "lockedIconLink": "l"}}, + {"achievement": {"name": "a2", "hidden": true, "unlockedDisplayName": "Secret", "lockedDisplayName": "", + "unlockedDescription": "", "lockedDescription": "", "unlockedIconLink": "", "lockedIconLink": ""}}, + {"achievement": {"name": ""}} + ]}}}}); + let progress = json!({"data": {"PlayerAchievement": {"playerAchievementGameRecordsBySandbox": {"records": [ + {"playerAchievements": [ + {"playerAchievement": {"achievementName": "a1", "unlocked": true, "unlockDate": "2024-05-01T12:00:00.000+0200"}}, + {"playerAchievement": {"achievementName": "a2", "unlocked": false, "unlockDate": ""}} + ]} + ]}}}}); + let list = merge(&defs, &unlocks_of(&progress)); + assert_eq!(list.len(), 2); + assert_eq!(list[0]["unlocked"], true); + assert_eq!(list[0]["name"], "Won"); + assert_eq!(list[0]["unlockedAt"], 1_714_557_600); + assert_eq!(list[1]["unlocked"], false); + assert_eq!(list[1]["name"], "Secret"); + assert_eq!(list[1]["hidden"], true); + assert!(merge(&json!({"data": null}), &HashMap::new()).is_empty()); + } +} diff --git a/app/src/main/rust/stores/src/epic/adaptive.rs b/app/src/main/rust/stores/src/epic/adaptive.rs deleted file mode 100644 index 7113723eaa..0000000000 --- a/app/src/main/rust/stores/src/epic/adaptive.rs +++ /dev/null @@ -1,1585 +0,0 @@ -//! Budgeted native install: ordered lookahead, shared peak schedule, fetch-core overlap and -//! verified per-file atomic swaps. The JNI snapshot is the Java delta pass's exact pending set. - -use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; -use serde_json::Value; -use sha1::{Digest, Sha1}; -use std::collections::{HashMap, HashSet}; -use std::fs::{self, OpenOptions}; -use std::io::{BufWriter, Write}; -use std::path::{Component, Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Condvar, Mutex}; -use std::time::Duration; - -const MIB: u64 = 1024 * 1024; -const RETAINED: u64 = 64 * MIB; - -#[derive(Clone, Debug)] -pub struct Chunk { - guid: String, - path: String, - size: u64, - wire: u64, - sha: Option<[u8; 20]>, - cached: Option, - exact: bool, -} -#[derive(Clone, Debug)] -pub struct Part { - chunk: usize, - offset: u64, - size: u64, -} -#[derive(Clone, Debug)] -pub struct Output { - path: String, - sha: Option<[u8; 20]>, - old: u64, - parts: Vec, -} -#[derive(Clone, Debug)] -pub struct Request { - pub install: PathBuf, - pub cache: PathBuf, - pub staging: PathBuf, - pub same_volume: bool, - pub free_game: Option, - pub free_cache: Option, - pub chunks: Vec, - pub files: Vec, - pub hosts: Vec, - pub max_workers: usize, -} - -fn uint(v: &Value, key: &str) -> Result { - v[key].as_u64().ok_or_else(|| format!("invalid {key}")) -} -fn text(v: &Value, key: &str) -> Result { - v[key] - .as_str() - .map(str::to_string) - .ok_or_else(|| format!("invalid {key}")) -} -fn digest(v: &Value) -> Result, String> { - let s = v.as_str().ok_or("invalid sha1")?; - if s.is_empty() { - return Ok(None); - } - if s.len() != 40 || !s.is_ascii() { - return Err("invalid sha1".into()); - } - let mut out = [0; 20]; - for (i, b) in out.iter_mut().enumerate() { - *b = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).map_err(|_| "invalid sha1")?; - } - Ok(out.iter().any(|b| *b != 0).then_some(out)) -} -fn relative(s: &str) -> Result<(), String> { - if s.is_empty() - || s.contains('\\') - || s.contains(':') - || Path::new(s) - .components() - .any(|c| !matches!(c, Component::Normal(_))) - { - return Err(format!("unsafe manifest path {s}")); - } - Ok(()) -} -impl Request { - pub fn parse(json: &str) -> Result { - let v: Value = serde_json::from_str(json).map_err(|e| e.to_string())?; - if uint(&v, "schema_version")? != 1 { - return Err("unsupported adaptive request schema".into()); - } - let mut chunks = Vec::new(); - let mut by_guid = HashMap::new(); - for c in v["chunks"].as_array().ok_or("missing chunks")? { - let guid = text(c, "guid")?; - if guid.len() != 32 - || !guid.bytes().all(|b| b.is_ascii_hexdigit()) - || by_guid.contains_key(&guid) - { - return Err("invalid or duplicate chunk GUID".into()); - } - let path = text(c, "path")?; - relative(&path)?; - by_guid.insert(guid.clone(), chunks.len()); - chunks.push(Chunk { - guid, - path, - size: uint(c, "size")?, - wire: uint(c, "wire_size")?, - sha: digest(&c["sha1"])?, - cached: c["cached"].as_u64(), - exact: c["exact_size"].as_bool().ok_or("missing exact_size")?, - }); - } - let mut files = Vec::new(); - for f in v["files"].as_array().ok_or("missing files")? { - let path = text(f, "path")?; - relative(&path)?; - if path.split('/').next() == Some(".droiddeck-epic-staging") { - return Err("manifest conflicts with owned staging".into()); - } - let mut parts = Vec::new(); - for p in f["parts"].as_array().ok_or("missing parts")? { - let chunk = *by_guid - .get(&text(p, "guid")?) - .ok_or("missing chunk metadata")?; - let offset = uint(p, "offset")?; - let size = uint(p, "size")?; - if offset - .checked_add(size) - .filter(|n| *n <= chunks[chunk].size) - .is_none() - { - return Err("part exceeds decoded window".into()); - } - parts.push(Part { - chunk, - offset, - size, - }); - } - files.push(Output { - path, - parts, - sha: digest(&f["sha1"])?, - old: uint(f, "old_size")?, - }); - } - let install = PathBuf::from(text(&v, "install_dir")?); - let staging = PathBuf::from(text(&v, "staging_dir")?); - Self::validate_staging(&install, &staging)?; - let hosts = v["hosts"] - .as_array() - .ok_or("missing hosts")? - .iter() - .map(|h| { - h.as_str() - .map(str::to_string) - .ok_or_else(|| "invalid host".to_string()) - }) - .collect::, _>>()?; - if hosts.is_empty() { - return Err("no Epic CDN".into()); - } - let cache = PathBuf::from(text(&v, "cache_dir")?); - let declared_same = v["same_volume"].as_bool().ok_or("missing same_volume")?; - let same_volume = declared_same || Self::same_filesystem(&install, &cache); - let game_probe = v["free_game"].as_u64(); - let cache_probe = v["free_cache"].as_u64(); - let free_game = if same_volume { - match (game_probe, cache_probe) { - (Some(a), Some(b)) => Some(a.min(b)), - (a, b) => a.or(b), - } - } else { - game_probe - }; - Ok(Self { - install, - staging, - cache, - chunks, - files, - hosts, - same_volume, - free_game, - free_cache: cache_probe, - max_workers: uint(&v, "max_workers")?.clamp(1, 32) as usize, - }) - } - - fn validate_staging(install: &Path, staging: &Path) -> Result<(), String> { - let expected = fs::canonicalize(install) - .map_err(|e| e.to_string())? - .join(".droiddeck-epic-staging"); - let metadata = fs::symlink_metadata(staging).map_err(|e| e.to_string())?; - if !metadata.file_type().is_dir() - || fs::canonicalize(staging).map_err(|e| e.to_string())? != expected - { - return Err("unsafe staging directory; refusing to traverse it".into()); - } - let owner = staging.join("owner"); - let metadata = fs::symlink_metadata(&owner).map_err(|e| e.to_string())?; - if !metadata.file_type().is_file() - || metadata.len() != 26 - || fs::canonicalize(&owner).map_err(|e| e.to_string())? != expected.join("owner") - || fs::read(&owner).map_err(|e| e.to_string())? != b"DroidDeck Epic staging v1\n" - { - return Err("unowned staging directory".into()); - } - for entry in fs::read_dir(staging).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - let name = entry.file_name(); - if name == "owner" { - continue; - } - if !entry.file_type().map_err(|e| e.to_string())?.is_file() - || !name.to_string_lossy().ends_with(".stage") - || fs::canonicalize(entry.path()).map_err(|e| e.to_string())? - != expected.join(&name) - { - return Err("unsafe staging entry".into()); - } - } - Ok(()) - } - - #[cfg(unix)] - fn same_filesystem(a: &Path, b: &Path) -> bool { - use std::os::unix::fs::MetadataExt; - match (fs::metadata(a), fs::metadata(b)) { - (Ok(a), Ok(b)) => a.dev() == b.dev(), - _ => false, - } - } - #[cfg(not(unix))] - fn same_filesystem(_a: &Path, _b: &Path) -> bool { - false - } - fn total(&self) -> u64 { - self.files - .iter() - .flat_map(|f| &f.parts) - .map(|p| p.size) - .sum() - } -} - -/// Resident includes full-size reservations for in-flight writes; only ready chunks can be read. -#[derive(Clone, Debug)] -struct Schedule { - order: Vec, - cursor: usize, - refs: Vec, - resident: Vec>, - oldest: Vec, - lookahead: usize, - retained: u64, -} -impl Schedule { - fn new(req: &Request, lookahead: usize, retained: u64) -> Self { - let order: Vec<_> = req - .files - .iter() - .flat_map(|f| f.parts.iter().map(|p| p.chunk)) - .collect(); - let mut refs = vec![0; req.chunks.len()]; - for &c in &order { - refs[c] += 1; - } - let resident: Vec<_> = req.chunks.iter().map(|c| c.cached).collect(); - let oldest = (0..resident.len()) - .filter(|&c| resident[c].is_some()) - .collect(); - Self { - order, - cursor: 0, - refs, - resident, - oldest, - lookahead, - retained, - } - } - fn window(&self) -> HashSet { - self.order[self.cursor..self.order.len().min(self.cursor + self.lookahead)] - .iter() - .copied() - .collect() - } - fn fill(&mut self, req: &Request) -> Vec { - let mut fetch = Vec::new(); - for &c in &self.order[self.cursor..self.order.len().min(self.cursor + self.lookahead)] { - if self.resident[c].is_none() { - self.resident[c] = Some(req.chunks[c].size); - self.oldest.push(c); - fetch.push(c); - } - } - fetch - } - fn consume(&mut self) -> Vec { - let c = self.order[self.cursor]; - self.cursor += 1; - self.refs[c] -= 1; - let mut release = Vec::new(); - if self.refs[c] == 0 { - self.resident[c] = None; - release.push(c); - } - let pinned = self.window(); - let mut reusable: u64 = self - .oldest - .iter() - .filter(|c| !pinned.contains(c)) - .map(|&c| self.resident[c].unwrap_or(0)) - .sum(); - for &c in &self.oldest { - if reusable <= self.retained { - break; - } - if pinned.contains(&c) { - continue; - } - if let Some(bytes) = self.resident[c].take() { - reusable -= bytes; - release.push(c); - } - } - self.oldest.retain(|&c| self.resident[c].is_some()); - release - } - fn bytes(&self) -> u64 { - self.oldest - .iter() - .map(|&c| self.resident[c].unwrap_or(0)) - .sum() - } -} - -#[derive(Clone, Copy, Debug)] -pub struct Budget { - pub lookahead: usize, - pub retained: u64, - pub game_peak: u64, - pub cache_peak: u64, -} -fn peaks(req: &Request, lookahead: usize, retained: u64) -> (u64, u64) { - let mut s = Schedule::new(req, lookahead, retained); - let initial = s.bytes() as i128; - let mut game = 0i128; - let mut game_peak = 0i128; - let mut cache_peak = 0i128; - for file in &req.files { - for part in &file.parts { - s.fill(req); - let cache = s.bytes() as i128 - initial; - cache_peak = cache_peak.max(cache); - game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); - game += part.size as i128; - game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); - s.consume(); - } - game -= file.old as i128; - } - (game_peak.max(0) as u64, cache_peak.max(0) as u64) -} -fn fits(needed: u64, free: Option) -> bool { - free.is_none_or(|n| needed <= n) -} -pub fn choose_budget(req: &Request) -> Result { - // Unknown probes deliberately choose a conservative bounded mode, never "0 free". - let unknown = req.free_game.is_none() || (!req.same_volume && req.free_cache.is_none()); - for lookahead in [128, 64, 32, 16, 8, 4, 2, 1] { - if unknown && lookahead > 8 { - continue; - } - for retained in [RETAINED, 16 * MIB, 0] { - let (game_peak, cache_peak) = peaks(req, lookahead, retained); - if fits(game_peak, req.free_game) - && (req.same_volume || fits(cache_peak, req.free_cache)) - { - return Ok(Budget { - lookahead, - retained, - game_peak, - cache_peak, - }); - } - } - } - let (game, cache) = peaks(req, 1, 0); - Err(format!("Not enough free space for safe staged Epic replacement: minimum game additional={game}, cache additional={cache}; free game={:?}, cache={:?}", req.free_game, req.free_cache)) -} - -fn remaining_peaks( - req: &Request, - schedule: &Schedule, - fi: usize, - pi: usize, - disk_cache: u64, -) -> (u64, u64) { - let mut s = schedule.clone(); - let mut game = 0i128; - let mut game_peak = 0i128; - let mut cache_peak = 0i128; - let outstanding = s.bytes() as i128 - disk_cache as i128; - cache_peak = cache_peak.max(outstanding); - if req.same_volume { - game_peak = game_peak.max(outstanding); - } - for (index, file) in req.files.iter().enumerate().skip(fi) { - for part in file.parts.iter().skip(if index == fi { pi } else { 0 }) { - s.fill(req); - let cache = s.bytes() as i128 - disk_cache as i128; - cache_peak = cache_peak.max(cache); - game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); - game += part.size as i128; - game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); - s.consume(); - } - game -= file.old as i128; - } - (game_peak.max(0) as u64, cache_peak.max(0) as u64) -} - -#[cfg(unix)] -fn available(path: &Path) -> Option { - use std::os::unix::ffi::OsStrExt; - let path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?; - let mut stat = std::mem::MaybeUninit::::uninit(); - if unsafe { libc::statvfs(path.as_ptr(), stat.as_mut_ptr()) } != 0 { - return None; - } - let stat = unsafe { stat.assume_init() }; - Some((stat.f_bavail as u64).saturating_mul(stat.f_frsize as u64)) -} -#[cfg(not(unix))] -fn available(_path: &Path) -> Option { - None -} - -/// Re-probe at admission boundaries. Already dispatched reservations cannot be withdrawn. -/// Shrink only unsent lookahead/reusable budget; never wait forever for unrelated space use. -fn live_admit( - req: &Request, - state: &mut State, - fi: usize, - pi: usize, - stage_remaining: u64, -) -> Result<(), String> { - let disk_cache = state - .schedule - .oldest - .iter() - .map(|&index| { - if state.ready[index] { - return state.disk_sizes[index]; - } - if !state.writing[index] { - return 0; - } - let c = &req.chunks[index]; - fs::metadata(req.cache.join(&c.guid)) - .map(|m| m.len()) - .unwrap_or(0) - + fs::metadata(super::chunk::part_path(&req.cache.join(&c.guid))) - .map(|m| m.len()) - .unwrap_or(0) - }) - .sum(); - let free_game = available(&req.install); - let free_cache = available(&req.cache); - admit_with_probes( - req, - state, - fi, - pi, - stage_remaining, - disk_cache, - free_game, - free_cache, - ) -} - -fn admit_with_probes( - req: &Request, - state: &mut State, - fi: usize, - pi: usize, - stage_remaining: u64, - disk_cache: u64, - free_game: Option, - free_cache: Option, -) -> Result<(), String> { - loop { - let incoming: u64 = state - .schedule - .window() - .iter() - .filter(|&&c| state.schedule.resident[c].is_none()) - .map(|&c| req.chunks[c].size) - .sum(); - let cache_now = state - .schedule - .bytes() - .saturating_add(incoming) - .saturating_sub(disk_cache); - let game_now = stage_remaining.saturating_add(if req.same_volume { cache_now } else { 0 }); - if fits(game_now, free_game) && (req.same_volume || fits(cache_now, free_cache)) { - return Ok(()); - } - // Full remaining simulation is only needed when live admission headroom is exhausted. - let (game, cache) = remaining_peaks(req, &state.schedule, fi, pi, disk_cache); - if fits(game, free_game) && (req.same_volume || fits(cache, free_cache)) { - return Ok(()); - } - if state.schedule.oldest.iter().any(|&c| !state.ready[c]) { - return Err("Not enough free space for already reserved Epic downloads".into()); - } - if state.schedule.retained > 0 { - state.schedule.retained = 0; - continue; - } - if state.schedule.lookahead > 1 { - state.schedule.lookahead /= 2; - continue; - } - return Err(format!("Not enough free space during Epic install: need additional game={game} cache={cache}, free game={free_game:?} cache={free_cache:?}")); - } -} - -struct State { - schedule: Schedule, - ready: Vec, - writing: Vec, - disk_sizes: Vec, - pending: Vec, - error: Option, - done: bool, -} -struct Shared { - state: Mutex, - changed: Condvar, - stop: AtomicBool, -} -struct Sink<'a> { - req: &'a Request, - shared: &'a Shared, -} -impl FetchSink for Sink<'_> { - fn process(&self, item: &FetchItem, body: Vec) -> Result { - if self.shared.stop.load(Ordering::Acquire) { - return Err(SinkError::Fatal("cancelled".into())); - } - let c = item.id as usize; - let chunk = &self.req.chunks[c]; - self.shared.state.lock().unwrap().writing[c] = true; - let written = super::chunk::write_verified_chunk_bounded( - &body, - chunk.sha.as_ref(), - &self.req.cache.join(&chunk.guid), - chunk.size, - chunk.exact, - ); - let mut state = self.shared.state.lock().unwrap(); - state.writing[c] = false; - let written = written.map_err(SinkError::Retry)?; - state.ready[c] = true; - state.disk_sizes[c] = written; - self.shared.changed.notify_all(); - Ok(chunk.wire) - } -} - -/// JNI owns this entire blocking run. All scoped workers join before completion is signalled. -pub fn run( - req: &Request, - budget: Budget, - cancel: &AtomicBool, - before_swap: &(dyn Fn() -> Result<(), String> + Sync), - progress: &(dyn Fn(u64, u64, u64, u64) + Sync), - log: &(dyn Fn(&str) + Sync), -) -> Result { - let shared = Shared { - state: Mutex::new(State { - schedule: Schedule::new(req, budget.lookahead, budget.retained), - ready: req.chunks.iter().map(|c| c.cached.is_some()).collect(), - writing: vec![false; req.chunks.len()], - disk_sizes: req.chunks.iter().map(|c| c.cached.unwrap_or(0)).collect(), - pending: Vec::new(), - error: None, - done: false, - }), - changed: Condvar::new(), - stop: AtomicBool::new(false), - }; - log(&format!( - "adaptive-native lookahead={} retained={} game_peak={} cache_peak={} workers={}", - budget.lookahead, - budget.retained, - budget.game_peak, - budget.cache_peak, - req.max_workers.min(budget.lookahead) - )); - let result = std::thread::scope(|scope| { - scope.spawn(|| { - crate::priority::background(); - let sink = Sink { - req, - shared: &shared, - }; - loop { - let batch = { - let mut state = shared.state.lock().unwrap(); - while state.pending.is_empty() - && !state.done - && !shared.stop.load(Ordering::Acquire) - { - state = shared - .changed - .wait_timeout(state, Duration::from_millis(100)) - .unwrap() - .0; - } - if state.done || shared.stop.load(Ordering::Acquire) { - break; - } - std::mem::take(&mut state.pending) - }; - let items = batch - .into_iter() - .map(|c| FetchItem { - id: c as u64, - urls: req - .hosts - .iter() - .map(|h| format!("{h}/{}", req.chunks[c].path)) - .collect(), - reserve: req.chunks[c].wire, - range: None, - }) - .collect(); - let opts = FetchOptions { - max_workers: req.max_workers.min(budget.lookahead), - per_host_cap: super::plan::per_host_cap( - req.max_workers.min(budget.lookahead), - req.hosts.len(), - ), - timeout: Duration::from_secs(60), - process_workers: 2, - headers: vec![("User-Agent".into(), super::USER_AGENT.into())], - label: "epic adaptive".into(), - ..Default::default() - }; - let fetched = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - run_fetch( - items, - &req.hosts, - &opts, - &sink, - &shared.stop, - &|_, _| {}, - log, - ) - })); - let outcome = match fetched { - Ok(outcome) => outcome, - Err(_) => { - let mut state = shared.state.lock().unwrap(); - state.error = Some("native fetch panicked".into()); - shared.stop.store(true, Ordering::Release); - shared.changed.notify_all(); - break; - } - }; - if !outcome.ok() { - let mut state = shared.state.lock().unwrap(); - state.error = Some(outcome.error.unwrap_or_else(|| "cancelled".into())); - shared.stop.store(true, Ordering::Release); - shared.changed.notify_all(); - break; - } - } - }); - // Cancel forwarding is independent of a writer blocked on disk I/O. - scope.spawn(|| { - while !shared.stop.load(Ordering::Acquire) { - if cancel.load(Ordering::Acquire) { - shared.stop.store(true, Ordering::Release); - shared.changed.notify_all(); - break; - } - std::thread::sleep(Duration::from_millis(50)); - } - }); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - assemble(req, &shared, cancel, before_swap, progress) - })) - .unwrap_or_else(|_| Err("native assembly panicked".into())); - shared.stop.store(true, Ordering::Release); - { - let mut state = shared.state.lock().unwrap(); - state.done = true; - shared.changed.notify_all(); - } - result - }); - // No .part writer survives this point; preserve verified resident chunks for resume. - let resident = shared.state.lock().unwrap().schedule.oldest.clone(); - for index in resident { - let _ = fs::remove_file(super::chunk::part_path( - &req.cache.join(&req.chunks[index].guid), - )); - } - if result.is_ok() { - fs::remove_dir_all(&req.cache).map_err(|e| format!("clear cache: {e}"))?; - } - result -} - -fn await_chunk( - req: &Request, - shared: &Shared, - cancel: &AtomicBool, - c: usize, - fi: usize, - pi: usize, - stage_remaining: u64, -) -> Result<(), String> { - let mut state = shared.state.lock().unwrap(); - if cancel.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - live_admit(req, &mut state, fi, pi, stage_remaining)?; - let fetch = state.schedule.fill(req); - state.pending.extend(fetch); - shared.changed.notify_all(); - while !state.ready[c] { - if cancel.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - if let Some(error) = &state.error { - return Err(error.clone()); - } - if shared.stop.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - state = shared - .changed - .wait_timeout(state, Duration::from_millis(100)) - .unwrap() - .0; - } - Ok(()) -} - -fn assemble( - req: &Request, - shared: &Shared, - cancel: &AtomicBool, - before_swap: &(dyn Fn() -> Result<(), String> + Sync), - progress: &(dyn Fn(u64, u64, u64, u64) + Sync), -) -> Result { - let mut done = 0; - let mut marked = false; - let total = req.total(); - for (fi, file) in req.files.iter().enumerate() { - Request::validate_staging(&req.install, &req.staging)?; - if cancel.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - let output = req.install.join(&file.path); - fs::create_dir_all(output.parent().ok_or("missing output parent")?) - .map_err(|e| e.to_string())?; - let stage = req - .staging - .join(format!("native-{}-{fi}.stage", std::process::id())); - let result = (|| { - let stage_file = OpenOptions::new() - .write(true) - .create_new(true) - .open(&stage) - .map_err(|e| e.to_string())?; - let mut out = BufWriter::with_capacity(64 * 1024, stage_file); - let mut sha = Sha1::new(); - let mut written = 0; - let file_total: u64 = file.parts.iter().map(|p| p.size).sum(); - for (pi, part) in file.parts.iter().enumerate() { - // A live free-space probe must include already buffered staging bytes. - out.flush().map_err(|e| e.to_string())?; - await_chunk( - req, - shared, - cancel, - part.chunk, - fi, - pi, - file_total - written, - )?; - written += super::assemble::copy_chunk_part( - &req.cache.join(&req.chunks[part.chunk].guid), - part.offset, - part.size, - &mut out, - &mut sha, - cancel, - )?; - let mut state = shared.state.lock().unwrap(); - for c in state.schedule.consume() { - if !state.ready[c] { - return Err("attempt to release in-flight chunk".into()); - } - fs::remove_file(req.cache.join(&req.chunks[c].guid)) - .map_err(|e| e.to_string())?; - state.ready[c] = false; - } - done += part.size; - progress(done, total, fi as u64, req.files.len() as u64); - } - out.flush().map_err(|e| e.to_string())?; - out.get_ref().sync_all().map_err(|e| e.to_string())?; - drop(out); - if written != file_total { - return Err("file size mismatch".into()); - } - if file - .sha - .is_some_and(|expected| sha.finalize().as_slice() != expected) - { - return Err(format!("SHA-1 mismatch for {}", file.path)); - } - if cancel.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - if !marked { - before_swap()?; - marked = true; - } - if cancel.load(Ordering::Acquire) { - return Err("cancelled".into()); - } - // Android/Linux rename replaces atomically on the same output volume. Never delete first. - fs::rename(&stage, &output) - .map_err(|e| format!("atomic replacement {}: {e}", file.path))?; - progress(done, total, fi as u64 + 1, req.files.len() as u64); - Ok(()) - })(); - let _ = fs::remove_file(&stage); - result?; - } - Ok(done) -} - -#[cfg(test)] -mod tests { - use super::super::chunk::test_support::{build_chunk_body, sha1_of}; - use super::*; - use std::io::Read; - use std::net::TcpListener; - use std::sync::atomic::{AtomicU64, AtomicUsize}; - use std::sync::{Arc, Barrier}; - use std::thread; - use std::time::{Instant, SystemTime, UNIX_EPOCH}; - - static SEQUENCE: AtomicU64 = AtomicU64::new(0); - struct Fixture { - root: PathBuf, - req: Request, - } - impl Fixture { - fn new(count: usize, size: usize) -> Self { - let root = PathBuf::from("target").join(format!( - "epic-adaptive-{}-{}-{}", - std::process::id(), - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(), - SEQUENCE.fetch_add(1, Ordering::Relaxed) - )); - let install = root.join("game"); - let cache = root.join("cache"); - let staging = install.join(".droiddeck-epic-staging"); - fs::create_dir_all(&staging).unwrap(); - fs::create_dir_all(&cache).unwrap(); - fs::write(staging.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); - let data = vec![42; size]; - let sha = sha1_of(&data); - let chunks: Vec<_> = (0..count) - .map(|c| Chunk { - guid: format!("{c:032X}"), - path: format!("chunks/{c}"), - size: size as u64, - wire: size as u64 + 41, - sha: Some(sha), - cached: None, - exact: true, - }) - .collect(); - let files = (0..count) - .map(|c| Output { - path: format!("file-{c}.bin"), - sha: Some(sha), - old: 0, - parts: vec![Part { - chunk: c, - offset: 0, - size: size as u64, - }], - }) - .collect(); - Self { - root, - req: Request { - install, - cache, - staging, - chunks, - files, - same_volume: false, - free_game: Some(u64::MAX), - free_cache: Some(u64::MAX), - hosts: vec!["http://127.0.0.1:1".into()], - max_workers: 8, - }, - } - } - } - impl Drop for Fixture { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.root); - } - } - - struct Server { - address: String, - requests: Arc, - active: Arc, - stop: Arc, - worker: Option>, - } - impl Server { - fn start(data: Vec, barrier_count: usize, stalled: bool) -> Self { - Self::start_mode(data, barrier_count, stalled.then_some(usize::MAX)) - } - - fn start_mode(data: Vec, barrier_count: usize, stall_chunk: Option) -> Self { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = format!("http://{}", listener.local_addr().unwrap()); - listener.set_nonblocking(true).unwrap(); - let requests = Arc::new(AtomicUsize::new(0)); - let active = Arc::new(AtomicUsize::new(0)); - let stop = Arc::new(AtomicBool::new(false)); - let requests2 = requests.clone(); - let active2 = active.clone(); - let stop2 = stop.clone(); - let worker = thread::spawn(move || { - let body = Arc::new(build_chunk_body(&data, false, 0, None)); - let barrier = Arc::new(Barrier::new(barrier_count.max(1))); - let mut handlers = Vec::new(); - while !stop2.load(Ordering::Acquire) { - let (mut socket, _) = match listener.accept() { - Ok(s) => s, - Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(5)); - continue; - } - Err(e) => panic!("{e}"), - }; - let body = body.clone(); - let requests = requests2.clone(); - let active = active2.clone(); - let stop = stop2.clone(); - let barrier = barrier.clone(); - handlers.push(thread::spawn(move || { - active.fetch_add(1, Ordering::Relaxed); - let _ = socket.set_read_timeout(Some(Duration::from_secs(5))); - let mut request = Vec::new(); - let mut byte = [0]; - while socket.read(&mut byte).unwrap_or(0) > 0 { - request.push(byte[0]); - if request.ends_with(b"\r\n\r\n") { - break; - } - } - let n = requests.fetch_add(1, Ordering::Relaxed); - if n < barrier_count { - barrier.wait(); - } - let _ = write!( - socket, - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - let stalled = stall_chunk.is_some_and(|c| { - c == usize::MAX - || String::from_utf8_lossy(&request) - .contains(&format!("/chunks/{c} ")) - }); - if stalled { - // Cancellation must close the native client's connection without data. - let _ = socket.set_read_timeout(Some(Duration::from_millis(100))); - loop { - if stop.load(Ordering::Acquire) { - break; - } - match socket.read(&mut byte) { - Ok(0) => break, - Ok(_) => {} - Err(e) - if e.kind() == std::io::ErrorKind::WouldBlock - || e.kind() == std::io::ErrorKind::TimedOut => {} - Err(_) => break, - } - } - } else { - let _ = socket.write_all(&body); - } - active.fetch_sub(1, Ordering::Relaxed); - })); - } - for handler in handlers { - handler.join().unwrap(); - } - }); - Self { - address, - requests, - active, - stop, - worker: Some(worker), - } - } - } - impl Drop for Server { - fn drop(&mut self) { - self.stop.store(true, Ordering::Release); - if let Some(worker) = self.worker.take() { - worker.join().unwrap(); - } - } - } - - #[test] - fn situation_table_selects_large_tight_minimal_and_rejects_no_fit() { - let mut f = Fixture::new(300, MIB as usize); - assert_eq!(choose_budget(&f.req).unwrap().lookahead, 128); - f.req.free_cache = Some(8 * MIB); - assert_eq!(choose_budget(&f.req).unwrap().lookahead, 8); - f.req.free_cache = Some(MIB); - assert_eq!(choose_budget(&f.req).unwrap().lookahead, 1); - f.req.free_cache = Some(MIB - 1); - assert!(choose_budget(&f.req) - .unwrap_err() - .contains("Not enough free space")); - f.req.free_cache = None; - assert_eq!(choose_budget(&f.req).unwrap().lookahead, 8); - f.req.free_game = Some(0); - assert!(choose_budget(&f.req).is_err()); - } - - #[test] - fn planner_counts_all_incoming_old_outputs_and_existing_cache_release() { - let mut f = Fixture::new(20, 100); - f.req.same_volume = true; - for file in &mut f.req.files { - file.old = 100; - } - assert_eq!(peaks(&f.req, 8, 0), (900, 800)); - for chunk in &mut f.req.chunks { - chunk.cached = Some(100); - } - assert_eq!(peaks(&f.req, 8, 0), (100, 0)); - f.req.same_volume = false; - assert_eq!(peaks(&f.req, 8, 0), (100, 0)); - } - - #[test] - fn shared_references_deduplicate_and_distant_chunks_refetch_under_budget() { - let mut f = Fixture::new(12, (40 * MIB) as usize); - f.req.files.extend(f.req.files.clone()); - let mut schedule = Schedule::new(&f.req, 8, RETAINED); - assert_eq!(schedule.fill(&f.req), (0..8).collect::>()); - assert!(schedule.fill(&f.req).is_empty()); - assert!(schedule.consume().is_empty()); - assert_eq!(schedule.fill(&f.req), vec![8]); - assert_eq!(schedule.consume(), vec![0]); - let mut refetched = false; - for _ in 2..24 { - refetched |= schedule.fill(&f.req).contains(&0); - assert!(schedule.bytes() <= RETAINED + 8 * 40 * MIB); - let removed = schedule.consume(); - assert!(removed.iter().all(|c| !schedule.window().contains(c))); - } - assert!(refetched); - assert_eq!(schedule.bytes(), 0); - } - - #[test] - fn native_fetch_overlaps_across_files_and_never_fetches_outside_window() { - let mut f = Fixture::new(30, 4096); - let server = Server::start(vec![42; 4096], 8, false); - f.req.hosts = vec![server.address.clone()]; - f.req.free_cache = Some(8 * 4096); - let budget = choose_budget(&f.req).unwrap(); - assert_eq!(budget.lookahead, 8); - let cancel = AtomicBool::new(false); - let swapped = AtomicUsize::new(0); - let progress = |_, _, files, _| { - if files == 0 { - cancel.store(true, Ordering::Release); - } - }; - let result = run( - &f.req, - budget, - &cancel, - &|| { - swapped.fetch_add(1, Ordering::Relaxed); - Ok(()) - }, - &progress, - &|_| {}, - ); - assert!(result.is_err()); - assert_eq!(server.requests.load(Ordering::Relaxed), 8); - assert_eq!(swapped.load(Ordering::Relaxed), 0); - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - assert!(fs::read_dir(&f.req.cache).unwrap().all(|e| !e - .unwrap() - .path() - .to_string_lossy() - .ends_with(".part"))); - } - - #[test] - fn native_success_stages_verifies_swaps_and_reclaims_both_cache_layouts() { - for same_volume in [false, true] { - let mut f = Fixture::new(20, 4096); - let server = Server::start(vec![42; 4096], 8, false); - f.req.hosts = vec![server.address.clone()]; - f.req.same_volume = same_volume; - if same_volume { - f.req.cache = f.req.install.join(".chunks"); - fs::create_dir_all(&f.req.cache).unwrap(); - } - for file in &mut f.req.files { - fs::write(f.req.install.join(&file.path), b"old").unwrap(); - file.old = 3; - } - let swaps = AtomicUsize::new(0); - let before = || { - assert_eq!(fs::read(f.req.install.join("file-0.bin")).unwrap(), b"old"); - swaps.fetch_add(1, Ordering::Relaxed); - Ok(()) - }; - assert_eq!( - run( - &f.req, - choose_budget(&f.req).unwrap(), - &AtomicBool::new(false), - &before, - &|_, _, _, _| {}, - &|_| {} - ) - .unwrap(), - 20 * 4096 - ); - assert_eq!(swaps.load(Ordering::Relaxed), 1); - assert!(!f.req.cache.exists()); - for file in &f.req.files { - assert_eq!( - fs::read(f.req.install.join(&file.path)).unwrap(), - vec![42; 4096] - ); - } - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - } - } - - #[test] - fn native_file_hash_failure_and_updating_failure_keep_old_output() { - for updating_failure in [false, true] { - let mut f = Fixture::new(1, 4096); - let server = Server::start(vec![42; 4096], 1, false); - f.req.hosts = vec![server.address.clone()]; - fs::write(f.req.install.join("file-0.bin"), b"old").unwrap(); - if !updating_failure { - f.req.files[0].sha = Some([1; 20]); - } - let before = || Err("updating record failed".into()); - let result = run( - &f.req, - choose_budget(&f.req).unwrap(), - &AtomicBool::new(false), - &before, - &|_, _, _, _| {}, - &|_| {}, - ); - assert!(result.is_err()); - assert_eq!(fs::read(f.req.install.join("file-0.bin")).unwrap(), b"old"); - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - } - } - - #[test] - fn native_cancellation_closes_stalled_parallel_requests_and_joins_before_return() { - let mut f = Fixture::new(30, 4096); - let server = Server::start(vec![42; 4096], 8, true); - f.req.hosts = vec![server.address.clone()]; - f.req.free_cache = Some(8 * 4096); - let cancel = AtomicBool::new(false); - thread::scope(|scope| { - scope.spawn(|| { - let until = Instant::now() + Duration::from_secs(5); - while server.requests.load(Ordering::Relaxed) < 8 && Instant::now() < until { - thread::sleep(Duration::from_millis(10)); - } - cancel.store(true, Ordering::Release); - }); - let started = Instant::now(); - assert!(run( - &f.req, - choose_budget(&f.req).unwrap(), - &cancel, - &|| Ok(()), - &|_, _, _, _| {}, - &|_| {} - ) - .is_err()); - assert!(started.elapsed() < Duration::from_secs(7)); - }); - let until = Instant::now() + Duration::from_secs(2); - while server.active.load(Ordering::Relaxed) > 0 && Instant::now() < until { - thread::sleep(Duration::from_millis(10)); - } - assert_eq!(server.active.load(Ordering::Relaxed), 0); - assert_eq!(server.requests.load(Ordering::Relaxed), 8); - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - assert_eq!(fs::read_dir(&f.req.cache).unwrap().count(), 0); - } - - #[test] - fn native_bounded_decode_never_publishes_oversized_or_truncated_chunks() { - let f = Fixture::new(1, 10); - let path = f.req.cache.join("chunk"); - let body = build_chunk_body(&[42; 11], true, 0, None); - assert!( - super::super::chunk::write_verified_chunk_bounded(&body, None, &path, 10, true) - .is_err() - ); - assert!(!path.exists()); - assert!(!super::super::chunk::part_path(&path).exists()); - let body = build_chunk_body(&[42; 10], false, 0, Some(11)); - assert!( - super::super::chunk::write_verified_chunk_bounded(&body, None, &path, 10, true) - .is_err() - ); - let data = vec![42; 512 * 1024]; - let mut compressed = build_chunk_body(&data, true, 0, None); - assert_eq!( - super::super::chunk::write_verified_chunk_bounded( - &compressed, - Some(&sha1_of(&data)), - &path, - data.len() as u64, - true - ) - .unwrap(), - data.len() as u64 - ); - fs::remove_file(&path).unwrap(); - compressed.pop(); - // A matching shorter payload header leaves the missing zlib stream end to detect failure. - let short = (compressed.len() - 41) as i32; - compressed[12..16].copy_from_slice(&short.to_le_bytes()); - assert!(super::super::chunk::write_verified_chunk_bounded( - &compressed, - None, - &path, - data.len() as u64, - false - ) - .is_err()); - assert!(!path.exists()); - } - - #[test] - fn live_admission_shrinks_unsent_work_but_never_withdraws_inflight_reservations() { - let f = Fixture::new(30, 4096); - let mut state = State { - schedule: Schedule::new(&f.req, 128, RETAINED), - ready: vec![false; 30], - writing: vec![false; 30], - disk_sizes: vec![0; 30], - pending: vec![], - error: None, - done: false, - }; - admit_with_probes( - &f.req, - &mut state, - 0, - 0, - 4096, - 0, - Some(u64::MAX), - Some(8 * 4096), - ) - .unwrap(); - assert_eq!(state.schedule.lookahead, 8); - assert_eq!(state.schedule.retained, 0); - assert_eq!(state.schedule.fill(&f.req).len(), 8); - assert!(admit_with_probes( - &f.req, - &mut state, - 0, - 0, - 4096, - 0, - Some(u64::MAX), - Some(4096) - ) - .is_err()); - assert_eq!(state.schedule.lookahead, 8); - assert!(state - .schedule - .oldest - .iter() - .all(|&c| state.schedule.resident[c].is_some())); - } - - #[test] - fn snapshot_schema_rejects_unowned_staging_bad_paths_and_bad_version() { - let f = Fixture::new(1, 10); - let mut snapshot = serde_json::json!({ - "schema_version": 1, - "install_dir": f.req.install, "cache_dir": f.req.cache, "staging_dir": f.req.staging, - "same_volume": false, "free_game": -1, "free_cache": 0, "max_workers": 32, - "hosts": ["http://127.0.0.1"], - "chunks": [{ "guid": "00000000000000000000000000000000", "path": "chunks/00/chunk", - "size": 10, "wire_size": 51, "sha1": "", "cached": -1, "exact_size": true }], - "files": [{ "path": "legitimate.part", "old_size": 0, "sha1": "", - "parts": [{ "guid": "00000000000000000000000000000000", "offset": 0, "size": 10 }] }] - }); - let parsed = Request::parse(&snapshot.to_string()).unwrap(); - assert_eq!( - parsed.free_game, - if parsed.same_volume { Some(0) } else { None } - ); - assert_eq!(parsed.free_cache, Some(0)); - assert_eq!(parsed.chunks[0].cached, None); - assert!(choose_budget(&parsed).is_err()); - snapshot["schema_version"] = 2.into(); - assert!(Request::parse(&snapshot.to_string()).is_err()); - snapshot["schema_version"] = 1.into(); - snapshot["files"][0]["path"] = "../outside".into(); - assert!(Request::parse(&snapshot.to_string()).is_err()); - snapshot["files"][0]["path"] = ".droiddeck-epic-staging/owner".into(); - assert!(Request::parse(&snapshot.to_string()).is_err()); - snapshot["files"][0]["path"] = "legitimate.part".into(); - fs::remove_file(f.req.staging.join("owner")).unwrap(); - assert!(Request::parse(&snapshot.to_string()).is_err()); - } - - #[test] - fn native_shared_chunk_survives_cancelled_file_boundary_and_is_reused_on_resume() { - let mut f = Fixture::new(3, 4096); - let server = Server::start(vec![42; 4096], 1, false); - f.req.hosts = vec![server.address.clone()]; - for file in &mut f.req.files { - file.parts[0].chunk = 0; - } - let cancel = AtomicBool::new(false); - let before = AtomicUsize::new(0); - let progress = |_, _, files, _| { - if files == 1 { - cancel.store(true, Ordering::Release); - } - }; - assert!(run( - &f.req, - choose_budget(&f.req).unwrap(), - &cancel, - &|| { - before.fetch_add(1, Ordering::Relaxed); - Ok(()) - }, - &progress, - &|_| {} - ) - .is_err()); - assert_eq!(before.load(Ordering::Relaxed), 1); - assert_eq!(server.requests.load(Ordering::Relaxed), 1); - assert_eq!( - fs::read(f.req.install.join("file-0.bin")).unwrap(), - vec![42; 4096] - ); - assert!(!f.req.install.join("file-1.bin").exists()); - assert!(f.req.cache.join(&f.req.chunks[0].guid).is_file()); - f.req.files.remove(0); - f.req.chunks[0].cached = Some(4096); - assert_eq!( - run( - &f.req, - choose_budget(&f.req).unwrap(), - &AtomicBool::new(false), - &|| Ok(()), - &|_, _, _, _| {}, - &|_| {} - ) - .unwrap(), - 8192 - ); - assert_eq!(server.requests.load(Ordering::Relaxed), 1); - assert!(!f.req.cache.exists()); - } - - #[test] - fn native_later_hash_failure_preserves_verified_swap_and_unreplaced_old_file() { - let mut f = Fixture::new(2, 4096); - let server = Server::start(vec![42; 4096], 2, false); - f.req.hosts = vec![server.address.clone()]; - for file in &mut f.req.files { - fs::write(f.req.install.join(&file.path), b"old").unwrap(); - file.old = 3; - } - f.req.files[1].sha = Some([1; 20]); - let marked = AtomicUsize::new(0); - let error = run( - &f.req, - choose_budget(&f.req).unwrap(), - &AtomicBool::new(false), - &|| { - marked.fetch_add(1, Ordering::Relaxed); - Ok(()) - }, - &|_, _, _, _| {}, - &|_| {}, - ) - .unwrap_err(); - assert!(error.contains("SHA-1 mismatch")); - assert_eq!(marked.load(Ordering::Relaxed), 1); - assert_eq!( - fs::read(f.req.install.join("file-0.bin")).unwrap(), - vec![42; 4096] - ); - assert_eq!(fs::read(f.req.install.join("file-1.bin")).unwrap(), b"old"); - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - } - - #[test] - fn actual_native_file_and_cache_growth_never_exceeds_the_shared_planned_peak() { - let mut f = Fixture::new(20, 4096); - let server = Server::start(vec![42; 4096], 8, false); - f.req.hosts = vec![server.address.clone()]; - f.req.same_volume = true; - f.req.free_game = Some(21 * 4096); - for file in &mut f.req.files { - file.old = 4096; - fs::write(f.req.install.join(&file.path), vec![1; 4096]).unwrap(); - } - let budget = choose_budget(&f.req).unwrap(); - let observed = AtomicU64::new(0); - let progress = |_, _, _, _| { - let game: u64 = f - .req - .files - .iter() - .map(|file| fs::metadata(f.req.install.join(&file.path)).unwrap().len()) - .sum(); - let stage: u64 = fs::read_dir(&f.req.staging) - .unwrap() - .filter_map(Result::ok) - .filter(|e| e.file_name() != "owner") - .map(|e| e.metadata().map(|m| m.len()).unwrap_or(0)) - .sum(); - let cache: u64 = fs::read_dir(&f.req.cache) - .unwrap() - .filter_map(Result::ok) - .map(|e| e.metadata().map(|m| m.len()).unwrap_or(0)) - .sum(); - let additional = (game + stage + cache).saturating_sub(20 * 4096); - observed.fetch_max(additional, Ordering::Relaxed); - assert!( - additional <= budget.game_peak, - "{additional} > planned {}", - budget.game_peak - ); - }; - assert_eq!( - run( - &f.req, - budget, - &AtomicBool::new(false), - &|| Ok(()), - &progress, - &|_| {} - ) - .unwrap(), - 20 * 4096 - ); - assert!(observed.load(Ordering::Relaxed) > 8 * 4096); - } - - #[test] - fn native_assembly_swaps_a_verified_file_while_a_future_chunk_is_still_fetching() { - let mut f = Fixture::new(20, 4096); - let server = Server::start_mode(vec![42; 4096], 8, Some(7)); - f.req.hosts = vec![server.address.clone()]; - f.req.free_cache = Some(8 * 4096); - let cancel = AtomicBool::new(false); - let overlap = AtomicBool::new(false); - let progress = |_, _, files, _| { - if files == 1 { - assert!(server.active.load(Ordering::Relaxed) > 0); - assert_eq!( - fs::read(f.req.install.join("file-0.bin")).unwrap(), - vec![42; 4096] - ); - overlap.store(true, Ordering::Release); - cancel.store(true, Ordering::Release); - } - }; - assert!(run( - &f.req, - choose_budget(&f.req).unwrap(), - &cancel, - &|| Ok(()), - &progress, - &|_| {} - ) - .is_err()); - assert!(overlap.load(Ordering::Acquire)); - assert_eq!(server.requests.load(Ordering::Relaxed), 8); - assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); - } - - #[cfg(unix)] - #[test] - fn staging_validation_rejects_symlink_namespace_owner_and_entries() { - use std::os::unix::fs::symlink; - let f = Fixture::new(1, 4096); - let outside = f.root.join("outside"); - fs::create_dir(&outside).unwrap(); - fs::write(outside.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); - fs::write(outside.join("keep.stage"), b"keep").unwrap(); - fs::remove_dir_all(&f.req.staging).unwrap(); - symlink(fs::canonicalize(&outside).unwrap(), &f.req.staging).unwrap(); - assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); - assert!(run( - &f.req, - choose_budget(&f.req).unwrap(), - &AtomicBool::new(false), - &|| Ok(()), - &|_, _, _, _| {}, - &|_| {} - ) - .is_err()); - assert_eq!(fs::read(outside.join("keep.stage")).unwrap(), b"keep"); - fs::remove_file(&f.req.staging).unwrap(); - fs::create_dir(&f.req.staging).unwrap(); - symlink( - fs::canonicalize(outside.join("owner")).unwrap(), - f.req.staging.join("owner"), - ) - .unwrap(); - assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); - fs::remove_file(f.req.staging.join("owner")).unwrap(); - fs::write(f.req.staging.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); - symlink( - fs::canonicalize(outside.join("keep.stage")).unwrap(), - f.req.staging.join("linked.stage"), - ) - .unwrap(); - assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); - assert_eq!(fs::read(outside.join("keep.stage")).unwrap(), b"keep"); - fs::remove_file(f.req.staging.join("linked.stage")).unwrap(); - } -} diff --git a/app/src/main/rust/stores/src/epic/assemble.rs b/app/src/main/rust/stores/src/epic/assemble.rs deleted file mode 100644 index 8094d64fce..0000000000 --- a/app/src/main/rust/stores/src/epic/assemble.rs +++ /dev/null @@ -1,499 +0,0 @@ - -use std::collections::HashMap; -use std::fs::{self, File}; -use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; - -use sha1::{Digest, Sha1}; - -use super::manifest::{parse_manifest, FileInfo, Manifest}; -use super::plan::resolve_cache_dir; - -/// Streaming assembly primitive used by the budgeted installer; never holds a decoded chunk -/// in memory and closes the reader before the caller releases the cache reservation. -pub fn copy_chunk_part( - cache_path: &Path, offset: u64, size: u64, output: &mut impl Write, - sha: &mut Sha1, cancel: &AtomicBool, -) -> Result { - let mut input = File::open(cache_path).map_err(|e| format!("read chunk: {e}"))?; - let length = input.metadata().map_err(|e| e.to_string())?.len(); - if offset.checked_add(size).filter(|end| *end <= length).is_none() { - return Err("chunk part exceeds cached data".into()); - } - input.seek(SeekFrom::Start(offset)).map_err(|e| e.to_string())?; - let mut left = size; - let mut buffer = [0u8; 64 * 1024]; - while left > 0 { - if cancel.load(Ordering::Acquire) { return Err("cancelled".into()); } - let take = left.min(buffer.len() as u64) as usize; - let count = input.read(&mut buffer[..take]) - .map_err(|e| e.to_string())?; - if count == 0 { return Err("short cached chunk".into()); } - output.write_all(&buffer[..count]).map_err(|e| e.to_string())?; - sha.update(&buffer[..count]); - left -= count as u64; - } - Ok(size) -} - -pub struct AssembleRequest { - pub manifest_bytes: Vec, - pub install_dir: String, - pub chunk_cache_dir: String, - pub file_indices: Vec, -} - -#[derive(Clone, Debug, Default)] -pub struct AssembleOutcome { - pub success: bool, - pub cancelled: bool, - pub error: String, - pub bytes_written: u64, - pub files_done: u64, - pub files_total: u64, - pub bytes_total: u64, - pub chunks_removed: u64, -} - -#[derive(Debug)] -pub struct AssemblePlan { - pub manifest: Manifest, - pub install_dir: PathBuf, - pub cache_dir: PathBuf, - pub scratch_cache: bool, - pub files: Vec, - pub bytes_total: u64, - pub refs: HashMap, -} - -pub fn build_assemble_plan(req: &AssembleRequest) -> Result { - if req.install_dir.is_empty() { - return Err("assemble: empty install dir".to_string()); - } - let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("assemble: {e}"))?; - for &i in &req.file_indices { - if i >= manifest.files.len() { - return Err(format!( - "assemble: file index {i} out of range ({} files)", - manifest.files.len() - )); - } - } - let refs = chunk_refs(&manifest, &req.file_indices); - let bytes_total = req - .file_indices - .iter() - .map(|&i| manifest.files[i].file_size()) - .sum(); - Ok(AssemblePlan { - install_dir: PathBuf::from(&req.install_dir), - cache_dir: resolve_cache_dir(&req.install_dir, &req.chunk_cache_dir), - scratch_cache: !req.chunk_cache_dir.is_empty(), - files: req.file_indices.clone(), - bytes_total, - refs, - manifest, - }) -} - -pub fn chunk_refs(m: &Manifest, file_indices: &[usize]) -> HashMap { - let mut refs: HashMap = HashMap::new(); - for &fi in file_indices { - let Some(f) = m.files.get(fi) else { - continue; - }; - for g in distinct_guids(f) { - *refs.entry(g).or_insert(0) += 1; - } - } - refs -} - -fn distinct_guids(f: &FileInfo) -> Vec { - let mut out: Vec = Vec::with_capacity(f.parts.len()); - for p in &f.parts { - let g = p.guid_str(); - if !out.contains(&g) { - out.push(g); - } - } - out -} - -pub fn relative_path(f: &FileInfo) -> String { - f.filename.replace('\\', "/") -} - -fn expected_sha1(f: &FileInfo) -> Option<&[u8; 20]> { - f.sha1.as_ref().filter(|h| h.iter().any(|&b| b != 0)) -} - -struct ChunkReader { - cache_dir: PathBuf, - last: Option<(String, Vec)>, -} - -impl ChunkReader { - fn read(&mut self, guid: &str, rel_path: &str) -> Result<&[u8], String> { - if self.last.as_ref().map(|(g, _)| g.as_str()) != Some(guid) { - let path = self.cache_dir.join(guid); - if !path.exists() { - return Err(format!("Missing chunk {guid} for {rel_path}")); - } - let data = fs::read(&path).map_err(|e| format!("read chunk {guid}: {e}"))?; - self.last = Some((guid.to_string(), data)); - } - Ok(self.last.as_ref().map(|(_, d)| d.as_slice()).unwrap_or(&[])) - } - - fn forget(&mut self, guid: &str) { - if self.last.as_ref().map(|(g, _)| g.as_str()) == Some(guid) { - self.last = None; - } - } -} - -fn write_file(reader: &mut ChunkReader, f: &FileInfo, out_path: &Path) -> Result { - let rel_path = relative_path(f); - if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent).map_err(|e| format!("mkdirs {}: {e}", parent.display()))?; - } - let result = (|| { - let file = File::create(out_path).map_err(|e| format!("create {rel_path}: {e}"))?; - let mut out = BufWriter::with_capacity(1024 * 1024, file); - let mut sha = expected_sha1(f).map(|_| Sha1::new()); - let mut written: u64 = 0; - for part in &f.parts { - let guid = part.guid_str(); - let data = reader.read(&guid, &rel_path)?; - let start = usize::try_from(part.offset).map_err(|_| { - format!("Chunk {guid} part offset {} invalid for {rel_path}", part.offset) - })?; - let len = part.size as u32 as usize; - let end = start - .checked_add(len) - .filter(|&e| e <= data.len()) - .ok_or_else(|| { - format!( - "Chunk {guid} too short for {rel_path}: need {start}+{len}, have {}", - data.len() - ) - })?; - let slice = &data[start..end]; - out.write_all(slice).map_err(|e| format!("write {rel_path}: {e}"))?; - if let Some(sha) = sha.as_mut() { - sha.update(slice); - } - written += len as u64; - } - out.flush().map_err(|e| format!("flush {rel_path}: {e}"))?; - drop(out); - if let (Some(expected), Some(sha)) = (expected_sha1(f), sha) { - if sha.finalize().as_slice() != &expected[..] { - return Err(format!("SHA-1 mismatch for {rel_path}")); - } - } - Ok(written) - })(); - if result.is_err() { - let _ = fs::remove_file(out_path); - } - result -} - -pub fn run_assemble( - plan: &AssemblePlan, - cancel: &AtomicBool, - progress: &(dyn Fn(u64, u64, u64, u64) + Sync), - log: &(dyn Fn(&str) + Sync), -) -> AssembleOutcome { - let files_total = plan.files.len() as u64; - let mut outcome = AssembleOutcome { - files_total, - bytes_total: plan.bytes_total, - ..AssembleOutcome::default() - }; - log(&format!( - "assemble cache_dir={} scratch={} {} files={files_total} bytes={} chunks_referenced={}", - plan.cache_dir.display(), - plan.scratch_cache, - crate::priority::log_field(), - plan.bytes_total, - plan.refs.len() - )); - - let mut refs = plan.refs.clone(); - let mut reader = ChunkReader { - cache_dir: plan.cache_dir.clone(), - last: None, - }; - for &fi in &plan.files { - if cancel.load(Ordering::Relaxed) { - outcome.cancelled = true; - outcome.error = "cancelled".to_string(); - log(&format!( - "cancelled during assembly ({}/{files_total} files)", - outcome.files_done - )); - return outcome; - } - let f = &plan.manifest.files[fi]; - let out_path = plan.install_dir.join(relative_path(f)); - match write_file(&mut reader, f, &out_path) { - Ok(written) => outcome.bytes_written += written, - Err(err) => { - outcome.error = err; - log(&format!( - "FAIL {} ({}/{files_total} files ok)", - outcome.error, outcome.files_done - )); - return outcome; - } - } - if plan.scratch_cache { - for guid in distinct_guids(f) { - let Some(count) = refs.get_mut(&guid) else { - continue; - }; - *count = count.saturating_sub(1); - if *count == 0 { - reader.forget(&guid); - if fs::remove_file(plan.cache_dir.join(&guid)).is_ok() { - outcome.chunks_removed += 1; - } - } - } - } - outcome.files_done += 1; - progress( - outcome.bytes_written, - plan.bytes_total, - outcome.files_done, - files_total, - ); - } - - outcome.success = true; - match fs::remove_dir_all(&plan.cache_dir) { - Ok(()) => log(&format!("cache removed {}", plan.cache_dir.display())), - Err(e) => log(&format!( - "cache not removed {}: {e}", - plan.cache_dir.display() - )), - } - log(&format!( - "assembled files={} bytes={} chunks_removed_early={}", - outcome.files_done, outcome.bytes_written, outcome.chunks_removed - )); - outcome -} - -#[cfg(test)] -mod tests { - use super::super::chunk::test_support::temp_dir; - use super::super::manifest::test_support::*; - use super::*; - use std::sync::Mutex; - - const A: [u32; 4] = [1, 2, 3, 4]; - const B: [u32; 4] = [5, 6, 7, 8]; - - fn chunk_a() -> Vec { - (0..300u32).map(|i| (i % 251) as u8).collect() - } - - fn chunk_b() -> Vec { - (0..100u32).map(|i| (200 - i) as u8).collect() - } - - fn sha(bytes: &[u8]) -> [u8; 20] { - let mut out = [0u8; 20]; - out.copy_from_slice(&Sha1::digest(bytes)); - out - } - - fn expected_files() -> (Vec, Vec) { - let (a, b) = (chunk_a(), chunk_b()); - let mut f0 = a[0..100].to_vec(); - f0.extend_from_slice(&b[10..60]); - (f0, a[100..300].to_vec()) - } - - fn manifest_bytes(file0_sha: [u8; 20]) -> Vec { - let (_, f1) = expected_files(); - let chunks = vec![ - TestChunk { guid: A, hash: 1, sha1: [0x11; 20], group: 1, window: 300, file_size: 300 }, - TestChunk { guid: B, hash: 2, sha1: [0x22; 20], group: 2, window: 100, file_size: 100 }, - ]; - let files = vec![ - TestFile { - name: "Game\\Binaries\\Game.exe".to_string(), - sha1: file0_sha, - tags: vec![], - parts: vec![(A, 0, 100), (B, 10, 50)], - }, - TestFile { - name: "Game/Content/data.pak".to_string(), - sha1: sha(&f1), - tags: vec![], - parts: vec![(A, 100, 200)], - }, - ]; - build_manifest(&chunks, &files, 21, false) - } - - fn guid(g: [u32; 4]) -> String { - super::super::manifest::guid_to_string(&g) - } - - fn fill_cache(cache: &Path) { - fs::create_dir_all(cache).unwrap(); - fs::write(cache.join(guid(A)), chunk_a()).unwrap(); - fs::write(cache.join(guid(B)), chunk_b()).unwrap(); - } - - fn request(install: &Path, cache: &str) -> AssembleRequest { - let (f0, _) = expected_files(); - AssembleRequest { - manifest_bytes: manifest_bytes(sha(&f0)), - install_dir: install.to_str().unwrap().to_string(), - chunk_cache_dir: cache.to_string(), - file_indices: vec![0, 1], - } - } - - fn run(plan: &AssemblePlan, cancel: bool) -> (AssembleOutcome, Vec<(u64, u64, u64, u64)>) { - let calls = Mutex::new(Vec::new()); - let progress = |b: u64, bt: u64, f: u64, ft: u64| calls.lock().unwrap().push((b, bt, f, ft)); - let log = |_: &str| {}; - let out = run_assemble(plan, &AtomicBool::new(cancel), &progress, &log); - (out, calls.into_inner().unwrap()) - } - - #[test] - fn default_cache_assembles_like_the_kotlin_loop_and_removes_the_cache_at_the_end() { - let dir = temp_dir("asm-default"); - let install = dir.join("install"); - let cache = install.join(".chunks"); - fill_cache(&cache); - let plan = build_assemble_plan(&request(&install, "")).unwrap(); - assert!(!plan.scratch_cache); - assert_eq!(plan.refs[&guid(A)], 2); - assert_eq!(plan.refs[&guid(B)], 1); - let (out, calls) = run(&plan, false); - assert!(out.success, "{}", out.error); - let (f0, f1) = expected_files(); - assert_eq!(fs::read(install.join("Game/Binaries/Game.exe")).unwrap(), f0); - assert_eq!(fs::read(install.join("Game/Content/data.pak")).unwrap(), f1); - assert_eq!(out.bytes_written, 350); - assert_eq!(calls, vec![(150, 350, 1, 2), (350, 350, 2, 2)]); - assert_eq!(out.chunks_removed, 0, "no early deletion without a scratch cache"); - assert!(!cache.exists(), "the cache goes at the end, as the Kotlin loop's deleteDir"); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn scratch_cache_frees_each_chunk_when_its_last_file_is_done() { - let dir = temp_dir("asm-scratch"); - let install = dir.join("install"); - let cache = dir.join("cache"); - fill_cache(&cache); - let plan = build_assemble_plan(&request(&install, cache.to_str().unwrap())).unwrap(); - assert!(plan.scratch_cache); - let seen = Mutex::new(Vec::new()); - let progress = |_: u64, _: u64, files_done: u64, _: u64| { - if files_done == 1 { - seen.lock().unwrap().push((cache.join(guid(A)).exists(), cache.join(guid(B)).exists())); - } - }; - let log = |_: &str| {}; - let out = run_assemble(&plan, &AtomicBool::new(false), &progress, &log); - assert!(out.success, "{}", out.error); - assert_eq!(*seen.lock().unwrap(), vec![(true, false)]); - assert_eq!(out.chunks_removed, 2); - assert!(!cache.exists()); - let (f0, f1) = expected_files(); - assert_eq!(fs::read(install.join("Game/Binaries/Game.exe")).unwrap(), f0); - assert_eq!(fs::read(install.join("Game/Content/data.pak")).unwrap(), f1); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn a_missing_chunk_fails_with_the_kotlin_wording_and_leaves_the_cache() { - let dir = temp_dir("asm-missing"); - let install = dir.join("install"); - let cache = dir.join("cache"); - fill_cache(&cache); - fs::remove_file(cache.join(guid(B))).unwrap(); - let plan = build_assemble_plan(&request(&install, cache.to_str().unwrap())).unwrap(); - let (out, calls) = run(&plan, false); - assert!(!out.success && !out.cancelled); - assert_eq!(out.error, format!("Missing chunk {} for Game/Binaries/Game.exe", guid(B))); - assert!(calls.is_empty()); - assert!(!install.join("Game/Binaries/Game.exe").exists(), "no partial file survives"); - assert!(cache.join(guid(A)).exists(), "failure leaves the cache for the next run"); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn a_sha1_mismatch_deletes_the_file_and_fails() { - let dir = temp_dir("asm-sha"); - let install = dir.join("install"); - let cache = dir.join("cache"); - fill_cache(&cache); - let mut req = request(&install, cache.to_str().unwrap()); - req.manifest_bytes = manifest_bytes([0x99; 20]); - let plan = build_assemble_plan(&req).unwrap(); - let (out, _) = run(&plan, false); - assert!(!out.success); - assert_eq!(out.error, "SHA-1 mismatch for Game/Binaries/Game.exe"); - assert!(!install.join("Game/Binaries/Game.exe").exists()); - assert!(cache.join(guid(B)).exists(), "nothing was freed for a file that failed"); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn an_all_zero_manifest_hash_means_no_check() { - let dir = temp_dir("asm-nohash"); - let install = dir.join("install"); - let cache = install.join(".chunks"); - fill_cache(&cache); - let mut req = request(&install, ""); - req.manifest_bytes = manifest_bytes([0u8; 20]); - let plan = build_assemble_plan(&req).unwrap(); - let (out, _) = run(&plan, false); - assert!(out.success, "{}", out.error); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn cancel_before_the_first_file_writes_nothing_and_keeps_the_cache() { - let dir = temp_dir("asm-cancel"); - let install = dir.join("install"); - let cache = dir.join("cache"); - fill_cache(&cache); - let plan = build_assemble_plan(&request(&install, cache.to_str().unwrap())).unwrap(); - let (out, calls) = run(&plan, true); - assert!(out.cancelled && !out.success); - assert!(calls.is_empty()); - assert!(cache.join(guid(A)).exists() && cache.join(guid(B)).exists()); - assert!(!install.join("Game").exists()); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn bad_inputs_fail_before_anything_is_written() { - let dir = temp_dir("asm-inputs"); - let mut req = request(&dir, ""); - req.file_indices = vec![0, 2]; - assert!(build_assemble_plan(&req).unwrap_err().contains("out of range")); - req.file_indices = vec![0]; - req.install_dir.clear(); - assert!(build_assemble_plan(&req).unwrap_err().contains("empty install dir")); - req.install_dir = dir.to_str().unwrap().to_string(); - req.manifest_bytes = vec![1, 2, 3]; - assert!(build_assemble_plan(&req).is_err()); - let _ = fs::remove_dir_all(&dir); - } -} diff --git a/app/src/main/rust/stores/src/epic/build_info.rs b/app/src/main/rust/stores/src/epic/build_info.rs new file mode 100644 index 0000000000..b70ccc9a74 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/build_info.rs @@ -0,0 +1,210 @@ +use std::time::Duration; + +use serde_json::{json, Value}; + +use super::net::{self, Request}; + +const LAUNCHER_HOST: &str = "https://launcher-public-service-prod06.ol.epicgames.com"; +const EXCLUDED_CDN: &str = "cloudflare.epicgamescdn.com"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cdn { + pub base: String, + pub dir: String, + pub auth: String, +} + +impl Cdn { + pub fn prefix(&self) -> String { + format!("{}{}", self.base, self.dir) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BuildInfo { + pub cdns: Vec, + pub manifest_file: String, + pub build_version: String, + pub deployment_id: String, +} + +impl BuildInfo { + pub fn to_json(&self) -> Value { + json!({ + "v": 1, + "manifest": self.manifest_file, + "buildVersion": self.build_version, + "deploymentId": self.deployment_id, + "cdns": cdns_to_json(&self.cdns), + }) + } + + pub fn from_json(v: &Value) -> Option { + Some(BuildInfo { + cdns: cdns_from_json(v.get("cdns")?), + manifest_file: net::str_of(v, "manifest").to_string(), + build_version: net::str_of(v, "buildVersion").to_string(), + deployment_id: net::str_of(v, "deploymentId").to_string(), + }) + } +} + +pub fn cdns_to_json(cdns: &[Cdn]) -> Value { + Value::Array(cdns.iter().map(|c| json!({"base": c.base, "dir": c.dir, "auth": c.auth})).collect()) +} + +pub fn cdns_from_json(v: &Value) -> Vec { + v.as_array() + .map(|a| { + a.iter() + .map(|c| Cdn { + base: net::str_of(c, "base").to_string(), + dir: net::str_of(c, "dir").to_string(), + auth: net::str_of(c, "auth").to_string(), + }) + .filter(|c| !c.base.is_empty()) + .collect() + }) + .unwrap_or_default() +} + +pub fn cdn_of(entry: &Value) -> Option { + let uri = net::str_of(entry, "uri"); + let at = uri.find("/Builds")?; + let base = &uri[..at]; + if !base.starts_with("http") || base.contains(EXCLUDED_CDN) { + return None; + } + let path = &uri[at..]; + let path = path.split('?').next().unwrap_or(path); + let dir = &path[..path.rfind('/').unwrap_or(path.len())]; + let params: Vec = entry + .get("queryParams") + .and_then(|q| q.as_array()) + .map(|a| { + a.iter() + .map(|p| format!("{}={}", net::str_of(p, "name"), net::str_of(p, "value"))) + .collect() + }) + .unwrap_or_default(); + let auth = if params.is_empty() { String::new() } else { format!("?{}", params.join("&")) }; + Some(Cdn { base: base.to_string(), dir: dir.to_string(), auth }) +} + +pub fn parse_answer(answer: &Value) -> Option { + let element = answer.get("elements")?.as_array()?.first()?; + let manifests = element.get("manifests").and_then(|m| m.as_array()); + let cdns: Vec = manifests.map(|m| m.iter().filter_map(cdn_of).collect()).unwrap_or_default(); + let manifest_file = manifests + .and_then(|m| m.first()) + .map(|m| net::str_of(m, "uri")) + .map(|u| u.split('?').next().unwrap_or(u)) + .map(|u| u.rsplit('/').next().unwrap_or("").to_string()) + .unwrap_or_default(); + let build_version = Some(net::str_of(element, "buildVersion")) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| net::str_of(element, "versionId")) + .to_string(); + let deployment_id = element + .get("sidecar") + .and_then(|s| s.get("config")) + .and_then(|c| c.as_str()) + .and_then(|c| serde_json::from_str::(c).ok()) + .map(|c| net::str_of(&c, "deploymentId").to_string()) + .unwrap_or_default(); + Some(BuildInfo { cdns, manifest_file, build_version, deployment_id }) +} + +pub fn fetch(access_token: &str, namespace: &str, catalog_item_id: &str, app_name: &str) -> Option { + let url = format!( + "{LAUNCHER_HOST}/launcher/api/public/assets/v2/platform/Windows/namespace/{}/catalogItem/{}/app/{}/label/Live", + net::percent_encode(namespace), + net::percent_encode(catalog_item_id), + net::percent_encode(app_name) + ); + let reply = net::send(Request::get(&url).bearer(access_token)).ok()?; + if !reply.ok() { + return None; + } + parse_answer(&reply.json()?) +} + +pub fn chunk_prefixes(cdns: &[Cdn]) -> Vec { + let mut out: Vec = Vec::new(); + for c in cdns { + let p = c.prefix().replace("\\/", "/").replace('\\', ""); + let p = p.trim_end_matches('/').to_string(); + if !p.is_empty() && !out.contains(&p) { + out.push(p); + } + } + out +} + +pub fn download_manifest(info: &BuildInfo, cdns: &[Cdn], log: &dyn Fn(&str)) -> Option> { + if info.manifest_file.is_empty() { + return None; + } + for cdn in cdns { + let url = format!("{}{}/{}{}", cdn.base, cdn.dir, info.manifest_file, cdn.auth); + match net::send(Request::get(&url).read_timeout(Duration::from_secs(120))) { + Ok(reply) if reply.status == 200 && reply.body.len() > 4 => return Some(reply.body), + Ok(reply) => log(&format!("manifest from {}: HTTP {}", cdn.base, reply.status)), + Err(e) => log(&format!("manifest from {}: {e}", cdn.base)), + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn answer() -> Value { + json!({"elements": [{ + "appName": "Fig", "buildVersion": "1.0.5", + "manifests": [ + {"uri": "https://download.epicgames.com/Builds/Org/o-x/abc/default/HASH.manifest?ignored=1", + "queryParams": [{"name": "f_token", "value": "tok=="}, {"name": "x", "value": "1"}]}, + {"uri": "https://cloudflare.epicgamescdn.com/Builds/Org/o-x/abc/default/HASH.manifest"}, + {"uri": "https://fastly-download.epicgames.com/Builds/Org/o-x/abc/default/HASH.manifest"}, + {"uri": "no-builds-here"} + ], + "sidecar": {"config": "{\"deploymentId\":\"dep42\"}"} + }]}) + } + + #[test] + fn cdn_list_and_manifest_name() { + let info = parse_answer(&answer()).unwrap(); + assert_eq!(info.manifest_file, "HASH.manifest"); + assert_eq!(info.build_version, "1.0.5"); + assert_eq!(info.deployment_id, "dep42"); + assert_eq!(info.cdns.len(), 2); + assert_eq!(info.cdns[0].base, "https://download.epicgames.com"); + assert_eq!(info.cdns[0].dir, "/Builds/Org/o-x/abc/default"); + assert_eq!(info.cdns[0].auth, "?f_token=tok==&x=1"); + assert_eq!(info.cdns[1].auth, ""); + let back = BuildInfo::from_json(&info.to_json()).unwrap(); + assert_eq!(back, info); + } + + #[test] + fn missing_elements_is_no_answer() { + assert!(parse_answer(&json!({"elements": []})).is_none()); + assert!(parse_answer(&json!({})).is_none()); + let v = parse_answer(&json!({"elements": [{"versionId": "v9"}]})).unwrap(); + assert_eq!(v.build_version, "v9"); + assert!(v.cdns.is_empty()); + } + + #[test] + fn chunk_prefixes_are_normalised() { + let cdns = vec![ + Cdn { base: "https://a.example".into(), dir: "\\/Builds\\/x/".into(), auth: String::new() }, + Cdn { base: "https://a.example".into(), dir: "/Builds/x".into(), auth: "?s=1".into() }, + Cdn { base: "https://b.example".into(), dir: "/Builds/x".into(), auth: String::new() }, + ]; + assert_eq!(chunk_prefixes(&cdns), vec!["https://a.example/Builds/x", "https://b.example/Builds/x"]); + } +} diff --git a/app/src/main/rust/stores/src/epic/catalog.rs b/app/src/main/rust/stores/src/epic/catalog.rs new file mode 100644 index 0000000000..4589ce276b --- /dev/null +++ b/app/src/main/rust/stores/src/epic/catalog.rs @@ -0,0 +1,128 @@ +use serde_json::{json, Value}; + +use super::net::{self, Request}; + +const CATALOG_HOST: &str = "https://catalog-public-service-prod06.ol.epicgames.com"; + +fn items_url(namespace: &str, item_id: &str, with_relations: bool) -> String { + let mut url = format!( + "{CATALOG_HOST}/catalog/api/shared/namespace/{}/bulk/items?id={}", + net::percent_encode(namespace), + net::percent_encode(item_id) + ); + if with_relations { + url.push_str("&includeDLCDetails=true&includeMainGameDetails=true"); + } + url.push_str("&country=US"); + url +} + +pub fn pick_entry<'v>(answer: &'v Value, item_id: &str) -> Option<&'v Value> { + let map = answer.as_object()?; + map.get(item_id).or_else(|| map.values().next()) +} + +fn fetch(access_token: &str, namespace: &str, item_id: &str, relations: bool) -> Result, ()> { + let url = items_url(namespace, item_id, relations); + let reply = net::send(Request::get(&url).bearer(access_token)).map_err(|_| ())?; + if !reply.ok() { + return Err(()); + } + let answer = reply.json().ok_or(())?; + Ok(pick_entry(&answer, item_id).cloned()) +} + +fn image(entry: &Value, types: &[&str]) -> String { + let Some(list) = entry.get("keyImages").and_then(|v| v.as_array()) else { + return String::new(); + }; + list.iter() + .find(|img| types.contains(&net::str_of(img, "type")) && !net::str_of(img, "url").is_empty()) + .map(|img| net::str_of(img, "url").to_string()) + .unwrap_or_default() +} + +fn attribute_of(entry: &Value, name: &str) -> Option { + entry + .get("customAttributes") + .and_then(|a| a.get(name)) + .and_then(|a| a.get("value")) + .and_then(|v| v.as_str()) + .map(str::to_string) +} + +pub fn details_of(entry: &Value) -> Value { + let publisher = attribute_of(entry, "publisherName") + .map(|p| p.trim().to_string()) + .filter(|p| !p.is_empty()) + .unwrap_or_else(|| { + entry.get("seller").map(|s| net::str_of(s, "name").trim().to_string()).unwrap_or_default() + }); + let release = Some(net::str_of(entry, "viewableDate")) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| net::str_of(entry, "effectiveDate")); + let offline = attribute_of(entry, "CanRunOffline").map(|v| !v.eq_ignore_ascii_case("false")).unwrap_or(true); + let main_game = entry.get("mainGameItem"); + json!({ + "title": net::str_of(entry, "title"), + "developer": net::str_of(entry, "developer"), + "description": net::str_of(entry, "description"), + "publisher": publisher, + "releaseDate": release, + "tall": image(entry, &["DieselGameBoxTall"]), + "wide": image(entry, &["DieselGameBox", "Thumbnail"]), + "canRunOffline": offline, + "dlc": main_game.is_some_and(|m| !m.is_null()), + "mainGameId": main_game.map(|m| net::str_of(m, "id")).unwrap_or(""), + }) +} + +pub fn details(access_token: &str, namespace: &str, item_id: &str) -> Option { + fetch(access_token, namespace, item_id, true).ok().flatten().map(|e| details_of(&e)) +} + +pub fn attribute(access_token: &str, namespace: &str, item_id: &str, name: &str) -> Option { + match fetch(access_token, namespace, item_id, false) { + Err(()) => None, + Ok(None) => Some(String::new()), + Ok(Some(entry)) => Some(attribute_of(&entry, name).unwrap_or_default()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn details_from_an_entry() { + let answer = json!({ + "other": {"title": "Wrong"}, + "cat1": { + "title": "Game", "developer": "Dev", "description": "Text", + "keyImages": [ + {"type": "Thumbnail", "url": "thumb"}, + {"type": "DieselGameBox", "url": ""}, + {"type": "DieselGameBoxTall", "url": "tall"} + ], + "customAttributes": {"CanRunOffline": {"value": "FALSE"}, "publisherName": {"value": " Pub "}}, + "seller": {"name": "Seller"}, + "effectiveDate": "2020-01-01T00:00:00.000Z" + } + }); + let d = details_of(pick_entry(&answer, "cat1").unwrap()); + assert_eq!(d["title"], "Game"); + assert_eq!(d["tall"], "tall"); + assert_eq!(d["wide"], "thumb"); + assert_eq!(d["publisher"], "Pub"); + assert_eq!(d["canRunOffline"], false); + assert_eq!(d["releaseDate"], "2020-01-01T00:00:00.000Z"); + assert_eq!(d["dlc"], false); + let dlc = details_of(&json!({"mainGameItem": {"id": "base"}, "seller": {"name": "S"}, "viewableDate": "v"})); + assert_eq!(dlc["dlc"], true); + assert_eq!(dlc["mainGameId"], "base"); + assert_eq!(dlc["publisher"], "S"); + assert_eq!(dlc["releaseDate"], "v"); + assert_eq!(dlc["canRunOffline"], true); + assert_eq!(pick_entry(&json!({"x": {"title": "Only"}}), "y").unwrap()["title"], "Only"); + } +} diff --git a/app/src/main/rust/stores/src/epic/chunk.rs b/app/src/main/rust/stores/src/epic/chunk.rs index e07c26c8ca..ea09ccd60b 100644 --- a/app/src/main/rust/stores/src/epic/chunk.rs +++ b/app/src/main/rust/stores/src/epic/chunk.rs @@ -1,478 +1,156 @@ - -use std::fs::{self, File}; -use std::io::{self, BufWriter, Write}; -use std::path::{Path, PathBuf}; - +use flate2::{Decompress, FlushDecompress, Status}; use sha1::{Digest, Sha1}; +use super::manifest::ChunkInfo; + pub const CHUNK_MAGIC: u32 = 0xB1FE_3AA2; +const MIN_HEADER: usize = 41; -pub const CHUNK_HEADER_PREFIX: usize = 41; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ChunkHeader { - pub header_version: i32, - pub header_size: i32, - pub compressed_size: i32, - pub stored_as: u8, +fn le_u32(b: &[u8], at: usize) -> u32 { + u32::from_le_bytes([b[at], b[at + 1], b[at + 2], b[at + 3]]) } -impl ChunkHeader { - pub fn is_compressed(&self) -> bool { - (self.stored_as & 1) != 0 +pub fn decode(body: &[u8], info: &ChunkInfo) -> Result, String> { + if body.len() < MIN_HEADER { + return Err(format!("chunk body of {} bytes", body.len())); } - - pub fn payload_start(&self) -> usize { - if self.header_size > CHUNK_HEADER_PREFIX as i32 { - self.header_size as usize - } else { - CHUNK_HEADER_PREFIX - } + if le_u32(body, 0) != CHUNK_MAGIC { + return Err("not a chunk file".into()); } -} - -pub fn parse_chunk_header(body: &[u8]) -> Result { - if body.len() < CHUNK_HEADER_PREFIX { - return Err(format!( - "Stream ended after {}/{} bytes", - body.len(), - CHUNK_HEADER_PREFIX - )); + let header = (le_u32(body, 8) as i32).max(MIN_HEADER as i32) as usize; + let stored = (le_u32(body, 12) as i32).max(0) as usize; + let flags = body[40]; + if header > body.len() { + return Err("chunk header runs past the body".into()); } - let le = |o: usize| i32::from_le_bytes([body[o], body[o + 1], body[o + 2], body[o + 3]]); - let magic = le(0) as u32; - if magic != CHUNK_MAGIC { - return Err(format!("Bad chunk magic (streaming): 0x{magic:x}")); - } - Ok(ChunkHeader { - header_version: le(4), - header_size: le(8), - compressed_size: le(12), - stored_as: body[40], - }) -} - -pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8], String> { - let start = hdr.payload_start(); - if start > body.len() { - return Err("Stream ended during skip".to_string()); - } - let want = if hdr.compressed_size <= 0 { - 0 + let limit = info.window as usize; + let data = if flags & 1 != 0 { + let end = header.saturating_add(stored).min(body.len()); + inflate(&body[header..end], limit)? } else { - hdr.compressed_size as usize + let end = header + .checked_add(stored) + .filter(|&e| e <= body.len()) + .ok_or("stored chunk payload is short")?; + body[header..end].to_vec() }; - let avail = body.len() - start; - let take = want.min(avail); - Ok(&body[start..start + take]) + if data.len() > limit { + return Err(format!("chunk decodes to {} bytes, more than {limit}", data.len())); + } + if info.window_exact && data.len() != limit { + return Err(format!("chunk decodes to {} bytes, manifest says {limit}", data.len())); + } + if let Some(expected) = info.sha1 { + let got: [u8; 20] = Sha1::digest(&data).into(); + if got != expected { + return Err("chunk SHA-1 mismatch".into()); + } + } + Ok(data) } -struct HashingWriter { - inner: W, - sha: Option, - written: u64, - limit: u64, -} - -impl Write for HashingWriter { - fn write(&mut self, buf: &[u8]) -> io::Result { - if self.written.saturating_add(buf.len() as u64) > self.limit { - return Err(io::Error::other("chunk exceeds reserved decoded window")); +fn inflate(input: &[u8], limit: usize) -> Result, String> { + let mut z = Decompress::new(true); + let mut out = Vec::with_capacity(limit + 1); + loop { + let (in_before, out_before) = (z.total_in(), out.len()); + let consumed = in_before as usize; + let status = z + .decompress_vec(&input[consumed..], &mut out, FlushDecompress::Finish) + .map_err(|e| format!("chunk does not inflate: {e}"))?; + if status == Status::StreamEnd { + return Ok(out); } - let n = self.inner.write(buf)?; - if let Some(sha) = self.sha.as_mut() { - sha.update(&buf[..n]); + if out.len() > limit { + return Err(format!("chunk inflates past {limit} bytes")); } - self.written += n as u64; - Ok(n) - } - - fn flush(&mut self) -> io::Result<()> { - self.inner.flush() - } -} - -pub fn part_path(final_path: &Path) -> PathBuf { - let mut s = final_path.as_os_str().to_os_string(); - s.push(".part"); - PathBuf::from(s) -} - -pub fn write_verified_chunk( - body: &[u8], - expected_sha1: Option<&[u8; 20]>, - final_path: &Path, -) -> Result { - let tmp = part_path(final_path); - let _ = fs::remove_file(&tmp); - - let result = write_part(body, expected_sha1, &tmp, final_path, u64::MAX, false); - if result.is_err() { - let _ = fs::remove_file(&tmp); - } - - result -} - -pub fn write_verified_chunk_bounded( - body: &[u8], expected_sha1: Option<&[u8; 20]>, final_path: &Path, - decoded_limit: u64, exact_size: bool, -) -> Result { - let tmp = part_path(final_path); - let result = write_part(body, expected_sha1, &tmp, final_path, decoded_limit, exact_size); - if result.is_err() { let _ = fs::remove_file(tmp); } - result -} - -fn write_part( - body: &[u8], - expected_sha1: Option<&[u8; 20]>, - tmp: &Path, - final_path: &Path, - limit: u64, - exact_size: bool, -) -> Result { - let hdr = parse_chunk_header(body)?; - let payload = chunk_payload(body, &hdr)?; - if limit != u64::MAX && (hdr.compressed_size < 0 || payload.len() != hdr.compressed_size as usize) { - return Err("truncated chunk body".into()); - } - - let file = File::create(tmp).map_err(|e| format!("open {}: {e}", tmp.display()))?; - let mut out = HashingWriter { - inner: BufWriter::with_capacity(128 * 1024, file), - sha: expected_sha1.map(|_| Sha1::new()), - written: 0, - limit, - }; - - if hdr.is_compressed() { - if limit == u64::MAX { - let mut dec = flate2::read::ZlibDecoder::new(payload); - io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?; - } else { - let mut dec = flate2::Decompress::new(true); - let mut buffer = [0u8; 128 * 1024]; - loop { - let input = dec.total_in() as usize; - let before = dec.total_out(); - let status = dec.decompress(&payload[input..], &mut buffer, flate2::FlushDecompress::None) - .map_err(|e| format!("inflate: {e}"))?; - let count = (dec.total_out() - before) as usize; - out.write_all(&buffer[..count]).map_err(|e| format!("inflate write: {e}"))?; - if status == flate2::Status::StreamEnd { break; } - if count == 0 && dec.total_in() as usize == input { - return Err("truncated compressed Epic chunk".into()); - } - } + if out.len() == out.capacity() { + out.reserve(64 * 1024); + continue; } - } else { - out.write_all(payload).map_err(|e| format!("write: {e}"))?; - } - out.flush().map_err(|e| format!("flush: {e}"))?; - let written = out.written; - let digest = out.sha.take().map(|s| s.finalize()); - drop(out); - if exact_size && written != limit { - return Err("chunk decoded size mismatch".into()); - } - - if let (Some(expected), Some(actual)) = (expected_sha1, digest) { - if actual.as_slice() != &expected[..] { - return Err("Chunk SHA-1 mismatch (streaming)".to_string()); + if z.total_in() == in_before && out.len() == out_before { + return Err("chunk zlib stream is truncated".into()); } } - - fs::rename(tmp, final_path).map_err(|e| format!("Chunk rename failed: {e}"))?; - Ok(written) -} - -/// The window a chunk decompresses to: the manifest's, or 1 MiB for a legacy JSON manifest that -/// names none (`EpicDownloadManager.chunkWindowSize`). The streaming installer also plans its -/// shared-chunk cache with this size. -pub fn chunk_window_bytes(window_size: i32) -> u64 { - if window_size > 0 { - window_size as u64 - } else { - 1024 * 1024 - } -} - -/// One downloaded body → the verified, decompressed chunk in memory, for the streaming installer -/// (`stream.rs`), which writes most chunks straight into the files that use them and never puts -/// them in a cache. The checks are the bounded Java writer's (`downloadChunkStreaming`), which is -/// stricter than [`write_verified_chunk`]: a stored payload shorter than `compressedSize` is -/// truncated, a zlib stream must reach its end, the output may not grow past the chunk's window -/// and must fill it exactly when the manifest names one, and the SHA-1 must match when the -/// manifest has one. `Err` = "try another CDN", exactly like a cache write that failed. -pub fn decode_verified_chunk( - body: &[u8], - window_size: i32, - expected_sha1: Option<&[u8; 20]>, -) -> Result, String> { - let hdr = parse_chunk_header(body)?; - let start = hdr.payload_start(); - if start > body.len() { - return Err("Stream ended during skip".to_string()); - } - let want = if hdr.compressed_size <= 0 { - 0 - } else { - hdr.compressed_size as usize - }; - let avail = body.len() - start; - let cap = chunk_window_bytes(window_size) as usize; - let out = if hdr.is_compressed() { - // Java stops reading once the inflater finishes, so a body may carry less than - // `compressedSize` as long as the stream itself is complete. - let payload = &body[start..start + want.min(avail)]; - let mut out: Vec = Vec::with_capacity(cap + 1); - let mut inflater = flate2::Decompress::new(true); - let status = inflater - .decompress_vec(payload, &mut out, flate2::FlushDecompress::Finish) - .map_err(|e| format!("inflate: {e}"))?; - if out.len() > cap { - return Err("chunk exceeds expected window size".to_string()); - } - if status != flate2::Status::StreamEnd { - return Err("truncated compressed Epic chunk".to_string()); - } - out - } else { - if avail < want { - return Err("truncated Epic chunk".to_string()); - } - if want > cap { - return Err("chunk exceeds expected window size".to_string()); - } - body[start..start + want].to_vec() - }; - if window_size > 0 && out.len() != window_size as usize { - return Err(format!( - "chunk is {} bytes, window is {window_size}", - out.len() - )); - } - if let Some(expected) = expected_sha1 { - if Sha1::digest(&out).as_slice() != &expected[..] { - return Err("Chunk SHA-1 mismatch (streaming)".to_string()); - } - } - Ok(out) } #[cfg(test)] -pub(crate) mod test_support { - use super::*; +pub mod testing { + use super::CHUNK_MAGIC; + use std::io::Write; - pub fn build_chunk_body( - decompressed: &[u8], - compress: bool, - extra_header: usize, - declared_compressed_size: Option, - ) -> Vec { + pub fn encode(data: &[u8], compress: bool) -> Vec { let payload = if compress { - let mut enc = - flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); - enc.write_all(decompressed).unwrap(); + let mut enc = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + enc.write_all(data).unwrap(); enc.finish().unwrap() } else { - decompressed.to_vec() + data.to_vec() }; - let header_size = (CHUNK_HEADER_PREFIX + extra_header) as i32; let mut out = Vec::new(); out.extend_from_slice(&CHUNK_MAGIC.to_le_bytes()); out.extend_from_slice(&3i32.to_le_bytes()); - out.extend_from_slice(&header_size.to_le_bytes()); - out.extend_from_slice( - &declared_compressed_size - .unwrap_or(payload.len() as i32) - .to_le_bytes(), - ); - out.extend_from_slice(&[0xAB; 16]); - out.extend_from_slice(&[0xCD; 8]); + out.extend_from_slice(&66i32.to_le_bytes()); + out.extend_from_slice(&(payload.len() as i32).to_le_bytes()); + out.extend_from_slice(&[0u8; 16]); + out.extend_from_slice(&0u64.to_le_bytes()); out.push(if compress { 1 } else { 0 }); - out.extend(std::iter::repeat(0x5A).take(extra_header)); - assert_eq!(out.len(), header_size as usize); + out.extend_from_slice(&[0u8; 25]); out.extend_from_slice(&payload); out } - - pub fn sha1_of(data: &[u8]) -> [u8; 20] { - let mut h = Sha1::new(); - h.update(data); - let d = h.finalize(); - let mut out = [0u8; 20]; - out.copy_from_slice(&d); - out - } - - pub fn temp_dir(tag: &str) -> PathBuf { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let dir = std::env::temp_dir().join(format!( - "epic-{tag}-{}-{nanos}", - std::process::id() - )); - fs::create_dir_all(&dir).unwrap(); - dir - } } #[cfg(test)] mod tests { - use super::test_support::*; + use super::testing::encode; use super::*; + use crate::epic::manifest::testing::chunk; - fn sample_data() -> Vec { - (0..200_000u32).map(|i| (i % 251) as u8).collect() - } - - #[test] - fn header_parse_matches_java_fields() { - let body = build_chunk_body(b"hello", true, 5, None); - let h = parse_chunk_header(&body).unwrap(); - assert_eq!(h.header_version, 3); - assert_eq!(h.header_size, 46); - assert!(h.is_compressed()); - assert_eq!(h.payload_start(), 46); - let short = ChunkHeader { - header_version: 0, - header_size: 20, - compressed_size: 1, - stored_as: 0, - }; - assert_eq!(short.payload_start(), 41, "headerSize < 41 → payload still starts at 41"); - } - - #[test] - fn short_or_wrong_magic_bodies_are_rejected() { - assert!(parse_chunk_header(&[0u8; 40]).is_err()); - let mut body = build_chunk_body(b"x", false, 0, None); - body[0] ^= 0xFF; - assert!(parse_chunk_header(&body).unwrap_err().contains("Bad chunk magic")); - } - - #[test] - fn payload_is_truncated_to_the_body_not_an_error() { - let body = build_chunk_body(b"abcdef", false, 0, Some(100)); - let h = parse_chunk_header(&body).unwrap(); - assert_eq!(chunk_payload(&body, &h).unwrap(), b"abcdef"); - let neg = build_chunk_body(b"abcdef", false, 0, Some(-1)); - let h = parse_chunk_header(&neg).unwrap(); - assert_eq!(chunk_payload(&neg, &h).unwrap(), b""); - let mut far = build_chunk_body(b"abcdef", false, 0, None); - far[8..12].copy_from_slice(&5000i32.to_le_bytes()); - let h = parse_chunk_header(&far).unwrap(); - assert!(chunk_payload(&far, &h).is_err()); - } - - #[test] - fn compressed_chunk_is_inflated_verified_and_renamed() { - let dir = temp_dir("ok"); - let data = sample_data(); - let body = build_chunk_body(&data, true, 3, None); - let final_path = dir.join("00000001000000020000000300000004"); - let n = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap(); - assert_eq!(n, data.len() as u64); - assert_eq!(fs::read(&final_path).unwrap(), data); - assert!(!part_path(&final_path).exists(), ".part renamed away"); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn stored_chunk_is_copied_as_is() { - let dir = temp_dir("stored"); - let data = b"stored-as-is payload".to_vec(); - let body = build_chunk_body(&data, false, 0, None); - let final_path = dir.join("chunk"); - let n = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap(); - assert_eq!(n, data.len() as u64); - assert_eq!(fs::read(&final_path).unwrap(), data); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn sha1_mismatch_leaves_nothing_behind() { - let dir = temp_dir("sha"); - let data = sample_data(); - let body = build_chunk_body(&data, true, 0, None); - let final_path = dir.join("chunk"); - let err = write_verified_chunk(&body, Some(&[0x42; 20]), &final_path).unwrap_err(); - assert!(err.contains("SHA-1 mismatch"), "{err}"); - assert!(!final_path.exists()); - assert!(!part_path(&final_path).exists()); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn unverifiable_chunk_is_accepted_without_a_hash() { - let dir = temp_dir("nosha"); - let data = sample_data(); - let body = build_chunk_body(&data, true, 0, None); - let final_path = dir.join("chunk"); - assert_eq!( - write_verified_chunk(&body, None, &final_path).unwrap(), - data.len() as u64 - ); - assert_eq!(fs::read(&final_path).unwrap(), data); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn corrupt_zlib_fails_and_truncated_zlib_fails_the_hash() { - let dir = temp_dir("zlib"); - let data = sample_data(); - let mut body = build_chunk_body(&data, true, 0, None); - let final_path = dir.join("chunk"); - let cut = body.len() - 1000; - let err = write_verified_chunk(&body[..cut], Some(&sha1_of(&data)), &final_path).unwrap_err(); - assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}"); - assert!(!final_path.exists()); - for b in body.iter_mut().skip(60).take(64) { - *b ^= 0x55; + fn info_for(data: &[u8], with_hash: bool) -> ChunkInfo { + let mut info = chunk(1, data.len() as u32); + if with_hash { + info.sha1 = Some(Sha1::digest(data).into()); } - let err = write_verified_chunk(&body, Some(&sha1_of(&data)), &final_path).unwrap_err(); - assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}"); - assert!(!final_path.exists()); - assert!(!part_path(&final_path).exists()); - let _ = fs::remove_dir_all(&dir); + info } #[test] - fn decode_in_memory_applies_the_bounded_writer_checks() { - let data = sample_data(); - let window = data.len() as i32; - let sha = sha1_of(&data); - let zipped = build_chunk_body(&data, true, 2, None); - assert_eq!(decode_verified_chunk(&zipped, window, Some(&sha)).unwrap(), data); - let stored = build_chunk_body(&data, false, 0, None); - assert_eq!(decode_verified_chunk(&stored, window, Some(&sha)).unwrap(), data); - // No window and no hash (legacy JSON manifest): accepted up to 1 MiB. - assert_eq!(decode_verified_chunk(&zipped, 0, None).unwrap(), data); - // Window mismatch, SHA-1 mismatch, truncation, oversize. - assert!(decode_verified_chunk(&zipped, window + 1, None).is_err()); - let err = decode_verified_chunk(&zipped, window, Some(&[7; 20])).unwrap_err(); - assert!(err.contains("SHA-1 mismatch"), "{err}"); - let cut = zipped.len() - 500; - assert!(decode_verified_chunk(&zipped[..cut], window, None).is_err()); - assert!(decode_verified_chunk(&stored[..stored.len() - 1], 0, None) - .unwrap_err() - .contains("truncated")); - assert!(decode_verified_chunk(&zipped, window - 1, None).is_err()); + fn compressed_and_stored_chunks_decode() { + let data: Vec = (0..5000u32).map(|i| (i % 251) as u8).collect(); + for compress in [true, false] { + assert_eq!(decode(&encode(&data, compress), &info_for(&data, true)).unwrap(), data); + } } #[test] - fn stale_part_file_is_cleared_before_the_attempt() { - let dir = temp_dir("stale"); - let final_path = dir.join("chunk"); - fs::write(part_path(&final_path), b"stale").unwrap(); - let body = [0u8; 10]; - assert!(write_verified_chunk(&body, None, &final_path).is_err()); - assert!(!part_path(&final_path).exists()); - assert!(!final_path.exists()); - let _ = fs::remove_dir_all(&dir); + fn bad_downloads_are_refused() { + let data = vec![7u8; 4096]; + let info = info_for(&data, true); + let good = encode(&data, true); + assert!(decode(&good[..good.len() - 3], &info).is_err()); + let mut wrong_magic = good.clone(); + wrong_magic[0] ^= 1; + assert!(decode(&wrong_magic, &info).is_err()); + let mut other = info.clone(); + other.sha1 = Some([9; 20]); + assert!(decode(&good, &other).is_err()); + let mut longer = info.clone(); + longer.window = 4097; + assert!(decode(&good, &longer).is_err()); + let stored = encode(&data, false); + assert!(decode(&stored[..stored.len() - 1], &info).is_err()); + assert!(decode(&good[..40], &info).is_err()); + } + + #[test] + fn an_inexact_window_is_an_upper_bound() { + let data = vec![1u8; 100]; + let mut info = info_for(&data, false); + info.window = 1024; + info.window_exact = false; + assert_eq!(decode(&encode(&data, true), &info).unwrap().len(), 100); + info.window = 50; + assert!(decode(&encode(&data, true), &info).is_err()); } } diff --git a/app/src/main/rust/stores/src/epic/driver.rs b/app/src/main/rust/stores/src/epic/driver.rs deleted file mode 100644 index f72118b874..0000000000 --- a/app/src/main/rust/stores/src/epic/driver.rs +++ /dev/null @@ -1,435 +0,0 @@ - -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; - -use super::manifest::{parse_manifest, Manifest}; -use super::plan::{ - cached_chunk_path, chunk_url, distinct_prefixes, per_host_cap, resolve_cache_dir, - total_credit_bytes, unique_chunks_for_files, -}; - -pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60); - -pub struct EpicRequest { - pub manifest_bytes: Vec, - pub install_dir: String, - pub chunk_cache_dir: String, - pub cdn_prefixes: Vec, - pub pending_file_indices: Vec, - pub expected_chunks: Option, - pub expected_bytes: Option, - pub ca_bundle_path: String, - pub max_workers: usize, - pub process_workers: usize, - pub label: String, -} - -#[derive(Debug)] -pub struct EpicPlan { - pub manifest: Manifest, - pub cache_dir: PathBuf, - pub scratch_cache: bool, - pub needed: Vec, - pub total_bytes: u64, - pub hosts: Vec, -} - -#[derive(Clone, Debug, Default)] -pub struct EpicOutcome { - pub success: bool, - pub cancelled: bool, - pub error: String, - pub bytes_credited: u64, - pub chunks_done: u64, - pub chunks_total: u64, - pub bytes_total: u64, - pub decompressed_written: u64, -} - -pub fn build_plan(req: &EpicRequest) -> Result { - let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("plan: {e}"))?; - for &i in &req.pending_file_indices { - if i >= manifest.files.len() { - return Err(format!( - "plan: pending file index {i} out of range ({} files)", - manifest.files.len() - )); - } - } - let needed = unique_chunks_for_files(&manifest, &req.pending_file_indices); - let total_bytes = total_credit_bytes(&manifest, &needed); - if let Some(expected) = req.expected_chunks { - if expected != needed.len() as u64 { - return Err(format!( - "plan: chunk count mismatch java={expected} rust={}", - needed.len() - )); - } - } - if let Some(expected) = req.expected_bytes { - if expected != total_bytes { - return Err(format!( - "plan: byte total mismatch java={expected} rust={total_bytes}" - )); - } - } - let hosts = distinct_prefixes(&req.cdn_prefixes); - if hosts.is_empty() { - return Err("plan: no CDN prefixes".to_string()); - } - let cache_dir = resolve_cache_dir(&req.install_dir, &req.chunk_cache_dir); - std::fs::create_dir_all(&cache_dir) - .map_err(|e| format!("plan: mkdirs {}: {e}", cache_dir.display()))?; - Ok(EpicPlan { - manifest, - cache_dir, - scratch_cache: !req.chunk_cache_dir.is_empty(), - needed, - total_bytes, - hosts, - }) -} - -struct ChunkCacheSink<'a> { - plan: &'a EpicPlan, - fetch_chunks: &'a [usize], - decompressed: AtomicU64, -} - -impl<'a> FetchSink for ChunkCacheSink<'a> { - fn process(&self, item: &FetchItem, body: Vec) -> Result { - let Some(&ci) = self.fetch_chunks.get(item.id as usize) else { - return Err(SinkError::Fatal(format!("item id {} out of range", item.id))); - }; - let chunk = &self.plan.manifest.unique_chunks[ci]; - let final_path = cached_chunk_path(&self.plan.cache_dir, chunk); - match super::chunk::write_verified_chunk(&body, chunk.verifiable_sha1(), &final_path) { - Ok(written) => { - self.decompressed.fetch_add(written, Ordering::Relaxed); - Ok(chunk.credit_bytes()) - } - Err(reason) => Err(SinkError::Retry(format!("{} {reason}", chunk.guid_str()))), - } - } -} - -struct SpeedMeter { - start: Instant, - last_at: Instant, - last_bytes: u64, - peak_bps: f64, -} - -impl SpeedMeter { - fn new() -> Self { - let now = Instant::now(); - Self { - start: now, - last_at: now, - last_bytes: 0, - peak_bps: 0.0, - } - } - - fn sample(&mut self, bytes_now: u64) { - let now = Instant::now(); - let dt = now.duration_since(self.last_at); - if dt >= Duration::from_millis(500) { - let delta = bytes_now.saturating_sub(self.last_bytes) as f64; - let bps = delta / dt.as_secs_f64(); - if bps > self.peak_bps { - self.peak_bps = bps; - } - self.last_at = now; - self.last_bytes = bytes_now; - } - } - - fn summary(&self, bytes: u64, decompressed: u64, skipped: u64) -> String { - let secs = self.start.elapsed().as_secs_f64().max(0.001); - let avg_bps = bytes as f64 / secs; - format!( - "summary bytes={bytes} decompressed={decompressed} skipped_chunks={skipped} elapsed={secs:.1}s avg_mbps={:.2} peak_mbps={:.2} avg_MBps={:.2}", - avg_bps * 8.0 / 1_000_000.0, - self.peak_bps * 8.0 / 1_000_000.0, - avg_bps / (1024.0 * 1024.0) - ) - } -} - -pub fn run_plan( - plan: &EpicPlan, - req: &EpicRequest, - cancel: &AtomicBool, - progress: &(dyn Fn(u64, u64) + Sync), - log: &(dyn Fn(&str) + Sync), -) -> EpicOutcome { - let chunks_total = plan.needed.len() as u64; - let mut outcome = EpicOutcome { - chunks_total, - bytes_total: plan.total_bytes, - ..EpicOutcome::default() - }; - let host_cap = per_host_cap(req.max_workers, plan.hosts.len()); - log(&format!( - "plan chunk_dir={} cache_dir={} scratch={} {} files_pending={} chunks={} bytes={} hosts={} workers={} per_host_cap={host_cap} process_workers={}", - plan.manifest.chunk_dir, - plan.cache_dir.display(), - plan.scratch_cache, - crate::priority::log_field(), - req.pending_file_indices.len(), - chunks_total, - plan.total_bytes, - plan.hosts.len(), - req.max_workers, - req.process_workers - )); - - let mut fetch_chunks: Vec = Vec::with_capacity(plan.needed.len()); - let mut pre_bytes: u64 = 0; - let mut pre_chunks: u64 = 0; - for &ci in &plan.needed { - if cancel.load(Ordering::Relaxed) { - outcome.cancelled = true; - outcome.error = "cancelled".to_string(); - outcome.bytes_credited = pre_bytes; - outcome.chunks_done = pre_chunks; - log(&format!( - "cancelled before fetch ({pre_chunks}/{chunks_total} chunks)" - )); - return outcome; - } - let chunk = &plan.manifest.unique_chunks[ci]; - if cached_chunk_path(&plan.cache_dir, chunk).exists() { - pre_bytes += chunk.credit_bytes(); - pre_chunks += 1; - progress(pre_bytes, pre_chunks); - } else { - fetch_chunks.push(ci); - } - } - for (i, h) in plan.hosts.iter().enumerate() { - log(&format!("host[{i}]={h}")); - } - log(&format!( - "skip cached={pre_chunks} bytes={pre_bytes} to_fetch={}", - fetch_chunks.len() - )); - - if fetch_chunks.is_empty() { - outcome.success = true; - outcome.bytes_credited = pre_bytes; - outcome.chunks_done = pre_chunks; - log("summary bytes=0 decompressed=0 elapsed=0.0s avg_mbps=0.00 peak_mbps=0.00 (nothing to fetch)"); - return outcome; - } - - let items: Vec = fetch_chunks - .iter() - .enumerate() - .map(|(id, &ci)| { - let chunk = &plan.manifest.unique_chunks[ci]; - FetchItem { - id: id as u64, - urls: plan - .hosts - .iter() - .map(|h| chunk_url(h, &plan.manifest.chunk_dir, chunk)) - .collect(), - reserve: if chunk.file_size > 0 { - chunk.file_size as u64 - } else { - 0 - }, - range: None, - } - }) - .collect(); - - let opts = FetchOptions { - max_workers: req.max_workers.max(1), - per_host_cap: host_cap, - timeout: REQUEST_TIMEOUT, - headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())], - ca_bundle_path: req.ca_bundle_path.clone(), - process_workers: req.process_workers.max(1), - label: req.label.clone(), - ..FetchOptions::default() - }; - - let sink = ChunkCacheSink { - plan, - fetch_chunks: &fetch_chunks, - decompressed: AtomicU64::new(0), - }; - - let meter = Mutex::new(SpeedMeter::new()); - let progress_adapter = |bytes: u64, items_ok: u64| { - if let Ok(mut m) = meter.lock() { - m.sample(bytes); - } - progress(pre_bytes + bytes, pre_chunks + items_ok); - }; - - let fetched = run_fetch( - items, - &plan.hosts, - &opts, - &sink, - cancel, - &progress_adapter, - log, - ); - - let decompressed = sink.decompressed.load(Ordering::Relaxed); - outcome.bytes_credited = pre_bytes + fetched.bytes_credited; - outcome.chunks_done = pre_chunks + fetched.items_ok; - outcome.decompressed_written = decompressed; - if let Ok(m) = meter.lock() { - log(&m.summary(fetched.bytes_credited, decompressed, pre_chunks)); - } - - if fetched.cancelled { - outcome.cancelled = true; - outcome.error = "cancelled".to_string(); - log(&format!( - "cancelled during chunk download ({}/{chunks_total} chunks)", - outcome.chunks_done - )); - return outcome; - } - if let Some(err) = fetched.error { - outcome.error = err; - log(&format!( - "FAIL {} ({}/{chunks_total} chunks ok)", - outcome.error, outcome.chunks_done - )); - return outcome; - } - if fetched.items_ok != fetch_chunks.len() as u64 { - outcome.error = format!( - "fetch ended with {}/{} chunks", - fetched.items_ok, - fetch_chunks.len() - ); - log(&format!("FAIL {}", outcome.error)); - return outcome; - } - outcome.success = true; - log(&format!("chunksOK={}", outcome.chunks_done)); - outcome -} - -#[cfg(test)] -mod tests { - use super::super::manifest::test_support::*; - use super::*; - - fn request(dir: &str, pending: Vec) -> EpicRequest { - EpicRequest { - manifest_bytes: build_manifest(&sample_chunks(), &sample_files(), 21, true), - install_dir: dir.to_string(), - chunk_cache_dir: String::new(), - cdn_prefixes: vec![ - "https://fastly-download.epicgames.com/Builds/o/x/default".to_string(), - "https://download.epicgames.com/Builds/o/x/default".to_string(), - "https://fastly-download.epicgames.com/Builds/o/x/default".to_string(), - ], - pending_file_indices: pending, - expected_chunks: None, - expected_bytes: None, - ca_bundle_path: String::new(), - max_workers: super::super::JAVA_POOL_THREADS, - process_workers: 2, - label: "epic test".to_string(), - } - } - - #[test] - fn plan_matches_java_for_the_pending_set() { - let dir = super::super::chunk::test_support::temp_dir("plan"); - let req = request(dir.to_str().unwrap(), vec![0, 1]); - let plan = build_plan(&req).unwrap(); - assert_eq!(plan.needed, vec![0, 1]); - assert_eq!(plan.total_bytes, 700_000 + 1); - assert_eq!(plan.hosts.len(), 2, "duplicate CDN prefix collapsed"); - assert!(plan.cache_dir.ends_with(".chunks")); - assert!(plan.cache_dir.is_dir()); - assert!(!plan.scratch_cache); - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn a_named_cache_dir_is_created_and_marked_scratch() { - let dir = super::super::chunk::test_support::temp_dir("scratch"); - let cache = dir.join("cache").join("Game"); - let mut req = request(dir.join("install").to_str().unwrap(), vec![0, 1]); - req.chunk_cache_dir = cache.to_str().unwrap().to_string(); - let plan = build_plan(&req).unwrap(); - assert_eq!(plan.cache_dir, cache); - assert!(plan.cache_dir.is_dir()); - assert!(plan.scratch_cache); - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn plan_cross_check_rejects_mismatches() { - let dir = super::super::chunk::test_support::temp_dir("xcheck"); - let mut req = request(dir.to_str().unwrap(), vec![0, 1]); - req.expected_chunks = Some(3); - assert!(build_plan(&req).unwrap_err().contains("chunk count mismatch")); - req.expected_chunks = Some(2); - req.expected_bytes = Some(5); - assert!(build_plan(&req).unwrap_err().contains("byte total mismatch")); - req.expected_bytes = Some(700_001); - assert!(build_plan(&req).is_ok()); - req.pending_file_indices = vec![7]; - assert!(build_plan(&req).unwrap_err().contains("out of range")); - req.pending_file_indices = vec![0]; - req.cdn_prefixes.clear(); - assert!(build_plan(&req).unwrap_err().contains("no CDN")); - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn fully_cached_plan_completes_without_fetching() { - let dir = super::super::chunk::test_support::temp_dir("cached"); - let req = request(dir.to_str().unwrap(), vec![0]); - let plan = build_plan(&req).unwrap(); - for &ci in &plan.needed { - std::fs::write( - cached_chunk_path(&plan.cache_dir, &plan.manifest.unique_chunks[ci]), - b"cached", - ) - .unwrap(); - } - let cancel = AtomicBool::new(false); - let calls = Mutex::new(Vec::new()); - let progress = |b: u64, c: u64| calls.lock().unwrap().push((b, c)); - let log = |_: &str| {}; - let out = run_plan(&plan, &req, &cancel, &progress, &log); - assert!(out.success); - assert_eq!(out.chunks_done, 2); - assert_eq!(out.bytes_credited, 700_001); - assert_eq!(*calls.lock().unwrap(), vec![(700_000, 1), (700_001, 2)]); - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn cancel_before_fetch_reports_cancelled() { - let dir = super::super::chunk::test_support::temp_dir("cancel"); - let req = request(dir.to_str().unwrap(), vec![0]); - let plan = build_plan(&req).unwrap(); - let cancel = AtomicBool::new(true); - let progress = |_: u64, _: u64| {}; - let log = |_: &str| {}; - let out = run_plan(&plan, &req, &cancel, &progress, &log); - assert!(out.cancelled); - assert!(!out.success); - let _ = std::fs::remove_dir_all(&dir); - } -} diff --git a/app/src/main/rust/stores/src/epic/ffi.rs b/app/src/main/rust/stores/src/epic/ffi.rs new file mode 100644 index 0000000000..31dc463e10 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/ffi.rs @@ -0,0 +1,305 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use jni::objects::{GlobalRef, JByteArray, JClass, JObject, JString, JValue}; +use jni::sys::{jbyteArray, jlong, jstring, JNI_FALSE, JNI_TRUE}; +use jni::{JNIEnv, JavaVM}; + +use super::install::{self, Events, Stop}; +use super::{net, service}; + +pub const LOG_TAG: &str = "EpicNative"; + +#[cfg(target_os = "android")] +#[link(name = "log")] +unsafe extern "C" { + fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32; +} + +pub fn android_log(message: &str) { + #[cfg(target_os = "android")] + { + use std::ffi::CString; + let (Ok(tag), Ok(text)) = (CString::new(LOG_TAG), CString::new(message.replace('\0', " "))) else { + return; + }; + unsafe { + let _ = __android_log_write(4, tag.as_ptr().cast(), text.as_ptr().cast()); + } + } + #[cfg(not(target_os = "android"))] + { + let _ = message; + } +} + +fn read_string(env: &mut JNIEnv, value: &JString) -> String { + if value.is_null() { + return String::new(); + } + env.get_string(value) + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +fn new_string(env: &mut JNIEnv, text: &str) -> jstring { + env.new_string(text).map(|s| s.into_raw()).unwrap_or(std::ptr::null_mut()) +} + +fn guarded(fallback: T, f: impl FnOnce() -> T) -> T { + catch_unwind(AssertUnwindSafe(f)).unwrap_or_else(|_| { + android_log("a native Epic call panicked"); + fallback + }) +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeConfigure( + mut env: JNIEnv, + _class: JClass, + app_version: JString, +) { + let version = read_string(&mut env, &app_version); + net::set_app_version(&version); +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCall( + mut env: JNIEnv, + _class: JClass, + op: JString, + request: JString, +) -> jstring { + let op = read_string(&mut env, &op); + let request = read_string(&mut env, &request); + let answer = guarded(None, || Some(service::call(&op, &request))); + match answer { + Some(text) => new_string(&mut env, &text), + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeFetch( + mut env: JNIEnv, + _class: JClass, + op: JString, + request: JString, +) -> jbyteArray { + let op = read_string(&mut env, &op); + let request = read_string(&mut env, &request); + match guarded(None, || service::fetch(&op, &request)) { + Some(bytes) => env + .byte_array_from_slice(&bytes) + .map(|a| a.into_raw()) + .unwrap_or(std::ptr::null_mut()), + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeSend( + mut env: JNIEnv, + _class: JClass, + op: JString, + request: JString, + body: JByteArray, +) -> jstring { + let op = read_string(&mut env, &op); + let request = read_string(&mut env, &request); + let bytes = if body.is_null() { Vec::new() } else { env.convert_byte_array(&body).unwrap_or_default() }; + let answer = guarded(None, || Some(service::send(&op, &request, bytes))); + match answer { + Some(text) => new_string(&mut env, &text), + None => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInspect( + mut env: JNIEnv, + _class: JClass, + manifest: JByteArray, + request: JString, +) -> jstring { + let request = read_string(&mut env, &request); + let bytes = if manifest.is_null() { Vec::new() } else { env.convert_byte_array(&manifest).unwrap_or_default() }; + let answer = guarded(None, || Some(service::inspect(&bytes, &request))); + match answer { + Some(text) => new_string(&mut env, &text), + None => std::ptr::null_mut(), + } +} + +struct Listener { + vm: JavaVM, + target: GlobalRef, +} + +impl Listener { + fn with_env(&self, fallback: R, f: impl FnOnce(&mut JNIEnv) -> R) -> R { + let Ok(mut env) = self.vm.attach_current_thread_as_daemon() else { + return fallback; + }; + let out = f(&mut env); + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } + out + } + + fn call_text(&self, method: &str, text: &str) { + self.with_env((), |env| { + let Ok(s) = env.new_string(text) else { return }; + let s = JObject::from(s); + let _ = env.call_method(self.target.as_obj(), method, "(Ljava/lang/String;)V", &[JValue::Object(&s)]); + let _ = env.delete_local_ref(s); + }); + } + + fn call_counts(&self, method: &str, a: u64, b: u64, c: u32, d: u32) { + self.with_env((), |env| { + let _ = env.call_method( + self.target.as_obj(), + method, + "(JJII)V", + &[ + JValue::Long(a.min(i64::MAX as u64) as i64), + JValue::Long(b.min(i64::MAX as u64) as i64), + JValue::Int(c.min(i32::MAX as u32) as i32), + JValue::Int(d.min(i32::MAX as u32) as i32), + ], + ); + }); + } + + fn complete(&self, success: bool, cancelled: bool, error: &str, result: &str) { + self.with_env((), |env| { + let (Ok(e), Ok(r)) = (env.new_string(error), env.new_string(result)) else { return }; + let (e, r) = (JObject::from(e), JObject::from(r)); + let _ = env.call_method( + self.target.as_obj(), + "onComplete", + "(ZZLjava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), + JValue::Bool(if cancelled { JNI_TRUE } else { JNI_FALSE }), + JValue::Object(&e), + JValue::Object(&r), + ], + ); + let _ = env.delete_local_ref(e); + let _ = env.delete_local_ref(r); + }); + } +} + +impl Events for Listener { + fn log(&self, line: &str) { + android_log(line); + self.call_text("onLog", line); + } + + fn status(&self, text: &str) { + self.call_text("onStatus", text); + } + + fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32) { + self.call_counts("onCheck", done_bytes, total_bytes, files, files_total); + } + + fn sizes(&self, pending_bytes: u64, disk_bytes: u64) { + self.with_env((), |env| { + let _ = env.call_method( + self.target.as_obj(), + "onSizes", + "(JJ)V", + &[JValue::Long(pending_bytes as i64), JValue::Long(disk_bytes as i64)], + ); + }); + } + + fn progress(&self, written: u64, total: u64, files: u32, files_total: u32) { + self.call_counts("onProgress", written, total, files, files_total); + } + + fn installing(&self, done: u64, total: u64, files: u32, files_total: u32) { + self.call_counts("onInstall", done, total, files, files_total); + } + + fn before_replace(&self) -> bool { + self.with_env(false, |env| { + match env.call_method(self.target.as_obj(), "beforeReplace", "()Z", &[]) { + Ok(v) => v.z().unwrap_or(false), + Err(_) => false, + } + }) + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstall( + mut env: JNIEnv, + _class: JClass, + request: JString, + listener: JObject, +) -> jlong { + if listener.is_null() { + return 0; + } + let Ok(vm) = env.get_java_vm() else { return 0 }; + let Ok(target) = env.new_global_ref(&listener) else { return 0 }; + let listener = Listener { vm, target }; + let text = read_string(&mut env, &request); + let parsed = serde_json::from_str::(&text) + .map_err(|_| "The install request could not be read".to_string()) + .and_then(|v| install::Request::from_json(&v)); + let request = match parsed { + Ok(r) => r, + Err(e) => { + listener.complete(false, false, &e, ""); + return 0; + } + }; + let cancel = Arc::new(AtomicBool::new(false)); + let worker_cancel = Arc::clone(&cancel); + let spawned = std::thread::Builder::new().name("epic-install".into()).spawn(move || { + crate::priority::background(); + let result = catch_unwind(AssertUnwindSafe(|| install::run(&request, &worker_cancel, &listener))) + .unwrap_or_else(|_| Err(Stop::Failed("Native Epic install failed: internal error".into()))); + match result { + Ok(done) => listener.complete(true, false, "", &done.to_json().to_string()), + Err(Stop::Cancelled) => listener.complete(false, true, "cancelled", ""), + Err(Stop::Failed(e)) => listener.complete(false, false, &e, ""), + } + }); + if spawned.is_err() { + return 0; + } + Box::into_raw(Box::new(cancel)) as jlong +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCancel( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + let ptr = handle as *const Arc; + if let Some(cancel) = unsafe { ptr.as_ref() } { + cancel.store(true, Ordering::SeqCst); + } +} + +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeRelease( + _env: JNIEnv, + _class: JClass, + handle: jlong, +) { + let ptr = handle as *mut Arc; + if !ptr.is_null() { + drop(unsafe { Box::from_raw(ptr) }); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/cache.rs b/app/src/main/rust/stores/src/epic/install/cache.rs new file mode 100644 index 0000000000..bcd8c7cccd --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/cache.rs @@ -0,0 +1,162 @@ +use std::collections::HashMap; +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use sha1::{Digest, Sha1}; + +use crate::epic::manifest::{guid_from_hex, guid_text, ChunkInfo}; + +pub fn entry_name(chunk: &ChunkInfo, copy: u32) -> String { + if copy == 0 { + guid_text(&chunk.guid) + } else { + format!("{}.{copy}", guid_text(&chunk.guid)) + } +} + +pub fn store(dir: &Path, name: &str, data: &[u8]) -> Result { + fs::create_dir_all(dir).map_err(|e| format!("chunk cache: {e}"))?; + let path = dir.join(name); + let part = dir.join(format!("{name}.part")); + let result = (|| -> std::io::Result<()> { + let mut f = fs::File::create(&part)?; + f.write_all(data)?; + f.flush()?; + fs::rename(&part, &path) + })(); + if let Err(e) = result { + let _ = fs::remove_file(&part); + return Err(format!("chunk cache write: {e}")); + } + Ok(path) +} + +pub fn remove_partials(dir: &Path) { + if let Ok(entries) = fs::read_dir(dir) { + for e in entries.flatten() { + if e.file_name().to_string_lossy().ends_with(".part") { + let _ = fs::remove_file(e.path()); + } + } + } +} + +fn remove_entry(path: &Path, meta: &fs::Metadata) { + if meta.is_dir() && !meta.file_type().is_symlink() { + let _ = fs::remove_dir_all(path); + } else { + let _ = fs::remove_file(path); + } +} + +fn sha1_of_file(path: &Path, cancel: &AtomicBool) -> Option<[u8; 20]> { + let mut f = fs::File::open(path).ok()?; + let mut h = Sha1::new(); + let mut buf = vec![0u8; 256 * 1024]; + loop { + if cancel.load(Ordering::Relaxed) { + return None; + } + let n = f.read(&mut buf).ok()?; + if n == 0 { + break; + } + h.update(&buf[..n]); + } + Some(h.finalize().into()) +} + +pub fn prepare( + dir: &Path, + chunks: &[ChunkInfo], + needed: &HashMap, + budget: u64, + cancel: &AtomicBool, +) -> Vec { + let Ok(entries) = fs::read_dir(dir) else { + return Vec::new(); + }; + let by_name: HashMap = needed + .keys() + .map(|&c| (guid_text(&chunks[c as usize].guid), c)) + .collect(); + let mut candidates: Vec<(u32, u32, PathBuf)> = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + let Ok(meta) = fs::symlink_metadata(&path) else { continue }; + let name = entry.file_name().to_string_lossy().into_owned(); + let chunk = by_name.get(&name).copied().filter(|_| name.len() == 32 && guid_from_hex(&name).is_some()); + let usable = meta.is_file() && !meta.file_type().is_symlink(); + match chunk { + Some(c) if usable => { + let info = &chunks[c as usize]; + let len_ok = if info.window_exact { + meta.len() == info.window as u64 + } else { + meta.len() > 0 && meta.len() <= info.window as u64 + }; + if len_ok { + candidates.push((needed[&c], c, path)); + } else { + remove_entry(&path, &meta); + } + } + _ => remove_entry(&path, &meta), + } + } + candidates.sort_unstable_by_key(|(first, _, _)| *first); + let mut kept = Vec::new(); + let mut used = 0u64; + for (_, c, path) in candidates { + let info = &chunks[c as usize]; + let size = fs::metadata(&path).map(|m| m.len()).unwrap_or(u64::MAX); + let fits = used.saturating_add(size) <= budget; + let hash_ok = fits + && match info.sha1 { + Some(expected) => sha1_of_file(&path, cancel) == Some(expected), + None => true, + }; + if hash_ok { + used += size; + kept.push(c); + } else { + let _ = fs::remove_file(&path); + } + } + kept +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::epic::manifest::testing::chunk; + + #[test] + fn keeps_only_valid_needed_entries_within_budget() { + let dir = std::env::temp_dir().join(format!("dd-epic-cache-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(dir.join("subdir")).unwrap(); + let mut chunks = vec![chunk(1, 4), chunk(2, 4), chunk(3, 4), chunk(4, 4)]; + chunks[1].sha1 = Some(Sha1::digest(b"good").into()); + chunks[2].sha1 = Some(Sha1::digest(b"good").into()); + store(&dir, &entry_name(&chunks[0], 0), b"aaaa").unwrap(); + store(&dir, &entry_name(&chunks[1], 0), b"good").unwrap(); + store(&dir, &entry_name(&chunks[2], 0), b"bad!").unwrap(); + store(&dir, &entry_name(&chunks[3], 0), b"dddd").unwrap(); + store(&dir, &entry_name(&chunks[0], 2), b"aaaa").unwrap(); + fs::write(dir.join("junk.part"), b"x").unwrap(); + fs::write(dir.join(entry_name(&chunk(9, 4), 0)), b"zzzz").unwrap(); + let needed: HashMap = [(0, 5), (1, 1), (2, 2), (3, 9)].into_iter().collect(); + let cancel = AtomicBool::new(false); + let kept = prepare(&dir, &chunks, &needed, 8, &cancel); + assert_eq!(kept, vec![1, 0]); + let mut left: Vec = fs::read_dir(&dir).unwrap().flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + left.sort(); + let mut want = vec![guid_text(&chunks[0].guid), guid_text(&chunks[1].guid)]; + want.sort(); + assert_eq!(left, want); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/delta.rs b/app/src/main/rust/stores/src/epic/install/delta.rs new file mode 100644 index 0000000000..d860e72fcf --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/delta.rs @@ -0,0 +1,72 @@ +use std::fs; +use std::io::Read; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; + +use sha1::{Digest, Sha1}; + +use crate::epic::manifest::Sha1 as Hash; + +pub enum Check { + UpToDate, + Pending { old_size: u64 }, + Cancelled, +} + +fn hash_file(path: &Path, cancel: &AtomicBool) -> Option> { + let Ok(mut f) = fs::File::open(path) else { return Some(None) }; + let mut h = Sha1::new(); + let mut buf = vec![0u8; 1024 * 1024]; + loop { + if cancel.load(Ordering::Relaxed) { + return None; + } + match f.read(&mut buf) { + Ok(0) => break, + Ok(n) => h.update(&buf[..n]), + Err(_) => return Some(None), + } + } + Some(Some(h.finalize().into())) +} + +pub fn check(dest: &Path, size: u64, sha1: Option, cancel: &AtomicBool) -> Check { + let old_size = match fs::metadata(dest) { + Ok(m) if m.is_file() => m.len(), + _ => return Check::Pending { old_size: 0 }, + }; + let Some(expected) = sha1 else { + return Check::Pending { old_size }; + }; + if old_size != size { + return Check::Pending { old_size }; + } + match hash_file(dest, cancel) { + None => Check::Cancelled, + Some(Some(got)) if got == expected => Check::UpToDate, + Some(_) => Check::Pending { old_size }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn up_to_date_needs_size_and_hash() { + let dir = std::env::temp_dir().join(format!("dd-epic-delta-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let f = dir.join("a.bin"); + fs::write(&f, b"hello").unwrap(); + let good: Hash = Sha1::digest(b"hello").into(); + let cancel = AtomicBool::new(false); + assert!(matches!(check(&f, 5, Some(good), &cancel), Check::UpToDate)); + assert!(matches!(check(&f, 5, None, &cancel), Check::Pending { old_size: 5 })); + assert!(matches!(check(&f, 6, Some(good), &cancel), Check::Pending { old_size: 5 })); + assert!(matches!(check(&f, 5, Some([0; 20]), &cancel), Check::Pending { .. })); + assert!(matches!(check(&dir.join("missing"), 5, Some(good), &cancel), Check::Pending { old_size: 0 })); + cancel.store(true, Ordering::Relaxed); + assert!(matches!(check(&f, 5, Some(good), &cancel), Check::Cancelled)); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/mod.rs b/app/src/main/rust/stores/src/epic/install/mod.rs new file mode 100644 index 0000000000..426e3bdcf9 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/mod.rs @@ -0,0 +1,321 @@ +pub mod cache; +pub mod delta; +pub mod schedule; +pub mod space; +pub mod staging; +pub mod writer; + +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use serde_json::{json, Value}; + +use super::build_info::{self, BuildInfo}; +use super::locale; +use super::manifest::{self, Manifest}; +use super::net; +use schedule::{PlanFile, PlanPart}; + +pub const BESIDE_CACHE_DIR: &str = ".chunks"; +const CHECK_PROGRESS_EVERY: usize = 16; + +pub trait Events: Sync { + fn log(&self, line: &str); + fn status(&self, text: &str); + fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32); + fn sizes(&self, pending_bytes: u64, disk_bytes: u64); + fn progress(&self, written: u64, total: u64, files: u32, files_total: u32); + fn installing(&self, done: u64, total: u64, files: u32, files_total: u32); + fn before_replace(&self) -> bool; +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Stop { + Cancelled, + Failed(String), +} + +pub struct Request { + pub info: BuildInfo, + pub game_dir: PathBuf, + pub scratch_dir: Option, + pub storage_label: String, + pub tags: Vec, + pub max_workers: usize, + pub process_workers: usize, + pub ca_bundle: String, +} + +impl Request { + pub fn from_json(v: &Value) -> Result { + let info = v.get("info").and_then(BuildInfo::from_json).ok_or("The manifest could not be fetched")?; + let game_dir = net::str_of(v, "gameDir"); + if game_dir.is_empty() || !game_dir.starts_with('/') { + return Err("No install folder was given".into()); + } + let scratch = net::str_of(v, "scratchDir"); + let tags = v + .get("tags") + .and_then(|t| t.as_array()) + .map(|a| a.iter().filter_map(|t| t.as_str()).map(str::to_string).collect()) + .unwrap_or_default(); + Ok(Request { + info, + game_dir: PathBuf::from(game_dir), + scratch_dir: (!scratch.is_empty()).then(|| PathBuf::from(scratch)), + storage_label: Some(net::str_of(v, "storageLabel")).filter(|s| !s.is_empty()).unwrap_or("storage").to_string(), + tags, + max_workers: v.get("maxWorkers").and_then(|n| n.as_u64()).unwrap_or(32) as usize, + process_workers: v.get("processWorkers").and_then(|n| n.as_u64()).unwrap_or(2) as usize, + ca_bundle: net::str_of(v, "caBundle").to_string(), + }) + } +} + +#[derive(Debug, Default)] +pub struct Finished { + pub build_version: String, + pub launch_exe: String, + pub prereq_name: String, + pub prereq_path: String, + pub prereq_read: bool, + pub deployment_id: String, + pub disk_bytes: u64, + pub bytes_written: u64, + pub files_written: usize, +} + +impl Finished { + pub fn to_json(&self) -> Value { + json!({ + "buildVersion": self.build_version, + "launchExe": self.launch_exe, + "prereqName": self.prereq_name, + "prereqPath": self.prereq_path, + "prereqRead": self.prereq_read, + "deploymentId": self.deployment_id, + "diskBytes": self.disk_bytes, + "bytesWritten": self.bytes_written, + "filesWritten": self.files_written, + }) + } +} + +pub fn safe_relative(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && path.split('/').all(|seg| !seg.is_empty() && seg != "." && seg != "..") +} + +fn dir_has_entries(dir: &Path) -> bool { + fs::read_dir(dir).map(|mut e| e.next().is_some()).unwrap_or(false) +} + +fn remove_tree(dir: &Path) { + if fs::symlink_metadata(dir).map(|m| m.file_type().is_symlink()).unwrap_or(false) { + let _ = fs::remove_file(dir); + } else { + let _ = fs::remove_dir_all(dir); + } +} + +fn cancelled(cancel: &AtomicBool) -> Result<(), Stop> { + if cancel.load(Ordering::SeqCst) { + Err(Stop::Cancelled) + } else { + Ok(()) + } +} + +pub fn fetch_manifest(info: &BuildInfo, log: &dyn Fn(&str)) -> Result { + if info.cdns.is_empty() { + return Err("No CDN in the manifest answer".into()); + } + let bytes = build_info::download_manifest(info, &info.cdns, log).ok_or("The manifest could not be downloaded")?; + manifest::parse(&bytes).map_err(|e| { + log(&format!("manifest parse: {e}")); + "The manifest could not be parsed".to_string() + }) +} + +pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result { + let failed = |m: String| Stop::Failed(m); + ev.status("Downloading the manifest…"); + let log = |line: &str| ev.log(line); + let manifest = fetch_manifest(&req.info, &log).map_err(failed)?; + cancelled(cancel)?; + ev.status("Reading the manifest…"); + + let selected = locale::select(&manifest.files, Some(&req.tags)); + for &i in &selected { + let path = &manifest.files[i].path; + if !safe_relative(path) { + return Err(failed(format!("The manifest has an unsafe path: {path}"))); + } + if staging::conflicts(path) { + return Err(failed("The manifest conflicts with the staging folder".into())); + } + } + let disk_bytes: u64 = selected.iter().map(|&i| manifest.files[i].size).sum(); + ev.log(&format!( + "manifest build={} version={} chunk_dir={} files={} selected={} tags={} disk_bytes={disk_bytes}", + manifest.build_version, + manifest.version, + manifest.chunk_folder(), + manifest.files.len(), + selected.len(), + req.tags.join("|"), + )); + + let stage_dir = staging::recover(&req.game_dir).map_err(failed)?; + cancelled(cancel)?; + + ev.status("Checking files…"); + let total_selected = selected.len() as u32; + let mut pending: Vec<(usize, u64)> = Vec::new(); + let mut checked_bytes = 0u64; + for (n, &i) in selected.iter().enumerate() { + let f = &manifest.files[i]; + match delta::check(&req.game_dir.join(&f.path), f.size, f.sha1, cancel) { + delta::Check::Cancelled => return Err(Stop::Cancelled), + delta::Check::UpToDate => {} + delta::Check::Pending { old_size } => pending.push((i, old_size)), + } + checked_bytes += f.size; + if (n + 1) % CHECK_PROGRESS_EVERY == 0 || n + 1 == selected.len() { + ev.checking(checked_bytes, disk_bytes, (n + 1) as u32, total_selected); + } + cancelled(cancel)?; + } + let pending_bytes: u64 = pending.iter().map(|&(i, _)| manifest.files[i].size).sum(); + ev.sizes(pending_bytes, disk_bytes); + + let mut finished = Finished { + build_version: manifest.build_version.clone(), + launch_exe: manifest.launch_exe.clone(), + prereq_name: manifest.prereq_name.clone(), + prereq_path: manifest.prereq_path.clone(), + prereq_read: manifest.prereq_read, + deployment_id: req.info.deployment_id.clone(), + disk_bytes, + ..Default::default() + }; + let beside = req.game_dir.join(BESIDE_CACHE_DIR); + let clean_caches = || { + remove_tree(&beside); + if let Some(s) = &req.scratch_dir { + remove_tree(s); + } + staging::remove_if_idle(&stage_dir); + }; + if pending.is_empty() { + ev.log("every selected file is already up to date"); + clean_caches(); + ev.installing(disk_bytes, disk_bytes, total_selected, total_selected); + return Ok(finished); + } + + let mut plan_files = Vec::with_capacity(pending.len()); + let mut engine_files = Vec::with_capacity(pending.len()); + for &(i, old_size) in &pending { + let f = &manifest.files[i]; + let mut parts = Vec::with_capacity(f.parts.len()); + let mut offset = 0u64; + for part in &f.parts { + let Some(&c) = manifest.chunk_index.get(&part.guid) else { + return Err(failed(format!("The manifest is missing chunk data for {}", f.path))); + }; + let info = &manifest.chunks[c]; + if info.window_exact && part.offset as u64 + part.size as u64 > info.window as u64 { + return Err(failed(format!("A chunk is too short for {}", f.path))); + } + parts.push(PlanPart { chunk: c as u32, chunk_offset: part.offset, size: part.size, file_offset: offset }); + offset += part.size as u64; + } + plan_files.push(PlanFile { size: f.size, parts: parts.clone() }); + engine_files.push(writer::EngineFile { + path: f.path.clone(), + dest: req.game_dir.join(&f.path), + size: f.size, + sha1: f.sha1, + replacing: old_size > 0, + parts, + }); + } + + let scratch = req + .scratch_dir + .clone() + .filter(|_| !dir_has_entries(&beside)) + .filter(|s| !space::same_volume(s, &req.game_dir)); + let sizes: Vec = pending + .iter() + .map(|&(i, old)| space::PendingSize { new_size: manifest.files[i].size, old_size: old }) + .collect(); + let free_game = space::free_bytes(&req.game_dir); + let scratch_space = scratch.as_ref().map(|s| space::Scratch { free: space::free_bytes(s) }); + let plan = space::plan(&sizes, free_game, scratch_space, &req.storage_label).map_err(failed)?; + let cache_dir = match (&scratch, plan.cache_on_scratch) { + (Some(s), true) => s.clone(), + _ => beside.clone(), + }; + cancelled(cancel)?; + + let mut first_use: HashMap = HashMap::new(); + let mut position = 0u32; + for f in &plan_files { + for p in &f.parts { + first_use.entry(p.chunk).or_insert(position); + position += 1; + } + } + let reused = cache::prepare(&cache_dir, &manifest.chunks, &first_use, plan.cache_budget, cancel); + cancelled(cancel)?; + let windows: Vec = manifest.chunks.iter().map(|c| c.window).collect(); + let schedule = schedule::build(&plan_files, &windows, plan.cache_budget, &reused); + + let prefixes = build_info::chunk_prefixes(&req.info.cdns); + if prefixes.is_empty() { + return Err(failed("No CDN in the manifest answer".into())); + } + ev.log(&format!( + "plan mode={} pending_files={} pending_bytes={pending_bytes} fetches={} reused={} cache={} \ +cache_budget_mib={} overlap_mib={} free_game={free_game} workers={} process_workers={}{}", + if plan.narrow { "narrow" } else { "adaptive" }, + pending.len(), + schedule.fetch.len(), + reused.len(), + if plan.cache_on_scratch { "scratch" } else { "beside-game" }, + plan.cache_budget / space::MIB, + plan.overlap.map(|o| (o / space::MIB).to_string()).unwrap_or_else(|| "unlimited".into()), + req.max_workers, + req.process_workers, + if plan.cache_moved_beside_game { " note=internal-storage-full" } else { "" }, + )); + + let input = writer::EngineInput { + manifest: &manifest, + files: engine_files, + schedule, + stage_dir: stage_dir.clone(), + cache_dir, + prefixes, + overlap: plan.overlap, + narrow: plan.narrow, + max_workers: req.max_workers, + process_workers: req.process_workers, + ca_bundle: req.ca_bundle.clone(), + user_agent: net::user_agent(), + }; + let stats = writer::run(&input, cancel, ev)?; + finished.bytes_written = stats.bytes_written; + finished.files_written = stats.files_written; + clean_caches(); + ev.installing(disk_bytes, disk_bytes, total_selected, total_selected); + Ok(finished) +} + +#[cfg(test)] +mod tests; diff --git a/app/src/main/rust/stores/src/epic/install/schedule.rs b/app/src/main/rust/stores/src/epic/install/schedule.rs new file mode 100644 index 0000000000..005a31dbd6 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/schedule.rs @@ -0,0 +1,237 @@ +use std::collections::{BTreeSet, HashMap}; + +pub const NONE: u32 = u32::MAX; + +#[derive(Clone, Debug)] +pub struct PlanPart { + pub chunk: u32, + pub chunk_offset: u32, + pub size: u32, + pub file_offset: u64, +} + +#[derive(Clone, Debug)] +pub struct PlanFile { + pub size: u64, + pub parts: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Producer { + pub chunk: u32, + pub anchor: u32, + pub preloaded: bool, + pub copy: u32, + pub uses: Vec, +} + +#[derive(Debug)] +pub struct Schedule { + pub file_start: Vec, + pub use_file: Vec, + pub use_producer: Vec, + pub producers: Vec, + pub fetch: Vec, + pub peak_cache: u64, + pub refetches: u32, +} + +impl Schedule { + pub fn uses_of_file(&self, file: usize) -> std::ops::Range { + self.file_start[file] as usize..self.file_start[file + 1] as usize + } +} + +pub fn build(files: &[PlanFile], windows: &[u32], budget: u64, cached: &[u32]) -> Schedule { + let mut file_start = Vec::with_capacity(files.len() + 1); + let mut use_chunk = Vec::new(); + let mut use_file = Vec::new(); + for (f, file) in files.iter().enumerate() { + file_start.push(use_chunk.len() as u32); + for part in &file.parts { + use_chunk.push(part.chunk); + use_file.push(f as u32); + } + } + file_start.push(use_chunk.len() as u32); + let total = use_chunk.len(); + + let mut next_use = vec![NONE; total]; + let mut first_use: HashMap = HashMap::new(); + { + let mut seen: HashMap = HashMap::new(); + for u in (0..total).rev() { + let c = use_chunk[u]; + if let Some(&n) = seen.get(&c) { + next_use[u] = n; + } + seen.insert(c, u as u32); + } + for (c, u) in seen { + first_use.insert(c, u); + } + } + + let window = |c: u32| windows.get(c as usize).copied().unwrap_or(0) as u64; + let mut producers: Vec = Vec::new(); + let mut fetch = Vec::new(); + let mut use_producer = vec![NONE; total]; + let mut active: HashMap = HashMap::new(); + let mut copies: HashMap = HashMap::new(); + let mut resident: HashMap = HashMap::new(); + let mut by_next: BTreeSet<(u32, u32)> = BTreeSet::new(); + let mut occupancy = 0u64; + let mut peak = 0u64; + + let mut preload: Vec<(u32, u32)> = cached + .iter() + .filter_map(|&c| first_use.get(&c).map(|&u| (u, c))) + .collect(); + preload.sort_unstable(); + preload.dedup_by_key(|(_, c)| *c); + for (u, c) in preload { + if occupancy + window(c) > budget { + continue; + } + occupancy += window(c); + let id = producers.len() as u32; + producers.push(Producer { chunk: c, anchor: NONE, preloaded: true, copy: 0, uses: Vec::new() }); + copies.insert(c, 1); + active.insert(c, id); + resident.insert(c, u); + by_next.insert((u, c)); + } + peak = peak.max(occupancy); + + for u in 0..total { + let (c, f) = (use_chunk[u], use_file[u]); + let valid = active + .get(&c) + .copied() + .filter(|&p| producers[p as usize].anchor == f || resident.contains_key(&c)); + let p = match valid { + Some(p) => p, + None => { + let id = producers.len() as u32; + let copy = copies.entry(c).or_insert(0); + producers.push(Producer { chunk: c, anchor: f, preloaded: false, copy: *copy, uses: Vec::new() }); + *copy += 1; + fetch.push(id); + active.insert(c, id); + id + } + }; + producers[p as usize].uses.push(u as u32); + use_producer[u] = p; + + let next = next_use[u]; + if next == NONE { + if let Some(pos) = resident.remove(&c) { + by_next.remove(&(pos, c)); + occupancy -= window(c); + } + active.remove(&c); + continue; + } + if let Some(pos) = resident.get_mut(&c) { + by_next.remove(&(*pos, c)); + *pos = next; + by_next.insert((next, c)); + continue; + } + if use_file[next as usize] == f { + continue; + } + loop { + if occupancy + window(c) <= budget { + occupancy += window(c); + resident.insert(c, next); + by_next.insert((next, c)); + peak = peak.max(occupancy); + break; + } + match by_next.iter().next_back().copied() { + Some((far, victim)) if far > next => { + by_next.remove(&(far, victim)); + resident.remove(&victim); + occupancy -= window(victim); + } + _ => break, + } + } + } + + let refetches = producers.iter().filter(|p| !p.preloaded && p.copy > 0).count() as u32; + Schedule { file_start, use_file, use_producer, producers, fetch, peak_cache: peak, refetches } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(chunks: &[u32]) -> PlanFile { + let mut off = 0; + PlanFile { + size: chunks.len() as u64 * 10, + parts: chunks + .iter() + .map(|&c| { + let p = PlanPart { chunk: c, chunk_offset: 0, size: 10, file_offset: off }; + off += 10; + p + }) + .collect(), + } + } + + #[test] + fn single_use_chunks_are_fetched_once_and_never_cached() { + let s = build(&[file(&[0, 1]), file(&[2])], &[10, 10, 10], 1000, &[]); + assert_eq!(s.fetch.len(), 3); + assert_eq!(s.peak_cache, 0); + assert_eq!(s.refetches, 0); + assert_eq!(s.uses_of_file(1), 2..3); + } + + #[test] + fn repeats_inside_one_file_need_no_cache() { + let s = build(&[file(&[0, 0, 0])], &[10], 0, &[]); + assert_eq!(s.fetch.len(), 1); + assert_eq!(s.producers[0].uses, vec![0, 1, 2]); + } + + #[test] + fn a_chunk_shared_across_files_is_kept_within_budget() { + let s = build(&[file(&[0, 1]), file(&[0])], &[10, 10], 10, &[]); + assert_eq!(s.fetch.len(), 2); + assert_eq!(s.peak_cache, 10); + assert_eq!(s.use_producer[2], s.use_producer[0]); + } + + #[test] + fn without_budget_a_later_file_refetches() { + let s = build(&[file(&[0, 1]), file(&[0])], &[10, 10], 0, &[]); + assert_eq!(s.fetch.len(), 3); + assert_eq!(s.refetches, 1); + assert_eq!(s.producers[2].copy, 1); + assert_eq!(s.producers[2].anchor, 1); + } + + #[test] + fn eviction_drops_the_chunk_needed_furthest_away() { + let files = [file(&[0, 1]), file(&[1]), file(&[1]), file(&[0])]; + let s = build(&files, &[10, 10], 10, &[]); + let fetched: Vec = s.fetch.iter().map(|&p| s.producers[p as usize].chunk).collect(); + assert_eq!(fetched, vec![0, 1, 0]); + assert_eq!(s.peak_cache, 10); + } + + #[test] + fn cached_chunks_are_reused_soonest_first() { + let s = build(&[file(&[0]), file(&[1]), file(&[2])], &[10, 10, 10], 20, &[2, 0, 1, 7]); + assert!(s.fetch.is_empty() || s.fetch.iter().all(|&p| s.producers[p as usize].chunk == 2)); + let preloaded: Vec = s.producers.iter().filter(|p| p.preloaded).map(|p| p.chunk).collect(); + assert_eq!(preloaded, vec![0, 1]); + assert_eq!(s.fetch.len(), 1); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/space.rs b/app/src/main/rust/stores/src/epic/install/space.rs new file mode 100644 index 0000000000..2302f35513 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/space.rs @@ -0,0 +1,266 @@ +use std::path::Path; + +pub const MIB: u64 = 1024 * 1024; +pub const TRANSIENT: u64 = 64 * MIB; +pub const CACHE_CAP: u64 = 1024 * MIB; +pub const CACHE_WHEN_UNKNOWN: u64 = 256 * MIB; +pub const MIN_USEFUL_SCRATCH: u64 = 64 * MIB; + +pub fn free_bytes(path: &Path) -> i64 { + let mut probe = Some(path); + while let Some(p) = probe { + if p.exists() { + return statvfs_available(p); + } + probe = p.parent(); + } + -1 +} + +#[cfg(unix)] +fn statvfs_available(path: &Path) -> i64 { + use std::os::unix::ffi::OsStrExt; + let Ok(c) = std::ffi::CString::new(path.as_os_str().as_bytes()) else { + return -1; + }; + let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut st) } != 0 { + return -1; + } + (st.f_bavail as u64).saturating_mul(st.f_frsize as u64).min(i64::MAX as u64) as i64 +} + +#[cfg(not(unix))] +fn statvfs_available(_path: &Path) -> i64 { + -1 +} + +pub fn same_volume(a: &Path, b: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let dev = |p: &Path| { + let mut probe = Some(p); + while let Some(q) = probe { + if let Ok(m) = std::fs::metadata(q) { + return Some(m.dev()); + } + probe = q.parent(); + } + None + }; + matches!((dev(a), dev(b)), (Some(x), Some(y)) if x == y) + } + #[cfg(not(unix))] + { + let _ = (a, b); + false + } +} + +pub fn human(bytes: u64) -> String { + const GIB: f64 = 1024.0 * 1024.0 * 1024.0; + let b = bytes as f64; + if b >= GIB { + format!("{:.1} GB", b / GIB) + } else if b >= MIB as f64 { + format!("{:.1} MB", b / MIB as f64) + } else if b >= 1024.0 { + format!("{:.0} KB", b / 1024.0) + } else { + format!("{bytes} B") + } +} + +#[derive(Clone, Copy, Debug)] +pub struct PendingSize { + pub new_size: u64, + pub old_size: u64, +} + +#[derive(Clone, Copy, Debug)] +pub struct Scratch { + pub free: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SpacePlan { + pub cache_on_scratch: bool, + pub cache_budget: u64, + pub overlap: Option, + pub narrow: bool, + pub cache_moved_beside_game: bool, + pub game_need: u64, + pub net_peak: u64, +} + +fn fits(need: u64, free: i64) -> bool { + free < 0 || need <= free as u64 +} + +fn half_of_rest(free: i64, other: u64) -> u64 { + if free < 0 { + return CACHE_WHEN_UNKNOWN; + } + ((free as u64).saturating_sub(other).saturating_sub(TRANSIENT) / 2).min(CACHE_CAP) +} + +pub fn net_peak(pending: &[PendingSize]) -> u64 { + let mut running: i128 = 0; + let mut peak: i128 = 0; + for p in pending { + peak = peak.max(running + p.new_size as i128); + running += p.new_size as i128 - p.old_size as i128; + } + peak.max(0) as u64 +} + +pub fn plan(pending: &[PendingSize], free_game: i64, scratch: Option, label: &str) -> Result { + let peak = net_peak(pending); + let largest_replaced = pending.iter().filter(|p| p.old_size > 0).map(|p| p.new_size).max(); + + let mut cache_on_scratch = false; + let mut moved = false; + let mut budget = 0; + if let Some(s) = scratch { + let b = half_of_rest(s.free, 0); + if s.free < 0 || b >= MIN_USEFUL_SCRATCH { + cache_on_scratch = true; + budget = b; + } else { + moved = true; + } + } + if !cache_on_scratch { + budget = half_of_rest(free_game, peak); + } + let cache_on_game = if cache_on_scratch { 0 } else { budget + TRANSIENT }; + let overlap = largest_replaced.map(|largest| { + if free_game < 0 { + largest + } else { + largest.max((free_game as u64).saturating_sub(peak).saturating_sub(cache_on_game)) + } + }); + let game_need = peak + overlap.unwrap_or(0) + cache_on_game; + + if let Some(s) = scratch.filter(|_| cache_on_scratch) { + let need = budget + TRANSIENT; + if !fits(need, s.free) { + return Err(format!( + "Not enough internal space for the download cache: it needs {}, {} is free", + human(need), + human(s.free.max(0) as u64) + )); + } + } + if fits(game_need, free_game) { + return Ok(SpacePlan { + cache_on_scratch, + cache_budget: budget, + overlap, + narrow: false, + cache_moved_beside_game: moved, + game_need, + net_peak: peak, + }); + } + if fits(peak, free_game) { + return Ok(SpacePlan { + cache_on_scratch: false, + cache_budget: 0, + overlap: Some(0), + narrow: true, + cache_moved_beside_game: moved, + game_need: peak, + net_peak: peak, + }); + } + let free = free_game.max(0) as u64; + let mut message = format!( + "Not enough space on {label} for this install: it needs {}, {} is free ({} short)", + human(peak), + human(free), + human(peak.saturating_sub(free)) + ); + if moved { + message.push_str(". The download cache had to move beside the game because internal storage is full"); + } + Err(message) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GB: u64 = 1024 * MIB; + + fn fresh(sizes: &[u64]) -> Vec { + sizes.iter().map(|&s| PendingSize { new_size: s, old_size: 0 }).collect() + } + + #[test] + fn net_peak_credits_old_copies_at_their_swap() { + assert_eq!(net_peak(&fresh(&[10, 20])), 30); + let update = vec![ + PendingSize { new_size: 100, old_size: 90 }, + PendingSize { new_size: 50, old_size: 60 }, + ]; + assert_eq!(net_peak(&update), 100); + assert_eq!(net_peak(&[PendingSize { new_size: 5, old_size: 500 }]), 5); + assert_eq!(net_peak(&[]), 0); + } + + #[test] + fn a_fresh_install_has_no_overlap_limit() { + let p = plan(&fresh(&[2 * GB]), (10 * GB) as i64, None, "Internal").unwrap(); + assert_eq!(p.overlap, None); + assert!(!p.narrow && !p.cache_on_scratch); + assert_eq!(p.cache_budget, CACHE_CAP); + } + + #[test] + fn unknown_free_space_is_never_no_space() { + let p = plan(&fresh(&[50 * GB]), -1, Some(Scratch { free: -1 }), "Card").unwrap(); + assert!(p.cache_on_scratch); + assert_eq!(p.cache_budget, CACHE_WHEN_UNKNOWN); + let update = vec![PendingSize { new_size: GB, old_size: GB }]; + assert_eq!(plan(&update, -1, None, "Card").unwrap().overlap, Some(GB)); + } + + #[test] + fn scratch_too_small_moves_the_cache_beside_the_game() { + let p = plan(&fresh(&[GB]), (20 * GB) as i64, Some(Scratch { free: (100 * MIB) as i64 }), "Card").unwrap(); + assert!(!p.cache_on_scratch && p.cache_moved_beside_game); + } + + #[test] + fn tight_space_falls_back_to_one_file_at_a_time() { + let pending = fresh(&[GB, GB]); + let free = (2 * GB + 10 * MIB) as i64; + let p = plan(&pending, free, None, "Internal").unwrap(); + assert!(p.narrow); + assert_eq!(p.cache_budget, 0); + assert_eq!(p.game_need, 2 * GB); + } + + #[test] + fn not_fitting_names_the_storage_and_shortfall() { + let err = plan(&fresh(&[3 * GB]), GB as i64, None, "SD card").unwrap_err(); + assert!(err.contains("SD card"), "{err}"); + assert!(err.contains("2.0 GB short"), "{err}"); + assert!(plan(&fresh(&[1]), 0, None, "x").is_err()); + assert!(plan(&fresh(&[0]), 0, None, "x").is_ok()); + } + + #[test] + fn updates_get_an_overlap_budget_of_at_least_the_largest_replaced_file() { + let update = vec![ + PendingSize { new_size: GB, old_size: GB }, + PendingSize { new_size: 2 * GB, old_size: 2 * GB }, + ]; + let p = plan(&update, (40 * GB) as i64, None, "Internal").unwrap(); + assert!(p.overlap.unwrap() >= 2 * GB); + assert!(!p.narrow); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/staging.rs b/app/src/main/rust/stores/src/epic/install/staging.rs new file mode 100644 index 0000000000..31b0b4ab61 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/staging.rs @@ -0,0 +1,174 @@ +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +pub const DIR_NAME: &str = ".droiddeck-epic-staging"; +pub const OWNER_FILE: &str = "owner"; +pub const OWNER_MARKER: &[u8] = b"DroidDeck Epic staging v1\n"; +const SUFFIX: &str = ".stage"; + +static SEQUENCE: AtomicU64 = AtomicU64::new(0); + +pub fn conflicts(manifest_path: &str) -> bool { + let first = manifest_path.trim_start_matches('/').split('/').next().unwrap_or(""); + first.eq_ignore_ascii_case(DIR_NAME) +} + +pub fn unique_name() -> String { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos()) + .unwrap_or(0); + format!( + "epic-{}-{}-{nanos}{SUFFIX}", + std::process::id(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + ) +} + +fn write_marker(path: &Path) -> Result<(), String> { + let mut f = fs::File::create(path).map_err(|e| format!("staging marker: {e}"))?; + f.write_all(OWNER_MARKER).map_err(|e| format!("staging marker: {e}"))?; + f.sync_all().map_err(|e| format!("staging marker: {e}"))?; + Ok(()) +} + +pub fn recover(game_dir: &Path) -> Result { + fs::create_dir_all(game_dir).map_err(|e| format!("game folder: {e}"))?; + let game = fs::canonicalize(game_dir).map_err(|e| format!("game folder: {e}"))?; + let expected = game.join(DIR_NAME); + let dir = game_dir.join(DIR_NAME); + let refuse = |why: &str| Err(format!("The staging folder {DIR_NAME} is not usable ({why}); nothing was changed")); + + match fs::symlink_metadata(&dir) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(&dir).map_err(|e| format!("staging folder: {e}"))?; + write_marker(&expected.join(OWNER_FILE))?; + return Ok(expected); + } + Err(e) => return Err(format!("staging folder: {e}")), + Ok(meta) => { + if meta.file_type().is_symlink() || !meta.is_dir() { + return refuse("not a plain directory"); + } + } + } + let canonical = fs::canonicalize(&dir).map_err(|e| format!("staging folder: {e}"))?; + if canonical != expected { + return refuse("it resolves elsewhere"); + } + + let mut stale = Vec::new(); + let mut owner_ok = false; + let mut others = 0usize; + let entries = fs::read_dir(&canonical).map_err(|e| format!("staging folder: {e}"))?; + for entry in entries { + let entry = entry.map_err(|e| format!("staging folder: {e}"))?; + let name = entry.file_name().to_string_lossy().into_owned(); + let path = canonical.join(&name); + let meta = fs::symlink_metadata(&path).map_err(|e| format!("staging folder: {e}"))?; + if meta.file_type().is_symlink() || !meta.is_file() { + return refuse("it holds something that is not a plain file"); + } + if fs::canonicalize(&path).ok().as_deref().and_then(Path::parent) != Some(canonical.as_path()) { + return refuse("an entry resolves elsewhere"); + } + if name == OWNER_FILE { + match fs::read(&path) { + Ok(content) if content == OWNER_MARKER => owner_ok = true, + _ => return refuse("its owner marker is not ours"), + } + } else if name.ends_with(SUFFIX) { + stale.push(path); + others += 1; + } else { + return refuse("it holds an unexpected file"); + } + } + if !owner_ok { + if others == 0 { + write_marker(&canonical.join(OWNER_FILE))?; + return Ok(canonical); + } + return refuse("its owner marker is missing"); + } + for path in stale { + fs::remove_file(&path).map_err(|e| format!("removing an old stage file: {e}"))?; + } + Ok(canonical) +} + +pub fn remove_if_idle(dir: &Path) { + let Ok(entries) = fs::read_dir(dir) else { return }; + let names: Vec = entries.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect(); + if names.iter().all(|n| n == OWNER_FILE) { + let marker = dir.join(OWNER_FILE); + if names.is_empty() || fs::read(&marker).map(|c| c == OWNER_MARKER).unwrap_or(false) { + let _ = fs::remove_file(&marker); + let _ = fs::remove_dir(dir); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("dd-epic-staging-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn creates_validates_and_cleans() { + let game = temp("ok"); + let dir = recover(&game).unwrap(); + assert_eq!(fs::read(dir.join(OWNER_FILE)).unwrap(), OWNER_MARKER); + fs::write(dir.join("file-1.stage"), b"x").unwrap(); + fs::write(dir.join("stream-1-2-3.stage"), b"y").unwrap(); + fs::write(game.join("keep.stage"), b"z").unwrap(); + recover(&game).unwrap(); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1); + assert!(game.join("keep.stage").exists()); + remove_if_idle(&dir); + assert!(!dir.exists()); + let _ = fs::remove_dir_all(&game); + } + + #[test] + fn foreign_contents_are_left_alone() { + let game = temp("foreign"); + let dir = game.join(DIR_NAME); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join(OWNER_FILE), b"someone else\n").unwrap(); + fs::write(dir.join("a.stage"), b"x").unwrap(); + assert!(recover(&game).is_err()); + assert!(dir.join("a.stage").exists()); + fs::write(dir.join(OWNER_FILE), OWNER_MARKER).unwrap(); + fs::write(dir.join("notes.txt"), b"x").unwrap(); + assert!(recover(&game).is_err()); + assert!(dir.join("a.stage").exists()); + let _ = fs::remove_dir_all(&game); + } + + #[test] + fn a_file_in_place_of_the_folder_is_refused() { + let game = temp("file"); + fs::write(game.join(DIR_NAME), b"x").unwrap(); + assert!(recover(&game).is_err()); + let _ = fs::remove_dir_all(&game); + } + + #[test] + fn manifest_conflicts_and_names() { + assert!(conflicts(".droiddeck-epic-staging/x")); + assert!(conflicts(".DroidDeck-Epic-Staging")); + assert!(!conflicts("Game/.droiddeck-epic-staging")); + let (a, b) = (unique_name(), unique_name()); + assert_ne!(a, b); + assert!(a.ends_with(".stage")); + } +} diff --git a/app/src/main/rust/stores/src/epic/install/tests.rs b/app/src/main/rust/stores/src/epic/install/tests.rs new file mode 100644 index 0000000000..2a75fb8a66 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/tests.rs @@ -0,0 +1,270 @@ +use std::collections::HashMap; +use std::io::{BufRead, BufReader, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use sha1::{Digest, Sha1}; + +use super::*; +use crate::epic::build_info::Cdn; +use crate::epic::chunk::testing::encode; +use crate::epic::manifest::testing::{build, chunk, TestFile, TestManifest}; +use crate::epic::manifest::{guid_text, PartRef}; + +fn serve(files: HashMap>, hits: Arc) -> String { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let files = Arc::new(files); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let files = Arc::clone(&files); + let hits = Arc::clone(&hits); + std::thread::spawn(move || answer(stream, &files, &hits)); + } + }); + format!("http://{addr}") +} + +fn answer(mut stream: TcpStream, files: &HashMap>, hits: &AtomicUsize) { + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + if reader.read_line(&mut first).is_err() { + return; + } + loop { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) | Err(_) => break, + Ok(_) if line == "\r\n" => break, + Ok(_) => {} + } + } + let path = first.split_whitespace().nth(1).unwrap_or("/").split('?').next().unwrap_or("/").to_string(); + hits.fetch_add(1, Ordering::Relaxed); + let (status, body) = match files.get(&path) { + Some(b) => ("200 OK", b.clone()), + None => ("404 Not Found", Vec::new()), + }; + let head = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + let _ = stream.flush(); +} + +#[derive(Default)] +struct Recorder { + logs: Mutex>, + replaced: AtomicUsize, + refuse: bool, + last_progress: Mutex<(u64, u64)>, + sizes: Mutex<(u64, u64)>, +} + +impl Events for Recorder { + fn log(&self, line: &str) { + self.logs.lock().unwrap().push(line.to_string()); + } + fn status(&self, _text: &str) {} + fn checking(&self, _d: u64, _t: u64, _f: u32, _ft: u32) {} + fn sizes(&self, pending: u64, disk: u64) { + *self.sizes.lock().unwrap() = (pending, disk); + } + fn progress(&self, written: u64, total: u64, _f: u32, _ft: u32) { + let mut p = self.last_progress.lock().unwrap(); + if written >= p.0 { + *p = (written, total); + } + } + fn installing(&self, _d: u64, _t: u64, _f: u32, _ft: u32) {} + fn before_replace(&self) -> bool { + self.replaced.fetch_add(1, Ordering::SeqCst); + !self.refuse + } +} + +struct Fixture { + base: String, + hits: Arc, + expected: Vec<(String, Vec)>, +} + +fn pattern(seed: u8, len: usize) -> Vec { + (0..len).map(|i| seed.wrapping_mul(31).wrapping_add((i * 7 % 251) as u8)).collect() +} + +fn fixture() -> Fixture { + let mut c1 = chunk(1, 4096); + let mut c2 = chunk(2, 2048); + let mut c3 = chunk(3, 1000); + let d1 = pattern(1, 4096); + let d2 = pattern(2, 2048); + let d3 = pattern(3, 1000); + c1.sha1 = Some(Sha1::digest(&d1).into()); + c2.sha1 = Some(Sha1::digest(&d2).into()); + c3.sha1 = Some(Sha1::digest(&d3).into()); + + let a: Vec = [&d1[..3000], &d2[100..1100]].concat(); + let c: Vec = [&d1[3000..], &d3[..]].concat(); + let d: Vec = d2[..100].to_vec(); + let sha = |b: &[u8]| -> [u8; 20] { Sha1::digest(b).into() }; + let files = vec![ + TestFile { + path: "a.bin".into(), + sha1: sha(&a), + tags: vec![], + parts: vec![ + PartRef { guid: c1.guid, offset: 0, size: 3000 }, + PartRef { guid: c2.guid, offset: 100, size: 1000 }, + ], + }, + TestFile { + path: "b\\c.bin".into(), + sha1: sha(&c), + tags: vec!["English".into()], + parts: vec![ + PartRef { guid: c1.guid, offset: 3000, size: 1096 }, + PartRef { guid: c3.guid, offset: 0, size: 1000 }, + ], + }, + TestFile { + path: "d.bin".into(), + sha1: sha(&d), + tags: vec![], + parts: vec![PartRef { guid: c2.guid, offset: 0, size: 100 }], + }, + TestFile { path: "e.txt".into(), sha1: [0; 20], tags: vec![], parts: vec![] }, + TestFile { + path: "fr.txt".into(), + sha1: [0; 20], + tags: vec!["French".into()], + parts: vec![PartRef { guid: c3.guid, offset: 0, size: 10 }], + }, + ]; + let m = TestManifest { + version: 21, + build: "9.9".into(), + exe: "a.bin".into(), + prereq: Some(("Redist".into(), "redist\\setup.exe".into())), + chunks: vec![c1.clone(), c2.clone(), c3.clone()], + files, + compress: true, + meta_tail: vec![], + }; + let manifest_bytes = build(&m); + let parsed = crate::epic::manifest::parse(&manifest_bytes).unwrap(); + let mut served = HashMap::new(); + served.insert("/Builds/x/test.manifest".to_string(), manifest_bytes); + for (info, data) in [(&c1, &d1), (&c2, &d2), (&c3, &d3)] { + served.insert(format!("/Builds/x/{}", parsed.chunk_path(info)), encode(data, true)); + } + let hits = Arc::new(AtomicUsize::new(0)); + let base = serve(served, Arc::clone(&hits)); + Fixture { + base, + hits, + expected: vec![ + ("a.bin".into(), a), + ("b/c.bin".into(), c), + ("d.bin".into(), d), + ("e.txt".into(), Vec::new()), + ], + } +} + +fn request(fx: &Fixture, game: &Path) -> Request { + Request { + info: BuildInfo { + cdns: vec![Cdn { base: fx.base.clone(), dir: "/Builds/x".into(), auth: String::new() }], + manifest_file: "test.manifest".into(), + build_version: "9.9".into(), + deployment_id: "dep".into(), + }, + game_dir: game.to_path_buf(), + scratch_dir: None, + storage_label: "Test storage".into(), + tags: vec!["English".into(), "en-US".into(), "en".into()], + max_workers: 4, + process_workers: 2, + ca_bundle: String::new(), + } +} + +fn temp_game(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("dd-epic-install-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + dir +} + +#[test] +fn fresh_install_then_rerun_then_repair() { + let fx = fixture(); + let game = temp_game("e2e"); + let cancel = AtomicBool::new(false); + + let events = Recorder::default(); + let done = run(&request(&fx, &game), &cancel, &events).unwrap(); + for (path, bytes) in &fx.expected { + assert_eq!(&fs::read(game.join(path)).unwrap(), bytes, "{path}"); + } + assert!(!game.join("fr.txt").exists()); + assert_eq!(done.build_version, "9.9"); + assert_eq!(done.launch_exe, "a.bin"); + assert_eq!(done.prereq_path, "redist/setup.exe"); + assert!(done.prereq_read); + assert_eq!(done.deployment_id, "dep"); + assert_eq!(done.disk_bytes, 4000 + 2096 + 100); + assert_eq!(events.replaced.load(Ordering::SeqCst), 1); + assert_eq!(*events.last_progress.lock().unwrap(), (6196, 6196)); + assert!(!game.join(BESIDE_CACHE_DIR).exists()); + assert!(!game.join(staging::DIR_NAME).exists()); + + let hits_before = fx.hits.load(Ordering::Relaxed); + let again = Recorder::default(); + run(&request(&fx, &game), &cancel, &again).unwrap(); + assert_eq!(fx.hits.load(Ordering::Relaxed) - hits_before, 1, "only the manifest is fetched"); + assert_eq!(again.sizes.lock().unwrap().0, 0); + assert_eq!(again.replaced.load(Ordering::SeqCst), 1); + + fs::write(game.join("b/c.bin"), b"damaged").unwrap(); + let repair = Recorder::default(); + run(&request(&fx, &game), &cancel, &repair).unwrap(); + assert_eq!(fs::read(game.join("b/c.bin")).unwrap(), fx.expected[1].1); + assert_eq!(repair.sizes.lock().unwrap().0, 2096 + 0, "the damaged file and the hashless empty file"); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn a_refused_update_switch_replaces_nothing() { + let fx = fixture(); + let game = temp_game("refuse"); + fs::create_dir_all(&game).unwrap(); + fs::write(game.join("a.bin"), b"old contents").unwrap(); + let cancel = AtomicBool::new(false); + let events = Recorder { refuse: true, ..Default::default() }; + let err = run(&request(&fx, &game), &cancel, &events).unwrap_err(); + assert!(matches!(err, Stop::Failed(_)), "{err:?}"); + assert_eq!(fs::read(game.join("a.bin")).unwrap(), b"old contents"); + let stage = game.join(staging::DIR_NAME); + let leftovers = fs::read_dir(&stage).map(|e| e.flatten().filter(|e| e.file_name() != "owner").count()).unwrap_or(0); + assert_eq!(leftovers, 0); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn a_cancelled_run_reports_cancel() { + let fx = fixture(); + let game = temp_game("cancel"); + let cancel = AtomicBool::new(true); + let events = Recorder::default(); + assert_eq!(run(&request(&fx, &game), &cancel, &events).unwrap_err(), Stop::Cancelled); + let _ = fs::remove_dir_all(&game); +} + +#[test] +fn guid_names_are_cache_names() { + let c = chunk(5, 10); + assert_eq!(cache::entry_name(&c, 0), guid_text(&c.guid)); + assert_eq!(cache::entry_name(&c, 2), format!("{}.2", guid_text(&c.guid))); +} diff --git a/app/src/main/rust/stores/src/epic/install/writer.rs b/app/src/main/rust/stores/src/epic/install/writer.rs new file mode 100644 index 0000000000..3a7e93a0e1 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/install/writer.rs @@ -0,0 +1,593 @@ +use std::fs::{self, File, OpenOptions}; +use std::os::unix::fs::FileExt; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use sha1::{Digest, Sha1}; + +use super::cache; +use super::schedule::{PlanPart, Schedule, NONE}; +use super::{staging, Events, Stop}; +use crate::epic::chunk; +use crate::epic::manifest::{Manifest, Sha1 as Hash}; +use crate::fetch_core::{run_fetch_gated, FetchItem, FetchOptions, FetchSink, SinkError}; + +const ADMIT_AHEAD: usize = 48; +const PROGRESS_EVERY_MS: u64 = 50; +const STUCK_AFTER: Duration = Duration::from_secs(10); + +pub struct EngineFile { + pub path: String, + pub dest: PathBuf, + pub size: u64, + pub sha1: Option, + pub replacing: bool, + pub parts: Vec, +} + +pub struct EngineInput<'a> { + pub manifest: &'a Manifest, + pub files: Vec, + pub schedule: Schedule, + pub stage_dir: PathBuf, + pub cache_dir: PathBuf, + pub prefixes: Vec, + pub overlap: Option, + pub narrow: bool, + pub max_workers: usize, + pub process_workers: usize, + pub ca_bundle: String, + pub user_agent: String, +} + +#[derive(Debug, Default)] +pub struct Stats { + pub bytes_written: u64, + pub files_written: usize, +} + +struct Slot { + file: Option>, + stage: Option, + written: Vec, + next_hash: usize, + hasher: Option, + left: usize, + done: bool, +} + +struct Coord { + next_file: usize, + open: usize, + open_replacing: u64, + processed: Vec, + remaining: Vec, + cached: Vec, + served: Vec, + fetch_done: bool, +} + +struct Engine<'a> { + input: &'a EngineInput<'a>, + slots: Vec>, + coord: Mutex, + wake: Condvar, + admitted: AtomicUsize, + hint: AtomicUsize, + stop: &'a AtomicBool, + failure: Mutex>, + written: AtomicU64, + files_done: AtomicUsize, + last_emit: AtomicU64, + started: Instant, + replace_ok: Mutex>, + events: &'a dyn Events, + total_bytes: u64, +} + +enum Source<'d> { + Memory(&'d [u8]), + Cache(PathBuf), +} + +impl<'a> Engine<'a> { + fn fail(&self, message: String) -> String { + let mut slot = self.failure.lock().expect("failure lock"); + if slot.is_none() { + *slot = Some(message.clone()); + } + drop(slot); + self.stop.store(true, Ordering::SeqCst); + self.wake.notify_all(); + message + } + + fn cache_path(&self, producer: usize) -> PathBuf { + let p = &self.input.schedule.producers[producer]; + let info = &self.input.manifest.chunks[p.chunk as usize]; + self.input.cache_dir.join(cache::entry_name(info, p.copy)) + } + + fn emit(&self, force: bool) { + let now = self.started.elapsed().as_millis() as u64; + let last = self.last_emit.load(Ordering::Relaxed); + if !force && now < last + PROGRESS_EVERY_MS { + return; + } + if self.last_emit.compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed).is_err() && !force { + return; + } + self.events.progress( + self.written.load(Ordering::Relaxed), + self.total_bytes, + self.files_done.load(Ordering::Relaxed) as u32, + self.input.files.len() as u32, + ); + } + + fn admissible(&self, c: &Coord, f: usize) -> bool { + let file = &self.input.files[f]; + if self.input.narrow { + return c.open == 0; + } + if !file.replacing { + return true; + } + match self.input.overlap { + None => true, + Some(budget) => c.open_replacing == 0 || c.open_replacing + file.size <= budget, + } + } + + fn admit_target(&self, fetch_done: bool) -> usize { + let fetch = &self.input.schedule.fetch; + if fetch_done || fetch.is_empty() { + return self.input.files.len(); + } + let at = (self.hint.load(Ordering::Relaxed) + ADMIT_AHEAD).min(fetch.len() - 1); + self.input.schedule.producers[fetch[at] as usize].anchor as usize + 1 + } + + fn write_use(&self, u: usize, source: &Source<'_>) -> Result<(), String> { + let schedule = &self.input.schedule; + let f = schedule.use_file[u] as usize; + let j = u - schedule.file_start[f] as usize; + let file = &self.input.files[f]; + let part = &file.parts[j]; + let loaded; + let data: &[u8] = match source { + Source::Memory(d) => d, + Source::Cache(path) => { + loaded = fs::read(path).map_err(|e| format!("chunk cache read for {}: {e}", file.path))?; + &loaded + } + }; + let start = part.chunk_offset as usize; + let end = start + part.size as usize; + if end > data.len() { + return Err(format!("A chunk is too short for {}", file.path)); + } + let piece = &data[start..end]; + let handle = self.slots[f].lock().expect("slot lock").file.clone(); + let Some(handle) = handle else { + return Err(format!("internal: {} is not open", file.path)); + }; + handle + .write_all_at(piece, part.file_offset) + .map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))?; + self.written.fetch_add(piece.len() as u64, Ordering::Relaxed); + + let complete = { + let mut slot = self.slots[f].lock().expect("slot lock"); + slot.written[j] = true; + if slot.hasher.is_some() { + while slot.next_hash < file.parts.len() && slot.written[slot.next_hash] { + let k = slot.next_hash; + if k == j { + slot.hasher.as_mut().expect("hasher").update(piece); + } else { + let other = &file.parts[k]; + let mut buf = vec![0u8; other.size as usize]; + handle + .read_exact_at(&mut buf, other.file_offset) + .map_err(|e| format!("Could not read back {}: {e}", file.path))?; + slot.hasher.as_mut().expect("hasher").update(&buf); + } + slot.next_hash += 1; + } + } + slot.left -= 1; + slot.left == 0 + }; + + let p = schedule.use_producer[u] as usize; + let drop_cache = { + let mut c = self.coord.lock().expect("coord lock"); + c.remaining[p] -= 1; + c.remaining[p] == 0 && c.cached[p] + }; + if drop_cache { + let _ = fs::remove_file(self.cache_path(p)); + } + if complete { + self.finish_file(f)?; + } + self.emit(false); + Ok(()) + } + + fn finish_file(&self, f: usize) -> Result<(), String> { + let file = &self.input.files[f]; + let (handle, stage, hash) = { + let mut slot = self.slots[f].lock().expect("slot lock"); + slot.done = true; + (slot.file.take(), slot.stage.clone(), slot.hasher.take().map(|h| -> Hash { h.finalize().into() })) + }; + let stage = stage.ok_or_else(|| format!("internal: {} has no stage file", file.path))?; + let result = (|| -> Result<(), String> { + let handle = handle.ok_or_else(|| format!("internal: {} is not open", file.path))?; + handle + .sync_all() + .map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))?; + let len = handle.metadata().map(|m| m.len()).unwrap_or(u64::MAX); + drop(handle); + if len != file.size { + return Err(format!("Size mismatch for {}", file.path)); + } + if let (Some(expected), Some(got)) = (file.sha1, hash) { + if expected != got { + return Err(format!("SHA-1 mismatch for {}", file.path)); + } + } + if self.stop.load(Ordering::SeqCst) { + return Err(String::new()); + } + { + let mut ok = self.replace_ok.lock().expect("replace lock"); + if ok.is_none() { + *ok = Some(self.events.before_replace()); + } + if *ok != Some(true) { + return Err("The install record could not be switched to updating; nothing was replaced".into()); + } + } + if let Some(parent) = file.dest.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))?; + } + fs::rename(&stage, &file.dest) + .map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path)) + })(); + if let Err(e) = result { + let _ = fs::remove_file(&stage); + return Err(e); + } + { + let mut c = self.coord.lock().expect("coord lock"); + c.open -= 1; + if file.replacing { + c.open_replacing = c.open_replacing.saturating_sub(file.size); + } + } + self.files_done.fetch_add(1, Ordering::Relaxed); + self.wake.notify_all(); + self.emit(true); + Ok(()) + } + + fn open_stage(&self, f: usize) -> Result<(), String> { + let path = self.input.stage_dir.join(staging::unique_name()); + let handle = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .map_err(|e| format!("Could not stage {}: {e}", self.input.files[f].path))?; + let mut slot = self.slots[f].lock().expect("slot lock"); + slot.file = Some(Arc::new(handle)); + slot.stage = Some(path); + Ok(()) + } + + fn admit_loop(&self) { + let n = self.input.files.len(); + let mut last_change = Instant::now(); + let mut last_seen = (usize::MAX, usize::MAX); + loop { + if self.stop.load(Ordering::SeqCst) { + return; + } + let done = self.files_done.load(Ordering::Relaxed); + if done == n { + return; + } + let candidate = { + let c = self.coord.lock().expect("coord lock"); + let target = self.admit_target(c.fetch_done); + if c.next_file < target.min(n) && self.admissible(&c, c.next_file) { + Some(c.next_file) + } else { + if c.fetch_done && c.next_file == n { + let seen = (done, self.written.load(Ordering::Relaxed) as usize); + if seen != last_seen { + last_seen = seen; + last_change = Instant::now(); + } else if last_change.elapsed() > STUCK_AFTER { + drop(c); + self.fail("internal: some files never completed".into()); + return; + } + } + let _ = self.wake.wait_timeout(c, Duration::from_millis(20)).expect("coord lock"); + None + } + }; + let Some(f) = candidate else { continue }; + if let Err(e) = self.admit(f) { + self.fail(e); + return; + } + } + } + + fn admit(&self, f: usize) -> Result<(), String> { + self.open_stage(f)?; + let schedule = &self.input.schedule; + let file = &self.input.files[f]; + let claims: Vec = { + let mut c = self.coord.lock().expect("coord lock"); + c.next_file += 1; + c.open += 1; + if file.replacing { + c.open_replacing += file.size; + } + let mut out = Vec::new(); + for u in schedule.uses_of_file(f) { + let p = schedule.use_producer[u] as usize; + if c.processed[p] && !c.served[u] { + c.served[u] = true; + out.push(u); + } + } + out + }; + self.admitted.store(f + 1, Ordering::SeqCst); + for u in claims { + let p = schedule.use_producer[u] as usize; + self.write_use(u, &Source::Cache(self.cache_path(p)))?; + } + if file.parts.is_empty() { + self.finish_file(f)?; + } + Ok(()) + } + + fn deliver(&self, fetch_index: usize, body: Vec) -> Result { + if self.stop.load(Ordering::SeqCst) { + return Ok(0); + } + let schedule = &self.input.schedule; + let p = schedule.fetch[fetch_index] as usize; + let producer = &schedule.producers[p]; + let info = &self.input.manifest.chunks[producer.chunk as usize]; + let data = chunk::decode(&body, info).map_err(SinkError::Retry)?; + drop(body); + let needs_cache = { + let c = self.coord.lock().expect("coord lock"); + producer.uses.iter().any(|&u| schedule.use_file[u as usize] as usize >= c.next_file) + }; + if needs_cache { + cache::store(&self.input.cache_dir, &cache::entry_name(info, producer.copy), &data) + .map_err(|e| SinkError::Fatal(self.fail(e)))?; + } + let claims: Vec = { + let mut c = self.coord.lock().expect("coord lock"); + c.processed[p] = true; + c.cached[p] = needs_cache; + let mut out = Vec::new(); + for &u in &producer.uses { + let u = u as usize; + if (schedule.use_file[u] as usize) < c.next_file && !c.served[u] { + c.served[u] = true; + out.push(u); + } + } + out + }; + for u in claims { + self.write_use(u, &Source::Memory(&data)).map_err(|e| SinkError::Fatal(self.fail(e)))?; + } + Ok(0) + } +} + +struct Sink<'e, 'a>(&'e Engine<'a>); + +impl FetchSink for Sink<'_, '_> { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + self.0.deliver(item.id as usize, body) + } +} + +pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> Result { + let schedule = &input.schedule; + let total_uses = schedule.use_file.len(); + let mut remaining = vec![0u32; schedule.producers.len()]; + for &p in &schedule.use_producer { + if p != NONE { + remaining[p as usize] += 1; + } + } + let mut processed = vec![false; schedule.producers.len()]; + let mut cached = vec![false; schedule.producers.len()]; + for (i, p) in schedule.producers.iter().enumerate() { + if p.preloaded { + processed[i] = true; + cached[i] = true; + } + } + let engine = Engine { + input, + slots: input + .files + .iter() + .map(|f| { + Mutex::new(Slot { + file: None, + stage: None, + written: vec![false; f.parts.len()], + next_hash: 0, + hasher: f.sha1.map(|_| Sha1::new()), + left: f.parts.len(), + done: false, + }) + }) + .collect(), + coord: Mutex::new(Coord { + next_file: 0, + open: 0, + open_replacing: 0, + processed, + remaining: remaining.clone(), + cached, + served: vec![false; total_uses], + fetch_done: false, + }), + wake: Condvar::new(), + admitted: AtomicUsize::new(0), + hint: AtomicUsize::new(0), + stop, + failure: Mutex::new(None), + written: AtomicU64::new(0), + files_done: AtomicUsize::new(0), + last_emit: AtomicU64::new(0), + started: Instant::now(), + replace_ok: Mutex::new(None), + events, + total_bytes: input.files.iter().map(|f| f.size).sum(), + }; + for (i, p) in schedule.producers.iter().enumerate() { + if p.preloaded && remaining[i] == 0 { + let _ = fs::remove_file(engine.cache_path(i)); + } + } + + let hosts: Vec = input.prefixes.iter().map(|p| crate::epic::net::origin(p)).collect(); + let items: Vec = schedule + .fetch + .iter() + .enumerate() + .map(|(i, &p)| { + let info = &input.manifest.chunks[schedule.producers[p as usize].chunk as usize]; + let path = input.manifest.chunk_path(info); + FetchItem { + id: i as u64, + urls: input.prefixes.iter().map(|prefix| format!("{prefix}/{path}")).collect(), + reserve: info.cdn_size, + range: None, + } + }) + .collect(); + let distinct = crate::fetch_core::distinct_host_count(&hosts); + let workers = input.max_workers.clamp(1, 128); + let options = FetchOptions { + max_workers: workers, + per_host_cap: 6usize.max(workers.div_ceil(distinct.max(1))), + timeout: Duration::from_secs(60), + headers: vec![("User-Agent".to_string(), input.user_agent.clone())], + ca_bundle_path: input.ca_bundle.clone(), + process_workers: input.process_workers.max(1), + label: "epic".to_string(), + stream: false, + }; + + events.progress(0, engine.total_bytes, 0, input.files.len() as u32); + let log = |line: &str| events.log(line); + let gate = |i: usize| -> bool { + engine.hint.fetch_max(i, Ordering::Relaxed); + let p = schedule.fetch[i] as usize; + let ready = (schedule.producers[p].anchor as usize) < engine.admitted.load(Ordering::SeqCst); + if !ready { + engine.wake.notify_all(); + } + ready + }; + let outcome = std::thread::scope(|scope| { + let admitter = std::thread::Builder::new() + .name("epic-admit".into()) + .spawn_scoped(scope, || { + crate::priority::background(); + engine.admit_loop(); + }); + if let Err(e) = admitter { + engine.fail(format!("could not start the install thread: {e}")); + return None; + } + let outcome = if items.is_empty() { + None + } else { + Some(run_fetch_gated(items, &hosts, &options, &Sink(&engine), stop, &|_: u64, _: u64| {}, &log, &gate)) + }; + { + let mut c = engine.coord.lock().expect("coord lock"); + c.fetch_done = true; + } + if outcome.as_ref().is_some_and(|o| !o.ok()) { + stop.store(true, Ordering::SeqCst); + } + engine.wake.notify_all(); + outcome + }); + + let failure = engine.failure.lock().expect("failure lock").take().filter(|m| !m.is_empty()); + let fetch_error = outcome.as_ref().and_then(|o| o.error.clone()); + let complete = failure.is_none() && engine.files_done.load(Ordering::Relaxed) == input.files.len(); + + if !complete { + for slot in &engine.slots { + let mut s = slot.lock().expect("slot lock"); + if !s.done { + s.file.take(); + if let Some(stage) = s.stage.take() { + let _ = fs::remove_file(stage); + } + } + } + cache::remove_partials(&input.cache_dir); + } + + let elapsed = engine.started.elapsed().as_secs_f64().max(0.001); + let written = engine.written.load(Ordering::Relaxed); + events.log(&format!( + "summary mode={} files={}/{} bytes_written={written} elapsed_s={elapsed:.1} avg_mbps={:.1} \ +fetches={} refetches={} peak_cache_mib={} hosts={}", + if input.narrow { "narrow" } else { "adaptive" }, + engine.files_done.load(Ordering::Relaxed), + input.files.len(), + written as f64 / 1_048_576.0 / elapsed, + schedule.fetch.len(), + schedule.refetches, + schedule.peak_cache / 1_048_576, + hosts.join(","), + )); + + if let Some(message) = failure { + return Err(Stop::Failed(message)); + } + if complete { + return Ok(Stats { bytes_written: written, files_written: input.files.len() }); + } + if let Some(message) = fetch_error { + return Err(Stop::Failed(format!("Download failed: {}", tidy_fetch_error(&message)))); + } + if stop.load(Ordering::SeqCst) { + return Err(Stop::Cancelled); + } + Err(Stop::Failed("The download ended before every file was written".into())) +} + +fn tidy_fetch_error(message: &str) -> String { + message.strip_prefix("epic: ").unwrap_or(message).to_string() +} diff --git a/app/src/main/rust/stores/src/epic/jni.rs b/app/src/main/rust/stores/src/epic/jni.rs deleted file mode 100644 index 29edbe4b7b..0000000000 --- a/app/src/main/rust/stores/src/epic/jni.rs +++ /dev/null @@ -1,708 +0,0 @@ - -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread; - -use jni::objects::{GlobalRef, JByteArray, JClass, JIntArray, JObject, JObjectArray, JString, JValue}; -use jni::sys::{jint, jlong, JNI_FALSE, JNI_TRUE}; -use jni::{JNIEnv, JavaVM}; - -use super::assemble::{build_assemble_plan, run_assemble, AssembleRequest}; -use super::driver::{build_plan, run_plan, EpicRequest}; -use super::stream::{build_stream_plan, run_stream, Hooks, StreamProgress, StreamRequest}; - -#[cfg(target_os = "android")] -#[link(name = "log")] -unsafe extern "C" { - fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32; -} - -fn android_log(message: &str) { - #[cfg(target_os = "android")] - { - use std::ffi::CString; - let Ok(tag) = CString::new(super::LOG_TAG) else { - return; - }; - let sanitized = message.replace('\0', " "); - let Ok(text) = CString::new(sanitized) else { - return; - }; - unsafe { - let _ = __android_log_write(4, tag.as_ptr().cast(), text.as_ptr().cast()); - } - } - #[cfg(not(target_os = "android"))] - { - let _ = message; - } -} - -pub struct EpicDownloadHandle { - cancel: Arc, - worker: Option>, -} - -fn clear_pending_exception(env: &mut JNIEnv) { - if env.exception_check().unwrap_or(false) { - let _ = env.exception_clear(); - } -} - -fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option { - env.get_string(value) - .ok() - .map(|s| s.to_string_lossy().into_owned()) -} - -fn string_array_to_vec(env: &mut JNIEnv, array: &JObjectArray) -> Vec { - let len = env.get_array_length(array).unwrap_or(0); - let mut out = Vec::with_capacity(len.max(0) as usize); - let mut i = 0; - while i < len { - match env.get_object_array_element(array, i) { - Ok(obj) => { - if obj.is_null() { - out.push(String::new()); - } else { - let js = JString::from(obj); - out.push(jstring_to_string(env, &js).unwrap_or_default()); - } - } - Err(_) => { - clear_pending_exception(env); - out.push(String::new()); - } - } - i += 1; - } - out -} - -fn int_array_to_vec(env: &JNIEnv, array: &JIntArray) -> Vec { - let len = env.get_array_length(array).unwrap_or(0); - if len <= 0 { - return Vec::new(); - } - let mut values = vec![0i32; len as usize]; - if env.get_int_array_region(array, 0, &mut values).is_err() { - return Vec::new(); - } - values -} - -fn call_on_log(env: &mut JNIEnv, listener: &JObject, line: &str) { - if listener.is_null() { - return; - } - let Ok(s) = env.new_string(line) else { - clear_pending_exception(env); - return; - }; - let obj = JObject::from(s); - let _ = env.call_method( - listener, - "onLog", - "(Ljava/lang/String;)V", - &[JValue::Object(&obj)], - ); - clear_pending_exception(env); -} - -fn call_on_plan(env: &mut JNIEnv, listener: &JObject, chunks_total: u64, bytes_total: u64, chunk_dir: &str) { - if listener.is_null() { - return; - } - let Ok(s) = env.new_string(chunk_dir) else { - clear_pending_exception(env); - return; - }; - let obj = JObject::from(s); - let _ = env.call_method( - listener, - "onPlan", - "(IJLjava/lang/String;)V", - &[ - JValue::Int(chunks_total.min(i32::MAX as u64) as jint), - JValue::Long(bytes_total as jlong), - JValue::Object(&obj), - ], - ); - clear_pending_exception(env); -} - -fn call_on_progress( - env: &mut JNIEnv, - listener: &JObject, - bytes_done: u64, - bytes_total: u64, - chunks_done: u64, - chunks_total: u64, -) { - if listener.is_null() { - return; - } - let _ = env.call_method( - listener, - "onProgress", - "(JJII)V", - &[ - JValue::Long(bytes_done as jlong), - JValue::Long(bytes_total as jlong), - JValue::Int(chunks_done.min(i32::MAX as u64) as jint), - JValue::Int(chunks_total.min(i32::MAX as u64) as jint), - ], - ); - clear_pending_exception(env); -} - -fn call_on_complete(env: &mut JNIEnv, listener: &JObject, success: bool, error: &str, bytes: u64) { - if listener.is_null() { - return; - } - let Ok(s) = env.new_string(error) else { - clear_pending_exception(env); - return; - }; - let obj = JObject::from(s); - let _ = env.call_method( - listener, - "onComplete", - "(ZLjava/lang/String;J)V", - &[ - JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }), - JValue::Object(&obj), - JValue::Long(bytes as jlong), - ], - ); - clear_pending_exception(env); -} - -fn log_both(env: &mut JNIEnv, listener: &JObject, line: &str) { - android_log(line); - call_on_log(env, listener, line); -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart( - mut env: JNIEnv, - _class: JClass, - manifest: JByteArray, - install_dir: JString, - chunk_cache_dir: JString, - cdn_prefixes: JObjectArray, - pending_file_idx: JIntArray, - expected_chunks: jint, - expected_bytes: jlong, - ca_bundle_path: JString, - max_workers: jint, - process_workers: jint, - listener: JObject, -) -> jlong { - if listener.is_null() { - android_log("nativeStart: null listener"); - return 0; - } - let Ok(manifest_bytes) = env.convert_byte_array(manifest) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "manifest bytes unreadable", 0); - return 0; - }; - let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); - let chunk_cache_dir = jstring_to_string(&mut env, &chunk_cache_dir).unwrap_or_default(); - let cdn_prefixes = string_array_to_vec(&mut env, &cdn_prefixes); - let pending_file_indices: Vec = int_array_to_vec(&env, &pending_file_idx) - .into_iter() - .map(|i| i.max(0) as usize) - .collect(); - let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); - let max_workers = max_workers.max(1) as usize; - let process_workers = process_workers.max(1) as usize; - let label = { - let base = std::path::Path::new(&install_dir) - .file_name() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_default(); - format!("epic app={base}") - }; - - let host_count = super::plan::distinct_prefixes(&cdn_prefixes).len(); - let host_cap = super::plan::per_host_cap(max_workers, host_count); - log_both( - &mut env, - &listener, - &format!( - "engine=rust label=\"{label}\" install_dir={install_dir} cache_dir={} cdns={} hosts={host_count} pending_files={} workers={max_workers} per_host_cap={host_cap} process_workers={process_workers} manifest_bytes={}", - if chunk_cache_dir.is_empty() { "/.chunks" } else { chunk_cache_dir.as_str() }, - cdn_prefixes.len(), - pending_file_indices.len(), - manifest_bytes.len() - ), - ); - - let req = EpicRequest { - manifest_bytes, - install_dir, - chunk_cache_dir, - cdn_prefixes, - pending_file_indices, - expected_chunks: if expected_chunks >= 0 { - Some(expected_chunks as u64) - } else { - None - }, - expected_bytes: if expected_bytes >= 0 { - Some(expected_bytes as u64) - } else { - None - }, - ca_bundle_path, - max_workers, - process_workers, - label, - }; - - let plan = match build_plan(&req) { - Ok(plan) => plan, - Err(err) => { - log_both(&mut env, &listener, &format!("not started: {err}")); - call_on_complete(&mut env, &listener, false, &err, 0); - return 0; - } - }; - call_on_plan( - &mut env, - &listener, - plan.needed.len() as u64, - plan.total_bytes, - &plan.manifest.chunk_dir, - ); - - let Ok(vm) = env.get_java_vm() else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); - return 0; - }; - let Ok(listener) = env.new_global_ref(&listener) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "listener ref failed", 0); - return 0; - }; - - let cancel = Arc::new(AtomicBool::new(false)); - let thread_cancel = Arc::clone(&cancel); - let worker = thread::spawn(move || { - crate::priority::background(); - run_on_thread(vm, listener, plan, req, thread_cancel); - }); - let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); - - Box::into_raw(handle) as jlong -} - -fn run_on_thread( - vm: JavaVM, - listener: GlobalRef, - plan: super::driver::EpicPlan, - req: EpicRequest, - cancel: Arc, -) { - let bytes_total = plan.total_bytes; - let chunks_total = plan.needed.len() as u64; - - let progress = |bytes_done: u64, chunks_done: u64| { - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_progress( - &mut env, - listener.as_obj(), - bytes_done, - bytes_total, - chunks_done, - chunks_total, - ); - }; - let log = |line: &str| { - android_log(line); - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_log(&mut env, listener.as_obj(), line); - }; - - let outcome = run_plan(&plan, &req, cancel.as_ref(), &progress, &log); - - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_complete( - &mut env, - listener.as_obj(), - outcome.success, - &outcome.error, - outcome.bytes_credited, - ); -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble( - mut env: JNIEnv, - _class: JClass, - manifest: JByteArray, - install_dir: JString, - chunk_cache_dir: JString, - file_idx: JIntArray, - listener: JObject, -) -> jlong { - if listener.is_null() { - android_log("nativeAssemble: null listener"); - return 0; - } - let Ok(manifest_bytes) = env.convert_byte_array(manifest) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "manifest bytes unreadable", 0); - return 0; - }; - let req = AssembleRequest { - manifest_bytes, - install_dir: jstring_to_string(&mut env, &install_dir).unwrap_or_default(), - chunk_cache_dir: jstring_to_string(&mut env, &chunk_cache_dir).unwrap_or_default(), - file_indices: int_array_to_vec(&env, &file_idx) - .into_iter() - .map(|i| i.max(0) as usize) - .collect(), - }; - let plan = match build_assemble_plan(&req) { - Ok(plan) => plan, - Err(err) => { - log_both(&mut env, &listener, &format!("not started: {err}")); - call_on_complete(&mut env, &listener, false, &err, 0); - return 0; - } - }; - let Ok(vm) = env.get_java_vm() else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); - return 0; - }; - let Ok(listener) = env.new_global_ref(&listener) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "listener ref failed", 0); - return 0; - }; - - let cancel = Arc::new(AtomicBool::new(false)); - let thread_cancel = Arc::clone(&cancel); - let worker = thread::spawn(move || { - crate::priority::background(); - let progress = |bytes_done: u64, bytes_total: u64, files_done: u64, files_total: u64| { - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_progress(&mut env, listener.as_obj(), bytes_done, bytes_total, files_done, files_total); - }; - let log = |line: &str| { - android_log(line); - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_log(&mut env, listener.as_obj(), line); - }; - let outcome = run_assemble(&plan, thread_cancel.as_ref(), &progress, &log); - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_complete( - &mut env, - listener.as_obj(), - outcome.success, - &outcome.error, - outcome.bytes_written, - ); - }); - let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); - - Box::into_raw(handle) as jlong -} - -fn call_on_stream_progress(env: &mut JNIEnv, listener: &JObject, p: &StreamProgress) { - if listener.is_null() { - return; - } - let _ = env.call_method( - listener, - "onStreamProgress", - "(JJIIJ)V", - &[ - JValue::Long(p.bytes_written as jlong), - JValue::Long(p.bytes_total as jlong), - JValue::Int(p.files_done.min(i32::MAX as u64) as jint), - JValue::Int(p.files_total.min(i32::MAX as u64) as jint), - JValue::Long(p.files_bytes as jlong), - ], - ); - clear_pending_exception(env); -} - -/// `boolean onBeforeReplace()`: the Java side marks an existing install `updating`. A thrown -/// exception or `false` is a refusal. -fn call_on_before_replace(env: &mut JNIEnv, listener: &JObject) -> Result<(), String> { - if listener.is_null() { - return Ok(()); - } - let result = env - .call_method(listener, "onBeforeReplace", "()Z", &[]) - .and_then(|v| v.z()); - let threw = env.exception_check().unwrap_or(false); - clear_pending_exception(env); - match result { - Ok(true) if !threw => Ok(()), - _ => Err("could not mark the install as updating".to_string()), - } -} - -/// `EpicNative.nativeStream(...)`: the adaptive install (see `stream.rs`). Parse, plan and cache -/// preparation run on the calling thread, so a refusal returns 0 (after a synchronous -/// `onComplete(false, reason, 0)`) before anything is fetched or written, and the manager runs -/// the bounded Java writer instead. The run itself is on a new thread; `onComplete` fires after -/// every unfinished staging file of a cancelled or failed run is gone. -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream( - mut env: JNIEnv, - _class: JClass, - manifest: JByteArray, - install_dir: JString, - chunk_cache_dir: JString, - staging_dir: JString, - cdn_prefixes: JObjectArray, - pending_file_idx: JIntArray, - expected_bytes: jlong, - cache_budget: jlong, - overlap_budget: jlong, - ca_bundle_path: JString, - max_workers: jint, - process_workers: jint, - listener: JObject, -) -> jlong { - if listener.is_null() { - android_log("nativeStream: null listener"); - return 0; - } - let Ok(manifest_bytes) = env.convert_byte_array(manifest) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "manifest bytes unreadable", 0); - return 0; - }; - let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); - let chunk_cache_dir = jstring_to_string(&mut env, &chunk_cache_dir).unwrap_or_default(); - let staging_dir = jstring_to_string(&mut env, &staging_dir).unwrap_or_default(); - let cdn_prefixes = string_array_to_vec(&mut env, &cdn_prefixes); - let pending_file_indices: Vec = int_array_to_vec(&env, &pending_file_idx) - .into_iter() - .map(|i| i.max(0) as usize) - .collect(); - let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); - let max_workers = max_workers.max(1) as usize; - let process_workers = process_workers.max(1) as usize; - let label = { - let base = std::path::Path::new(&install_dir) - .file_name() - .map(|s| s.to_string_lossy().into_owned()) - .unwrap_or_default(); - format!("epic app={base}") - }; - log_both( - &mut env, - &listener, - &format!( - "engine=rust-stream label=\"{label}\" install_dir={install_dir} cache_dir={} cdns={} pending_files={} cache_budget={} workers={max_workers} process_workers={process_workers} manifest_bytes={}", - if chunk_cache_dir.is_empty() { "/.chunks" } else { chunk_cache_dir.as_str() }, - cdn_prefixes.len(), - pending_file_indices.len(), - cache_budget.max(0), - manifest_bytes.len() - ), - ); - - let req = StreamRequest { - manifest_bytes, - install_dir, - chunk_cache_dir, - staging_dir, - cdn_prefixes, - pending_file_indices, - expected_bytes: if expected_bytes >= 0 { - Some(expected_bytes as u64) - } else { - None - }, - cache_budget: cache_budget.max(0) as u64, - // Negative = no limit (nothing the manager knows of is being replaced). - replace_overlap_budget: if overlap_budget < 0 { - u64::MAX - } else { - overlap_budget as u64 - }, - ca_bundle_path, - max_workers, - process_workers, - label, - }; - let plan = match build_stream_plan(&req) { - Ok(plan) => plan, - Err(err) => { - log_both(&mut env, &listener, &format!("not started: {err}")); - call_on_complete(&mut env, &listener, false, &err, 0); - return 0; - } - }; - let Ok(vm) = env.get_java_vm() else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); - return 0; - }; - let Ok(listener) = env.new_global_ref(&listener) else { - clear_pending_exception(&mut env); - call_on_complete(&mut env, &listener, false, "listener ref failed", 0); - return 0; - }; - - let cancel = Arc::new(AtomicBool::new(false)); - let thread_cancel = Arc::clone(&cancel); - let worker = thread::spawn(move || { - crate::priority::background(); - let progress = |p: &StreamProgress| { - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_stream_progress(&mut env, listener.as_obj(), p); - }; - let before_replace = || { - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return Err("JavaVM attach failed".to_string()); - }; - call_on_before_replace(&mut env, listener.as_obj()) - }; - let log = |line: &str| { - android_log(line); - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_log(&mut env, listener.as_obj(), line); - }; - let hooks = Hooks { - progress: &progress, - before_replace: &before_replace, - log: &log, - }; - let outcome = run_stream(&plan, &req, thread_cancel.as_ref(), &hooks); - let Ok(mut env) = vm.attach_current_thread_as_daemon() else { - return; - }; - call_on_complete( - &mut env, - listener.as_obj(), - outcome.success, - &outcome.error, - outcome.bytes_written, - ); - }); - let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); - - Box::into_raw(handle) as jlong -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCancel( - _env: JNIEnv, - _class: JClass, - handle: jlong, -) { - if handle == 0 { - return; - } - let handle = unsafe { &*(handle as *const EpicDownloadHandle) }; - handle.cancel.store(true, Ordering::Relaxed); - android_log("cancel requested"); -} - -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeRelease( - _env: JNIEnv, - _class: JClass, - handle: jlong, -) { - if handle == 0 { - return; - } - let mut handle = unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) }; - if let Some(worker) = handle.worker.take() { let _ = worker.join(); } -} - -/// ABI v1 snapshot: selected chunks/files, verified cache lengths, old output lengths and both -/// measured free-space probes. Parse/budget failures are terminal, never fallback requests. -#[no_mangle] -pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive( - mut env: JNIEnv, _class: JClass, request: JString, listener: JObject, -) -> jlong { - if listener.is_null() { return 0; } - let parsed = jstring_to_string(&mut env, &request).ok_or("unreadable adaptive request".to_string()) - .and_then(|s| super::adaptive::Request::parse(&s)) - .and_then(|req| super::adaptive::choose_budget(&req).map(|budget| (req, budget))); - let (req, budget) = match parsed { - Ok(plan) => plan, - Err(error) => { - call_on_complete(&mut env, &listener, false, &error, 0); - return 0; - } - }; - call_on_plan(&mut env, &listener, req.chunks.len() as u64, 0, - &format!("lookahead={} retained={} game_peak={} cache_peak={}", - budget.lookahead, budget.retained, budget.game_peak, budget.cache_peak)); - let Ok(vm) = env.get_java_vm() else { - call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); - return 0; - }; - let Ok(listener) = env.new_global_ref(&listener) else { - call_on_complete(&mut env, &listener, false, "listener ref failed", 0); - return 0; - }; - let cancel = Arc::new(AtomicBool::new(false)); - let thread_cancel = cancel.clone(); - let worker = thread::spawn(move || { - crate::priority::background(); - let progress = |done, total, files, files_total| { - if let Ok(mut env) = vm.attach_current_thread_as_daemon() { - call_on_progress(&mut env, listener.as_obj(), done, total, files, files_total); - } - }; - let before_swap = || -> Result<(), String> { - let mut env = vm.attach_current_thread_as_daemon().map_err(|e| e.to_string())?; - match env.call_method(listener.as_obj(), "beforeSwap", "()Z", &[]) { - Ok(value) => if value.z().unwrap_or(false) { Ok(()) } else { Err("Could not mark installed record updating before replacement".into()) }, - _ => { - clear_pending_exception(&mut env); - Err("Could not mark installed record updating before replacement".into()) - } - } - }; - let log = |line: &str| { - android_log(line); - if let Ok(mut env) = vm.attach_current_thread_as_daemon() { - call_on_log(&mut env, listener.as_obj(), line); - } - }; - let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| - super::adaptive::run(&req, budget, &thread_cancel, &before_swap, &progress, &log))) - .unwrap_or_else(|_| Err("Native Epic install panicked".into())); - let (success, error, bytes) = match outcome { - Ok(bytes) => (true, String::new(), bytes), - Err(error) => (false, error, 0), - }; - if let Ok(mut env) = vm.attach_current_thread_as_daemon() { - call_on_complete(&mut env, listener.as_obj(), success, &error, bytes); - } - }); - Box::into_raw(Box::new(EpicDownloadHandle { cancel, worker: Some(worker) })) as jlong -} diff --git a/app/src/main/rust/stores/src/epic/launch.rs b/app/src/main/rust/stores/src/epic/launch.rs new file mode 100644 index 0000000000..481e9b8255 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/launch.rs @@ -0,0 +1,77 @@ +#[derive(Clone, Debug, Default)] +pub struct LaunchData { + pub display_name: String, + pub account_id: String, + pub namespace: String, + pub locale: String, + pub app_name: String, + pub deployment_id: String, + pub additional: String, +} + +fn clean(value: &str) -> String { + value.chars().filter(|c| !matches!(c, '\r' | '\n' | '\0')).collect() +} + +pub fn arguments(d: &LaunchData) -> Vec { + let name = clean(&d.display_name).replace('"', ""); + let name = if name.trim().is_empty() { "EpicUser".to_string() } else { name }; + let account = Some(clean(&d.account_id)).filter(|a| !a.is_empty()).unwrap_or_else(|| "0".into()); + let locale = Some(clean(&d.locale)).filter(|l| !l.is_empty()).unwrap_or_else(|| "en".into()); + let mut args = vec![ + "-EpicPortal".to_string(), + format!("-epicusername=\"{name}\""), + format!("-epicuserid={account}"), + format!("-epicsandboxid={}", clean(&d.namespace)), + format!("-epiclocale={locale}"), + format!("-epicapp={}", clean(&d.app_name)), + "-epicenv=Prod".to_string(), + ]; + let deployment = clean(&d.deployment_id); + if !deployment.is_empty() { + args.push(format!("-epicdeploymentid={deployment}")); + } + let extra: String = d.additional.chars().filter(|&c| c >= ' ' && c != '\u{7f}').collect(); + let extra = extra.trim(); + if !extra.is_empty() { + args.push(extra.to_string()); + } + args +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arguments_in_order() { + let args = arguments(&LaunchData { + display_name: "Some \"One\"\r\n".into(), + account_id: "acc".into(), + namespace: "ns".into(), + locale: "fr".into(), + app_name: "Fig".into(), + deployment_id: "dep".into(), + additional: " -nosplash\t-dx12 ".into(), + }); + assert_eq!( + args, + vec![ + "-EpicPortal", + "-epicusername=\"Some One\"", + "-epicuserid=acc", + "-epicsandboxid=ns", + "-epiclocale=fr", + "-epicapp=Fig", + "-epicenv=Prod", + "-epicdeploymentid=dep", + "-nosplash-dx12", + ] + ); + let bare = arguments(&LaunchData { namespace: "ns".into(), app_name: "A".into(), ..Default::default() }); + assert_eq!(bare[1], "-epicusername=\"EpicUser\""); + assert_eq!(bare[2], "-epicuserid=0"); + assert_eq!(bare[4], "-epiclocale=en"); + assert_eq!(bare.len(), 7); + } +} diff --git a/app/src/main/rust/stores/src/epic/library.rs b/app/src/main/rust/stores/src/epic/library.rs new file mode 100644 index 0000000000..80f06e3a4b --- /dev/null +++ b/app/src/main/rust/stores/src/epic/library.rs @@ -0,0 +1,116 @@ +use serde_json::Value; + +use super::net::{self, Request}; + +const ITEMS_URL: &str = + "https://library-service.live.use1a.on.epicgames.com/library/api/public/items?includeMetadata=true"; +const MAX_PAGES: usize = 2000; +const MARKETPLACE_NAMESPACES: &[&str] = &["ue", "89efe5924d3d467c839449ab6ab52e7f"]; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Owned { + pub app_name: String, + pub namespace: String, + pub catalog_item_id: String, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ListError { + SignIn, + Failed(String), +} + +pub fn keep_record(record: &Value) -> Option { + let app_name = net::str_of(record, "appName"); + let namespace = net::str_of(record, "namespace"); + if app_name.is_empty() || app_name == "1" || MARKETPLACE_NAMESPACES.contains(&namespace) { + return None; + } + if net::str_of(record, "sandboxType").eq_ignore_ascii_case("PRIVATE") { + return None; + } + if let Some(platforms) = record.get("platform").and_then(|p| p.as_array()) { + let names: Vec<&str> = platforms.iter().filter_map(|p| p.as_str()).collect(); + if !names.is_empty() && !names.iter().any(|p| *p == "Windows" || *p == "Win32") { + return None; + } + } + Some(Owned { + app_name: app_name.to_string(), + namespace: namespace.to_string(), + catalog_item_id: net::str_of(record, "catalogItemId").to_string(), + }) +} + +pub fn read_page(body: &Value, cursor: Option<&str>) -> (Vec, Option) { + let records = body + .get("records") + .and_then(|r| r.as_array()) + .map(|a| a.iter().filter_map(keep_record).collect()) + .unwrap_or_default(); + let next = body + .get("responseMetadata") + .map(|m| net::str_of(m, "nextCursor")) + .filter(|c| !c.is_empty() && Some(*c) != cursor) + .map(str::to_string); + (records, next) +} + +pub fn list_owned(access_token: &str) -> Result, ListError> { + let mut out = Vec::new(); + let mut cursor: Option = None; + for _ in 0..MAX_PAGES { + let url = match &cursor { + Some(c) => format!("{ITEMS_URL}&cursor={}", net::percent_encode(c)), + None => ITEMS_URL.to_string(), + }; + let reply = net::send(Request::get(&url).bearer(access_token)).map_err(ListError::Failed)?; + if reply.status == 401 { + return Err(ListError::SignIn); + } + if !reply.ok() { + return Err(ListError::Failed(format!("Epic library answered HTTP {}", reply.status))); + } + let body = reply + .json() + .ok_or_else(|| ListError::Failed("Epic library answer could not be read".into()))?; + let (records, next) = read_page(&body, cursor.as_deref()); + out.extend(records); + match next { + Some(n) => cursor = Some(n), + None => return Ok(out), + } + } + Err(ListError::Failed("Epic library has too many pages".into())) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn record_filtering() { + let ok = json!({"appName":"Fig","namespace":"ns","catalogItemId":"cat","sandboxType":"PUBLIC","platform":["Windows","Mac"]}); + assert_eq!(keep_record(&ok).unwrap().catalog_item_id, "cat"); + assert!(keep_record(&json!({"appName":"Fig","namespace":"ns","platform":[]})).is_some()); + assert!(keep_record(&json!({"appName":"Fig","namespace":"ns","platform":["Win32"]})).is_some()); + assert!(keep_record(&json!({"appName":"Fig","namespace":"ns","platform":["Mac"]})).is_none()); + assert!(keep_record(&json!({"appName":"Fig","namespace":"ns","sandboxType":"PRIVATE"})).is_none()); + assert!(keep_record(&json!({"appName":"1","namespace":"ns"})).is_none()); + assert!(keep_record(&json!({"appName":"","namespace":"ns"})).is_none()); + assert!(keep_record(&json!({"appName":"Asset","namespace":"ue"})).is_none()); + assert!(keep_record(&json!({"appName":"Asset","namespace":"89efe5924d3d467c839449ab6ab52e7f"})).is_none()); + } + + #[test] + fn page_cursor_rules() { + let page = json!({"records":[{"appName":"A","namespace":"n"}],"responseMetadata":{"nextCursor":"c2"}}); + let (records, next) = read_page(&page, Some("c1")); + assert_eq!(records.len(), 1); + assert_eq!(next.as_deref(), Some("c2")); + assert_eq!(read_page(&page, Some("c2")).1, None); + assert_eq!(read_page(&json!({"records":[]}), None).1, None); + assert_eq!(read_page(&json!({"responseMetadata":{"nextCursor":""}}), None).1, None); + } +} diff --git a/app/src/main/rust/stores/src/epic/locale.rs b/app/src/main/rust/stores/src/epic/locale.rs new file mode 100644 index 0000000000..4da00e17bc --- /dev/null +++ b/app/src/main/rust/stores/src/epic/locale.rs @@ -0,0 +1,165 @@ +use super::manifest::FileEntry; + +const LANGUAGE_TAGS: &[(&str, &[&str])] = &[ + ("ar", &["Arabic", "ar"]), + ("bg", &["Bulgarian", "bg-BG", "bg"]), + ("zh-Hans", &["Chinese", "ChineseSimplified", "zh-Hans", "zh_Hans", "zh"]), + ("zh-Hant", &["ChineseTraditional", "zh-Hant", "zh_Hant"]), + ("cs", &["Czech", "cs-CZ", "cs"]), + ("da", &["Danish", "da-DK", "da"]), + ("nl", &["Dutch", "nl-NL", "nl"]), + ("en", &["English", "en-US", "en"]), + ("fi", &["Finnish", "fi-FI", "fi"]), + ("fr", &["French", "fr-FR", "fr"]), + ("de", &["German", "de-DE", "de"]), + ("el", &["Greek", "el-GR", "el"]), + ("hu", &["Hungarian", "hu-HU", "hu"]), + ("it", &["Italian", "it-IT", "it"]), + ("ja", &["Japanese", "ja-JP", "ja"]), + ("ko", &["Korean", "ko-KR", "ko"]), + ("no", &["Norwegian", "nb-NO", "no"]), + ("pl", &["Polish", "pl-PL", "pl"]), + ("pt", &["Portuguese", "pt-PT", "pt"]), + ("pt-BR", &["PortugueseBrazilian", "Brazilian", "pt-BR", "br"]), + ("ro", &["Romanian", "ro-RO", "ro"]), + ("ru", &["Russian", "ru-RU", "ru"]), + ("es", &["Spanish", "es-ES", "es"]), + ("es-419", &["SpanishLatinAmerica", "Latam", "es-MX", "es_mx"]), + ("sv", &["Swedish", "sv-SE", "sv"]), + ("th", &["Thai", "th-TH", "th"]), + ("tr", &["Turkish", "tr-TR", "tr"]), + ("uk", &["Ukrainian", "uk-UA", "uk"]), + ("vi", &["Vietnamese", "vi-VN", "vi"]), +]; + +pub fn language_key(language: &str, region: &str) -> &'static str { + let lang = language.trim().to_ascii_lowercase(); + let region = region.trim().to_ascii_uppercase(); + match lang.as_str() { + "zh" => { + if matches!(region.as_str(), "TW" | "HK" | "MO") { + "zh-Hant" + } else { + "zh-Hans" + } + } + "pt" => { + if region == "BR" { + "pt-BR" + } else { + "pt" + } + } + "es" => { + if region.is_empty() || region == "ES" { + "es" + } else { + "es-419" + } + } + "nb" | "nn" => "no", + other => LANGUAGE_TAGS + .iter() + .map(|(k, _)| *k) + .find(|k| *k == other) + .unwrap_or("en"), + } +} + +pub fn tags_for_key(key: &str) -> Vec { + LANGUAGE_TAGS + .iter() + .find(|(k, _)| *k == key) + .or_else(|| LANGUAGE_TAGS.iter().find(|(k, _)| *k == "en")) + .map(|(_, tags)| tags.iter().map(|t| t.to_string()).collect()) + .unwrap_or_default() +} + +pub fn launch_locale(key: &str) -> String { + let head = key.split('-').next().unwrap_or("").trim(); + if head.is_empty() { + "en".to_string() + } else { + head.to_string() + } +} + +pub fn select(files: &[FileEntry], selected: Option<&[String]>) -> Vec { + let all = || (0..files.len()).collect::>(); + let required = || { + let untagged: Vec = (0..files.len()).filter(|&i| files[i].tags.is_empty()).collect(); + if untagged.is_empty() { + all() + } else { + untagged + } + }; + let Some(selected) = selected else { + return all(); + }; + if selected.is_empty() { + return required(); + } + let picked: Vec = (0..files.len()) + .filter(|&i| files[i].tags.is_empty() || files[i].tags.iter().any(|t| selected.contains(t))) + .collect(); + if picked.is_empty() { + required() + } else { + picked + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(path: &str, tags: &[&str]) -> FileEntry { + FileEntry { + path: path.into(), + sha1: None, + tags: tags.iter().map(|t| t.to_string()).collect(), + parts: vec![], + size: 1, + } + } + + #[test] + fn locale_keys() { + assert_eq!(language_key("zh", "TW"), "zh-Hant"); + assert_eq!(language_key("zh", "HK"), "zh-Hant"); + assert_eq!(language_key("zh", "CN"), "zh-Hans"); + assert_eq!(language_key("pt", "BR"), "pt-BR"); + assert_eq!(language_key("pt", "PT"), "pt"); + assert_eq!(language_key("es", ""), "es"); + assert_eq!(language_key("es", "ES"), "es"); + assert_eq!(language_key("es", "MX"), "es-419"); + assert_eq!(language_key("nb", "NO"), "no"); + assert_eq!(language_key("nn", ""), "no"); + assert_eq!(language_key("de", "AT"), "de"); + assert_eq!(language_key("xx", "YY"), "en"); + assert_eq!(launch_locale("zh-Hant"), "zh"); + assert_eq!(launch_locale("es-419"), "es"); + assert_eq!(launch_locale("fr"), "fr"); + assert_eq!(tags_for_key("pt-BR"), vec!["PortugueseBrazilian", "Brazilian", "pt-BR", "br"]); + assert_eq!(tags_for_key("nope"), vec!["English", "en-US", "en"]); + } + + #[test] + fn selection_rules() { + let files = vec![ + file("base", &[]), + file("fr", &["French", "fr"]), + file("de", &["German"]), + file("opt", &["HighResTextures"]), + ]; + let fr = tags_for_key("fr"); + assert_eq!(select(&files, Some(&fr)), vec![0, 1]); + assert_eq!(select(&files, None), vec![0, 1, 2, 3]); + assert_eq!(select(&files, Some(&[])), vec![0]); + assert_eq!(select(&files, Some(&["french".to_string()])), vec![0]); + let tagged_only = vec![file("a", &["X"]), file("b", &["Y"])]; + assert_eq!(select(&tagged_only, Some(&["Z".to_string()])), vec![0, 1]); + assert_eq!(select(&tagged_only, Some(&["Y".to_string()])), vec![1]); + } +} diff --git a/app/src/main/rust/stores/src/epic/manifest.rs b/app/src/main/rust/stores/src/epic/manifest.rs index d0b38fdd27..2690e0f19d 100644 --- a/app/src/main/rust/stores/src/epic/manifest.rs +++ b/app/src/main/rust/stores/src/epic/manifest.rs @@ -1,156 +1,142 @@ - use std::collections::HashMap; use std::io::Read; -pub const MANIFEST_MAGIC: u32 = 0x44BE_C00C; +pub const BINARY_MAGIC: u32 = 0x44BE_C00C; +pub const DEFAULT_WINDOW: u32 = 1024 * 1024; +const MIN_HEADER: usize = 41; +const MAX_BODY: usize = 1 << 30; -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ChunkInfo { - pub guid: [u32; 4], - pub hash: u64, - pub sha1: Option<[u8; 20]>, - pub group_num: i32, - pub window_size: i32, - pub file_size: i64, -} +pub type Guid = [u32; 4]; +pub type Sha1 = [u8; 20]; -impl ChunkInfo { - pub fn guid_str(&self) -> String { - guid_to_string(&self.guid) - } - - pub fn path(&self, chunk_dir: &str) -> String { - format!( - "{}/{:02}/{:016X}_{}.chunk", - chunk_dir, - self.group_num, - self.hash, - self.guid_str() - ) - } - - pub fn verifiable_sha1(&self) -> Option<&[u8; 20]> { - match &self.sha1 { - Some(h) if h.iter().any(|b| *b != 0) => Some(h), - _ => None, - } - } - - pub fn credit_bytes(&self) -> u64 { - if self.file_size < 1 { - 1 - } else { - self.file_size as u64 - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ChunkPart { - pub guid: [u32; 4], - pub offset: i32, - pub size: i32, -} - -impl ChunkPart { - pub fn guid_str(&self) -> String { - guid_to_string(&self.guid) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct FileInfo { - pub filename: String, - pub parts: Vec, - pub install_tags: Vec, - pub sha1: Option<[u8; 20]>, -} - -impl FileInfo { - pub fn file_size(&self) -> u64 { - self.parts.iter().map(|p| p.size as u32 as u64).sum() - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Manifest { - pub chunk_dir: String, - pub unique_chunks: Vec, - pub files: Vec, -} - -impl Manifest { - pub fn chunk_index_by_guid(&self) -> HashMap { - let mut map = HashMap::with_capacity(self.unique_chunks.len()); - for (i, c) in self.unique_chunks.iter().enumerate() { - map.insert(c.guid_str(), i); - } - map - } -} - -pub fn guid_to_string(guid: &[u32; 4]) -> String { +pub fn guid_text(guid: &Guid) -> String { format!("{:08X}{:08X}{:08X}{:08X}", guid[0], guid[1], guid[2], guid[3]) } -pub fn chunk_dir_for_binary_version(version: i32) -> &'static str { - if version >= 15 { - "ChunksV4" - } else if version >= 6 { - "ChunksV3" - } else if version >= 3 { - "ChunksV2" - } else { - "Chunks" +pub fn guid_from_hex(text: &str) -> Option { + let head = text.get(..32)?; + if !head.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let mut out = [0u32; 4]; + for (i, slot) in out.iter_mut().enumerate() { + *slot = u32::from_str_radix(&head[i * 8..i * 8 + 8], 16).ok()?; + } + Some(out) +} + +pub fn sha1_hex(hash: &Sha1) -> String { + hash.iter().map(|b| format!("{b:02x}")).collect() +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ChunkInfo { + pub guid: Guid, + pub rolling_hash: u64, + pub sha1: Option, + pub group: u64, + pub window: u32, + pub window_exact: bool, + pub cdn_size: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PartRef { + pub guid: Guid, + pub offset: u32, + pub size: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileEntry { + pub path: String, + pub sha1: Option, + pub tags: Vec, + pub parts: Vec, + pub size: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Variant { + Binary, + Json, +} + +#[derive(Clone, Debug)] +pub struct Manifest { + pub variant: Variant, + pub version: i64, + pub build_version: String, + pub launch_exe: String, + pub prereq_name: String, + pub prereq_path: String, + pub prereq_read: bool, + pub chunks: Vec, + pub chunk_index: HashMap, + pub files: Vec, +} + +impl Manifest { + pub fn chunk(&self, guid: &Guid) -> Option<&ChunkInfo> { + self.chunk_index.get(guid).map(|&i| &self.chunks[i]) + } + + pub fn chunk_folder(&self) -> &'static str { + chunk_folder(self.variant, self.version) + } + + pub fn chunk_path(&self, chunk: &ChunkInfo) -> String { + format!( + "{}/{:02}/{:016X}_{}.chunk", + self.chunk_folder(), + chunk.group, + chunk.rolling_hash, + guid_text(&chunk.guid) + ) } } -pub fn chunk_dir_for_json_version(version: i32) -> &'static str { - if version >= 15 { - "ChunksV4" - } else if version >= 6 { - "ChunksV3" - } else if version >= 3 { - "ChunksV2" - } else { - "ChunksV4" +pub fn chunk_folder(variant: Variant, version: i64) -> &'static str { + match version { + v if v >= 15 => "ChunksV4", + v if v >= 6 => "ChunksV3", + v if v >= 3 => "ChunksV2", + _ => match variant { + Variant::Binary => "Chunks", + Variant::Json => "ChunksV4", + }, } } +pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() < 4 { + return Err("manifest too short".into()); + } + if u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) == BINARY_MAGIC { + parse_binary(bytes) + } else { + parse_json(bytes) + } +} -struct Cursor<'a> { +fn normalise_path(raw: &str) -> String { + raw.replace('\\', "/") +} + +struct Reader<'a> { buf: &'a [u8], pos: usize, } -impl<'a> Cursor<'a> { +impl<'a> Reader<'a> { fn new(buf: &'a [u8]) -> Self { - Self { buf, pos: 0 } - } - - fn position(&self) -> usize { - self.pos - } - - fn set_position(&mut self, p: i64) -> Result<(), String> { - if p < 0 || p as u64 > self.buf.len() as u64 { - return Err(format!("position {p} out of range (limit {})", self.buf.len())); - } - self.pos = p as usize; - Ok(()) + Reader { buf, pos: 0 } } fn take(&mut self, n: usize) -> Result<&'a [u8], String> { - let end = self - .pos - .checked_add(n) - .ok_or_else(|| "buffer overflow".to_string())?; + let end = self.pos.checked_add(n).ok_or("length overflow")?; if end > self.buf.len() { - return Err(format!( - "buffer underflow: need {n} at {} (limit {})", - self.pos, - self.buf.len() - )); + return Err(format!("read past the end at {} (+{n})", self.pos)); } let out = &self.buf[self.pos..end]; self.pos = end; @@ -161,795 +147,749 @@ impl<'a> Cursor<'a> { Ok(self.take(1)?[0]) } - fn i32(&mut self) -> Result { + fn u32(&mut self) -> Result { let b = self.take(4)?; - Ok(i32::from_le_bytes([b[0], b[1], b[2], b[3]])) + Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) } - fn u32(&mut self) -> Result { - Ok(self.i32()? as u32) + fn i32(&mut self) -> Result { + Ok(self.u32()? as i32) } fn u64(&mut self) -> Result { let b = self.take(8)?; - Ok(u64::from_le_bytes([ - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - ])) + let mut a = [0u8; 8]; + a.copy_from_slice(b); + Ok(u64::from_le_bytes(a)) } - fn sha1(&mut self) -> Result<[u8; 20], String> { - let b = self.take(20)?; + fn i64(&mut self) -> Result { + Ok(self.u64()? as i64) + } + + fn guid(&mut self) -> Result { + Ok([self.u32()?, self.u32()?, self.u32()?, self.u32()?]) + } + + fn sha1(&mut self) -> Result { let mut out = [0u8; 20]; - out.copy_from_slice(b); + out.copy_from_slice(self.take(20)?); Ok(out) } - fn skip(&mut self, n: i64) -> Result<(), String> { - let target = self.pos as i64 + n; - self.set_position(target) - } - - fn remaining(&self) -> &'a [u8] { - &self.buf[self.pos..] + fn count(&mut self, min_item_bytes: usize) -> Result { + let n = self.i32()?; + if n < 0 { + return Err(format!("negative count {n}")); + } + let n = n as usize; + let left = self.buf.len() - self.pos; + if n.saturating_mul(min_item_bytes.max(1)) > left { + return Err(format!("count {n} does not fit the section")); + } + Ok(n) } fn fstring(&mut self) -> Result { - let length = self.i32()?; - if length == 0 { + let len = self.i32()? as i64; + if len == 0 { return Ok(String::new()); } - if length < 0 { - let chars = length.wrapping_neg().wrapping_sub(1); - let byte_len = chars.checked_mul(2).ok_or("fstring length overflow")?; - if byte_len < 0 { - return Err("negative fstring length".to_string()); - } - let bytes = self.take(byte_len as usize)?; - self.take(2)?; - let units: Vec = bytes - .chunks_exact(2) - .map(|p| u16::from_le_bytes([p[0], p[1]])) - .collect(); - Ok(String::from_utf16_lossy(&units)) - } else { - let bytes = self.take((length - 1) as usize)?; - self.take(1)?; - - Ok(bytes + if len > 0 { + let raw = self.take(len as usize)?; + let text = &raw[..raw.len() - 1]; + Ok(text .iter() .map(|&b| if b < 0x80 { b as char } else { '\u{FFFD}' }) .collect()) + } else { + let units = (-len) as usize; + let raw = self.take(units.checked_mul(2).ok_or("length overflow")?)?; + let mut code_units: Vec = raw + .chunks_exact(2) + .map(|p| u16::from_le_bytes([p[0], p[1]])) + .collect(); + code_units.pop(); + Ok(String::from_utf16_lossy(&code_units)) } } } - -pub fn parse_manifest(bytes: &[u8]) -> Result { - let mut cur = Cursor::new(bytes); - let magic = cur.u32()?; - if magic != MANIFEST_MAGIC { - return parse_json_manifest(bytes); +fn section<'a>(body: &'a [u8], start: usize) -> Result<(&'a [u8], usize), String> { + let mut r = Reader::new(&body[start.min(body.len())..]); + let size = r.i32()?; + if size < 4 { + return Err(format!("section at {start} has size {size}")); } - parse_binary_after_magic(&mut cur) + let end = start + .checked_add(size as usize) + .filter(|&e| e <= body.len()) + .ok_or_else(|| format!("section at {start} runs past the body"))?; + Ok((&body[start + 4..end], end)) } -fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result { - let header_size = cur.i32()?; - let size_uncompressed = cur.i32()?; - let _size_compressed = cur.i32()?; - cur.skip(20)?; - let stored_as = cur.u8()?; - let version = cur.i32()?; - let chunk_dir = chunk_dir_for_binary_version(version); - - cur.set_position(header_size as i64)?; - let body_slice = cur.remaining(); - +fn parse_binary(bytes: &[u8]) -> Result { + if bytes.len() < MIN_HEADER { + return Err("binary manifest header too short".into()); + } + let mut h = Reader::new(bytes); + h.u32()?; + let header_size = h.i32()?; + let raw_size = h.i32()?; + let _stored_size = h.i32()?; + let _body_sha1 = h.sha1()?; + let flags = h.u8()?; + let version = h.i32()? as i64; + if header_size < MIN_HEADER as i32 || header_size as usize > bytes.len() { + return Err(format!("bad header size {header_size}")); + } + if raw_size < 0 || raw_size as usize > MAX_BODY { + return Err(format!("bad body size {raw_size}")); + } + if flags & 2 != 0 { + return Err("encrypted manifests are not supported".into()); + } + let stored = &bytes[header_size as usize..]; let inflated; - let body: &[u8] = if (stored_as & 1) != 0 { - if size_uncompressed < 0 { - return Err("negative sizeUncompressed".to_string()); - } - let mut out = Vec::with_capacity(size_uncompressed as usize); - let mut dec = flate2::read::ZlibDecoder::new(body_slice).take(size_uncompressed as u64); - dec.read_to_end(&mut out) - .map_err(|e| format!("manifest inflate failed: {e}"))?; - if out.len() != size_uncompressed as usize { - return Err(format!( - "decomp size mismatch: expected {size_uncompressed} got {}", - out.len() - )); + let body: &[u8] = if flags & 1 != 0 { + let want = raw_size as usize; + let mut out = Vec::with_capacity(want); + flate2::read::ZlibDecoder::new(stored) + .take(want as u64 + 1) + .read_to_end(&mut out) + .map_err(|e| format!("manifest body does not inflate: {e}"))?; + if out.len() != want { + return Err(format!("manifest body inflated to {} bytes, header says {want}", out.len())); } inflated = out; &inflated } else { - body_slice + stored }; - let mut b = Cursor::new(body); - - let meta_size = b.i32()?; - let meta_end = b.position() as i64 - 4 + meta_size as i64; - b.set_position(meta_end)?; - - let cdl_start = b.position(); - let cdl_size = b.i32()?; - let _cdl_version = b.u8()?; - let chunk_count = b.i32()?; - if chunk_count < 0 { - return Err("negative chunkCount".to_string()); - } - let chunk_count = chunk_count as usize; - let mut chunks: Vec = (0..chunk_count) - .map(|_| ChunkInfo { - guid: [0; 4], - hash: 0, - sha1: None, - group_num: 0, - window_size: 0, - file_size: 0, - }) - .collect(); - for c in chunks.iter_mut() { - c.guid = [b.u32()?, b.u32()?, b.u32()?, b.u32()?]; - } - for c in chunks.iter_mut() { - c.hash = b.u64()?; - } - for c in chunks.iter_mut() { - c.sha1 = Some(b.sha1()?); - } - for c in chunks.iter_mut() { - c.group_num = b.u8()? as i32; - } - for c in chunks.iter_mut() { - c.window_size = b.i32()?; - } - for c in chunks.iter_mut() { - c.file_size = b.u64()? as i64; - } - b.set_position(cdl_start as i64 + cdl_size as i64)?; - - let fml_start = b.position(); - let fml_size = b.i32()?; - let _fml_version = b.u8()?; - let file_count = b.i32()?; - if file_count < 0 { - return Err("negative fileCount".to_string()); - } - let file_count = file_count as usize; - let mut files: Vec = (0..file_count).map(|_| FileInfo::default()).collect(); - for f in files.iter_mut() { - f.filename = b.fstring()?; - } - for _ in 0..file_count { - b.fstring()?; - } - for f in files.iter_mut() { - f.sha1 = Some(b.sha1()?); - } - b.skip(file_count as i64)?; - for f in files.iter_mut() { - let tag_count = b.i32()?; - let mut j = 0; - while j < tag_count { - let tag = b.fstring()?; - if !tag.is_empty() { - f.install_tags.push(tag); - } - j += 1; - } - } - for f in files.iter_mut() { - let part_count = b.i32()?; - let mut j = 0; - while j < part_count { - let part_start = b.position(); - let part_struct_size = b.i32()?; - let guid = [b.u32()?, b.u32()?, b.u32()?, b.u32()?]; - let offset = b.i32()?; - let size = b.i32()?; - f.parts.push(ChunkPart { guid, offset, size }); - b.set_position(part_start as i64 + part_struct_size as i64)?; - j += 1; - } - } - b.set_position(fml_start as i64 + fml_size as i64)?; + let (meta, after_meta) = section(body, 0)?; + let meta = read_meta(meta); + let (chunk_list, after_chunks) = section(body, after_meta)?; + let (chunks, chunk_index) = read_chunk_list(chunk_list)?; + let (file_list, _after_files) = section(body, after_chunks)?; + let files = read_file_list(file_list)?; Ok(Manifest { - chunk_dir: chunk_dir.to_string(), - unique_chunks: dedup_last_wins_first_position(chunks), + variant: Variant::Binary, + version, + build_version: meta.build_version, + launch_exe: meta.launch_exe, + prereq_name: meta.prereq_name, + prereq_path: meta.prereq_path, + prereq_read: meta.complete, + chunks, + chunk_index, files, }) } -fn dedup_last_wins_first_position(chunks: Vec) -> Vec { - let mut order: Vec = Vec::with_capacity(chunks.len()); - let mut index: HashMap = HashMap::with_capacity(chunks.len()); - for c in chunks { - let key = c.guid_str(); - match index.get(&key).copied() { - Some(i) => order[i] = c, +#[derive(Default)] +struct Meta { + build_version: String, + launch_exe: String, + prereq_name: String, + prereq_path: String, + complete: bool, +} + +fn read_meta(data: &[u8]) -> Meta { + let mut meta = Meta::default(); + let mut r = Reader::new(data); + let _ = (|| -> Result<(), String> { + r.u8()?; + r.u32()?; + r.u8()?; + r.u32()?; + r.fstring()?; + meta.build_version = r.fstring()?; + meta.launch_exe = normalise_path(&r.fstring()?); + r.fstring()?; + let ids = r.count(4)?; + for _ in 0..ids { + r.fstring()?; + } + meta.prereq_name = r.fstring()?; + meta.prereq_path = normalise_path(&r.fstring()?); + r.fstring()?; + meta.complete = true; + Ok(()) + })(); + meta +} + +fn read_chunk_list(data: &[u8]) -> Result<(Vec, HashMap), String> { + let mut r = Reader::new(data); + r.u8()?; + let count = r.count(16 + 8 + 20 + 1 + 4 + 8)?; + let mut guids = Vec::with_capacity(count); + for _ in 0..count { + guids.push(r.guid()?); + } + let mut rolling = Vec::with_capacity(count); + for _ in 0..count { + rolling.push(r.u64()?); + } + let mut hashes = Vec::with_capacity(count); + for _ in 0..count { + hashes.push(r.sha1()?); + } + let mut groups = Vec::with_capacity(count); + for _ in 0..count { + groups.push(r.u8()?); + } + let mut windows = Vec::with_capacity(count); + for _ in 0..count { + windows.push(r.i32()?); + } + let mut sizes = Vec::with_capacity(count); + for _ in 0..count { + sizes.push(r.i64()?); + } + let mut chunks: Vec = Vec::with_capacity(count); + let mut index: HashMap = HashMap::with_capacity(count); + for i in 0..count { + let window = windows[i]; + let info = ChunkInfo { + guid: guids[i], + rolling_hash: rolling[i], + sha1: Some(hashes[i]).filter(|h| h.iter().any(|&b| b != 0)), + group: groups[i] as u64, + window: if window > 0 { window as u32 } else { DEFAULT_WINDOW }, + window_exact: window > 0, + cdn_size: sizes[i].max(0) as u64, + }; + match index.get(&info.guid) { + Some(&at) => chunks[at] = info, None => { - index.insert(key, order.len()); - order.push(c); + index.insert(info.guid, chunks.len()); + chunks.push(info); } } } - order + Ok((chunks, index)) } - -fn json_to_string(v: &serde_json::Value) -> String { - match v { - serde_json::Value::String(s) => s.clone(), - serde_json::Value::Null => "null".to_string(), - other => other.to_string(), +fn read_file_list(data: &[u8]) -> Result, String> { + let mut r = Reader::new(data); + r.u8()?; + let count = r.count(4 + 4 + 20 + 1 + 4 + 4)?; + let mut names = Vec::with_capacity(count); + for _ in 0..count { + names.push(normalise_path(&r.fstring()?)); } + for _ in 0..count { + r.fstring()?; + } + let mut hashes = Vec::with_capacity(count); + for _ in 0..count { + hashes.push(r.sha1()?); + } + for _ in 0..count { + r.u8()?; + } + let mut tags = Vec::with_capacity(count); + for _ in 0..count { + let n = r.count(4)?; + let mut list = Vec::with_capacity(n); + for _ in 0..n { + let tag = r.fstring()?; + if !tag.is_empty() { + list.push(tag); + } + } + tags.push(list); + } + let mut files = Vec::with_capacity(count); + for (i, (path, tag_list)) in names.into_iter().zip(tags).enumerate() { + let n = r.count(28)?; + let mut parts = Vec::with_capacity(n); + let mut size = 0u64; + for _ in 0..n { + let start = r.pos; + let record = r.i32()?; + if record < 28 { + return Err(format!("chunk part record of {record} bytes")); + } + let guid = r.guid()?; + let offset = r.u32()?; + let len = r.u32()?; + let end = start + record as usize; + if end > data.len() { + return Err("chunk part record runs past the section".into()); + } + r.pos = end; + size += len as u64; + parts.push(PartRef { guid, offset, size: len }); + } + files.push(FileEntry { + path, + sha1: Some(hashes[i]).filter(|h| h.iter().any(|&b| b != 0)), + tags: tag_list, + parts, + size, + }); + } + Ok(files) } -pub fn blob_to_num(s: &str) -> Option { - let bytes = s.as_bytes(); - let mut num: u64 = 0; - for (i, group) in bytes.chunks(3).enumerate() { - if !group.iter().all(u8::is_ascii_digit) { - return None; - } - let byte: u64 = std::str::from_utf8(group).ok()?.parse().ok()?; - let shift = i * 8; - if shift < 64 { - num = num.wrapping_add(byte << shift); +pub fn blob(text: &str) -> Option { + if !text.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let mut value = 0u64; + for (i, group) in text.as_bytes().chunks(3).take(8).enumerate() { + let n: u64 = std::str::from_utf8(group).ok()?.parse().ok()?; + value = value.wrapping_add(n.wrapping_shl(8 * i as u32)); + } + Some(value) +} + +fn blob_of(value: Option<&serde_json::Value>) -> Option { + value.and_then(|v| v.as_str()).and_then(blob) +} + +fn guid_map(obj: Option<&serde_json::Value>) -> HashMap { + let mut out = HashMap::new(); + if let Some(map) = obj.and_then(|v| v.as_object()) { + for (k, v) in map { + if let Some(g) = guid_from_hex(k) { + out.insert(g, v.as_str().and_then(blob).unwrap_or(0)); + } } } - Some(num) + out } -fn opt_string(obj: &serde_json::Map, key: &str, fallback: &str) -> String { - match obj.get(key) { - Some(v) => json_to_string(v), - None => fallback.to_string(), - } -} +fn parse_json(bytes: &[u8]) -> Result { + let doc: serde_json::Value = + serde_json::from_slice(bytes).map_err(|e| format!("manifest is neither binary nor JSON: {e}"))?; + let str_field = |k: &str| doc.get(k).and_then(|v| v.as_str()).unwrap_or("").to_string(); + let version = blob_of(doc.get("ManifestFileVersion")).unwrap_or(0) as i64; + let launch = doc + .get("LaunchExeString") + .and_then(|v| v.as_str()) + .or_else(|| doc.get("LaunchExe").and_then(|v| v.as_str())) + .unwrap_or(""); -fn parse_guid_quarter(s: &str) -> Result { - let v = i64::from_str_radix(s, 16).map_err(|e| format!("bad guid hex '{s}': {e}"))?; - Ok(v as i32 as u32) -} - -fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> { - if s.len() < 32 || !s.as_bytes()[..32].is_ascii() { - return Err(format!("bad guid hex '{s}'")); - } - Ok([ - parse_guid_quarter(&s[0..8])?, - parse_guid_quarter(&s[8..16])?, - parse_guid_quarter(&s[16..24])?, - parse_guid_quarter(&s[24..32])?, - ]) -} - -pub fn parse_json_manifest(bytes: &[u8]) -> Result { - let text = String::from_utf8_lossy(bytes); - let root: serde_json::Value = - serde_json::from_str(&text).map_err(|e| format!("JSON manifest: {e}"))?; - let root = root - .as_object() - .ok_or_else(|| "JSON manifest: root is not an object".to_string())?; - - let manifest_version: i32 = blob_to_num(&opt_string(root, "ManifestFileVersion", "0")) - .map_or(0, |v| v as i32); - let chunk_dir = chunk_dir_for_json_version(manifest_version); - - let chunk_hash_list = root + let hashes = doc .get("ChunkHashList") .and_then(|v| v.as_object()) - .ok_or_else(|| "JSON manifest: no ChunkHashList".to_string())?; - let data_group_list = root.get("DataGroupList").and_then(|v| v.as_object()); - let chunk_filesize_list = root.get("ChunkFilesizeList").and_then(|v| v.as_object()); - - let mut chunk_order: Vec = Vec::new(); - let mut chunk_index: HashMap = HashMap::new(); - for (guid_hex, hash_value) in chunk_hash_list.iter() { - if guid_hex.len() < 32 || !guid_hex.is_char_boundary(32) { + .ok_or("JSON manifest has no ChunkHashList")?; + let groups = guid_map(doc.get("DataGroupList")); + let sizes = guid_map(doc.get("ChunkFilesizeList")); + let mut chunks = Vec::with_capacity(hashes.len()); + let mut chunk_index = HashMap::with_capacity(hashes.len()); + for (key, value) in hashes { + if key.len() < 32 { continue; } - let guid = parse_guid_from_hex(guid_hex)?; - let hash = blob_to_num(&json_to_string(hash_value)).unwrap_or(0); - let group_num = match data_group_list { - Some(dgl) => blob_to_num(&opt_string(dgl, guid_hex, "0")).map_or(0, |v| v as i32), - None => 0, - }; - let file_size = match chunk_filesize_list { - Some(cfl) => blob_to_num(&opt_string(cfl, guid_hex, "0")).map_or(0, |v| v as i64), - None => 0, - }; - let c = ChunkInfo { + let guid = guid_from_hex(key).ok_or_else(|| format!("chunk key {key} is not hex"))?; + let info = ChunkInfo { guid, - hash, + rolling_hash: value.as_str().and_then(blob).unwrap_or(0), sha1: None, - group_num, - window_size: 0, - file_size, + group: groups.get(&guid).copied().unwrap_or(0), + window: DEFAULT_WINDOW, + window_exact: false, + cdn_size: sizes.get(&guid).copied().unwrap_or(0), }; - match chunk_index.get(guid_hex).copied() { - Some(i) => chunk_order[i] = c, + match chunk_index.get(&guid) { + Some(&at) => chunks[at] = info, None => { - chunk_index.insert(guid_hex.clone(), chunk_order.len()); - chunk_order.push(c); + chunk_index.insert(guid, chunks.len()); + chunks.push(info); } } } - let file_list = root + let list = doc .get("FileManifestList") .and_then(|v| v.as_array()) - .ok_or_else(|| "JSON manifest: no FileManifestList".to_string())?; - - let mut files = Vec::with_capacity(file_list.len()); - for entry in file_list { - let file_obj = entry - .as_object() - .ok_or_else(|| "JSON manifest: FileManifestList entry is not an object".to_string())?; - let mut fi = FileInfo { - filename: opt_string(file_obj, "Filename", ""), - ..FileInfo::default() - }; - if let Some(tags) = file_obj.get("InstallTags").and_then(|v| v.as_array()) { - for t in tags { - let tag = json_to_string(t); - if !tag.is_empty() { - fi.install_tags.push(tag); - } + .ok_or("JSON manifest has no FileManifestList")?; + let mut files = Vec::with_capacity(list.len()); + for entry in list { + let path = normalise_path(entry.get("Filename").and_then(|v| v.as_str()).unwrap_or("")); + let tags = entry + .get("InstallTags") + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|t| t.as_str()) + .filter(|t| !t.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + let mut parts = Vec::new(); + let mut size = 0u64; + if let Some(list) = entry.get("FileChunkParts").and_then(|v| v.as_array()) { + for part in list { + let guid = part + .get("Guid") + .and_then(|v| v.as_str()) + .and_then(guid_from_hex) + .unwrap_or([0; 4]); + let offset = blob_of(part.get("Offset")).unwrap_or(0) as u32; + let len = blob_of(part.get("Size")).unwrap_or(0) as u32; + size += len as u64; + parts.push(PartRef { guid, offset, size: len }); } } - if let Some(parts) = file_obj.get("FileChunkParts").and_then(|v| v.as_array()) { - for p in parts { - let part_obj = p.as_object().ok_or_else(|| { - "JSON manifest: FileChunkParts entry is not an object".to_string() - })?; - let mut part = ChunkPart { - guid: [0; 4], - offset: 0, - size: 0, - }; - let part_guid = opt_string(part_obj, "Guid", ""); - if part_guid.len() >= 32 && part_guid.is_char_boundary(32) { - part.guid = parse_guid_from_hex(&part_guid)?; - } - part.offset = blob_to_num(&opt_string(part_obj, "Offset", "0")).map_or(0, |v| v as i32); - part.size = blob_to_num(&opt_string(part_obj, "Size", "0")).map_or(0, |v| v as i32); - fi.parts.push(part); - } - } - files.push(fi); + files.push(FileEntry { path, sha1: None, tags, parts, size }); } Ok(Manifest { - chunk_dir: chunk_dir.to_string(), - unique_chunks: chunk_order, + variant: Variant::Json, + version, + build_version: str_field("BuildVersionString"), + launch_exe: normalise_path(launch), + prereq_name: str_field("PrereqName"), + prereq_path: normalise_path(&str_field("PrereqPath")), + prereq_read: true, + chunks, + chunk_index, files, }) } - #[cfg(test)] -pub(crate) mod test_support { - +pub mod testing { use super::*; use std::io::Write; - pub struct TestChunk { - pub guid: [u32; 4], - pub hash: u64, - pub sha1: [u8; 20], - pub group: u8, - pub window: i32, - pub file_size: u64, + pub fn fstring(out: &mut Vec, s: &str) { + if s.is_empty() { + out.extend_from_slice(&0i32.to_le_bytes()); + } else if s.is_ascii() { + out.extend_from_slice(&((s.len() + 1) as i32).to_le_bytes()); + out.extend_from_slice(s.as_bytes()); + out.push(0); + } else { + let units: Vec = s.encode_utf16().collect(); + out.extend_from_slice(&(-((units.len() + 1) as i32)).to_le_bytes()); + for u in units { + out.extend_from_slice(&u.to_le_bytes()); + } + out.extend_from_slice(&[0, 0]); + } + } + + fn with_size(mut payload: Vec) -> Vec { + let mut out = ((payload.len() + 4) as i32).to_le_bytes().to_vec(); + out.append(&mut payload); + out } pub struct TestFile { - pub name: String, - pub sha1: [u8; 20], + pub path: String, + pub sha1: Sha1, pub tags: Vec, - pub parts: Vec<([u32; 4], i32, i32)>, + pub parts: Vec, } - pub fn fstring_ascii(s: &str) -> Vec { - let mut v = Vec::new(); - if s.is_empty() { - v.extend_from_slice(&0i32.to_le_bytes()); - return v; + pub struct TestManifest { + pub version: i32, + pub build: String, + pub exe: String, + pub prereq: Option<(String, String)>, + pub chunks: Vec, + pub files: Vec, + pub compress: bool, + pub meta_tail: Vec, + } + + pub fn build(m: &TestManifest) -> Vec { + let mut meta = vec![0u8]; + meta.extend_from_slice(&18u32.to_le_bytes()); + meta.push(1); + meta.extend_from_slice(&0u32.to_le_bytes()); + fstring(&mut meta, "App"); + fstring(&mut meta, &m.build); + fstring(&mut meta, &m.exe); + fstring(&mut meta, ""); + if let Some((name, path)) = &m.prereq { + meta.extend_from_slice(&1i32.to_le_bytes()); + fstring(&mut meta, "prereq-id"); + fstring(&mut meta, name); + fstring(&mut meta, path); + fstring(&mut meta, "/quiet"); } - v.extend_from_slice(&((s.len() as i32) + 1).to_le_bytes()); - v.extend_from_slice(s.as_bytes()); - v.push(0); - v - } + meta.extend_from_slice(&m.meta_tail); - pub fn fstring_utf16(s: &str) -> Vec { - let units: Vec = s.encode_utf16().collect(); - let mut v = Vec::new(); - v.extend_from_slice(&(-((units.len() as i32) + 1)).to_le_bytes()); - for u in units { - v.extend_from_slice(&u.to_le_bytes()); - } - v.extend_from_slice(&[0, 0]); - v - } - - pub fn build_body(chunks: &[TestChunk], files: &[TestFile], meta_padding: usize) -> Vec { - let mut body = Vec::new(); - let meta_size = 4 + meta_padding; - body.extend_from_slice(&(meta_size as i32).to_le_bytes()); - body.extend(std::iter::repeat(0xEEu8).take(meta_padding)); - - let mut cdl = Vec::new(); - cdl.push(0u8); - cdl.extend_from_slice(&(chunks.len() as i32).to_le_bytes()); - for c in chunks { - for g in c.guid { - cdl.extend_from_slice(&g.to_le_bytes()); + let mut cl = vec![2u8]; + cl.extend_from_slice(&(m.chunks.len() as i32).to_le_bytes()); + for c in &m.chunks { + for v in c.guid { + cl.extend_from_slice(&v.to_le_bytes()); } } - for c in chunks { - cdl.extend_from_slice(&c.hash.to_le_bytes()); + for c in &m.chunks { + cl.extend_from_slice(&c.rolling_hash.to_le_bytes()); } - for c in chunks { - cdl.extend_from_slice(&c.sha1); + for c in &m.chunks { + cl.extend_from_slice(&c.sha1.unwrap_or([0; 20])); } - for c in chunks { - cdl.push(c.group); + for c in &m.chunks { + cl.push(c.group as u8); } - for c in chunks { - cdl.extend_from_slice(&c.window.to_le_bytes()); + for c in &m.chunks { + cl.extend_from_slice(&(c.window as i32).to_le_bytes()); } - for c in chunks { - cdl.extend_from_slice(&c.file_size.to_le_bytes()); + for c in &m.chunks { + cl.extend_from_slice(&(c.cdn_size as i64).to_le_bytes()); } - cdl.extend_from_slice(&[0xAA, 0xBB, 0xCC]); - body.extend_from_slice(&((cdl.len() + 4) as i32).to_le_bytes()); - body.extend_from_slice(&cdl); + cl.extend_from_slice(&[9, 9, 9]); - let mut fml = Vec::new(); - fml.push(0u8); - fml.extend_from_slice(&(files.len() as i32).to_le_bytes()); - for f in files { - if f.name.is_ascii() { - fml.extend_from_slice(&fstring_ascii(&f.name)); - } else { - fml.extend_from_slice(&fstring_utf16(&f.name)); - } + let mut fl = vec![0u8]; + fl.extend_from_slice(&(m.files.len() as i32).to_le_bytes()); + for f in &m.files { + fstring(&mut fl, &f.path); } - for _ in files { - fml.extend_from_slice(&fstring_ascii("")); + for _ in &m.files { + fstring(&mut fl, ""); } - for f in files { - fml.extend_from_slice(&f.sha1); + for f in &m.files { + fl.extend_from_slice(&f.sha1); } - for _ in files { - fml.push(0); + for _ in &m.files { + fl.push(0); } - for f in files { - fml.extend_from_slice(&(f.tags.len() as i32).to_le_bytes()); + for f in &m.files { + fl.extend_from_slice(&(f.tags.len() as i32).to_le_bytes()); for t in &f.tags { - fml.extend_from_slice(&fstring_ascii(t)); + fstring(&mut fl, t); } } - for f in files { - fml.extend_from_slice(&(f.parts.len() as i32).to_le_bytes()); - for (guid, offset, size) in &f.parts { - fml.extend_from_slice(&30i32.to_le_bytes()); - for g in guid { - fml.extend_from_slice(&g.to_le_bytes()); + for f in &m.files { + fl.extend_from_slice(&(f.parts.len() as i32).to_le_bytes()); + for p in &f.parts { + fl.extend_from_slice(&32i32.to_le_bytes()); + for v in p.guid { + fl.extend_from_slice(&v.to_le_bytes()); } - fml.extend_from_slice(&offset.to_le_bytes()); - fml.extend_from_slice(&size.to_le_bytes()); - fml.extend_from_slice(&[0x11, 0x22]); + fl.extend_from_slice(&p.offset.to_le_bytes()); + fl.extend_from_slice(&p.size.to_le_bytes()); + fl.extend_from_slice(&[0, 0, 0, 0]); } } - body.extend_from_slice(&((fml.len() + 4) as i32).to_le_bytes()); - body.extend_from_slice(&fml); - body - } - pub fn build_manifest( - chunks: &[TestChunk], - files: &[TestFile], - version: i32, - compress: bool, - ) -> Vec { - let body = build_body(chunks, files, 7); - let payload = if compress { - let mut enc = - flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + let mut body = with_size(meta); + body.extend(with_size(cl)); + body.extend(with_size(fl)); + body.extend(with_size(vec![0, 0, 0, 0, 0])); + + let stored = if m.compress { + let mut enc = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); enc.write_all(&body).unwrap(); enc.finish().unwrap() } else { body.clone() }; - let header_size: i32 = 41 + 3; + let header_size = 45i32; let mut out = Vec::new(); - out.extend_from_slice(&MANIFEST_MAGIC.to_le_bytes()); + out.extend_from_slice(&BINARY_MAGIC.to_le_bytes()); out.extend_from_slice(&header_size.to_le_bytes()); out.extend_from_slice(&(body.len() as i32).to_le_bytes()); - out.extend_from_slice(&(payload.len() as i32).to_le_bytes()); + out.extend_from_slice(&(stored.len() as i32).to_le_bytes()); out.extend_from_slice(&[0u8; 20]); - out.push(if compress { 1 } else { 0 }); - out.extend_from_slice(&version.to_le_bytes()); - out.extend_from_slice(&[0xF0, 0xF1, 0xF2]); - assert_eq!(out.len(), header_size as usize); - out.extend_from_slice(&payload); + out.push(if m.compress { 1 } else { 0 }); + out.extend_from_slice(&m.version.to_le_bytes()); + out.extend_from_slice(&[7, 7, 7, 7]); + out.extend_from_slice(&stored); out } - pub fn sample_chunks() -> Vec { - vec![ - TestChunk { - guid: [0x0000_0001, 0x0000_0002, 0x0000_0003, 0x0000_0004], - hash: 0x0123_4567_89AB_CDEF, - sha1: [0x11; 20], - group: 5, - window: 1_048_576, - file_size: 700_000, - }, - TestChunk { - guid: [0xDEAD_BEEF, 0xFFFF_FFFF, 0x0000_0000, 0x8000_0000], - hash: 0xFFFF_FFFF_FFFF_FFFE, - sha1: [0x22; 20], - group: 123, - window: 4096, - file_size: 0, - }, - TestChunk { - guid: [0x0000_00AA, 0x0000_00BB, 0x0000_00CC, 0x0000_00DD], - hash: 0x10, - sha1: [0u8; 20], - group: 0, - window: 10, - file_size: 9, - }, - ] - } - - pub fn sample_files() -> Vec { - let c = sample_chunks(); - vec![ - TestFile { - name: "Game/Binaries/Win64/Game.exe".to_string(), - sha1: [0x51; 20], - tags: vec![], - parts: vec![(c[0].guid, 0, 1_048_576), (c[1].guid, 100, 4000)], - }, - TestFile { - name: "Game\\Content\\Localization\\de.pak".to_string(), - sha1: [0x52; 20], - tags: vec!["German".to_string(), "de-DE".to_string()], - parts: vec![(c[1].guid, 0, 4096)], - }, - TestFile { - name: "Game/Content/Ünïcode.pak".to_string(), - sha1: [0x53; 20], - tags: vec!["French".to_string()], - parts: vec![(c[2].guid, 0, 10), (c[0].guid, 5, 7)], - }, - ] + pub fn chunk(n: u32, window: u32) -> ChunkInfo { + ChunkInfo { + guid: [n, n + 1, 0, 0x8000_0000], + rolling_hash: 0x0123_4567_89AB_CDEF ^ n as u64, + sha1: None, + group: (n % 100) as u64, + window, + window_exact: true, + cdn_size: 1000, + } } } #[cfg(test)] mod tests { - use super::test_support::*; + use super::testing::*; use super::*; - #[test] - fn binary_manifest_round_trips_uncompressed() { - let bytes = build_manifest(&sample_chunks(), &sample_files(), 21, false); - let m = parse_manifest(&bytes).expect("parse"); - assert_eq!(m.chunk_dir, "ChunksV4"); - assert_eq!(m.unique_chunks.len(), 3); - assert_eq!(m.files.len(), 3); - - let c0 = &m.unique_chunks[0]; - assert_eq!(c0.guid_str(), "00000001000000020000000300000004"); - assert_eq!(c0.hash, 0x0123_4567_89AB_CDEF); - assert_eq!(c0.sha1, Some([0x11; 20])); - assert_eq!(c0.group_num, 5); - assert_eq!(c0.window_size, 1_048_576); - assert_eq!(c0.file_size, 700_000); - assert_eq!( - c0.path("ChunksV4"), - "ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" - ); - - let c1 = &m.unique_chunks[1]; - assert_eq!(c1.guid_str(), "DEADBEEFFFFFFFFF0000000080000000"); - assert_eq!( - c1.path("ChunksV4"), - "ChunksV4/123/FFFFFFFFFFFFFFFE_DEADBEEFFFFFFFFF0000000080000000.chunk" - ); - assert_eq!(c1.credit_bytes(), 1, "Math.max(fileSize, 1)"); - assert!(c1.verifiable_sha1().is_some()); - assert!(m.unique_chunks[2].verifiable_sha1().is_none(), "all-zero SHA-1 → unverifiable"); - - let f0 = &m.files[0]; - assert_eq!(f0.filename, "Game/Binaries/Win64/Game.exe"); - assert!(f0.install_tags.is_empty()); - assert_eq!(f0.sha1, Some([0x51; 20])); - assert_eq!(f0.parts.len(), 2); - assert_eq!(f0.parts[1].offset, 100); - assert_eq!(f0.parts[1].size, 4000); - assert_eq!(f0.file_size(), 1_048_576 + 4000); - - let f1 = &m.files[1]; - assert_eq!(f1.filename, "Game\\Content\\Localization\\de.pak"); - assert_eq!(f1.install_tags, vec!["German".to_string(), "de-DE".to_string()]); - - let f2 = &m.files[2]; - assert_eq!(f2.filename, "Game/Content/Ünïcode.pak", "UTF-16LE fstring"); - assert_eq!(f2.parts[1].guid_str(), c0.guid_str()); - } - - #[test] - fn binary_manifest_round_trips_compressed() { - let bytes = build_manifest(&sample_chunks(), &sample_files(), 17, true); - let m = parse_manifest(&bytes).expect("parse"); - assert_eq!(m.unique_chunks.len(), 3); - assert_eq!(m.files.len(), 3); - assert_eq!(m.files[2].filename, "Game/Content/Ünïcode.pak"); - } - - #[test] - fn chunk_dir_follows_version_thresholds() { - for (v, dir) in [(2, "Chunks"), (3, "ChunksV2"), (6, "ChunksV3"), (15, "ChunksV4")] { - let bytes = build_manifest(&sample_chunks(), &sample_files(), v, false); - assert_eq!(parse_manifest(&bytes).unwrap().chunk_dir, dir, "version {v}"); + fn sample(compress: bool, prereq: bool) -> TestManifest { + let c1 = chunk(1, 16); + let c2 = chunk(2, 8); + TestManifest { + version: 21, + build: "1.2.3-CL-99".into(), + exe: "Binaries\\Win64\\Game.exe".into(), + prereq: prereq.then(|| ("UE4 Prerequisites (x64)".into(), "Engine\\Extras\\UE4PrereqSetup_x64.exe".into())), + chunks: vec![c1.clone(), c2.clone()], + files: vec![ + TestFile { + path: "Game\\Content\\a.pak".into(), + sha1: [1; 20], + tags: vec![], + parts: vec![ + PartRef { guid: c1.guid, offset: 0, size: 10 }, + PartRef { guid: c2.guid, offset: 2, size: 6 }, + ], + }, + TestFile { + path: "Lang/日本語.txt".into(), + sha1: [0; 20], + tags: vec!["Japanese".into(), "".into()], + parts: vec![PartRef { guid: c1.guid, offset: 10, size: 6 }], + }, + TestFile { path: "empty.txt".into(), sha1: [0; 20], tags: vec![], parts: vec![] }, + ], + compress, + meta_tail: vec![1, 2, 3], } - assert_eq!(chunk_dir_for_json_version(1), "ChunksV4"); - assert_eq!(chunk_dir_for_json_version(4), "ChunksV2"); } #[test] - fn duplicate_guid_keeps_first_position_last_value() { - let mut chunks = sample_chunks(); - let dup = TestChunk { - guid: chunks[0].guid, - hash: 0x99, - sha1: [0x77; 20], - group: 9, - window: 1, - file_size: 1, - }; - chunks.push(dup); - let bytes = build_manifest(&chunks, &sample_files(), 21, false); - let m = parse_manifest(&bytes).unwrap(); - assert_eq!(m.unique_chunks.len(), 3); - assert_eq!(m.unique_chunks[0].hash, 0x99); - assert_eq!(m.unique_chunks[0].group_num, 9); + fn binary_manifest_round_trip_compressed_and_stored() { + for compress in [true, false] { + let m = parse(&build(&sample(compress, true))).unwrap(); + assert_eq!(m.variant, Variant::Binary); + assert_eq!(m.version, 21); + assert_eq!(m.chunk_folder(), "ChunksV4"); + assert_eq!(m.build_version, "1.2.3-CL-99"); + assert_eq!(m.launch_exe, "Binaries/Win64/Game.exe"); + assert_eq!(m.prereq_name, "UE4 Prerequisites (x64)"); + assert_eq!(m.prereq_path, "Engine/Extras/UE4PrereqSetup_x64.exe"); + assert!(m.prereq_read); + assert_eq!(m.chunks.len(), 2); + assert_eq!(m.files.len(), 3); + let a = &m.files[0]; + assert_eq!(a.path, "Game/Content/a.pak"); + assert_eq!(a.size, 16); + assert_eq!(a.sha1, Some([1; 20])); + assert_eq!(a.parts[1].offset, 2); + assert_eq!(m.files[1].path, "Lang/日本語.txt"); + assert_eq!(m.files[1].tags, vec!["Japanese".to_string()]); + assert_eq!(m.files[1].sha1, None); + assert_eq!(m.files[2].size, 0); + } } #[test] - fn truncated_manifest_is_a_parse_failure() { - let bytes = build_manifest(&sample_chunks(), &sample_files(), 21, false); - assert!(parse_manifest(&bytes[..bytes.len() - 40]).is_err()); - assert!(parse_manifest(&bytes[..20]).is_err()); + fn a_short_meta_section_keeps_the_prereq_flag_unset() { + let mut s = sample(true, false); + s.meta_tail.clear(); + let m = parse(&build(&s)).unwrap(); + assert!(!m.prereq_read); + assert_eq!(m.build_version, "1.2.3-CL-99"); + assert_eq!(m.files.len(), 3); } #[test] - fn bad_uncompressed_size_is_a_parse_failure() { - let good = build_manifest(&sample_chunks(), &sample_files(), 21, true); - let real = i32::from_le_bytes([good[8], good[9], good[10], good[11]]); - let mut bigger = good.clone(); - bigger[8..12].copy_from_slice(&(real + 1).to_le_bytes()); - assert!(parse_manifest(&bigger).is_err()); - let mut smaller = good; - smaller[8..12].copy_from_slice(&(real - 1).to_le_bytes()); - assert!(parse_manifest(&smaller).is_err()); + fn duplicate_chunks_keep_first_position_and_last_values() { + let mut s = sample(false, true); + let mut again = s.chunks[0].clone(); + again.rolling_hash = 42; + s.chunks.push(again); + let m = parse(&build(&s)).unwrap(); + assert_eq!(m.chunks.len(), 2); + assert_eq!(m.chunks[0].rolling_hash, 42); } #[test] - fn fstring_decoding_matches_java() { - let mut v = fstring_ascii("abc"); - v.extend_from_slice(&fstring_utf16("dé")); - v.extend_from_slice(&fstring_ascii("")); - v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]); - let mut c = Cursor::new(&v); - assert_eq!(c.fstring().unwrap(), "abc"); - assert_eq!(c.fstring().unwrap(), "dé"); - assert_eq!(c.fstring().unwrap(), ""); - assert_eq!(c.fstring().unwrap(), "a\u{FFFD}b"); - } - - #[test] - fn blob_numbers_are_decimal_bytes_least_significant_first() { - assert_eq!(blob_to_num("001000000000"), Some(1)); - assert_eq!(blob_to_num("000001000000"), Some(0x100)); - assert_eq!(blob_to_num("001002003004005006007008"), Some(0x0807_0605_0403_0201)); - assert_eq!(blob_to_num("255255255255255255255255"), Some(u64::MAX)); - assert_eq!(blob_to_num("7"), Some(7)); - assert_eq!(blob_to_num(""), Some(0)); - assert_eq!(blob_to_num("0x1"), None); - assert_eq!(blob_to_num("-05"), None); - } - - #[test] - fn json_manifest_parses_like_java() { - let json = r#"{ - "ManifestFileVersion": "013", - "ChunkHashList": { - "0000000100000002000000030000000ff": "001002003004005006007008", - "short": "00", - "DEADBEEFFFFFFFFF0000000080000000": "zz" - }, - "DataGroupList": { "0000000100000002000000030000000ff": "7", "DEADBEEFFFFFFFFF0000000080000000": "x" }, - "ChunkFilesizeList": { "0000000100000002000000030000000ff": "255" }, - "FileManifestList": [ - { "Filename": "a.bin", "InstallTags": ["German", ""], - "FileChunkParts": [ { "Guid": "0000000100000002000000030000000ff", "Offset": "12", "Size": "34" }, - { "Guid": "tooshort", "Offset": "x", "Size": "-5" } ] }, - { "Filename": "b.bin" } - ] - }"#; - let m = parse_manifest(json.as_bytes()).expect("json parse"); - assert_eq!(m.chunk_dir, "ChunksV3"); - assert_eq!(m.unique_chunks.len(), 2, "keys shorter than 32 chars are skipped"); - let c0 = &m.unique_chunks[0]; - assert_eq!(c0.guid, [1, 2, 3, 0x0000_000F]); - assert_eq!(c0.hash, 0x0807_0605_0403_0201); - assert_eq!(c0.group_num, 7); - assert_eq!(c0.file_size, 0xFF); - assert_eq!(c0.window_size, 0); - assert!(c0.sha1.is_none()); - let c1 = &m.unique_chunks[1]; - assert_eq!(c1.hash, 0, "unparseable hash → 0"); - assert_eq!(c1.group_num, 0, "unparseable group → 0"); - assert_eq!(c1.file_size, 0, "missing size → 0"); - - assert_eq!(m.files.len(), 2); - assert_eq!(m.files[0].install_tags, vec!["German".to_string()]); - assert_eq!(m.files[0].parts.len(), 2); - assert_eq!(m.files[0].parts[0].offset, 12); - assert_eq!(m.files[0].parts[0].size, 34); - assert_eq!(m.files[0].parts[1].guid, [0; 4]); - assert_eq!(m.files[0].parts[1].offset, 0); - assert_eq!(m.files[0].parts[1].size, 0); - assert!(m.files[0].sha1.is_none()); - assert!(m.files[1].parts.is_empty()); - } - - #[test] - fn json_manifest_uses_blobs() { - let json = r#"{ - "ManifestFileVersion": "006000000000", - "ChunkHashList": { "0000000100000002000000030000000F": "001002003004005006007008" }, - "DataGroupList": { "0000000100000002000000030000000F": "007" }, - "ChunkFilesizeList": { "0000000100000002000000030000000F": "001002000000000000000000" }, - "FileManifestList": [ - { "Filename": "a.bin", - "FileChunkParts": [ { "Guid": "0000000100000002000000030000000F", "Offset": "001000000000", "Size": "000000016000" } ] } - ] - }"#; - let m = parse_manifest(json.as_bytes()).expect("json parse"); - assert_eq!(m.chunk_dir, "ChunksV3"); - let c = &m.unique_chunks[0]; - assert_eq!(c.file_size, 0x0201); + fn chunk_paths_and_guid_text() { + let m = parse(&build(&sample(false, true))).unwrap(); + let c = &m.chunks[0]; + assert_eq!(guid_text(&[0xDEADBEEF, 0xFFFFFFFF, 0, 0x80000000]), "DEADBEEFFFFFFFFF0000000080000000"); assert_eq!( - c.path(&m.chunk_dir), - "ChunksV3/07/0807060504030201_0000000100000002000000030000000F.chunk" + m.chunk_path(c), + format!("ChunksV4/01/{:016X}_00000001000000020000000080000000.chunk", c.rolling_hash) ); - let part = &m.files[0].parts[0]; - assert_eq!(part.offset, 1); - assert_eq!(part.size, 1_048_576); } #[test] - fn json_manifest_failures_match_java_nulls() { - assert!(parse_manifest(br#"{"FileManifestList": []}"#).is_err(), "no ChunkHashList"); - assert!(parse_manifest(br#"{"ChunkHashList": {}}"#).is_err(), "no FileManifestList"); - assert!( - parse_manifest(br#"{"ChunkHashList": {"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz": "0"}, "FileManifestList": []}"#) - .is_err(), - "invalid GUID hex throws out of the parse" - ); - assert!(parse_manifest(b"not json at all").is_err()); + fn truncated_or_inconsistent_bodies_are_rejected() { + let good = build(&sample(true, true)); + assert!(parse(&good[..good.len() - 10]).is_err()); + let mut lying = good.clone(); + lying[8..12].copy_from_slice(&5i32.to_le_bytes()); + assert!(parse(&lying).is_err()); + assert!(parse(&[0x0C, 0xC0]).is_err()); + } + + #[test] + fn chunk_folder_per_version_and_variant() { + assert_eq!(chunk_folder(Variant::Binary, 15), "ChunksV4"); + assert_eq!(chunk_folder(Variant::Binary, 14), "ChunksV3"); + assert_eq!(chunk_folder(Variant::Binary, 6), "ChunksV3"); + assert_eq!(chunk_folder(Variant::Binary, 5), "ChunksV2"); + assert_eq!(chunk_folder(Variant::Binary, 3), "ChunksV2"); + assert_eq!(chunk_folder(Variant::Binary, 2), "Chunks"); + assert_eq!(chunk_folder(Variant::Json, 2), "ChunksV4"); + assert_eq!(chunk_folder(Variant::Json, 0), "ChunksV4"); + } + + #[test] + fn blob_numbers() { + assert_eq!(blob("001000000000"), Some(1)); + assert_eq!(blob("000001000000"), Some(256)); + assert_eq!(blob("001002003004005006007008"), Some(0x0807060504030201)); + assert_eq!(blob("255255255255255255255255"), Some(u64::MAX)); + assert_eq!(blob("7"), Some(7)); + assert_eq!(blob(""), Some(0)); + assert_eq!(blob("000000016000"), Some(1_048_576)); + assert_eq!(blob("12a"), None); + } + + #[test] + fn json_manifest() { + let doc = r#"{ + "ManifestFileVersion": "013000000000", + "LaunchExeString": "Bin\\Game.exe", + "BuildVersionString": "++Game+Release-1.0", + "PrereqName": "Redist", "PrereqPath": "Redist\\setup.exe", + "ChunkHashList": { + "0000000A00000000000000000000000F": "001000000000", + "short": "001", + "0000000b00000000000000000000000f": "002000000000" + }, + "DataGroupList": { "0000000A00000000000000000000000F": "005" }, + "ChunkFilesizeList": { "0000000B00000000000000000000000F": "000001000000" }, + "FileManifestList": [ + { "Filename": "Bin\\Game.exe", "InstallTags": ["", "fr"], + "FileChunkParts": [ + { "Guid": "0000000A00000000000000000000000F", "Offset": "000000000000", "Size": "016000000000" }, + { "Guid": "0000000B00000000000000000000000F", "Offset": "004", "Size": "002" } ] } + ] + }"#; + let m = parse(doc.as_bytes()).unwrap(); + assert_eq!(m.variant, Variant::Json); + assert_eq!(m.version, 13); + assert_eq!(m.chunk_folder(), "ChunksV3"); + assert_eq!(m.launch_exe, "Bin/Game.exe"); + assert!(m.prereq_read); + assert_eq!(m.prereq_path, "Redist/setup.exe"); + assert_eq!(m.chunks.len(), 2); + assert_eq!(m.chunks[0].group, 5); + assert_eq!(m.chunks[1].cdn_size, 256); + assert_eq!(m.chunks[1].rolling_hash, 2); + assert!(!m.chunks[0].window_exact); + let f = &m.files[0]; + assert_eq!(f.path, "Bin/Game.exe"); + assert_eq!(f.tags, vec!["fr".to_string()]); + assert_eq!(f.size, 18); + assert!(m.chunk(&f.parts[1].guid).is_some()); + } + + #[test] + fn json_manifest_with_a_bad_chunk_key_is_rejected() { + let doc = r#"{"ChunkHashList": {"ZZZZZZZZ000000000000000000000000": "001"}, "FileManifestList": []}"#; + assert!(parse(doc.as_bytes()).is_err()); + assert!(parse(br#"{"FileManifestList": []}"#).is_err()); } } diff --git a/app/src/main/rust/stores/src/epic/mod.rs b/app/src/main/rust/stores/src/epic/mod.rs new file mode 100644 index 0000000000..932df65cd0 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/mod.rs @@ -0,0 +1,16 @@ +pub mod account; +pub mod achievements; +pub mod build_info; +pub mod catalog; +pub mod chunk; +pub mod ffi; +pub mod install; +pub mod launch; +pub mod library; +pub mod locale; +pub mod manifest; +pub mod net; +pub mod savesync; +pub mod service; +pub mod shop; +pub mod timefmt; diff --git a/app/src/main/rust/stores/src/epic/net.rs b/app/src/main/rust/stores/src/epic/net.rs new file mode 100644 index 0000000000..7421815a52 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/net.rs @@ -0,0 +1,279 @@ +use std::sync::OnceLock; +use std::time::Duration; + +pub const USER_AGENT_PRODUCT: &str = "DroidDeck"; + +static APP_VERSION: OnceLock = OnceLock::new(); + +pub fn set_app_version(version: &str) { + let clean: String = version.trim().chars().filter(|c| !c.is_control()).collect(); + if !clean.is_empty() { + let _ = APP_VERSION.set(clean); + } +} + +pub fn user_agent() -> String { + let version = APP_VERSION + .get() + .map(String::as_str) + .unwrap_or(crate::stores_jni::VERSION); + format!("{USER_AGENT_PRODUCT}/{version}") +} + +pub const EPIC_LAUNCHER_USER_AGENT: &str = + "EpicGamesLauncher/15.17.1-22692490+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Method { + Get, + Post, + Put, +} + +pub enum Body { + Empty, + Form(Vec<(String, String)>), + Json(String), + Bytes(Vec), +} + +pub struct Request<'a> { + pub method: Method, + pub url: &'a str, + pub bearer: Option<&'a str>, + pub basic: Option<(&'a str, &'a str)>, + pub body: Body, + pub read_timeout: Duration, + pub agent: Option<&'static str>, +} + +impl<'a> Request<'a> { + pub fn get(url: &'a str) -> Self { + Request { + method: Method::Get, + url, + bearer: None, + basic: None, + body: Body::Empty, + read_timeout: Duration::from_secs(30), + agent: None, + } + } + + pub fn post(url: &'a str, body: Body) -> Self { + Request { + method: Method::Post, + body, + ..Request::get(url) + } + } + + pub fn put(url: &'a str, body: Body) -> Self { + Request { + method: Method::Put, + body, + ..Request::get(url) + } + } + + pub fn agent(mut self, agent: &'static str) -> Self { + self.agent = Some(agent); + self + } + + pub fn bearer(mut self, token: &'a str) -> Self { + self.bearer = Some(token); + self + } + + pub fn basic(mut self, user: &'a str, password: &'a str) -> Self { + self.basic = Some((user, password)); + self + } + + pub fn read_timeout(mut self, timeout: Duration) -> Self { + self.read_timeout = timeout; + self + } +} + +pub struct Reply { + pub status: u16, + pub body: Vec, +} + +impl Reply { + pub fn ok(&self) -> bool { + (200..300).contains(&self.status) + } + + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } + + pub fn json(&self) -> Option { + if self.body.iter().all(|b| b.is_ascii_whitespace()) { + return Some(serde_json::Value::Object(Default::default())); + } + serde_json::from_slice(&self.body).ok() + } +} + +const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +const WHOLE_REQUEST_CAP: Duration = Duration::from_secs(600); + +pub fn send(request: Request<'_>) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| format!("runtime: {e}"))?; + runtime.block_on(send_async(request)) +} + +async fn send_async(request: Request<'_>) -> Result { + use reqwest::header::{ACCEPT, AUTHORIZATION, CONTENT_TYPE}; + let client = reqwest::Client::builder() + .user_agent(request.agent.map(str::to_string).unwrap_or_else(user_agent)) + .connect_timeout(CONNECT_TIMEOUT) + .read_timeout(request.read_timeout) + .timeout(WHOLE_REQUEST_CAP) + .build() + .map_err(|e| format!("http client: {}", e.without_url()))?; + let mut builder = match request.method { + Method::Get => client.get(request.url), + Method::Post => client.post(request.url), + Method::Put => client.put(request.url), + }; + if let Some(token) = request.bearer { + builder = builder.header(AUTHORIZATION, format!("Bearer {token}")); + } + if let Some((user, password)) = request.basic { + let pair = format!("{user}:{password}"); + builder = builder.header(AUTHORIZATION, format!("Basic {}", base64(pair.as_bytes()))); + } + builder = match request.body { + Body::Empty => builder, + Body::Form(pairs) => builder + .header(CONTENT_TYPE, "application/x-www-form-urlencoded") + .header(ACCEPT, "application/json") + .body(form_encode(&pairs)), + Body::Json(text) => builder + .header(CONTENT_TYPE, "application/json") + .header(ACCEPT, "application/json") + .body(text), + Body::Bytes(bytes) => builder + .header(CONTENT_TYPE, "application/octet-stream") + .body(bytes), + }; + let response = builder.send().await.map_err(describe)?; + let status = response.status().as_u16(); + let body = response.bytes().await.map_err(describe)?.to_vec(); + Ok(Reply { status, body }) +} + +fn describe(err: reqwest::Error) -> String { + let kind = if err.is_timeout() { + "timed out" + } else if err.is_connect() { + "could not connect" + } else if err.is_body() || err.is_decode() { + "body could not be read" + } else { + "request failed" + }; + format!("{kind}: {}", err.without_url()) +} + +pub fn percent_encode(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for b in text.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~') { + out.push(b as char); + } else { + out.push('%'); + out.push(HEX_UPPER[(b >> 4) as usize] as char); + out.push(HEX_UPPER[(b & 15) as usize] as char); + } + } + out +} + +const HEX_UPPER: &[u8; 16] = b"0123456789ABCDEF"; + +pub fn form_encode(pairs: &[(String, String)]) -> String { + pairs + .iter() + .map(|(k, v)| format!("{}={}", percent_encode(k), percent_encode(v))) + .collect::>() + .join("&") +} + +pub fn base64(data: &[u8]) -> String { + const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(data.len().div_ceil(3) * 4); + for group in data.chunks(3) { + let b0 = group[0] as u32; + let b1 = group.get(1).copied().unwrap_or(0) as u32; + let b2 = group.get(2).copied().unwrap_or(0) as u32; + let n = (b0 << 16) | (b1 << 8) | b2; + out.push(TABLE[(n >> 18) as usize & 63] as char); + out.push(TABLE[(n >> 12) as usize & 63] as char); + out.push(if group.len() > 1 { TABLE[(n >> 6) as usize & 63] as char } else { '=' }); + out.push(if group.len() > 2 { TABLE[n as usize & 63] as char } else { '=' }); + } + out +} + +pub fn origin(url: &str) -> String { + let after_scheme = url.find("://").map(|i| i + 3).unwrap_or(0); + let end = url[after_scheme..] + .find(['/', '?', '#']) + .map(|i| after_scheme + i) + .unwrap_or(url.len()); + url[..end].to_string() +} + +pub fn str_of<'v>(value: &'v serde_json::Value, key: &str) -> &'v str { + value.get(key).and_then(|v| v.as_str()).unwrap_or("") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percent_encoding_keeps_only_unreserved() { + assert_eq!(percent_encode("abc-_.~XYZ09"), "abc-_.~XYZ09"); + assert_eq!(percent_encode("a b+c/d=e&f"), "a%20b%2Bc%2Fd%3De%26f"); + assert_eq!(percent_encode("é"), "%C3%A9"); + } + + #[test] + fn form_values_are_encoded() { + let body = form_encode(&[ + ("grant_type".into(), "refresh_token".into()), + ("refresh_token".into(), "a+b/c=".into()), + ]); + assert_eq!(body, "grant_type=refresh_token&refresh_token=a%2Bb%2Fc%3D"); + } + + #[test] + fn base64_matches_rfc4648_vectors() { + assert_eq!(base64(b""), ""); + assert_eq!(base64(b"f"), "Zg=="); + assert_eq!(base64(b"fo"), "Zm8="); + assert_eq!(base64(b"foo"), "Zm9v"); + assert_eq!(base64(b"foobar"), "Zm9vYmFy"); + } + + #[test] + fn origin_is_scheme_and_host() { + assert_eq!(origin("https://cdn.example.com/Builds/x?y=1"), "https://cdn.example.com"); + assert_eq!(origin("https://cdn.example.com"), "https://cdn.example.com"); + } + + #[test] + fn user_agent_names_droiddeck() { + assert!(user_agent().starts_with("DroidDeck/")); + } +} diff --git a/app/src/main/rust/stores/src/epic/plan.rs b/app/src/main/rust/stores/src/epic/plan.rs deleted file mode 100644 index 5f96e338c9..0000000000 --- a/app/src/main/rust/stores/src/epic/plan.rs +++ /dev/null @@ -1,240 +0,0 @@ - -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; - -use super::manifest::{ChunkInfo, Manifest}; - -pub fn required_install_files(m: &Manifest) -> Vec { - let required: Vec = m - .files - .iter() - .enumerate() - .filter(|(_, f)| f.install_tags.is_empty()) - .map(|(i, _)| i) - .collect(); - if required.is_empty() { - (0..m.files.len()).collect() - } else { - required - } -} - -pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) -> Vec { - let Some(tags) = selected_tags else { - return (0..m.files.len()).collect(); - }; - if tags.is_empty() { - return required_install_files(m); - } - let want: HashSet<&str> = tags.iter().map(String::as_str).collect(); - let mut out = Vec::new(); - for (i, f) in m.files.iter().enumerate() { - if f.install_tags.is_empty() { - out.push(i); - continue; - } - if f.install_tags.iter().any(|t| want.contains(t.as_str())) { - out.push(i); - } - } - if out.is_empty() { - required_install_files(m) - } else { - out - } -} - -pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec { - let by_guid: HashMap = m.chunk_index_by_guid(); - let mut seen: HashSet = HashSet::new(); - let mut chunks = Vec::new(); - for &fi in file_indices { - let Some(f) = m.files.get(fi) else { - continue; - }; - for p in &f.parts { - let g = p.guid_str(); - if seen.insert(g.clone()) { - if let Some(&ci) = by_guid.get(&g) { - chunks.push(ci); - } - } - } - } - chunks -} - -pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 { - chunk_indices - .iter() - .filter_map(|&i| m.unique_chunks.get(i)) - .map(ChunkInfo::credit_bytes) - .sum() -} - -pub fn chunk_cache_dir(install_dir: &str) -> PathBuf { - Path::new(install_dir).join(super::CHUNK_CACHE_DIR) -} - -pub fn resolve_cache_dir(install_dir: &str, cache_dir: &str) -> PathBuf { - if cache_dir.is_empty() { - chunk_cache_dir(install_dir) - } else { - PathBuf::from(cache_dir) - } -} - -pub fn cached_chunk_path(cache_dir: &Path, chunk: &ChunkInfo) -> PathBuf { - cache_dir.join(chunk.guid_str()) -} - -pub fn is_chunk_cached(cache_dir: &Path, chunk: &ChunkInfo) -> bool { - cached_chunk_path(cache_dir, chunk).exists() -} - -pub fn chunk_url(cdn_prefix: &str, chunk_dir: &str, chunk: &ChunkInfo) -> String { - format!("{}/{}", cdn_prefix, chunk.path(chunk_dir)) -} - -pub fn normalize_prefix(raw: &str) -> String { - let unescaped = raw.replace("\\/", "/").replace('\\', ""); - unescaped.trim_end_matches('/').to_string() -} - -pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec { - let mut seen = HashSet::new(); - let mut out = Vec::new(); - for p in cdn_prefixes { - let n = normalize_prefix(p); - if n.is_empty() { - continue; - } - if seen.insert(n.clone()) { - out.push(n); - } - } - out -} - -pub fn per_host_cap(max_workers: usize, host_count: usize) -> usize { - let hosts = host_count.max(1); - let workers = max_workers.max(1); - let split = (workers + hosts - 1) / hosts; - split.max(6) -} - -#[cfg(test)] -mod tests { - use super::super::manifest::test_support::*; - use super::super::manifest::parse_manifest; - use super::*; - - fn manifest() -> Manifest { - parse_manifest(&build_manifest(&sample_chunks(), &sample_files(), 21, false)).unwrap() - } - - #[test] - fn install_tag_rules_match_java() { - let m = manifest(); - assert_eq!(resolve_install_files(&m, None), vec![0, 1, 2]); - assert_eq!(resolve_install_files(&m, Some(&[])), vec![0]); - let de = vec!["German".to_string(), "de-DE".to_string()]; - assert_eq!(resolve_install_files(&m, Some(&de)), vec![0, 1]); - let xx = vec!["Klingon".to_string()]; - assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0]); - } - - #[test] - fn manifest_without_tags_treats_every_file_as_required() { - let mut files = sample_files(); - for f in files.iter_mut() { - f.tags.clear(); - } - let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); - assert_eq!(required_install_files(&m), vec![0, 1, 2]); - assert_eq!(resolve_install_files(&m, Some(&[])), vec![0, 1, 2]); - } - - #[test] - fn manifest_where_every_file_is_tagged_falls_back_to_all() { - let mut files = sample_files(); - files[0].tags = vec!["English".to_string()]; - let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); - let xx = vec!["Klingon".to_string()]; - assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0, 1, 2]); - } - - #[test] - fn unique_chunks_follow_first_seen_part_order() { - let m = manifest(); - assert_eq!(unique_chunks_for_files(&m, &[2, 0]), vec![2, 0, 1]); - assert_eq!(unique_chunks_for_files(&m, &[1]), vec![1]); - assert!(unique_chunks_for_files(&m, &[]).is_empty()); - assert!(unique_chunks_for_files(&m, &[99]).is_empty(), "out-of-range index ignored"); - assert_eq!(total_credit_bytes(&m, &[0, 1, 2]), 700_000 + 1 + 9); - } - - #[test] - fn parts_referencing_unknown_guids_are_dropped() { - let mut files = sample_files(); - files[0] - .parts - .push(([0x1234, 0x5678, 0x9ABC, 0xDEF0], 0, 1)); - let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap(); - assert_eq!(unique_chunks_for_files(&m, &[0]), vec![0, 1]); - } - - #[test] - fn urls_and_cache_paths_match_java() { - let m = manifest(); - let c = &m.unique_chunks[0]; - assert_eq!( - chunk_url("https://fastly-download.epicgames.com/Builds/Org/o-x/y/default", "ChunksV4", c), - "https://fastly-download.epicgames.com/Builds/Org/o-x/y/default/ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" - ); - let cache = chunk_cache_dir("/data/x/imagefs/epic_games/Game"); - assert_eq!(cache, PathBuf::from("/data/x/imagefs/epic_games/Game/.chunks")); - assert_eq!(resolve_cache_dir("/data/x/imagefs/epic_games/Game", ""), cache); - assert_eq!( - resolve_cache_dir("/sdcard/Games/Game", "/data/x/cache/epic/Game"), - PathBuf::from("/data/x/cache/epic/Game") - ); - assert_eq!( - cached_chunk_path(&cache, c), - PathBuf::from("/data/x/imagefs/epic_games/Game/.chunks/00000001000000020000000300000004") - ); - let prefixes = vec!["a".to_string(), "b".to_string(), "a".to_string()]; - assert_eq!(distinct_prefixes(&prefixes), vec!["a".to_string(), "b".to_string()]); - } - - #[test] - fn prefixes_are_unescaped_from_the_java_json_scan() { - let raw = "https:\\/\\/egdownload.fastly-edge.com\\/Builds\\/Org\\/o-83e8\\/04b7\\/default\\"; - assert_eq!( - normalize_prefix(raw), - "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default" - ); - assert_eq!( - normalize_prefix("https://download.epicgames.com/Builds/x/default/"), - "https://download.epicgames.com/Builds/x/default" - ); - let m = manifest(); - let c = &m.unique_chunks[0]; - assert_eq!( - chunk_url(&normalize_prefix(raw), "ChunksV4", c), - "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default/ChunksV4/05/0123456789ABCDEF_00000001000000020000000300000004.chunk" - ); - let two = vec![raw.to_string(), "https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default".to_string(), String::new()]; - assert_eq!(distinct_prefixes(&two).len(), 1, "escaped and clean forms collapse; empty dropped"); - } - - #[test] - fn per_host_cap_splits_the_window_with_a_floor_of_six() { - assert_eq!(per_host_cap(32, 3), 11); - assert_eq!(per_host_cap(32, 1), 32); - assert_eq!(per_host_cap(32, 2), 16); - assert_eq!(per_host_cap(8, 3), 6, "floor"); - assert_eq!(per_host_cap(8, 1), 8); - assert_eq!(per_host_cap(0, 0), 6, "degenerate inputs"); - } -} diff --git a/app/src/main/rust/stores/src/epic/savesync.rs b/app/src/main/rust/stores/src/epic/savesync.rs new file mode 100644 index 0000000000..ac78dbb4dc --- /dev/null +++ b/app/src/main/rust/stores/src/epic/savesync.rs @@ -0,0 +1,119 @@ +use std::time::Duration; + +use serde_json::{json, Value}; + +use super::net::{self, Body, Request}; + +const STORAGE_HOST: &str = "https://datastorage-public-service-liveegs.live.use1a.on.epicgames.com"; + +fn base(account_id: &str, app_name: &str) -> String { + format!( + "{STORAGE_HOST}/api/v1/access/egstore/savesync/{}/{}/", + net::percent_encode(account_id), + net::percent_encode(app_name) + ) +} + +pub fn name_of(key: &str, app_name: &str) -> String { + let marker = format!("/{app_name}/"); + if let Some(at) = key.find(&marker) { + return key[at + marker.len()..].to_string(); + } + let lead = format!("{app_name}/"); + key.strip_prefix(&lead).unwrap_or(key).to_string() +} + +pub fn read_listing(answer: &Value, app_name: &str) -> Vec { + let Some(files) = answer.get("files").and_then(|f| f.as_object()) else { + return Vec::new(); + }; + files + .iter() + .map(|(key, f)| { + let modified = super::timefmt::iso_seconds(net::str_of(f, "lastModified")) + .map(|s| s * 1000) + .unwrap_or(-1); + json!({ + "name": name_of(key, app_name), + "modified": modified, + "hash": net::str_of(f, "hash"), + "readLink": net::str_of(f, "readLink"), + }) + }) + .filter(|f| !net::str_of(f, "name").is_empty()) + .collect() +} + +pub fn list(access_token: &str, account_id: &str, app_name: &str) -> Result, String> { + let url = base(account_id, app_name); + let reply = net::send(Request::get(&url).bearer(access_token))?; + if reply.status == 404 { + return Ok(Vec::new()); + } + if !reply.ok() { + return Err(format!("Epic cloud list HTTP {}", reply.status)); + } + let answer = reply.json().ok_or("Epic cloud list could not be read")?; + Ok(read_listing(&answer, app_name)) +} + +pub fn write_links(access_token: &str, account_id: &str, app_name: &str, names: &[String]) -> Result { + let url = base(account_id, app_name); + let body = json!({ "files": names }).to_string(); + let reply = net::send(Request::post(&url, Body::Json(body)).bearer(access_token))?; + if !reply.ok() { + return Err(format!("Epic cloud upload request HTTP {}", reply.status)); + } + let answer = reply.json().ok_or("Epic cloud upload answer could not be read")?; + let mut links = serde_json::Map::new(); + if let Some(files) = answer.get("files").and_then(|f| f.as_object()) { + for (key, f) in files { + let link = net::str_of(f, "writeLink"); + if !link.is_empty() { + links.insert(name_of(key, app_name), json!(link)); + } + } + } + Ok(Value::Object(links)) +} + +pub fn read(link: &str) -> Result, String> { + let reply = net::send(Request::get(link).read_timeout(Duration::from_secs(30)))?; + if !reply.ok() { + return Err(format!("Epic cloud download HTTP {}", reply.status)); + } + Ok(reply.body) +} + +pub fn write(link: &str, data: Vec) -> Result<(), String> { + let reply = net::send(Request::put(link, Body::Bytes(data)))?; + if !reply.ok() { + return Err(format!("Epic cloud upload HTTP {}", reply.status)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_from_keys() { + assert_eq!(name_of("acc/Fig/Saves/slot1.sav", "Fig"), "Saves/slot1.sav"); + assert_eq!(name_of("Fig/slot1.sav", "Fig"), "slot1.sav"); + assert_eq!(name_of("slot1.sav", "Fig"), "slot1.sav"); + } + + #[test] + fn listing() { + let answer = json!({"files": { + "acc/Fig/a.sav": {"lastModified": "2024-05-01T10:00:00.000Z", "hash": "abc", "readLink": "https://r/a"}, + "acc/Fig/": {"lastModified": "", "hash": "", "readLink": ""} + }}); + let files = read_listing(&answer, "Fig"); + assert_eq!(files.len(), 1); + assert_eq!(files[0]["name"], "a.sav"); + assert_eq!(files[0]["modified"], 1_714_557_600_000i64); + assert!(read_listing(&json!({}), "Fig").is_empty()); + } +} diff --git a/app/src/main/rust/stores/src/epic/service.rs b/app/src/main/rust/stores/src/epic/service.rs new file mode 100644 index 0000000000..67c1995bdf --- /dev/null +++ b/app/src/main/rust/stores/src/epic/service.rs @@ -0,0 +1,216 @@ +use serde_json::{json, Value}; + +use super::build_info::{self, BuildInfo}; +use super::net::str_of; +use super::{account, achievements, catalog, install, launch, library, locale, manifest, savesync, shop}; + +fn error(message: impl Into) -> Value { + json!({ "error": message.into() }) +} + +fn info_of(req: &Value) -> Option { + req.get("info").and_then(BuildInfo::from_json) +} + +pub fn call(op: &str, request: &str) -> String { + let req: Value = serde_json::from_str(request).unwrap_or(Value::Null); + dispatch(op, &req).to_string() +} + +fn dispatch(op: &str, req: &Value) -> Value { + let token = str_of(req, "accessToken"); + match op { + "auth.config" => json!({ + "startUrl": account::start_url(), + "redirectPrefix": account::REDIRECT_PREFIX, + "webViewUserAgent": account::WEBVIEW_USER_AGENT, + }), + "auth.codeFromPage" => match account::code_from_page_text(str_of(req, "text")) { + Some(code) => json!({ "code": code }), + None => json!({}), + }, + "auth.signIn" => match account::sign_in_with_code(str_of(req, "code")) { + Ok(t) => t.to_json(), + Err(e) => error(e), + }, + "auth.refresh" => match account::refresh(str_of(req, "refreshToken")) { + Ok(t) => t.to_json(), + Err(e) => error(e), + }, + "auth.exchange" => match account::exchange_code(token) { + Ok(code) => json!({ "code": code }), + Err(e) => error(e), + }, + "library.owned" => match library::list_owned(token) { + Ok(list) => json!({ "records": list.iter().map(|o| json!({ + "appName": o.app_name, "namespace": o.namespace, "catalogItemId": o.catalog_item_id, + })).collect::>() }), + Err(library::ListError::SignIn) => json!({ "error": "sign-in refused", "signIn": true }), + Err(library::ListError::Failed(e)) => error(e), + }, + "catalog.details" => match catalog::details(token, str_of(req, "namespace"), str_of(req, "catalogItemId")) { + Some(d) => json!({ "details": d }), + None => error("no catalog answer"), + }, + "catalog.attribute" => match catalog::attribute( + token, + str_of(req, "namespace"), + str_of(req, "catalogItemId"), + str_of(req, "name"), + ) { + Some(v) => json!({ "value": v }), + None => json!({ "unknown": true }), + }, + "build.info" => match build_info::fetch( + token, + str_of(req, "namespace"), + str_of(req, "catalogItemId"), + str_of(req, "appName"), + ) { + Some(info) => json!({ "info": info.to_json() }), + None => error("The manifest could not be fetched"), + }, + "build.cdns" => json!({ "cdns": info_of(req).map(|i| build_info::cdns_to_json(&i.cdns)).unwrap_or(json!([])) }), + "build.prereq" => match info_of(req).map(|i| install::fetch_manifest(&i, &|_: &str| {})) { + Some(Ok(m)) => json!({ "name": m.prereq_name, "path": m.prereq_path, "read": m.prereq_read }), + Some(Err(e)) => error(e), + None => error("no build information"), + }, + "shop.shelves" => shop::shelves(str_of(req, "country")), + "achievements" => match achievements::fetch( + token, + str_of(req, "namespace"), + str_of(req, "accountId"), + str_of(req, "locale"), + ) { + Some(list) => json!({ "list": list }), + None => error("unreachable"), + }, + "cloud.list" => match savesync::list(token, str_of(req, "accountId"), str_of(req, "appName")) { + Ok(files) => json!({ "files": files }), + Err(e) => error(e), + }, + "cloud.writeLinks" => { + let names: Vec = req + .get("names") + .and_then(|n| n.as_array()) + .map(|a| a.iter().filter_map(|n| n.as_str()).map(str::to_string).collect()) + .unwrap_or_default(); + match savesync::write_links(token, str_of(req, "accountId"), str_of(req, "appName"), &names) { + Ok(links) => json!({ "links": links }), + Err(e) => error(e), + } + } + "launch.args" => { + let data = launch::LaunchData { + display_name: str_of(req, "displayName").into(), + account_id: str_of(req, "accountId").into(), + namespace: str_of(req, "namespace").into(), + locale: str_of(req, "locale").into(), + app_name: str_of(req, "appName").into(), + deployment_id: str_of(req, "deploymentId").into(), + additional: str_of(req, "additional").into(), + }; + json!({ "args": launch::arguments(&data) }) + } + "locale" => { + let key = locale::language_key(str_of(req, "language"), str_of(req, "region")); + json!({ "key": key, "tags": locale::tags_for_key(key), "launchLocale": locale::launch_locale(key) }) + } + _ => error(format!("unknown operation {op}")), + } +} + +pub fn fetch(op: &str, request: &str) -> Option> { + let req: Value = serde_json::from_str(request).ok()?; + match op { + "manifest.download" => { + let info = info_of(&req)?; + let cdns = req.get("cdns").map(build_info::cdns_from_json).unwrap_or_else(|| info.cdns.clone()); + build_info::download_manifest(&info, &cdns, &|_: &str| {}) + } + "cloud.read" => savesync::read(str_of(&req, "url")).ok(), + _ => None, + } +} + +pub fn send(op: &str, request: &str, body: Vec) -> String { + let req: Value = serde_json::from_str(request).unwrap_or(Value::Null); + let out = match op { + "cloud.write" => match savesync::write(str_of(&req, "url"), body) { + Ok(()) => json!({ "ok": true }), + Err(e) => error(e), + }, + _ => error(format!("unknown operation {op}")), + }; + out.to_string() +} + +pub fn inspect(bytes: &[u8], request: &str) -> String { + let req: Value = serde_json::from_str(request).unwrap_or(Value::Null); + let m = match manifest::parse(bytes) { + Ok(m) => m, + Err(e) => return error(e).to_string(), + }; + let out = match str_of(&req, "op") { + "select" => { + let tags: Option> = req + .get("tags") + .and_then(|t| t.as_array()) + .map(|a| a.iter().filter_map(|t| t.as_str()).map(str::to_string).collect()); + json!({ "indices": locale::select(&m.files, tags.as_deref()) }) + } + _ => json!({ + "buildVersion": m.build_version, + "launchExe": m.launch_exe, + "prereqName": m.prereq_name, + "prereqPath": m.prereq_path, + "prereqRead": m.prereq_read, + "files": m.files.iter().map(|f| json!({ "path": f.path, "size": f.size, "tags": f.tags })).collect::>(), + }), + }; + out.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_operations() { + let v: Value = serde_json::from_str(&call("locale", r#"{"language":"pt","region":"BR"}"#)).unwrap(); + assert_eq!(v["key"], "pt-BR"); + assert_eq!(v["launchLocale"], "pt"); + let v: Value = serde_json::from_str(&call("auth.codeFromPage", r#"{"text":"{\"authorizationCode\":\"c1\"}"}"#)).unwrap(); + assert_eq!(v["code"], "c1"); + let v: Value = serde_json::from_str(&call("nope", "{}")).unwrap(); + assert!(v["error"].is_string()); + let v: Value = serde_json::from_str(&call("launch.args", r#"{"namespace":"ns","appName":"A"}"#)).unwrap(); + assert_eq!(v["args"][0], "-EpicPortal"); + } + + #[test] + fn inspect_lists_and_selects() { + use crate::epic::manifest::testing::*; + let m = TestManifest { + version: 21, + build: "b".into(), + exe: "x.exe".into(), + prereq: None, + chunks: vec![], + files: vec![ + TestFile { path: "x.exe".into(), sha1: [0; 20], tags: vec![], parts: vec![] }, + TestFile { path: "de.pak".into(), sha1: [0; 20], tags: vec!["German".into()], parts: vec![] }, + ], + compress: false, + meta_tail: vec![], + }; + let bytes = build(&m); + let v: Value = serde_json::from_str(&inspect(&bytes, "{}")).unwrap(); + assert_eq!(v["files"].as_array().unwrap().len(), 2); + let v: Value = serde_json::from_str(&inspect(&bytes, r#"{"op":"select","tags":["English"]}"#)).unwrap(); + assert_eq!(v["indices"], json!([0])); + let v: Value = serde_json::from_str(&inspect(b"nonsense", "{}")).unwrap(); + assert!(v["error"].is_string()); + } +} diff --git a/app/src/main/rust/stores/src/epic/shop.rs b/app/src/main/rust/stores/src/epic/shop.rs new file mode 100644 index 0000000000..a2a06592d0 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/shop.rs @@ -0,0 +1,317 @@ +use std::collections::HashSet; +use std::time::Duration; + +use serde_json::{json, Value}; + +use super::net::{self, Body, Request}; + +const GRAPHQL_URL: &str = "https://store.epicgames.com/graphql"; +const PROMOTIONS_URL: &str = "https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions"; +const SHELF_COUNT: u32 = 24; + +#[derive(Clone, Copy)] +pub enum Shelf { + New, + Deals, + Free, + Trending, +} + +fn country_code(raw: &str) -> String { + let c = raw.trim().to_ascii_uppercase(); + if c.len() == 2 && c.bytes().all(|b| b.is_ascii_uppercase()) { + c + } else { + "US".to_string() + } +} + +fn now_stamp() -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + let days = secs.div_euclid(86_400); + let rem = secs.rem_euclid(86_400); + let (y, m, d) = civil_from_days(days); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}.000Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +fn civil_from_days(z: i64) -> (i64, i64, i64) { + let z = z + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + (yoe + era * 400 + if m <= 2 { 1 } else { 0 }, m, d) +} + +pub fn shelf_query(shelf: Shelf, country: &str, now: &str) -> String { + let (sort_by, sort_dir, extra) = match shelf { + Shelf::New => ("releaseDate", "DESC", format!("releaseDate: \"[,{now}]\"")), + Shelf::Deals => ("currentPrice", "ASC", "onSale: true".to_string()), + Shelf::Free => ("relevancy", "DESC", "freeGame: true".to_string()), + Shelf::Trending => ("relevancy", "DESC", format!("releaseDate: \"[,{now}]\"")), + }; + format!( + "query {{ Catalog {{ searchStore(category: \"games/edition/base\", count: {SHELF_COUNT}, \ +country: \"{country}\", locale: \"en-US\", sortBy: \"{sort_by}\", sortDir: \"{sort_dir}\", \ +allowCountries: \"{country}\", {extra}) {{ elements {{ title id namespace description effectiveDate \ +releaseDate offerType productSlug urlSlug items {{ id namespace }} keyImages {{ type url }} \ +seller {{ name }} tags {{ name }} customAttributes {{ key value }} \ +catalogNs {{ mappings(pageType: \"productHome\") {{ pageSlug pageType }} }} \ +price(country: \"{country}\") {{ totalPrice {{ discountPrice originalPrice discount currencyCode \ +fmtPrice(locale: \"en-US\") {{ originalPrice discountPrice }} }} }} }} paging {{ total }} }} }} }}" + ) +} + +fn elements(answer: &Value) -> Vec { + answer + .pointer("/data/Catalog/searchStore/elements") + .and_then(|e| e.as_array()) + .cloned() + .unwrap_or_default() +} + +fn key_image(el: &Value, order: &[&[&str]]) -> String { + let list = el.get("keyImages").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + for group in order { + if let Some(img) = list + .iter() + .find(|i| group.contains(&net::str_of(i, "type")) && !net::str_of(i, "url").is_empty()) + { + return net::str_of(img, "url").to_string(); + } + } + String::new() +} + +fn custom_attribute(el: &Value, key: &str) -> String { + el.get("customAttributes") + .and_then(|a| a.as_array()) + .and_then(|a| a.iter().find(|kv| net::str_of(kv, "key") == key)) + .map(|kv| net::str_of(kv, "value").trim().to_string()) + .unwrap_or_default() +} + +fn store_slug(el: &Value) -> String { + let mapped = el + .pointer("/catalogNs/mappings") + .and_then(|m| m.as_array()) + .and_then(|m| m.iter().find(|x| net::str_of(x, "pageType") == "productHome")) + .map(|x| net::str_of(x, "pageSlug").to_string()) + .filter(|s| !s.is_empty()); + mapped + .or_else(|| { + let s = net::str_of(el, "productSlug"); + Some(s.trim_end_matches("/home").to_string()).filter(|s| !s.is_empty() && s != "null") + }) + .unwrap_or_default() +} + +pub fn item_of(el: &Value) -> Option { + let id = net::str_of(el, "id"); + let title = net::str_of(el, "title"); + if id.is_empty() || title.is_empty() { + return None; + } + let total = el.pointer("/price/totalPrice"); + let orig = total.and_then(|t| t.get("originalPrice")).and_then(|v| v.as_i64()); + let disc = total.and_then(|t| t.get("discountPrice")).and_then(|v| v.as_i64()); + let fmt = |k: &str| { + total + .and_then(|t| t.get("fmtPrice")) + .map(|f| net::str_of(f, k).trim().to_string()) + .unwrap_or_default() + }; + let has_price = orig.is_some() && disc.is_some(); + let free = disc == Some(0); + let discount = match (orig, disc) { + (Some(o), Some(d)) if o > 0 && d < o => ((o - d) * 100 / o) as i32, + _ => 0, + }; + let shown = if free { + "Free".to_string() + } else { + let d = fmt("discountPrice"); + if !d.is_empty() && d != "0" { + d + } else { + fmt("originalPrice") + } + }; + let names: Vec = el + .get("tags") + .and_then(|t| t.as_array()) + .map(|a| a.iter().map(|t| net::str_of(t, "name").to_string()).filter(|n| !n.is_empty()).collect()) + .unwrap_or_default(); + let tag_line = names.iter().filter(|n| *n != "Windows").take(3).cloned().collect::>().join(", "); + let items: Vec = el + .get("items") + .and_then(|i| i.as_array()) + .map(|a| a.iter().map(|i| net::str_of(i, "id").to_string()).filter(|s| !s.is_empty()).collect()) + .unwrap_or_default(); + let slug = store_slug(el); + let url = if slug.is_empty() { + format!("https://store.epicgames.com/en-US/browse?q={}", net::percent_encode(title)) + } else { + format!("https://store.epicgames.com/en-US/p/{slug}") + }; + let publisher = Some(custom_attribute(el, "publisherName")) + .filter(|p| !p.is_empty()) + .unwrap_or_else(|| el.get("seller").map(|s| net::str_of(s, "name").to_string()).unwrap_or_default()); + let release = Some(net::str_of(el, "releaseDate")) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| net::str_of(el, "effectiveDate")); + let mut extra = json!({ "namespace": net::str_of(el, "namespace") }); + if !items.is_empty() { + extra["items"] = json!(items.join(",")); + } + if !slug.is_empty() { + extra["slug"] = json!(slug); + } + Some(json!({ + "id": id, + "title": title, + "image": key_image(el, &[&["DieselGameBox"], &["OfferImageWide", "DieselStoreFrontWide"], &["Thumbnail"]]), + "tall": key_image(el, &[&["DieselGameBoxTall"], &["OfferImageTall", "DieselStoreFrontTall"], &["Thumbnail"]]), + "tags": tag_line, + "free": free, + "hasPrice": has_price, + "final": shown, + "orig": if discount > 0 { fmt("originalPrice") } else { String::new() }, + "disc": discount, + "url": url, + "dev": custom_attribute(el, "developerName"), + "pub": publisher, + "rel": release, + "desc": net::str_of(el, "description"), + "extra": extra, + "allTags": names, + })) +} + +pub fn free_now(el: &Value) -> Option { + let offer = el.pointer("/promotions/promotionalOffers/0/promotionalOffers/0")?; + let pct = offer.pointer("/discountSetting/discountPercentage")?.as_f64()?; + if pct != 0.0 { + return None; + } + let mut item = item_of(el)?; + let until: String = net::str_of(offer, "endDate").chars().take(10).collect(); + item["free"] = json!(true); + item["disc"] = json!(100); + item["final"] = json!("Free"); + if !until.is_empty() { + item["tags"] = json!(format!("Free until {until}")); + } + Some(item) +} + +fn post_query(query: String) -> Vec { + let body = json!({ "query": query, "variables": {} }).to_string(); + match net::send(Request::post(GRAPHQL_URL, Body::Json(body)).read_timeout(Duration::from_secs(25))) { + Ok(reply) if reply.ok() => reply.json().map(|a| elements(&a)).unwrap_or_default(), + _ => Vec::new(), + } +} + +fn promotions(country: &str) -> Vec { + let url = format!("{PROMOTIONS_URL}?locale=en-US&country={country}&allowCountries={country}"); + match net::send(Request::get(&url).read_timeout(Duration::from_secs(25))) { + Ok(reply) if reply.ok() => reply.json().map(|a| elements(&a)).unwrap_or_default(), + _ => Vec::new(), + } +} + +fn unique(items: impl IntoIterator) -> Vec { + let mut seen = HashSet::new(); + items.into_iter().filter(|i| seen.insert(net::str_of(i, "id").to_string())).collect() +} + +pub fn shelves(country: &str) -> Value { + let country = country_code(country); + let now = now_stamp(); + let shelf = |s: Shelf| -> Vec { + post_query(shelf_query(s, &country, &now)).iter().filter_map(item_of).collect() + }; + let new = unique(shelf(Shelf::New)); + let deals = unique(shelf(Shelf::Deals).into_iter().filter(|i| i["disc"].as_i64().unwrap_or(0) > 0)); + let promos: Vec = promotions(&country).iter().filter_map(free_now).collect(); + let free = unique(promos.into_iter().chain(shelf(Shelf::Free))); + let trending = unique(shelf(Shelf::Trending)); + json!({ "new": new, "deals": deals, "free": free, "trending": trending }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn element() -> Value { + json!({ + "id": "offer1", "title": "Fig & Co", "namespace": "ns", + "description": "

Hi

", "releaseDate": "2024-01-02T00:00:00.000Z", + "productSlug": "fig/home", "items": [{"id": "cat1"}, {"id": "cat2"}], + "keyImages": [{"type": "Thumbnail", "url": "t"}, {"type": "OfferImageWide", "url": "w"}, {"type": "DieselStoreFrontTall", "url": "tl"}], + "seller": {"name": "Seller"}, + "tags": [{"name": "Windows"}, {"name": "Action"}, {"name": "RPG"}, {"name": "Indie"}, {"name": "Mature"}], + "customAttributes": [{"key": "developerName", "value": "Dev"}], + "catalogNs": {"mappings": []}, + "price": {"totalPrice": {"originalPrice": 2000, "discountPrice": 1499, + "fmtPrice": {"originalPrice": "$20.00", "discountPrice": "$14.99"}}} + }) + } + + #[test] + fn shelf_item_fields() { + let i = item_of(&element()).unwrap(); + assert_eq!(i["image"], "w"); + assert_eq!(i["tall"], "tl"); + assert_eq!(i["tags"], "Action, RPG, Indie"); + assert_eq!(i["disc"], 25); + assert_eq!(i["final"], "$14.99"); + assert_eq!(i["orig"], "$20.00"); + assert_eq!(i["free"], false); + assert_eq!(i["hasPrice"], true); + assert_eq!(i["url"], "https://store.epicgames.com/en-US/p/fig"); + assert_eq!(i["pub"], "Seller"); + assert_eq!(i["dev"], "Dev"); + assert_eq!(i["extra"]["items"], "cat1,cat2"); + assert_eq!(i["allTags"].as_array().unwrap().len(), 5); + assert!(item_of(&json!({"id": "x"})).is_none()); + } + + #[test] + fn slugs_free_and_promotions() { + let mut el = element(); + el["catalogNs"] = json!({"mappings": [{"pageType": "productHome", "pageSlug": "fig-game"}]}); + el["price"] = json!({"totalPrice": {"originalPrice": 999, "discountPrice": 0, "fmtPrice": {"originalPrice": "$9.99", "discountPrice": "0"}}}); + let i = item_of(&el).unwrap(); + assert_eq!(i["url"], "https://store.epicgames.com/en-US/p/fig-game"); + assert_eq!(i["free"], true); + assert_eq!(i["final"], "Free"); + el["productSlug"] = json!("null"); + el["catalogNs"] = json!({}); + assert_eq!(item_of(&el).unwrap()["url"], "https://store.epicgames.com/en-US/browse?q=Fig%20%26%20Co"); + el["promotions"] = json!({"promotionalOffers": [{"promotionalOffers": [{"endDate": "2026-10-20T15:00:00.000Z", "discountSetting": {"discountPercentage": 0}}]}]}); + let p = free_now(&el).unwrap(); + assert_eq!(p["tags"], "Free until 2026-10-20"); + assert_eq!(p["disc"], 100); + el["promotions"]["promotionalOffers"][0]["promotionalOffers"][0]["discountSetting"]["discountPercentage"] = json!(50); + assert!(free_now(&el).is_none()); + } + + #[test] + fn query_and_dates() { + assert_eq!(country_code("de"), "DE"); + assert_eq!(country_code("419"), "US"); + assert_eq!(civil_from_days(0), (1970, 1, 1)); + assert_eq!(civil_from_days(19_723), (2024, 1, 1)); + let q = shelf_query(Shelf::Deals, "DE", "x"); + assert!(q.contains("onSale: true") && q.contains("sortBy: \"currentPrice\"") && q.contains("country: \"DE\"")); + assert!(shelf_query(Shelf::New, "US", "2026-01-01T00:00:00.000Z").contains("releaseDate: \"[,2026-01-01T00:00:00.000Z]\"")); + } +} diff --git a/app/src/main/rust/stores/src/epic/stream.rs b/app/src/main/rust/stores/src/epic/stream.rs deleted file mode 100644 index 5489801ddc..0000000000 --- a/app/src/main/rust/stores/src/epic/stream.rs +++ /dev/null @@ -1,1956 +0,0 @@ -//! The adaptive Epic install: chunks fetched in parallel and written straight into the files. -//! -//! The batch engine ([`super::driver`] + [`super::assemble`]) fetched every chunk into a cache -//! first and assembled the files afterwards: fast, but the cache and the game existed side by side -//! (twice the game's size on disk) and every byte was written twice. The bounded Java writer fixed -//! the space by fetching one chunk at a time, at the cost of the parallel fetch. This module keeps -//! the fetch core's parallel, adaptive download (window ramp, per-host caps, the in-flight byte -//! budget, retries, host rotation, stall handoff — none of it changes) and keeps the bounded -//! writer's safety rules, by deciding up front what each downloaded chunk is for: -//! -//! - **A chunk exactly one file part uses** is decompressed, verified and written straight into -//! that file's staging file at the part's offset (a positioned write, so chunks may arrive in -//! any order). It never touches a cache. -//! - **A chunk several parts use** is written into every part whose file is already open, and the -//! decompressed copy is kept in the cache (`/`, the same verified form and name the -//! batch engine and the Java writer use, so a resume on any engine reuses it) until its last -//! covered part is written; then it is deleted. Decompressed, not raw: a later part is then a -//! plain read and write with no second inflate or hash, and the file is exactly what a resume -//! already knows how to check. -//! - The cache has a **byte budget** (from the Java planner, sized from free space). The plan -//! simulates the whole install in file order and, when the budget would be exceeded, evicts the -//! cached chunk whose next use is furthest away (Belady's rule, the fewest refetches for that -//! budget). An evicted chunk is downloaded again for its later uses, as its own fetch item. -//! -//! **Files open in order.** The fetch items are queued in the order of their first use, so the -//! fetch core works through the game front to back. A file is opened (its staging file created) -//! only once the download has reached it: every file before the first chunk still to arrive, plus -//! the file of every chunk that has arrived. That keeps the set of half-written files to what the -//! fetch window spans, and it is also why the cache never holds more than the plan allowed: a -//! cached chunk is written into every newly reached file at once, so what stays in the cache is -//! exactly what the simulation kept at that point of the file order. -//! -//! **Updates are paced by a replacement-overlap budget.** A staged file that replaces an existing -//! one sits beside the old copy until its swap, so every replaced file open at once costs its full -//! new size in extra space (on a fresh install a staged file simply becomes the file). Files are -//! admitted in order; a replaced file is admitted only while the planned sizes of admitted, -//! not-yet-swapped replaced files stay within the budget the manager computed from free space, -//! and one is always admitted when none is open, so the install always progresses. The fetch core -//! dispatches a chunk only once the file of its first use is admitted ([`run_fetch_gated`]). -//! Admission never changes the order files are reached in, only when, so the cache simulation -//! below (which works in file order) stays exact: the cache holds at any moment what the plan holds -//! at that point of the order. Nothing waits on a gated file: every chunk an admitted file needs is -//! first used at or before that file, so it was dispatched before the gate closed. -//! -//! **A slow disk throttles the download by itself.** Every write happens in the fetch core's -//! process pool, inside [`FetchSink::process`]; the core frees a body's share of its in-flight -//! byte budget only when `process` returns, so while the disk is behind, the budget fills and no -//! new request goes out. Memory is that budget plus one decompressed chunk per pool thread. -//! -//! **Each file is replaced safely**, with the bounded Java writer's rules: written into a unique -//! `.stage` file in the manager's owned staging directory (`/.droiddeck-epic-staging`, -//! whose leftovers the manager recovers before the next run; a sibling `.droiddeck-epic-*.stage` -//! when no directory is given), never a manifest path, so a game file legitimately named `x.part` -//! is safe. Its SHA-1 computed as the bytes go out (parts written out of order are -//! read back once the bytes before them are in, while they are still in the page cache), then -//! synced, its size and hash checked, the install marked `updating` once (the Java callback, -//! before the first replacement), and the stage renamed over the destination. On cancel or failure -//! every unfinished stage is removed and the old files are untouched; verified files that were -//! already swapped stay, and the next run's delta pass skips them. The cache is left as it stands -//! for that run, and removed when the install completes. - -use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fs::{self, File, OpenOptions}; -use std::io::ErrorKind; -use std::os::unix::fs::FileExt; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; -use std::sync::Mutex; -use std::time::Instant; - -use sha1::{Digest, Sha1}; - -use crate::fetch_core::{run_fetch_gated, FetchItem, FetchOptions, FetchSink, SinkError}; - -use super::assemble::relative_path; -use super::chunk::{chunk_window_bytes, decode_verified_chunk, part_path}; -use super::driver::REQUEST_TIMEOUT; -use super::manifest::{parse_manifest, ChunkInfo, Manifest}; -use super::plan::{chunk_url, distinct_prefixes, per_host_cap, resolve_cache_dir}; - -/// "No further use" in the schedule's next-use table. -const NONE: usize = usize::MAX; - -/// Progress callbacks are rate-limited to one per this many milliseconds (a finished file always -/// reports): a large game has hundreds of thousands of parts, each a JNI call otherwise. -const PROGRESS_EVERY_MS: u64 = 50; - -// ───────────────────────────────────────────────────────────────────────────────────────────── -// The schedule: which download serves which file part -// ───────────────────────────────────────────────────────────────────────────────────────────── - -/// One download of a chunk (or one cached copy left by an earlier run) and the file parts it -/// serves. Positions index the install's part sequence: every pending file's parts, file by file. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ScheduledEvent { - /// Index into `manifest.unique_chunks`. - pub chunk: usize, - /// The part positions this copy serves, in order. - pub uses: Vec, - /// Already in the cache when the run started; not fetched. - pub preloaded: bool, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Schedule { - pub events: Vec, - /// Position → the event that serves it. - pub use_event: Vec, - /// Cached chunks left by an earlier run that this run will not use (no pending part needs - /// them, or keeping them would exceed the budget): deleted before anything is fetched. - pub discarded: Vec, - /// The most the cache holds at any point of the file order. - pub peak_cache_bytes: u64, -} - -/// Plans the downloads for the part sequence `chunk_of_use` (the chunk each part position reads) -/// with at most `budget` bytes of cached chunks (`chunk_bytes[c]` each). A chunk with one use is -/// a plain download. A chunk with more uses stays cached from one use to the next; when the cache -/// would exceed the budget, the cached chunk needed furthest in the future (possibly the one just -/// downloaded) is dropped, and its next use becomes a new download. `preloaded` chunks start in -/// the cache, the soonest needed first while they fit. -pub fn schedule( - chunk_of_use: &[usize], - chunk_bytes: &[u64], - preloaded: &[usize], - budget: u64, -) -> Schedule { - let n = chunk_of_use.len(); - let mut next_use = vec![NONE; n]; - let mut first_use: HashMap = HashMap::new(); - for p in (0..n).rev() { - let c = chunk_of_use[p]; - next_use[p] = first_use.get(&c).copied().unwrap_or(NONE); - first_use.insert(c, p); - } - - let mut out = Schedule { - use_event: vec![NONE; n], - ..Schedule::default() - }; - // Cached chunk → (its event, the position it is next needed at), and the same keyed by next - // use so the furthest one is the last entry. - let mut resident: HashMap = HashMap::new(); - let mut by_next: BTreeSet<(usize, usize)> = BTreeSet::new(); - let mut bytes: u64 = 0; - - let mut pre: Vec<(usize, usize)> = Vec::new(); - let mut seen_pre: HashSet = HashSet::new(); - for &c in preloaded { - if !seen_pre.insert(c) { - continue; - } - match first_use.get(&c) { - Some(&p) => pre.push((p, c)), - None => out.discarded.push(c), - } - } - pre.sort_unstable(); - for (p, c) in pre { - let size = chunk_bytes[c]; - if bytes + size <= budget { - let e = out.events.len(); - out.events.push(ScheduledEvent { - chunk: c, - uses: Vec::new(), - preloaded: true, - }); - resident.insert(c, (e, p)); - by_next.insert((p, c)); - bytes += size; - } else { - out.discarded.push(c); - } - } - out.peak_cache_bytes = bytes; - - for p in 0..n { - let c = chunk_of_use[p]; - let next = next_use[p]; - if let Some((e, _)) = resident.remove(&c) { - by_next.remove(&(p, c)); - out.events[e].uses.push(p); - out.use_event[p] = e; - if next == NONE { - bytes -= chunk_bytes[c]; - } else { - resident.insert(c, (e, next)); - by_next.insert((next, c)); - } - continue; - } - let e = out.events.len(); - out.events.push(ScheduledEvent { - chunk: c, - uses: vec![p], - preloaded: false, - }); - out.use_event[p] = e; - if next == NONE { - continue; - } - resident.insert(c, (e, next)); - by_next.insert((next, c)); - bytes += chunk_bytes[c]; - while bytes > budget { - let Some(&(far, victim)) = by_next.iter().next_back() else { - break; - }; - by_next.remove(&(far, victim)); - resident.remove(&victim); - bytes -= chunk_bytes[victim]; - } - out.peak_cache_bytes = out.peak_cache_bytes.max(bytes); - } - out -} - -// ───────────────────────────────────────────────────────────────────────────────────────────── -// The plan -// ───────────────────────────────────────────────────────────────────────────────────────────── - -/// Everything the JNI layer hands over for one run. -pub struct StreamRequest { - pub manifest_bytes: Vec, - pub install_dir: String, - /// `""` = `/.chunks`; a path = scratch cache (internal storage for a card install). - pub chunk_cache_dir: String, - /// The manager's owned staging directory on the install volume; `""` = stage beside each file. - pub staging_dir: String, - pub cdn_prefixes: Vec, - /// Indices into `manifest.files` of the files still to write (Java's delta pass), in order. - pub pending_file_indices: Vec, - /// Java's `Σ fileSize` over the pending files, for a cross-check (`None` = skip). - pub expected_bytes: Option, - /// Most bytes of shared chunks the cache may hold. - pub cache_budget: u64, - /// Most planned bytes of replaced files (destination already there) staged at once; one is - /// always allowed. Fresh files are not limited by it. - pub replace_overlap_budget: u64, - pub ca_bundle_path: String, - pub max_workers: usize, - pub process_workers: usize, - pub label: String, -} - -#[derive(Debug)] -pub struct PlanPart { - /// Index into `manifest.unique_chunks`. - pub chunk: usize, - pub chunk_offset: usize, - pub file_offset: u64, - pub size: usize, -} - -#[derive(Debug)] -pub struct PlanFile { - pub rel_path: String, - pub size: u64, - /// The manifest's SHA-1 when it has one (all-zero = none, as the Java delta pass treats it). - pub sha1: Option<[u8; 20]>, - pub parts: Vec, - /// Position of the file's first part in the install's part sequence. - pub first_use: usize, - /// The destination exists with data: staging this file costs space until its swap. - pub replaces: bool, -} - -#[derive(Debug)] -pub struct StreamPlan { - pub manifest: Manifest, - pub install_dir: PathBuf, - pub cache_dir: PathBuf, - pub scratch_cache: bool, - pub files: Vec, - /// Position → (file, part index in that file). - pub use_file: Vec, - pub use_part: Vec, - pub schedule: Schedule, - /// The events that are fetched, in fetch order; a fetch item's id indexes this. - pub fetch_events: Vec, - /// Event → the file of its first use (0 for a preloaded copy nothing ends up using). - pub event_first_file: Vec, - /// Event → it is its chunk's first copy, cached as `` (later copies get `.`, - /// so a copy being deleted can never remove the next one). - pub event_first_copy: Vec, - /// Every manifest file's destination: a staging name may never be one of them. - pub reserved: HashSet, - /// Where staging files go; `None` = beside each destination. - pub staging_dir: Option, - pub bytes_total: u64, - pub budget: u64, - pub overlap_budget: u64, - pub hosts: Vec, - pub direct_chunks: usize, - pub shared_chunks: usize, - pub refetches: usize, -} - -/// A cached chunk left by an earlier run is reused only when it is what the manifest says: -/// the window size when there is one, the SHA-1 when there is one (`isValidCachedChunk`). -pub fn cached_chunk_is_valid(path: &Path, chunk: &ChunkInfo) -> bool { - let Ok(meta) = fs::metadata(path) else { - return false; - }; - if !meta.is_file() { - return false; - } - if chunk.window_size > 0 && meta.len() != chunk.window_size as u64 { - return false; - } - let Some(expected) = chunk.verifiable_sha1() else { - return true; - }; - match fs::read(path) { - Ok(data) => Sha1::digest(&data).as_slice() == &expected[..], - Err(_) => false, - } -} - -/// Parse, check, clean the cache and schedule. `Err` = nothing was fetched or written (stale cache -/// entries may have been removed, as the Java writer's own preparation does); the manager then -/// runs the bounded Java writer instead. -pub fn build_stream_plan(req: &StreamRequest) -> Result { - if req.install_dir.is_empty() { - return Err("stream: empty install dir".to_string()); - } - let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("stream: {e}"))?; - let by_guid = manifest.chunk_index_by_guid(); - - let mut files: Vec = Vec::with_capacity(req.pending_file_indices.len()); - let mut use_file: Vec = Vec::new(); - let mut use_part: Vec = Vec::new(); - let mut chunk_of_use: Vec = Vec::new(); - let mut bytes_total: u64 = 0; - for &fi in &req.pending_file_indices { - let Some(f) = manifest.files.get(fi) else { - return Err(format!( - "stream: pending file index {fi} out of range ({} files)", - manifest.files.len() - )); - }; - let rel_path = relative_path(f); - let first_use = chunk_of_use.len(); - let mut parts = Vec::with_capacity(f.parts.len()); - let mut file_offset: u64 = 0; - for (k, part) in f.parts.iter().enumerate() { - let guid = part.guid_str(); - let Some(&ci) = by_guid.get(&guid) else { - return Err(format!("Missing chunk metadata {guid} for {rel_path}")); - }; - let size = part.size as u32 as usize; - let chunk_offset = usize::try_from(part.offset) - .map_err(|_| format!("Chunk {guid} is too short for {rel_path}"))?; - let window = chunk_window_bytes(manifest.unique_chunks[ci].window_size); - if (chunk_offset as u64).saturating_add(size as u64) > window { - return Err(format!("Chunk {guid} is too short for {rel_path}")); - } - parts.push(PlanPart { - chunk: ci, - chunk_offset, - file_offset, - size, - }); - file_offset += size as u64; - use_file.push(files.len()); - use_part.push(k); - chunk_of_use.push(ci); - } - bytes_total += file_offset; - let replaces = fs::metadata(Path::new(&req.install_dir).join(&rel_path)) - .map(|m| m.is_file() && m.len() > 0) - .unwrap_or(false); - files.push(PlanFile { - replaces, - rel_path, - size: file_offset, - sha1: f.sha1.filter(|h| h.iter().any(|&b| b != 0)), - parts, - first_use, - }); - } - if let Some(expected) = req.expected_bytes { - if expected != bytes_total { - return Err(format!( - "stream: byte total mismatch java={expected} rust={bytes_total}" - )); - } - } - let hosts = distinct_prefixes(&req.cdn_prefixes); - if hosts.is_empty() { - return Err("stream: no CDN prefixes".to_string()); - } - - let install_dir = PathBuf::from(&req.install_dir); - let staging_dir = if req.staging_dir.is_empty() { - None - } else { - let dir = PathBuf::from(&req.staging_dir); - if !dir.is_dir() { - return Err(format!("stream: staging dir {} missing", dir.display())); - } - Some(dir) - }; - let cache_dir = resolve_cache_dir(&req.install_dir, &req.chunk_cache_dir); - fs::create_dir_all(&cache_dir) - .map_err(|e| format!("stream: mkdirs {}: {e}", cache_dir.display()))?; - let reserved: HashSet = manifest - .files - .iter() - .map(|f| install_dir.join(relative_path(f))) - .collect(); - - // The cache keeps only verified chunks this install still needs, as the Java writer's - // `prepareBoundedCache` does: anything else (a stale chunk, a `.part`, a later copy from a - // cancelled run) would only take space the files need. - let needed: HashSet = chunk_of_use.iter().copied().collect(); - let mut needed_by_name: HashMap = HashMap::new(); - for &ci in &needed { - needed_by_name.insert(manifest.unique_chunks[ci].guid_str(), ci); - } - let entries = fs::read_dir(&cache_dir) - .map_err(|e| format!("stream: list {}: {e}", cache_dir.display()))?; - let mut preloaded: Vec = Vec::new(); - for entry in entries.flatten() { - let path = entry.path(); - let name = entry.file_name().to_string_lossy().into_owned(); - if let Some(&ci) = needed_by_name.get(&name) { - if cached_chunk_is_valid(&path, &manifest.unique_chunks[ci]) { - preloaded.push(ci); - continue; - } - } - remove_entry(&path); - } - - let chunk_bytes: Vec = manifest - .unique_chunks - .iter() - .map(|c| chunk_window_bytes(c.window_size)) - .collect(); - let schedule = schedule(&chunk_of_use, &chunk_bytes, &preloaded, req.cache_budget); - for &ci in &schedule.discarded { - remove_entry(&cache_dir.join(manifest.unique_chunks[ci].guid_str())); - } - - let mut fetch_events = Vec::new(); - let mut event_first_file = Vec::with_capacity(schedule.events.len()); - let mut event_first_copy = Vec::with_capacity(schedule.events.len()); - let mut copies: HashMap = HashMap::new(); - for (e, ev) in schedule.events.iter().enumerate() { - event_first_file.push(ev.uses.first().map(|&p| use_file[p]).unwrap_or(0)); - let count = copies.entry(ev.chunk).or_insert(0); - event_first_copy.push(*count == 0); - *count += 1; - if !ev.preloaded { - fetch_events.push(e); - } - } - let mut uses_per_chunk: HashMap = HashMap::new(); - for &c in &chunk_of_use { - *uses_per_chunk.entry(c).or_insert(0) += 1; - } - let direct_chunks = uses_per_chunk.values().filter(|&&n| n == 1).count(); - let shared_chunks = uses_per_chunk.len() - direct_chunks; - let refetches: usize = copies.values().map(|&n| n - 1).sum(); - - Ok(StreamPlan { - manifest, - install_dir, - cache_dir, - scratch_cache: !req.chunk_cache_dir.is_empty(), - files, - use_file, - use_part, - schedule, - fetch_events, - event_first_file, - event_first_copy, - reserved, - staging_dir, - bytes_total, - budget: req.cache_budget, - overlap_budget: req.replace_overlap_budget, - hosts, - direct_chunks, - shared_chunks, - refetches, - }) -} - -fn remove_entry(path: &Path) { - if path.is_dir() { - let _ = fs::remove_dir_all(path); - } else { - let _ = fs::remove_file(path); - } -} - -// ───────────────────────────────────────────────────────────────────────────────────────────── -// The run -// ───────────────────────────────────────────────────────────────────────────────────────────── - -/// What the Java side sees of a run's progress. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct StreamProgress { - /// File bytes written so far (into staging files). - pub bytes_written: u64, - pub bytes_total: u64, - pub files_done: u64, - pub files_total: u64, - /// Sizes of the files verified and swapped into place. - pub files_bytes: u64, -} - -/// The run's callbacks. All may be called from any engine thread. -pub struct Hooks<'a> { - pub progress: &'a (dyn Fn(&StreamProgress) + Sync), - /// Called once, before the first file replaces its destination: the Java side marks an - /// existing install `updating`. `Err` fails the run and nothing is replaced. - pub before_replace: &'a (dyn Fn() -> Result<(), String> + Sync), - pub log: &'a (dyn Fn(&str) + Sync), -} - -#[derive(Clone, Debug, Default)] -pub struct StreamOutcome { - pub success: bool, - pub cancelled: bool, - pub error: String, - pub bytes_written: u64, - pub files_done: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum EventState { - /// Not downloaded yet. - Pending, - /// Downloaded, its cache copy being written; uses are only claimed by the downloading thread. - Caching, - /// In the cache: newly reached files read it from there. - Cached, - /// Every use is claimed; no cache copy (left). - Done, -} - -struct Shared { - state: Vec, - /// Event → uses not written yet. - remaining: Vec, - /// Event → bytes its cache copy takes. - cached_size: Vec, - /// Position → a thread has taken this part to write. - claimed: Vec, - /// File → parts not written yet. - parts_left: Vec, - /// Files `[0, active)` have been reached and may be written. - active: usize, - /// One past the highest first-use file of a downloaded event. - reached: usize, - /// `fetch_events[..next_fetch]` have all been downloaded. - next_fetch: usize, - /// Files `[0, admitted)` passed the replacement-overlap budget; `active` never passes it. - admitted: usize, - /// Planned bytes of admitted replaced files not swapped in yet. - overlap_open: u64, - peak_overlap: u64, - cache_bytes: u64, - peak_cache: u64, -} - -#[derive(Default)] -struct FileWriter { - staging: Option, - file: Option, - sha: Option, - /// Parts `[0, hashed)` are in the SHA-1. - hashed: usize, - done: Vec, - /// Bytes written into the staging file. - written: u64, -} - -#[derive(Default)] -struct Stats { - written: AtomicU64, - direct: AtomicU64, - from_cache: AtomicU64, - cache_stored: AtomicU64, - staging_now: AtomicU64, - staging_peak: AtomicU64, - files_done: AtomicU64, - files_bytes: AtomicU64, - fetched: AtomicU64, - last_progress_ms: AtomicU64, -} - -/// Parts to write from cache copies (event → positions), and part-less files to finish. -#[derive(Default)] -struct Work { - from_cache: HashMap>, - empty_files: Vec, -} - -/// Unique part of every staging name this process creates. -static STAGE_SEQ: AtomicU64 = AtomicU64::new(0); - -/// The run's state; it is the fetch core's sink. -pub struct Engine<'a> { - plan: &'a StreamPlan, - cancel: &'a AtomicBool, - hooks: &'a Hooks<'a>, - shared: Mutex, - writers: Vec>, - replace_gate: Mutex>>, - stats: Stats, - started: Instant, - /// `Shared::admitted`, readable by the fetch driver without the lock. - admitted: AtomicUsize, -} - -fn fatal_write(rel: &str, err: impl std::fmt::Display) -> SinkError { - SinkError::Fatal(format!("Could not assemble or atomically replace {rel}: {err}")) -} - -impl<'a> Engine<'a> { - pub fn new(plan: &'a StreamPlan, cancel: &'a AtomicBool, hooks: &'a Hooks<'a>) -> Self { - let events = &plan.schedule.events; - let mut state = Vec::with_capacity(events.len()); - let mut cached_size = vec![0u64; events.len()]; - let mut cache_bytes = 0u64; - for (e, ev) in events.iter().enumerate() { - if !ev.preloaded { - state.push(EventState::Pending); - continue; - } - let path = cache_path(plan, e); - if ev.uses.is_empty() { - // Evicted before its first use in the plan: it only takes space. - let _ = fs::remove_file(&path); - state.push(EventState::Done); - continue; - } - let size = fs::metadata(&path) - .map(|m| m.len()) - .unwrap_or(plan_chunk_bytes(plan, ev.chunk)); - cached_size[e] = size; - cache_bytes += size; - state.push(EventState::Cached); - } - Engine { - plan, - cancel, - hooks, - shared: Mutex::new(Shared { - state, - remaining: events.iter().map(|ev| ev.uses.len() as u32).collect(), - cached_size, - claimed: vec![false; plan.use_file.len()], - parts_left: plan.files.iter().map(|f| f.parts.len() as u32).collect(), - active: 0, - reached: 0, - next_fetch: 0, - admitted: 0, - overlap_open: 0, - peak_overlap: 0, - cache_bytes, - peak_cache: cache_bytes, - }), - writers: plan.files.iter().map(|_| Mutex::new(FileWriter::default())).collect(), - replace_gate: Mutex::new(None), - stats: Stats::default(), - started: Instant::now(), - admitted: AtomicUsize::new(0), - } - } - - /// The fetch driver's gate: fetch item `id` may go out once its first file is admitted. - pub fn may_dispatch(&self, id: usize) -> bool { - let p = self.plan; - match p.fetch_events.get(id) { - Some(&e) => p.event_first_file[e] < self.admitted.load(Ordering::Acquire), - None => true, - } - } - - /// The most planned bytes of replaced files that were staged at once. - pub fn peak_replace_overlap(&self) -> u64 { - self.shared.lock().map(|st| st.peak_overlap).unwrap_or_default() - } - - /// Keep reaching files until nothing new can be written: a file that finishes can admit the - /// next replaced file, whose parts may already be in the cache. Iterative, so a long run of - /// cache-only files never deepens the stack. - fn settle(&self) -> Result<(), SinkError> { - loop { - let mut work = Work::default(); - { - let mut st = self.shared.lock().expect("stream state poisoned"); - self.advance(&mut st, &mut work); - } - if work.from_cache.is_empty() && work.empty_files.is_empty() { - return Ok(()); - } - self.run_work(work)?; - } - } - - /// Before the first fetch: the files before the first chunk to download can be written from - /// the cache an earlier run left (or are empty). - pub fn start(&self) -> Result<(), SinkError> { - self.settle() - } - - /// Reach every file that can be written now: all files before the first chunk still to - /// download, and every file up to the furthest first use of a chunk that has arrived. Parts - /// of newly reached files whose chunk is in the cache are claimed for writing. - fn advance(&self, st: &mut Shared, work: &mut Work) { - let p = self.plan; - while st.admitted < p.files.len() { - let file = &p.files[st.admitted]; - if file.replaces { - if st.overlap_open > 0 && st.overlap_open + file.size > p.overlap_budget { - break; - } - st.overlap_open += file.size; - st.peak_overlap = st.peak_overlap.max(st.overlap_open); - } - st.admitted += 1; - } - self.admitted.store(st.admitted, Ordering::Release); - while st.next_fetch < p.fetch_events.len() - && st.state[p.fetch_events[st.next_fetch]] != EventState::Pending - { - st.next_fetch += 1; - } - let frontier = match p.fetch_events.get(st.next_fetch) { - Some(&e) => p.event_first_file[e], - None => p.files.len(), - }; - let target = st.active.max(frontier).max(st.reached).min(st.admitted); - for f in st.active..target { - let file = &p.files[f]; - if file.parts.is_empty() { - work.empty_files.push(f); - continue; - } - for k in 0..file.parts.len() { - let pos = file.first_use + k; - let e = p.schedule.use_event[pos]; - if st.state[e] == EventState::Cached && !st.claimed[pos] { - st.claimed[pos] = true; - work.from_cache.entry(e).or_default().push(pos); - } - } - } - st.active = target; - } - - /// One downloaded chunk body (fetch item `id`). - fn on_body(&self, id: usize, body: Vec) -> Result { - let p = self.plan; - let Some(&e) = p.fetch_events.get(id) else { - return Err(SinkError::Fatal(format!("item id {id} out of range"))); - }; - let ev = &p.schedule.events[e]; - let chunk = &p.manifest.unique_chunks[ev.chunk]; - let data = decode_verified_chunk(&body, chunk.window_size, chunk.verifiable_sha1()) - .map_err(|reason| SinkError::Retry(format!("{} {reason}", chunk.guid_str())))?; - drop(body); - self.stats.fetched.fetch_add(1, Ordering::Relaxed); - - let mut work = Work::default(); - let mut now: Vec = Vec::new(); - let caching; - { - let mut st = self.shared.lock().expect("stream state poisoned"); - if st.state[e] != EventState::Pending { - return Ok(chunk.credit_bytes()); - } - // Provisional: until the cache copy exists, nobody else may claim this chunk's uses. - st.state[e] = EventState::Caching; - st.reached = st.reached.max(p.event_first_file[e] + 1); - self.advance(&mut st, &mut work); - for &pos in &ev.uses { - if p.use_file[pos] < st.active && !st.claimed[pos] { - st.claimed[pos] = true; - now.push(pos); - } - } - caching = now.len() < ev.uses.len(); - if !caching { - st.state[e] = EventState::Done; - } - } - self.write_from_memory(&data, &now)?; - self.run_work(work)?; - if caching { - let path = cache_path(p, e); - write_cache_file(&path, &data).map_err(|err| { - SinkError::Fatal(format!("cache {}: {err}", chunk.guid_str())) - })?; - self.stats - .cache_stored - .fetch_add(data.len() as u64, Ordering::Relaxed); - let mut more = Vec::new(); - { - let mut st = self.shared.lock().expect("stream state poisoned"); - st.state[e] = EventState::Cached; - st.cached_size[e] = data.len() as u64; - st.cache_bytes += data.len() as u64; - st.peak_cache = st.peak_cache.max(st.cache_bytes); - // Files reached while the copy was being written. - for &pos in &ev.uses { - if p.use_file[pos] < st.active && !st.claimed[pos] { - st.claimed[pos] = true; - more.push(pos); - } - } - } - self.write_from_memory(&data, &more)?; - } - self.settle()?; - Ok(chunk.credit_bytes()) - } - - fn write_from_memory(&self, data: &[u8], positions: &[usize]) -> Result<(), SinkError> { - for &pos in positions { - let written = self.write_use(pos, data)?; - self.stats.direct.fetch_add(written, Ordering::Relaxed); - self.complete_use(pos)?; - } - Ok(()) - } - - fn run_work(&self, work: Work) -> Result<(), SinkError> { - let mut groups: Vec<(usize, Vec)> = work.from_cache.into_iter().collect(); - groups.sort_unstable(); - for (e, positions) in groups { - let ev = &self.plan.schedule.events[e]; - let guid = self.plan.manifest.unique_chunks[ev.chunk].guid_str(); - let data = fs::read(cache_path(self.plan, e)) - .map_err(|err| SinkError::Fatal(format!("read cached chunk {guid}: {err}")))?; - for pos in positions { - let written = self.write_use(pos, &data)?; - self.stats.from_cache.fetch_add(written, Ordering::Relaxed); - self.complete_use(pos)?; - } - } - for f in work.empty_files { - self.finalize(f)?; - } - Ok(()) - } - - /// Write the part at `pos` from its chunk's bytes into its file's staging file. - fn write_use(&self, pos: usize, chunk_data: &[u8]) -> Result { - let p = self.plan; - let f = p.use_file[pos]; - let k = p.use_part[pos]; - let file = &p.files[f]; - let part = &file.parts[k]; - let guid = p.manifest.unique_chunks[part.chunk].guid_str(); - let end = part - .chunk_offset - .checked_add(part.size) - .filter(|&end| end <= chunk_data.len()) - .ok_or_else(|| { - SinkError::Fatal(format!("Chunk {guid} is too short for {}", file.rel_path)) - })?; - let slice = &chunk_data[part.chunk_offset..end]; - { - let mut guard = self.writers[f].lock().expect("writer poisoned"); - if guard.file.is_none() { - self.open_staging(f, &mut guard)?; - } - let FileWriter { - file: out, - sha, - hashed, - done, - written, - .. - } = &mut *guard; - let out = out.as_ref().expect("staging open"); - out.write_all_at(slice, part.file_offset) - .map_err(|err| fatal_write(&file.rel_path, err))?; - done[k] = true; - *written += slice.len() as u64; - // SHA-1 in file order: the part just written comes from memory, parts that arrived - // earlier out of order are read back from the staging file. - if let Some(sha) = sha.as_mut() { - while *hashed < file.parts.len() && done[*hashed] { - let h = *hashed; - if h == k { - sha.update(slice); - } else { - let q = &file.parts[h]; - let mut buf = vec![0u8; q.size]; - out.read_exact_at(&mut buf, q.file_offset) - .map_err(|err| fatal_write(&file.rel_path, err))?; - sha.update(&buf); - } - *hashed += 1; - } - } - } - let len = slice.len() as u64; - let staged = self.stats.staging_now.fetch_add(len, Ordering::Relaxed) + len; - self.stats.staging_peak.fetch_max(staged, Ordering::Relaxed); - self.stats.written.fetch_add(len, Ordering::Relaxed); - self.report(false); - Ok(len) - } - - /// A part is written: release its chunk copy after its last use, finish its file after its - /// last part. - fn complete_use(&self, pos: usize) -> Result<(), SinkError> { - let p = self.plan; - let e = p.schedule.use_event[pos]; - let f = p.use_file[pos]; - let (release, finish) = { - let mut st = self.shared.lock().expect("stream state poisoned"); - st.remaining[e] = st.remaining[e].saturating_sub(1); - let release = st.remaining[e] == 0 && st.state[e] == EventState::Cached; - if release { - st.state[e] = EventState::Done; - st.cache_bytes = st.cache_bytes.saturating_sub(st.cached_size[e]); - } - st.parts_left[f] = st.parts_left[f].saturating_sub(1); - (release, st.parts_left[f] == 0) - }; - if release { - let _ = fs::remove_file(cache_path(p, e)); - } - if finish { - self.finalize(f)?; - } - Ok(()) - } - - /// A unique `.stage` file in the owned staging directory (or beside the destination), never a - /// manifest path (`createStagingFile`). The destination's directory is created either way. - fn open_staging(&self, f: usize, w: &mut FileWriter) -> Result<(), SinkError> { - let file = &self.plan.files[f]; - let dest = self.plan.install_dir.join(&file.rel_path); - let parent = dest - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| self.plan.install_dir.clone()); - fs::create_dir_all(&parent).map_err(|err| fatal_write(&file.rel_path, err))?; - let (stage_dir, prefix) = match &self.plan.staging_dir { - Some(dir) => (dir.clone(), "stream-"), - None => (parent.clone(), ".droiddeck-epic-"), - }; - for _ in 0..64 { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.subsec_nanos()) - .unwrap_or(0); - let seq = STAGE_SEQ.fetch_add(1, Ordering::Relaxed); - let path = stage_dir.join(format!( - "{prefix}{}-{seq}-{nanos}.stage", - std::process::id() - )); - if self.plan.reserved.contains(&path) { - continue; - } - match OpenOptions::new() - .read(true) - .write(true) - .create_new(true) - .open(&path) - { - Ok(out) => { - w.staging = Some(path); - w.file = Some(out); - w.sha = file.sha1.map(|_| Sha1::new()); - w.hashed = 0; - w.done = vec![false; file.parts.len()]; - w.written = 0; - return Ok(()); - } - Err(err) if err.kind() == ErrorKind::AlreadyExists => continue, - Err(err) => return Err(fatal_write(&file.rel_path, err)), - } - } - Err(fatal_write(&file.rel_path, "no free staging name")) - } - - /// All parts are in: sync, check size and SHA-1, mark the install updating (once), rename - /// over the destination. On any failure the staging file goes and the destination stays. - fn finalize(&self, f: usize) -> Result<(), SinkError> { - let mut guard = self.writers[f].lock().expect("writer poisoned"); - if guard.file.is_none() { - self.open_staging(f, &mut guard)?; - } - let result = self.finish_file(f, &mut guard); - let staged = guard.written; - if result.is_err() { - guard.file = None; - if let Some(path) = guard.staging.take() { - let _ = fs::remove_file(&path); - } - } - drop(guard); - self.stats - .staging_now - .fetch_sub(staged.min(self.stats.staging_now.load(Ordering::Relaxed)), Ordering::Relaxed); - if result.is_ok() { - let size = self.plan.files[f].size; - if self.plan.files[f].replaces { - let mut st = self.shared.lock().expect("stream state poisoned"); - st.overlap_open = st.overlap_open.saturating_sub(size); - } - self.stats.files_done.fetch_add(1, Ordering::Relaxed); - self.stats.files_bytes.fetch_add(size, Ordering::Relaxed); - self.report(true); - } - result - } - - fn finish_file(&self, f: usize, w: &mut FileWriter) -> Result<(), SinkError> { - let file = &self.plan.files[f]; - let rel = file.rel_path.as_str(); - let out = w.file.take().expect("staging open"); - out.sync_all().map_err(|err| fatal_write(rel, err))?; - if self.cancel.load(Ordering::Relaxed) { - return Err(SinkError::Fatal("cancelled".to_string())); - } - let len = out.metadata().map_err(|err| fatal_write(rel, err))?.len(); - drop(out); - if len != file.size { - return Err(SinkError::Fatal(format!("Size mismatch for {rel}"))); - } - if let (Some(expected), Some(sha)) = (file.sha1.as_ref(), w.sha.take()) { - if w.hashed != file.parts.len() || sha.finalize().as_slice() != &expected[..] { - return Err(SinkError::Fatal(format!("SHA-1 mismatch for {rel}"))); - } - } - // Once a swap can mix versions, the old installed record is no longer truthful. - self.mark_updating().map_err(|err| fatal_write(rel, err))?; - if self.cancel.load(Ordering::Relaxed) { - return Err(SinkError::Fatal("cancelled".to_string())); - } - let staging = w.staging.clone().expect("staging path"); - let dest = self.plan.install_dir.join(&file.rel_path); - fs::rename(&staging, &dest).map_err(|err| fatal_write(rel, err))?; - w.staging = None; - Ok(()) - } - - fn mark_updating(&self) -> Result<(), String> { - let mut gate = self.replace_gate.lock().expect("replace gate poisoned"); - if let Some(result) = gate.as_ref() { - return result.clone(); - } - let result = (self.hooks.before_replace)(); - *gate = Some(result.clone()); - result - } - - fn report(&self, force: bool) { - let now_ms = self.started.elapsed().as_millis() as u64; - let last = self.stats.last_progress_ms.load(Ordering::Relaxed); - if !force { - if now_ms.saturating_sub(last) < PROGRESS_EVERY_MS { - return; - } - if self - .stats - .last_progress_ms - .compare_exchange(last, now_ms, Ordering::Relaxed, Ordering::Relaxed) - .is_err() - { - return; - } - } else { - self.stats.last_progress_ms.store(now_ms, Ordering::Relaxed); - } - (self.hooks.progress)(&self.snapshot()); - } - - fn snapshot(&self) -> StreamProgress { - StreamProgress { - bytes_written: self.stats.written.load(Ordering::Relaxed), - bytes_total: self.plan.bytes_total, - files_done: self.stats.files_done.load(Ordering::Relaxed), - files_total: self.plan.files.len() as u64, - files_bytes: self.stats.files_bytes.load(Ordering::Relaxed), - } - } - - /// Cancel or failure: every unfinished staging file goes; the destinations are untouched and - /// the cache stays for the next run. Call only when no other thread is writing. - pub fn discard_staging(&self) -> usize { - let mut removed = 0; - for writer in &self.writers { - let mut guard = writer.lock().unwrap_or_else(|e| e.into_inner()); - guard.file = None; - if let Some(path) = guard.staging.take() { - if fs::remove_file(&path).is_ok() { - removed += 1; - } - } - } - removed - } - - /// Files not yet swapped into place. - pub fn unfinished_files(&self) -> u64 { - self.plan.files.len() as u64 - self.stats.files_done.load(Ordering::Relaxed) - } - - pub fn summary(&self) -> String { - let secs = self.started.elapsed().as_secs_f64().max(0.001); - let written = self.stats.written.load(Ordering::Relaxed); - let (peak_cache, peak_overlap) = self - .shared - .lock() - .map(|st| (st.peak_cache, st.peak_overlap)) - .unwrap_or_default(); - format!( - "stream-summary mode=adaptive files={}/{} bytes_written={written} direct_bytes={} from_cache_bytes={} cache_stored_bytes={} peak_cache_bytes={peak_cache} budget={} plan_peak_cache_bytes={} peak_staging_bytes={} peak_replace_overlap_bytes={peak_overlap} overlap_budget={} fetched_chunks={} refetches={} elapsed={secs:.1}s avg_MBps={:.2}", - self.stats.files_done.load(Ordering::Relaxed), - self.plan.files.len(), - self.stats.direct.load(Ordering::Relaxed), - self.stats.from_cache.load(Ordering::Relaxed), - self.stats.cache_stored.load(Ordering::Relaxed), - self.plan.budget, - self.plan.schedule.peak_cache_bytes, - self.stats.staging_peak.load(Ordering::Relaxed), - self.plan.overlap_budget, - self.stats.fetched.load(Ordering::Relaxed), - self.plan.refetches, - written as f64 / secs / (1024.0 * 1024.0), - ) - } -} - -impl FetchSink for Engine<'_> { - fn process(&self, item: &FetchItem, body: Vec) -> Result { - self.on_body(item.id as usize, body) - } -} - -fn plan_chunk_bytes(plan: &StreamPlan, chunk: usize) -> u64 { - chunk_window_bytes(plan.manifest.unique_chunks[chunk].window_size) -} - -/// Where event `e`'s copy lives in the cache. -fn cache_path(plan: &StreamPlan, e: usize) -> PathBuf { - let guid = plan.manifest.unique_chunks[plan.schedule.events[e].chunk].guid_str(); - if plan.event_first_copy[e] { - plan.cache_dir.join(guid) - } else { - plan.cache_dir.join(format!("{guid}.{e}")) - } -} - -/// `.part` + rename, so a cache file under its final name is always complete. -fn write_cache_file(path: &Path, data: &[u8]) -> Result<(), String> { - let tmp = part_path(path); - let _ = fs::remove_file(&tmp); - let result = fs::write(&tmp, data) - .map_err(|e| format!("write {}: {e}", tmp.display())) - .and_then(|()| fs::rename(&tmp, path).map_err(|e| format!("rename: {e}"))); - if result.is_err() { - let _ = fs::remove_file(&tmp); - } - result -} - -/// Run the plan to the end: the fetch core downloads the chunks in order, the engine writes them. -pub fn run_stream( - plan: &StreamPlan, - req: &StreamRequest, - cancel: &AtomicBool, - hooks: &Hooks<'_>, -) -> StreamOutcome { - let log = hooks.log; - let host_cap = per_host_cap(req.max_workers, plan.hosts.len()); - log(&format!( - "stream-plan chunk_dir={} cache_dir={} scratch={} {} files={} bytes={} uses={} chunks_direct={} chunks_shared={} fetches={} preloaded={} refetches={} budget={} plan_peak_cache={} replaced_files={} overlap_budget={} hosts={} workers={} per_host_cap={host_cap} process_workers={}", - plan.manifest.chunk_dir, - plan.cache_dir.display(), - plan.scratch_cache, - crate::priority::log_field(), - plan.files.len(), - plan.bytes_total, - plan.use_file.len(), - plan.direct_chunks, - plan.shared_chunks, - plan.fetch_events.len(), - plan.schedule.events.len() - plan.fetch_events.len(), - plan.refetches, - plan.budget, - plan.schedule.peak_cache_bytes, - plan.files.iter().filter(|f| f.replaces).count(), - plan.overlap_budget, - plan.hosts.len(), - req.max_workers, - req.process_workers - )); - for (i, h) in plan.hosts.iter().enumerate() { - log(&format!("host[{i}]={h}")); - } - - let engine = Engine::new(plan, cancel, hooks); - let mut outcome = StreamOutcome::default(); - let mut error: Option = match engine.start() { - Ok(()) => None, - Err(SinkError::Fatal(m)) | Err(SinkError::Retry(m)) => Some(m), - }; - - if error.is_none() && !cancel.load(Ordering::Relaxed) && !plan.fetch_events.is_empty() { - let items: Vec = plan - .fetch_events - .iter() - .enumerate() - .map(|(id, &e)| { - let chunk = &plan.manifest.unique_chunks[plan.schedule.events[e].chunk]; - FetchItem { - id: id as u64, - urls: plan - .hosts - .iter() - .map(|h| chunk_url(h, &plan.manifest.chunk_dir, chunk)) - .collect(), - reserve: if chunk.file_size > 0 { - chunk.file_size as u64 - } else { - 0 - }, - range: None, - } - }) - .collect(); - let opts = FetchOptions { - max_workers: req.max_workers.max(1), - per_host_cap: host_cap, - timeout: REQUEST_TIMEOUT, - headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())], - ca_bundle_path: req.ca_bundle_path.clone(), - process_workers: req.process_workers.max(1), - label: req.label.clone(), - ..FetchOptions::default() - }; - // Progress is the engine's (file bytes written); the core's own credit only feeds its - // `fetch-end` line. - let no_progress = |_: u64, _: u64| {}; - let gate = |id: usize| engine.may_dispatch(id); - let fetched = run_fetch_gated( - items, - &plan.hosts, - &opts, - &engine, - cancel, - &no_progress, - log, - &gate, - ); - error = fetched.error; - } - - let progress = engine.snapshot(); - outcome.bytes_written = progress.bytes_written; - outcome.files_done = progress.files_done; - if cancel.load(Ordering::Relaxed) { - let removed = engine.discard_staging(); - outcome.cancelled = true; - outcome.error = "cancelled".to_string(); - log(&engine.summary()); - log(&format!( - "cancelled ({}/{} files, {removed} staging files removed)", - progress.files_done, progress.files_total - )); - return outcome; - } - if error.is_none() && engine.unfinished_files() > 0 { - error = Some(format!( - "stream ended with {} files unfinished", - engine.unfinished_files() - )); - } - if let Some(err) = error { - let removed = engine.discard_staging(); - outcome.error = err; - log(&engine.summary()); - log(&format!( - "FAIL {} ({}/{} files ok, {removed} staging files removed)", - outcome.error, progress.files_done, progress.files_total - )); - return outcome; - } - outcome.success = true; - log(&engine.summary()); - match fs::remove_dir_all(&plan.cache_dir) { - Ok(()) => log(&format!("cache removed {}", plan.cache_dir.display())), - Err(e) => log(&format!("cache not removed {}: {e}", plan.cache_dir.display())), - } - outcome -} - -#[cfg(test)] -mod tests { - use super::super::chunk::test_support::{build_chunk_body, sha1_of, temp_dir}; - use super::super::manifest::test_support::{build_manifest, TestChunk, TestFile}; - use super::*; - - // ── schedule ── - - #[test] - fn single_use_chunks_are_plain_downloads_and_shared_ones_stay_cached() { - // Chunk 0 is used at positions 0 and 2, chunk 1 once, chunk 2 once. - let s = schedule(&[0, 1, 0, 2], &[10, 10, 10], &[], 100); - assert_eq!(s.events.len(), 3); - assert_eq!(s.events[0].uses, vec![0, 2]); - assert_eq!(s.events[1].uses, vec![1]); - assert_eq!(s.events[2].uses, vec![3]); - assert_eq!(s.use_event, vec![0, 1, 0, 2]); - assert_eq!(s.peak_cache_bytes, 10, "only the shared chunk is ever cached"); - assert!(s.discarded.is_empty()); - } - - #[test] - fn a_chunk_bigger_than_the_budget_is_downloaded_for_every_use() { - let s = schedule(&[0, 1, 0, 0], &[10, 1], &[], 5); - let chunk0: Vec<&ScheduledEvent> = s.events.iter().filter(|e| e.chunk == 0).collect(); - assert_eq!(chunk0.len(), 3, "never cached → one download per use"); - assert!(chunk0.iter().all(|e| e.uses.len() == 1)); - assert_eq!(s.peak_cache_bytes, 0); - } - - #[test] - fn eviction_drops_the_chunk_needed_furthest_away() { - // Budget fits two chunks. At position 2 chunk 2 arrives (next use 6) while 0 (next 3) - // and 1 (next 4) are cached: chunk 2 itself is the furthest and is not kept. - let uses = [0, 1, 2, 0, 1, 9, 2]; - let s = schedule(&uses, &[4, 4, 4, 0, 0, 0, 0, 0, 0, 1], &[], 8); - let copies = |c: usize| s.events.iter().filter(|e| e.chunk == c).count(); - assert_eq!(copies(0), 1); - assert_eq!(copies(1), 1); - assert_eq!(copies(2), 2, "chunk 2 downloaded again for its later use"); - assert!(s.peak_cache_bytes <= 8); - // Same sequence with room for all three: no refetch. - let s = schedule(&uses, &[4, 4, 4, 0, 0, 0, 0, 0, 0, 1], &[], 12); - assert_eq!(s.events.iter().filter(|e| e.chunk == 2).count(), 1); - assert_eq!(s.peak_cache_bytes, 12); - } - - #[test] - fn preloaded_chunks_serve_their_uses_until_the_budget_says_otherwise() { - // Chunk 0 cached by an earlier run, chunk 1 too but nothing needs it, chunk 2 cached but - // over budget. - let s = schedule(&[0, 2, 0], &[4, 4, 4], &[0, 1, 2], 4); - assert!(s.events[0].preloaded); - assert_eq!(s.events[0].chunk, 0); - assert_eq!(s.events[0].uses, vec![0, 2]); - let mut discarded = s.discarded.clone(); - discarded.sort_unstable(); - assert_eq!(discarded, vec![1, 2]); - let fetched: Vec<&ScheduledEvent> = s.events.iter().filter(|e| !e.preloaded).collect(); - assert_eq!(fetched.len(), 1); - assert_eq!(fetched[0].chunk, 2); - } - - // ── engine (no network: bodies are handed to the sink directly) ── - - const A: [u32; 4] = [1, 0, 0, 0]; - const B: [u32; 4] = [2, 0, 0, 0]; - const C: [u32; 4] = [3, 0, 0, 0]; - - fn data(seed: u32, len: usize) -> Vec { - (0..len as u32).map(|i| ((i * 7 + seed) % 251) as u8).collect() - } - - struct Fixture { - dir: PathBuf, - install: PathBuf, - chunks: Vec<([u32; 4], Vec)>, - files: Vec<(String, Vec<([u32; 4], i32, i32)>)>, - } - - impl Fixture { - /// Three files: `a.bin` = A[0..100] + B[0..50]; `dir/b.bin` = A[100..300] (A is shared); - /// `c.bin` = C whole. - fn new(tag: &str) -> Self { - let dir = temp_dir(tag); - let install = dir.join("game"); - fs::create_dir_all(&install).unwrap(); - Fixture { - dir, - install, - chunks: vec![(A, data(1, 300)), (B, data(2, 50)), (C, data(3, 80))], - files: vec![ - ("a.bin".to_string(), vec![(A, 0, 100), (B, 0, 50)]), - ("dir\\b.bin".to_string(), vec![(A, 100, 200)]), - ("c.bin".to_string(), vec![(C, 0, 80)]), - ], - } - } - - fn chunk(&self, guid: [u32; 4]) -> &Vec { - &self.chunks.iter().find(|(g, _)| *g == guid).unwrap().1 - } - - fn expected(&self, file: usize) -> Vec { - let mut out = Vec::new(); - for (g, off, size) in &self.files[file].1 { - out.extend_from_slice(&self.chunk(*g)[*off as usize..(*off + *size) as usize]); - } - out - } - - fn manifest(&self) -> Vec { - let chunks: Vec = self - .chunks - .iter() - .map(|(g, d)| TestChunk { - guid: *g, - hash: g[0] as u64, - sha1: sha1_of(d), - group: 1, - window: d.len() as i32, - file_size: d.len() as u64, - }) - .collect(); - let files: Vec = (0..self.files.len()) - .map(|i| TestFile { - name: self.files[i].0.clone(), - sha1: sha1_of(&self.expected(i)), - tags: vec![], - parts: self.files[i].1.clone(), - }) - .collect(); - build_manifest(&chunks, &files, 21, true) - } - - fn request(&self, pending: Vec, budget: u64) -> StreamRequest { - StreamRequest { - manifest_bytes: self.manifest(), - install_dir: self.install.to_str().unwrap().to_string(), - chunk_cache_dir: self.dir.join("cache").to_str().unwrap().to_string(), - staging_dir: String::new(), - cdn_prefixes: vec!["http://127.0.0.1:9/Builds/x".to_string()], - pending_file_indices: pending, - expected_bytes: None, - cache_budget: budget, - replace_overlap_budget: u64::MAX, - ca_bundle_path: String::new(), - max_workers: 4, - process_workers: 2, - label: "epic test".to_string(), - } - } - - fn body_for(&self, plan: &StreamPlan, id: usize) -> Vec { - let ev = &plan.schedule.events[plan.fetch_events[id]]; - let guid = plan.manifest.unique_chunks[ev.chunk].guid; - build_chunk_body(self.chunk(guid), true, 0, None) - } - - fn stages(&self) -> Vec { - let mut out = Vec::new(); - let mut dirs = vec![self.install.clone()]; - while let Some(d) = dirs.pop() { - for entry in fs::read_dir(&d).unwrap().flatten() { - let path = entry.path(); - if path.is_dir() { - dirs.push(path); - } else if path.to_string_lossy().ends_with(".stage") { - out.push(path); - } - } - } - out - } - } - - impl Drop for Fixture { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.dir); - } - } - - struct Recorder { - marks: Mutex, - last: Mutex, - fail_mark: bool, - } - - impl Recorder { - fn new() -> Self { - Recorder { - marks: Mutex::new(0), - last: Mutex::new(StreamProgress::default()), - fail_mark: false, - } - } - } - - fn with_engine( - plan: &StreamPlan, - cancel: &AtomicBool, - rec: &Recorder, - body: impl FnOnce(&Engine) -> R, - ) -> R { - let progress = |p: &StreamProgress| *rec.last.lock().unwrap() = *p; - let before = || { - *rec.marks.lock().unwrap() += 1; - if rec.fail_mark { - Err("mark failed".to_string()) - } else { - Ok(()) - } - }; - let log = |_: &str| {}; - let hooks = Hooks { - progress: &progress, - before_replace: &before, - log: &log, - }; - let engine = Engine::new(plan, cancel, &hooks); - body(&engine) - } - - #[test] - fn plan_counts_direct_and_shared_chunks_and_orders_fetches_by_first_use() { - let fx = Fixture::new("stream-plan"); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); - assert_eq!(plan.direct_chunks, 2, "B and C"); - assert_eq!(plan.shared_chunks, 1, "A"); - assert_eq!(plan.fetch_events.len(), 3); - assert_eq!(plan.refetches, 0); - assert_eq!(plan.bytes_total, 150 + 200 + 80); - let order: Vec<[u32; 4]> = plan - .fetch_events - .iter() - .map(|&e| plan.manifest.unique_chunks[plan.schedule.events[e].chunk].guid) - .collect(); - assert_eq!(order, vec![A, B, C]); - let mut req = fx.request(vec![0, 1, 2], 1 << 20); - req.expected_bytes = Some(1); - assert!(build_stream_plan(&req).unwrap_err().contains("byte total mismatch")); - } - - #[test] - fn out_of_order_arrival_writes_verified_files_and_empties_the_cache() { - let fx = Fixture::new("stream-ooo"); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - // C, then B, then A: the shared chunk last. - for id in (0..plan.fetch_events.len()).rev() { - engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); - } - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..3 { - let rel = fx.files[i].0.replace('\\', "/"); - assert_eq!(fs::read(fx.install.join(&rel)).unwrap(), fx.expected(i), "{rel}"); - } - assert!(fx.stages().is_empty()); - assert_eq!(*rec.marks.lock().unwrap(), 1, "marked updating once"); - let last = *rec.last.lock().unwrap(); - assert_eq!(last.files_done, 3); - assert_eq!(last.bytes_written, 430); - assert_eq!(last.files_bytes, 430); - let leftovers = fs::read_dir(&plan.cache_dir).unwrap().count(); - assert_eq!(leftovers, 0, "the shared chunk is released after its last use"); - } - - #[test] - fn a_shared_chunk_arriving_first_is_cached_until_its_later_file_is_reached() { - let fx = Fixture::new("stream-cache"); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); // A - let cached = plan.cache_dir.join(guid_to(A)); - assert!(cached.is_file(), "A waits in the cache for dir/b.bin"); - engine.process(&item(1), fx.body_for(&plan, 1)).unwrap(); // B: reaches b.bin - assert!(!cached.exists(), "released after its last use"); - engine.process(&item(2), fx.body_for(&plan, 2)).unwrap(); // C - }); - assert_eq!(fs::read(fx.install.join("dir/b.bin")).unwrap(), fx.expected(1)); - } - - #[test] - fn a_zero_budget_refetches_shared_chunks_and_never_caches() { - let fx = Fixture::new("stream-budget"); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 0)).unwrap(); - assert_eq!(plan.refetches, 1, "A downloaded once per file"); - assert_eq!(plan.fetch_events.len(), 4); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - for id in [3, 1, 0, 2] { - engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); - assert_eq!(fs::read_dir(&plan.cache_dir).unwrap().count(), 0); - } - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..3 { - let rel = fx.files[i].0.replace('\\', "/"); - assert_eq!(fs::read(fx.install.join(&rel)).unwrap(), fx.expected(i)); - } - } - - #[test] - fn resume_writes_only_pending_files_and_reuses_a_cached_chunk() { - let fx = Fixture::new("stream-resume"); - // An earlier run left A verified in the cache, plus a stale entry and a later copy. - let cache = fx.dir.join("cache"); - fs::create_dir_all(&cache).unwrap(); - fs::write(cache.join(guid_to(A)), fx.chunk(A)).unwrap(); - fs::write(cache.join("FFFFFFFF000000000000000000000000"), b"stale").unwrap(); - fs::write(cache.join(format!("{}.7", guid_to(A))), fx.chunk(A)).unwrap(); - // a.bin was verified by the delta pass; only dir/b.bin and c.bin are pending. - let plan = build_stream_plan(&fx.request(vec![1, 2], 1 << 20)).unwrap(); - assert_eq!(plan.fetch_events.len(), 1, "only C is downloaded"); - assert_eq!(fs::read_dir(&cache).unwrap().count(), 1, "stale entries removed"); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - assert!(fx.install.join("dir/b.bin").is_file(), "written from the cache at start"); - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); - assert_eq!(engine.unfinished_files(), 0); - }); - assert!(!fx.install.join("a.bin").exists()); - assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), fx.expected(2)); - } - - #[test] - fn cancel_keeps_old_files_removes_staging_and_keeps_the_cache() { - let fx = Fixture::new("stream-cancel"); - fs::write(fx.install.join("a.bin"), b"old a").unwrap(); - fs::write(fx.install.join("c.bin"), b"old c").unwrap(); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - // A arrives: a.bin half written, A cached for dir/b.bin. - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); - assert_eq!(fx.stages().len(), 1); - cancel.store(true, Ordering::Relaxed); - // B completes a.bin, but a cancelled run never swaps it in. - assert!(engine.process(&item(1), fx.body_for(&plan, 1)).is_err()); - assert!(engine.discard_staging() <= 2); - }); - assert!(fx.stages().is_empty()); - assert_eq!(fs::read(fx.install.join("a.bin")).unwrap(), b"old a"); - assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); - assert_eq!(*rec.marks.lock().unwrap(), 0, "nothing was replaced"); - assert!(plan.cache_dir.join(guid_to(A)).is_file(), "kept for the resume"); - } - - #[test] - fn a_hash_failure_keeps_the_old_file_and_marks_nothing() { - let fx = Fixture::new("stream-hash"); - fs::write(fx.install.join("c.bin"), b"old c").unwrap(); - let mut plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); - // The manifest promises another hash than the bytes the chunk yields. - plan.files[0].sha1 = Some([9; 20]); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - let err = with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - let res = engine.process(&item(0), fx.body_for(&plan, 0)); - engine.discard_staging(); - res - }); - match err { - Err(SinkError::Fatal(m)) => assert!(m.contains("SHA-1 mismatch for c.bin"), "{m}"), - other => panic!("expected a fatal hash failure, got {other:?}"), - } - assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); - assert!(fx.stages().is_empty()); - assert_eq!(*rec.marks.lock().unwrap(), 0); - } - - #[test] - fn a_corrupt_body_is_a_retry_that_changes_nothing() { - let fx = Fixture::new("stream-corrupt"); - let plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - let mut body = fx.body_for(&plan, 0); - let n = body.len(); - body[n - 3] ^= 0xFF; - assert!(matches!(engine.process(&item(0), body), Err(SinkError::Retry(_)))); - assert!(fx.stages().is_empty()); - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); - }); - assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), fx.expected(2)); - } - - #[test] - fn a_failed_mark_stops_before_any_replacement() { - let fx = Fixture::new("stream-mark"); - fs::write(fx.install.join("c.bin"), b"old c").unwrap(); - let plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); - let cancel = AtomicBool::new(false); - let mut rec = Recorder::new(); - rec.fail_mark = true; - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - let err = engine.process(&item(0), fx.body_for(&plan, 0)).unwrap_err(); - assert!(matches!(err, SinkError::Fatal(ref m) if m.contains("atomically replace"))); - }); - assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); - assert!(fx.stages().is_empty()); - } - - #[test] - fn an_owned_staging_dir_holds_the_stages_and_is_left_empty() { - let fx = Fixture::new("stream-staging"); - let staging = fx.install.join(".droiddeck-epic-staging"); - fs::create_dir_all(&staging).unwrap(); - let mut req = fx.request(vec![0, 1, 2], 1 << 20); - req.staging_dir = staging.to_str().unwrap().to_string(); - let plan = build_stream_plan(&req).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); - let stages: Vec<_> = fs::read_dir(&staging).unwrap().flatten().collect(); - assert_eq!(stages.len(), 1, "a.bin is staged in the owned directory"); - assert!(stages[0].file_name().to_string_lossy().ends_with(".stage")); - for id in 1..plan.fetch_events.len() { - engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); - } - }); - assert_eq!(fs::read_dir(&staging).unwrap().count(), 0); - assert_eq!(fs::read(fx.install.join("dir/b.bin")).unwrap(), fx.expected(1)); - let mut missing = fx.request(vec![0], 1 << 20); - missing.staging_dir = fx.dir.join("nope").to_str().unwrap().to_string(); - assert!(build_stream_plan(&missing).unwrap_err().contains("staging dir")); - } - - #[test] - fn a_manifest_file_named_like_a_stage_is_never_used_for_staging() { - let fx = Fixture::new("stream-reserved"); - let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); - assert!(plan.reserved.contains(&fx.install.join("dir/b.bin"))); - assert!(plan.reserved.contains(&fx.install.join("a.bin"))); - } - - // ── replacement-overlap gate ── - - /// `n` files of `len` bytes, one chunk each. - fn big_fixture(tag: &str, n: usize, len: usize) -> Fixture { - let dir = temp_dir(tag); - let install = dir.join("game"); - fs::create_dir_all(&install).unwrap(); - let guid = |i: usize| [0x100 + i as u32, 0, 0, 0]; - Fixture { - dir, - install, - chunks: (0..n).map(|i| (guid(i), data(i as u32 + 10, len))).collect(), - files: (0..n) - .map(|i| (format!("f{i}.bin"), vec![(guid(i), 0, len as i32)])) - .collect(), - } - } - - /// Drive the engine the way the fetch core does: items in order while the gate allows, - /// bodies handed back newest first (the most out-of-order the core can be). - fn drive_gated(fx: &Fixture, plan: &StreamPlan, engine: &Engine) -> Result<(), SinkError> { - engine.start()?; - let mut next = 0; - let mut in_flight: Vec = Vec::new(); - while next < plan.fetch_events.len() || !in_flight.is_empty() { - while next < plan.fetch_events.len() && engine.may_dispatch(next) { - in_flight.push(next); - next += 1; - } - let id = in_flight.pop().expect("gate closed with nothing in flight: deadlock"); - engine.process(&item(id), fx.body_for(plan, id))?; - } - Ok(()) - } - - fn write_old(fx: &Fixture, files: &[usize]) { - for &i in files { - fs::write(fx.install.join(&fx.files[i].0), b"old").unwrap(); - } - } - - #[test] - fn replaced_files_stay_within_the_overlap_budget_and_complete() { - let fx = big_fixture("stream-overlap", 6, 1000); - write_old(&fx, &[0, 1, 2, 3, 4, 5]); - let mut req = fx.request((0..6).collect(), 1 << 20); - req.replace_overlap_budget = 2500; - let plan = build_stream_plan(&req).unwrap(); - assert!(plan.files.iter().all(|f| f.replaces)); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - assert!(engine.may_dispatch(1), "two 1000-byte files fit 2500"); - assert!(!engine.may_dispatch(2), "a third does not"); - drive_gated(&fx, &plan, engine).unwrap(); - assert_eq!(engine.peak_replace_overlap(), 2000); - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..6 { - assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); - } - assert!(fx.stages().is_empty()); - } - - #[test] - fn a_budget_below_the_largest_file_still_progresses_one_at_a_time() { - let fx = big_fixture("stream-overlap-one", 4, 800); - write_old(&fx, &[0, 1, 2, 3]); - let mut req = fx.request((0..4).collect(), 1 << 20); - req.replace_overlap_budget = 1; - let plan = build_stream_plan(&req).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - assert!(engine.may_dispatch(0)); - assert!(!engine.may_dispatch(1)); - drive_gated(&fx, &plan, engine).unwrap(); - assert_eq!(engine.peak_replace_overlap(), 800); - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..4 { - assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); - } - } - - #[test] - fn a_fresh_install_is_never_gated() { - let fx = big_fixture("stream-overlap-fresh", 5, 600); - let mut req = fx.request((0..5).collect(), 1 << 20); - req.replace_overlap_budget = 1; - let plan = build_stream_plan(&req).unwrap(); - assert!(plan.files.iter().all(|f| !f.replaces)); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - assert!((0..5).all(|id| engine.may_dispatch(id))); - drive_gated(&fx, &plan, engine).unwrap(); - assert_eq!(engine.peak_replace_overlap(), 0); - }); - assert_eq!(fs::read(fx.install.join("f4.bin")).unwrap(), fx.expected(4)); - } - - #[test] - fn mixed_fresh_and_replaced_files_gate_only_the_replacements() { - let fx = big_fixture("stream-overlap-mixed", 6, 700); - write_old(&fx, &[1, 3, 5]); - let mut req = fx.request((0..6).collect(), 1 << 20); - req.replace_overlap_budget = 700; - let plan = build_stream_plan(&req).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - // f0 fresh, f1 replaced (admitted), f2 fresh, f3 replaced (waits for f1's swap). - assert!(engine.may_dispatch(2)); - assert!(!engine.may_dispatch(3)); - drive_gated(&fx, &plan, engine).unwrap(); - assert_eq!(engine.peak_replace_overlap(), 700); - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..6 { - assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); - } - } - - #[test] - fn a_shared_chunk_for_a_gated_file_waits_in_the_cache() { - // f0 and f2 are replaced and share chunk A; f1 is fresh. With room for one replaced file, - // f2 is admitted only after f0's swap, and A is cached until then. - let dir = temp_dir("stream-overlap-shared"); - let install = dir.join("game"); - fs::create_dir_all(&install).unwrap(); - let fx = Fixture { - dir, - install, - chunks: vec![(A, data(1, 300)), (B, data(2, 50))], - files: vec![ - ("f0.bin".to_string(), vec![(A, 0, 100)]), - ("f1.bin".to_string(), vec![(B, 0, 50)]), - ("f2.bin".to_string(), vec![(A, 100, 200)]), - ], - }; - write_old(&fx, &[0, 2]); - let mut req = fx.request(vec![0, 1, 2], 1 << 20); - req.replace_overlap_budget = 150; - let plan = build_stream_plan(&req).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - drive_gated(&fx, &plan, engine).unwrap(); - assert!(engine.peak_replace_overlap() <= 200); - assert_eq!(engine.unfinished_files(), 0); - }); - for i in 0..3 { - assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); - } - assert_eq!(fs::read_dir(&plan.cache_dir).unwrap().count(), 0); - } - - #[test] - fn cancelling_a_gated_update_keeps_every_old_file() { - let fx = big_fixture("stream-overlap-cancel", 4, 900); - write_old(&fx, &[0, 1, 2, 3]); - let mut req = fx.request((0..4).collect(), 1 << 20); - req.replace_overlap_budget = 1000; - let plan = build_stream_plan(&req).unwrap(); - let cancel = AtomicBool::new(false); - let rec = Recorder::new(); - with_engine(&plan, &cancel, &rec, |engine| { - engine.start().unwrap(); - engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); // f0 swapped in - assert!(engine.may_dispatch(1)); - cancel.store(true, Ordering::Relaxed); - assert!(engine.process(&item(1), fx.body_for(&plan, 1)).is_err()); - engine.discard_staging(); - }); - assert_eq!(fs::read(fx.install.join("f0.bin")).unwrap(), fx.expected(0)); - for i in 1..4 { - assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), b"old"); - } - assert!(fx.stages().is_empty()); - } - - fn item(id: usize) -> FetchItem { - FetchItem { - id: id as u64, - urls: vec!["http://127.0.0.1:9/x".to_string()], - reserve: 0, - range: None, - } - } - - fn guid_to(g: [u32; 4]) -> String { - super::super::manifest::guid_to_string(&g) - } -} diff --git a/app/src/main/rust/stores/src/epic/timefmt.rs b/app/src/main/rust/stores/src/epic/timefmt.rs new file mode 100644 index 0000000000..119106a4c8 --- /dev/null +++ b/app/src/main/rust/stores/src/epic/timefmt.rs @@ -0,0 +1,85 @@ +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let y = if month <= 2 { year - 1 } else { year }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let mp = (month + 9) % 12; + let doy = (153 * mp + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +fn digits(s: &str, from: usize, len: usize) -> Option { + let part = s.get(from..from + len)?; + if !part.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + part.parse().ok() +} + +pub fn utc_prefix_seconds(text: &str) -> Option { + let s = text.trim(); + let bytes = s.as_bytes(); + if bytes.len() < 19 || bytes[4] != b'-' || bytes[7] != b'-' || bytes[13] != b':' || bytes[16] != b':' { + return None; + } + if bytes[10] != b'T' && bytes[10] != b't' && bytes[10] != b' ' { + return None; + } + let (y, mo, d) = (digits(s, 0, 4)?, digits(s, 5, 2)?, digits(s, 8, 2)?); + let (h, mi, se) = (digits(s, 11, 2)?, digits(s, 14, 2)?, digits(s, 17, 2)?); + if !(1..=12).contains(&mo) || !(1..=31).contains(&d) || h > 23 || mi > 59 || se > 60 { + return None; + } + Some(days_from_civil(y, mo, d) * 86_400 + h * 3600 + mi * 60 + se) +} + +pub fn iso_seconds(text: &str) -> Option { + let s = text.trim(); + let base = utc_prefix_seconds(s)?; + let mut rest = &s[19..]; + if let Some(stripped) = rest.strip_prefix('.') { + let n = stripped.bytes().take_while(|b| b.is_ascii_digit()).count(); + rest = &stripped[n..]; + } + if rest.is_empty() || rest.eq_ignore_ascii_case("z") { + return Some(base); + } + let sign = match rest.as_bytes()[0] { + b'+' => 1, + b'-' => -1, + _ => return None, + }; + let offset: String = rest[1..].chars().filter(|c| *c != ':').collect(); + if !offset.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let (hh, mm) = match offset.len() { + 2 => (offset.parse::().ok()?, 0), + 4 => (offset[..2].parse::().ok()?, offset[2..].parse::().ok()?), + _ => return None, + }; + Some(base - sign * (hh * 3600 + mm * 60)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn epoch_and_known_dates() { + assert_eq!(utc_prefix_seconds("1970-01-01T00:00:00"), Some(0)); + assert_eq!(utc_prefix_seconds("2000-03-01T00:00:00.000Z"), Some(951_868_800)); + assert_eq!(utc_prefix_seconds("2026-10-13T12:34:56.789Z"), Some(1_791_894_896)); + assert_eq!(utc_prefix_seconds("not a date"), None); + } + + #[test] + fn offsets_in_every_spelling() { + let z = iso_seconds("2024-05-01T10:00:00Z").unwrap(); + assert_eq!(iso_seconds("2024-05-01T12:00:00+02:00"), Some(z)); + assert_eq!(iso_seconds("2024-05-01T12:00:00+0200"), Some(z)); + assert_eq!(iso_seconds("2024-05-01T08:00:00.123-02:00"), Some(z)); + assert_eq!(iso_seconds("2024-05-01T10:00:00"), Some(z)); + assert_eq!(iso_seconds("2024-05-01T10:00:00 garbage"), None); + } +} diff --git a/app/src/main/rust/stores/src/fetch_core.rs b/app/src/main/rust/stores/src/fetch_core.rs index d0a5fa36c7..e0c10112ca 100644 --- a/app/src/main/rust/stores/src/fetch_core.rs +++ b/app/src/main/rust/stores/src/fetch_core.rs @@ -1,4 +1,3 @@ - use crate::fetch_tuning::{ budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError, FetchFailKind, BOOTSTRAP_WINDOW, MAX_CHUNK_ATTEMPTS, NOMINAL_CHUNK_RESERVE_BYTES, @@ -25,21 +24,18 @@ pub const MAX_ITEM_ATTEMPTS: u32 = MAX_CHUNK_ATTEMPTS; pub const THROUGHPUT_SAMPLE_INTERVAL_MS: u64 = 5_000; const DRIVER_POLL_MS: u64 = 250; - pub const LATENCY_MIN_SAMPLES: u32 = 3; pub const LATENCY_HOLD_FACTOR: f64 = 2.5; pub const LATENCY_MIN_EXCESS_MS: f64 = 250.0; pub const LATENCY_GAIN_EPS: f64 = 0.25; pub const LATENCY_SHRINK_FACTOR: f64 = 0.85; - pub const HOST_STALL_MS: u64 = if cfg!(test) { 1_500 } else { 10_000 }; pub const HOST_STALL_COOLDOWN_MS: u64 = 30_000; pub const HOST_STALL_COOLDOWN_MAX_MS: u64 = 300_000; const STALL_CHECK_MS: u64 = 250; pub const STREAM_IDLE_CAP: Duration = Duration::from_secs(30); - #[derive(Clone, Debug)] pub struct FetchItem { pub id: u64, @@ -121,11 +117,6 @@ pub fn run_fetch( run_fetch_gated(items, hosts, opts, sink, cancel, progress, log, &|_: usize| true) } -/// [`run_fetch`] with an admission gate: the next fresh item (by index) is only dispatched once -/// `gate(index)` says so. Items are still taken strictly in order, and retries are never held -/// back. While the gate is closed the driver idles as it does on a full window, so a gate that -/// opens as the sink makes progress (e.g. as files finish) paces the run without stalling the -/// requests already in flight. The adaptivity is unchanged. #[allow(clippy::too_many_arguments)] pub fn run_fetch_gated( items: Vec, @@ -446,7 +437,6 @@ avg_mbps={avg_mbps:.2} peak_mbps={peak_mbps:.2} result={result}", outcome } - pub fn reserve_bytes(item: &FetchItem) -> u64 { if item.reserve == 0 { NOMINAL_CHUNK_RESERVE_BYTES @@ -568,7 +558,6 @@ fn record_first_error(slot: &Mutex>, err: String) { } } - struct Http { client: reqwest::Client, headers: Vec<(reqwest::header::HeaderName, reqwest::header::HeaderValue)>, @@ -789,7 +778,6 @@ async fn fetch_stream( Ok(offset) } - const NO_ATTEMPT: u32 = u32::MAX; enum ProcessMsg { @@ -1256,7 +1244,6 @@ fn silent_since(last_rx_at: Option, watch_from: Instant) -> Instant { } } - pub struct BandwidthMeter { start: Instant, total_bytes: AtomicU64, @@ -1342,7 +1329,6 @@ impl BandwidthMeter { } } - #[derive(Clone, Debug, Default)] struct ServerHealth { ewma_bps: f64, @@ -1804,7 +1790,6 @@ stalled_hosts={}", } } - #[cfg(test)] mod tests { #[test] @@ -2287,7 +2272,7 @@ mod tests { let mut options = opts("stream app=stub"); options.stream = true; options.per_host_cap = 8; - options.headers.push(("User-Agent".to_string(), "nile/0.1 Amazon".to_string())); + options.headers.push(("User-Agent".to_string(), "DroidDeck/test".to_string())); let cancel = AtomicBool::new(false); let lines: Mutex> = Mutex::new(Vec::new()); let outcome = run_fetch( @@ -2309,7 +2294,7 @@ mod tests { assert_eq!(got[&3], vec![9u8; 20_000]); assert_eq!(got[&4].len(), 1000); assert_eq!(got[&4][0], 100u8); - assert_eq!(got[&5], b"nile/0.1 Amazon".to_vec()); + assert_eq!(got[&5], b"DroidDeck/test".to_vec()); let finished = sink.finished.lock().unwrap(); assert_eq!(finished.len(), 5); let written: u64 = got.values().map(|b| b.len() as u64).sum(); diff --git a/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt deleted file mode 100644 index 6382f9519a..0000000000 --- a/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt +++ /dev/null @@ -1,43 +0,0 @@ -package com.droiddeck.launcher.stores.amazon - -import java.io.File -import java.security.MessageDigest -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.RuntimeEnvironment -import org.robolectric.annotation.Config -import org.robolectric.shadows.ShadowStatFs -import com.droiddeck.launcher.stores.StoreInstallRoot - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [33]) -class AmazonResumeSpaceTest { - @Test fun remainingWorkRejectsMeasuredZeroButNotUnknownSpace() { - val root = File(RuntimeEnvironment.getApplication().filesDir, "amazon-zero").apply { mkdirs() } - ShadowStatFs.registerStats(root.path, 1024, 0, 0) - val files = listOf(AmazonManifest.ManifestFile("missing", 4, 0, null)) - val needed = AmazonDownloadManager.remainingInstallBytes(files, root) - assertEquals(4L, needed) - assertTrue(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.availableBytes(root))) - assertFalse(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.SPACE_UNKNOWN)) - } - - @Test fun remainingSpaceExcludesVerifiedFilesButIncludesCorruptAndPartialFiles() { - val root = File(RuntimeEnvironment.getApplication().filesDir, "amazon-resume").apply { mkdirs() } - val data = byteArrayOf(1, 2, 3, 4) - val hash = MessageDigest.getInstance("SHA-256").digest(data) - val files = listOf("good", "corrupt", "partial", "missing").map { - AmazonManifest.ManifestFile(it, 4, 0, hash) - } - File(root, "good").writeBytes(data) - File(root, "corrupt").writeBytes(ByteArray(4)) - File(root, "partial").writeBytes(byteArrayOf(1)) - assertEquals(12L, AmazonDownloadManager.remainingInstallBytes(files, root)) - for (file in files) File(root, file.unixPath()).writeBytes(data) - assertEquals(0L, AmazonDownloadManager.remainingInstallBytes(files, root)) - } -} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt deleted file mode 100644 index fdf46d0df5..0000000000 --- a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt +++ /dev/null @@ -1,107 +0,0 @@ -package com.droiddeck.launcher.stores.epic - -import java.io.File -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [33]) -class EpicAdaptivePlanTest { - @get:Rule val temp = TemporaryFolder() - private val mib = 1024L * 1024 - private val gib = 1024 * mib - - private fun file(name: String, size: Long) = EpicDownloadManager.FileInfo().apply { - filename = name - var left = size - while (left > 0) { - val part = minOf(left, Int.MAX_VALUE.toLong()) - parts.add(EpicDownloadManager.ChunkPart().apply { this.size = part.toInt() }) - left -= part - } - } - - @Test fun netPeakCreditsEachOldCopyAtItsSwapAndLargestReplacedIgnoresFreshFiles() { - val files = listOf(file("a", 10 * gib), file("b", 2 * gib), file("c", 4 * gib)) - val old = mapOf("a" to 9 * gib, "c" to 5 * gib) - // Running Σ(new − old): +1, +3, +2 GiB. - assertEquals(3 * gib, EpicDownloadManager.settledNetPeak(files, old)) - assertEquals(10 * gib, EpicDownloadManager.largestReplacedFile(files, old)) - assertEquals(16 * gib, EpicDownloadManager.settledNetPeak(files, emptyMap())) - assertEquals(0L, EpicDownloadManager.largestReplacedFile(files, emptyMap())) - assertEquals(0L, EpicDownloadManager.largestReplacedFile(files, mapOf("b" to 0L))) - } - - @Test fun cacheBudgetIsHalfTheHeadroomCappedAndDefaultsWhenSpaceIsUnknown() { - assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_DEFAULT_BYTES, EpicDownloadManager.adaptiveCacheBudget(-1, 0)) - assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_MAX_BYTES, EpicDownloadManager.adaptiveCacheBudget(100 * gib, 10 * gib)) - assertEquals(0L, EpicDownloadManager.adaptiveCacheBudget(gib, gib)) - val free = 1000 * mib - assertEquals((free - 400 * mib - EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES) / 2, EpicDownloadManager.adaptiveCacheBudget(free, 400 * mib)) - } - - @Test fun freshInstallsHaveNoOverlapLimit() { - val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 10 * gib) - assertNull(plan.error) - assertEquals(-1L, plan.overlapBudget) - assertEquals(20 * gib, plan.gameNeed) - } - - @Test fun updateOverlapBudgetIsTheLeftoverAfterNetGrowthAndCache() { - // Card install, cache on internal: the card's leftover after the net growth is the budget. - val roomy = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 20 * gib, true, 10 * gib) - assertNull(roomy.error) - assertFalse(roomy.cacheBesideGame) - assertEquals(19 * gib, roomy.overlapBudget) - assertEquals(20 * gib, roomy.gameNeed) - // Tight: floored at the largest replaced file, one at a time. - val tight = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 5 * gib, true, 10 * gib) - assertNull(tight.error) - assertEquals(4 * gib, tight.overlapBudget) - // Not even one replaced file fits: the shortfall is named. - val short = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 4 * gib, true, 10 * gib) - assertTrue(short.error!!.contains("game's storage")) - assertTrue(short.error!!.contains("short by 1.0 GB")) - // Unknown free space: one replaced file at a time. - assertEquals(4 * gib, EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, -1, true, -1).overlapBudget) - } - - @Test fun cacheBesideTheGameComesOutOfTheOverlapBudget() { - val plan = EpicDownloadManager.planAdaptiveSpace(gib, 2 * gib, 10 * gib, false, -1) - assertNull(plan.error) - assertTrue(plan.cacheBesideGame) - assertTrue(plan.cacheBudget > 0) - assertEquals(10 * gib - gib - plan.cacheBudget - EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.overlapBudget) - assertEquals(10 * gib, plan.gameNeed) - } - - @Test fun fullInternalStorageMovesTheCacheBesideTheGameInsteadOfFailing() { - val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 100 * mib) - assertNull(plan.error) - assertTrue(plan.cacheBesideGame) - assertEquals(0L, plan.scratchNeed) - assertTrue(plan.cacheBudget > 0) - assertEquals(20 * gib + plan.cacheBudget + EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.gameNeed) - } - - @Test fun ownedStagingIsRemovedOnlyWhenItHoldsJustItsMarker() { - val root = temp.newFolder("game") - val staging = EpicDownloadManager.recoverOwnedStaging(root, emptyList()) - File(staging, "keep.stage").writeText("x") - EpicDownloadManager.releaseOwnedStaging(root) - assertTrue(File(staging, "owner").exists()) - File(staging, "keep.stage").delete() - EpicDownloadManager.releaseOwnedStaging(root) - assertFalse(staging.exists()) - // A directory without the marker is not ours. - staging.mkdir() - File(staging, "owner").writeText("someone else") - EpicDownloadManager.releaseOwnedStaging(root) - assertTrue(staging.exists()) - } -} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt deleted file mode 100644 index 37dd7d849d..0000000000 --- a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt +++ /dev/null @@ -1,646 +0,0 @@ -package com.droiddeck.launcher.stores.epic - -import com.droiddeck.launcher.stores.Store -import com.droiddeck.launcher.stores.StoreGameSidecar -import com.droiddeck.launcher.stores.StoreInstalls -import java.io.File -import java.net.InetAddress -import java.net.ServerSocket -import java.net.SocketException -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.security.MessageDigest -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.CountDownLatch -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit -import java.net.Socket -import org.junit.After -import org.junit.Assert.* -import org.junit.Before -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config -import org.robolectric.shadows.ShadowStatFs - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [33]) -class EpicBoundedInstallTest { - @get:Rule val temp = TemporaryFolder() - private lateinit var server: ServerSocket - private lateinit var worker: Thread - private lateinit var manifest: EpicDownloadManager.Manifest - private lateinit var root: File - private val requests = AtomicInteger() - private val reject = AtomicBoolean() - private val rejectGuid = AtomicInteger() - private val handlers = Executors.newCachedThreadPool() - private val sockets = ConcurrentHashMap.newKeySet() - private var requestBarrier: CountDownLatch? = null - private var deliveryGate: CountDownLatch? = null - private var bytes = ByteArray(256 * 1024) { (it % 251).toByte() } - - @Before fun setUp() { - root = temp.newFolder("game") - ShadowStatFs.registerStats(root.path, 1_000_000_000, 1_000_000_000, 1_000_000_000) - server = ServerSocket(0, 8, InetAddress.getByName("127.0.0.1")) - worker = Thread { - while (!server.isClosed) { - val socket = try { server.accept() } catch (e: SocketException) { - if (server.isClosed) break else throw e - } - sockets.add(socket) - handlers.submit { - try { - socket.use { - it.soTimeout = 5000 - val reader = it.getInputStream().bufferedReader() - val request = reader.readLine() - while (!reader.readLine().isNullOrEmpty()) { } - requests.incrementAndGet() - requestBarrier?.let { barrier -> - barrier.countDown() - check(barrier.await(5, TimeUnit.SECONDS)) { "Requests did not overlap" } - } - val out = it.getOutputStream() - val rejectedGuid = rejectGuid.get() - if (reject.get() || (rejectedGuid != 0 && request.contains("_%08X".format(rejectedGuid)))) { - out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray()) - } else { - val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN) - .putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size) - .put(ByteArray(24)).put(0).array() - out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray()) - out.write(header) - out.flush() - deliveryGate?.await(5, TimeUnit.SECONDS) - out.write(bytes) - } - out.flush() - } - } catch (_: java.io.IOException) { - // Cancellation disconnects an owned request while this fixture serves it. - } finally { sockets.remove(socket) } - } - } - }.apply { start() } - val chunk = EpicDownloadManager.ChunkInfo().apply { - guid = intArrayOf(1, 0, 0, 0) - windowSize = bytes.size - fileSize = bytes.size.toLong() - sha1 = sha(bytes) - } - manifest = EpicDownloadManager.Manifest().apply { - chunkDir = "chunks" - uniqueChunks = arrayListOf(chunk) - cdnUrls = listOf(EpicDownloadManager.CdnUrl("http://127.0.0.1:${server.localPort}", "", "")) - files = arrayListOf(gameFile("game.bin", 0, bytes.size / 2), gameFile("game.bin.part", bytes.size / 2, bytes.size / 2)) - } - } - - @After fun tearDown() { - deliveryGate?.countDown() - server.close() - worker.join(6000) - sockets.forEach { it.close() } - handlers.shutdownNow() - assertTrue(handlers.awaitTermination(6, TimeUnit.SECONDS)) - assertFalse(worker.isAlive) - } - - private fun sha(data: ByteArray) = MessageDigest.getInstance("SHA-1").digest(data) - - private fun gameFile(name: String, offset: Int, size: Int) = EpicDownloadManager.FileInfo().apply { - filename = name - sha1 = sha(bytes.copyOfRange(offset, offset + size)) - parts.add(EpicDownloadManager.ChunkPart().apply { - guid = intArrayOf(1, 0, 0, 0) - this.offset = offset - this.size = size - }) - } - - private fun install(cancel: AtomicBoolean = AtomicBoolean(), callback: EpicDownloadManager.Callback = - EpicDownloadManager.Callback { _, _ -> }, cache: String = "") = - EpicDownloadManager.installManifest(manifest, root.path, cache, null, cancel, callback) - - private fun oldInstall() { - File(root, "game.bin").writeBytes(byteArrayOf(9, 8, 7)) - File(root, "game.bin.part").writeBytes(byteArrayOf(6, 5, 4)) - StoreGameSidecar(Store.EPIC, "game", "Game", "game.bin", installVersion = "old").write(root) - } - - private fun assertOldInstallIntact() { - assertArrayEquals(byteArrayOf(9, 8, 7), File(root, "game.bin").readBytes()) - assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) - assertEquals("old", StoreGameSidecar.read(root)?.installVersion) - assertTrue(StoreGameSidecar.read(root)?.isInstalled == true) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - } - - @Test fun networkFailureAfterAcceptedPreflightPreservesInstalledFilesAndRecord() { - oldInstall() - reject.set(true) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertTrue(requests.get() > 0) - assertOldInstallIntact() - } - - @Test fun hashFailureAfterAcceptedPreflightPreservesInstalledFilesAndRecord() { - oldInstall() - manifest.files.first().sha1 = ByteArray(20) { 1 } - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertOldInstallIntact() - } - - @Test fun cancellationWhileStagingPreservesInstalledFilesAndRecord() { - oldInstall() - val cancel = AtomicBoolean() - assertNull(install(cancel, EpicDownloadManager.Callback { message, _ -> - if (message.startsWith("Downloading chunks (")) cancel.set(true) - })) - assertOldInstallIntact() - } - - @Test fun laterFailureRetainsVerifiedSwapsAndMarksMixedUpdateResumable() { - oldInstall() - manifest.files.last().sha1 = ByteArray(20) { 1 } - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) - val record = StoreGameSidecar.read(root)!! - assertEquals(StoreGameSidecar.STATE_UPDATING, record.state) - assertFalse(record.isInstalled) - assertEquals("", record.installVersion) - StoreInstalls.begin(root, Store.EPIC, "game", "Game", null, null) - assertEquals("game.bin", StoreGameSidecar.read(root)?.exe) - assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) - } - - @Test fun laterNetworkFailurePreservesTheUnreplacedOldFile() { - oldInstall() - val secondChunk = EpicDownloadManager.ChunkInfo().apply { - guid = intArrayOf(2, 0, 0, 0) - windowSize = bytes.size - fileSize = bytes.size.toLong() - sha1 = sha(bytes) - } - manifest.uniqueChunks.add(secondChunk) - manifest.files.last().parts.single().guid = secondChunk.guid - // Prefetch may fail early, but replacement is still ordered by the writer. - rejectGuid.set(2) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) - assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - } - - @Test fun cancellationAfterFirstSwapLeavesAResumableMixedUpdate() { - oldInstall() - val cancel = AtomicBoolean() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) {} - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - if (stage == "stream" && items == 1) cancel.set(true) - } - } - assertNull(install(cancel, callback)) - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) - assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) - assertNotNull(install()) - assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) - // Registration, not an individual file swap, restores the installed state. - assertFalse(StoreGameSidecar.read(root)!!.isInstalled) - } - - @Test fun atomicRenameFailurePreservesDestinationAndCleansStagingWithoutDeleteFallback() { - val destination = File(root, "game.bin").apply { mkdir() } - File(destination, "old-content").writeText("keep") - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertTrue(error.message.orEmpty().contains("atomically replace")) - assertEquals("keep", File(destination, "old-content").readText()) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - } - - @Test fun safeStagingSpaceBoundaryRejectsWithoutReclaimingAllOldOutputs() { - // The first staged half-file must coexist with the shared chunk and BOTH old outputs. - val old = ByteArray(bytes.size / 2) { 9 } - File(root, "game.bin").writeBytes(old) - File(root, "game.bin.part").writeBytes(old) - val peak = bytes.size + old.size - ShadowStatFs.registerStats(root.path, 1024, peak / 4096 - 1, peak / 4096 - 1) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertEquals(0, requests.get()) - assertArrayEquals(old, File(root, "game.bin").readBytes()) - assertArrayEquals(old, File(root, "game.bin.part").readBytes()) - ShadowStatFs.registerStats(root.path, 1024, peak / 4096, peak / 4096) - assertNotNull(install()) - } - - @Test fun zeroSpaceRejectsPendingFilesWhileUnknownSpaceIsNotTreatedAsZero() { - oldInstall() - ShadowStatFs.registerStats(root.path, 1024, 0, 0) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertOldInstallIntact() - assertEquals(0, requests.get()) - assertTrue(com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(1, 0)) - assertFalse(com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(1, -1)) - } - - @Test fun zeroScratchSpaceRejectsBeforeFetchWithoutTouchingInstalledOutputs() { - oldInstall() - val cache = temp.newFolder("full-scratch") - ShadowStatFs.registerStats(cache.path, 1024, 0, 0) - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache.path) } - assertTrue(error.message.orEmpty().contains("Not enough internal space")) - assertOldInstallIntact() - assertEquals(0, requests.get()) - } - - @Test fun installsSharedChunksAndPreservesLegitimatePartFilename() { - assertNotNull(install()) - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) - assertEquals(1, requests.get()) - assertFalse(File(root, ".chunks").exists()) - assertNotNull(install()) - assertEquals("Verified files must not be downloaded again", 1, requests.get()) - } - - private fun oversizedUpdate() { - val file = manifest.files[0] - val part = file.parts[0] - file.parts.clear() - repeat(100_000) { - file.parts.add(EpicDownloadManager.ChunkPart().apply { - guid = part.guid - size = Int.MAX_VALUE - }) - } - } - - @Test fun insufficientInstallSpacePreservesExistingFilesOnBothCacheLayouts() { - val oldGame = byteArrayOf(9, 8, 7) - val oldPart = byteArrayOf(6, 5, 4) - File(root, "game.bin").writeBytes(oldGame) - File(root, "game.bin.part").writeBytes(oldPart) - oversizedUpdate() - assertTrue(manifest.files[0].fileSize() > root.usableSpace) - - for (cache in listOf("", temp.newFolder("scratch-rejected").path)) { - if (cache.isNotEmpty()) ShadowStatFs.registerStats(cache, 1_000_000_000, 1_000_000_000, 1_000_000_000) - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache) } - assertTrue(error.message.orEmpty().contains("Not enough free space")) - assertArrayEquals(oldGame, File(root, "game.bin").readBytes()) - assertArrayEquals(oldPart, File(root, "game.bin.part").readBytes()) - } - assertEquals(0, requests.get()) - } - - @Test fun acceptedUpdateReplacesOutdatedFilesWithVerifiedOutputs() { - oldInstall() - assertNotNull(install()) - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) - assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - } - - @Test fun pauseKeepsCompletedFilesAndResumeReusesSharedCache() { - val cancel = AtomicBoolean() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) {} - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - if (stage == "stream" && items == 1) cancel.set(true) - } - } - assertNull(install(cancel, callback)) - assertTrue(File(root, "game.bin").exists()) - assertFalse(File(root, "game.bin.part").exists()) - assertEquals(1, requests.get()) - assertNotNull(install()) - assertEquals(1, requests.get()) - } - - @Test fun cancellationDuringChunkDeliveryRemovesIncompleteOutput() { - val cancel = AtomicBoolean() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) { - if (message.startsWith("Downloading chunks (")) cancel.set(true) - } - } - assertNull(install(cancel, callback)) - assertFalse(File(root, "game.bin").exists()) - assertNotNull(install()) - assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) - } - - @Test fun fileHashFailureIsExplicitAndRetryRepairsIt() { - val expected = manifest.files[0].sha1 - manifest.files[0].sha1 = ByteArray(20) { 1 } - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertTrue(error.message.orEmpty().contains("SHA-1 mismatch")) - assertFalse(File(root, "game.bin").exists()) - manifest.files[0].sha1 = expected - assertNotNull(install()) - } - - @Test fun corruptCacheIsRefetchedAndScratchCacheIsRemovedOnSuccess() { - val cache = temp.newFolder("scratch") - ShadowStatFs.registerStats(cache.path, 1_000_000_000, 1_000_000_000, 1_000_000_000) - File(cache, manifest.uniqueChunks[0].guidStr()).writeBytes(ByteArray(bytes.size)) - assertNotNull(install(cache = cache.path)) - assertEquals(1, requests.get()) - assertFalse(cache.exists()) - } - - @Test fun networkFailureLeavesNoCompletedFileAndCanBeRetried() { - reject.set(true) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertFalse(File(root, "game.bin").exists()) - reject.set(false) - assertNotNull(install()) - } - - @Test fun boundedProgressNeverReentersDownloadAfterEnteringInstall() { - val events = mutableListOf() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) { - if (message.startsWith("Downloading chunks")) events.add("download") - if (message.startsWith("Writing")) events.add("install") - } - override fun onBytes(done: Long, total: Long, speedBps: Long) { events.add("download") } - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - if (stage == "install") { - assertEquals(total, done) - assertEquals(itemsTotal, items) - events.add("install") - } - } - } - assertNotNull(install(callback = callback)) - val firstInstall = events.indexOf("install") - assertTrue(firstInstall > 0) - assertFalse(events.drop(firstInstall).contains("download")) - } - - @Test fun sharedChunksDoNotCompleteProgressBeforeTheRemainingFilesAreWritten() { - val samples = mutableListOf>() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) {} - override fun onBytes(done: Long, total: Long, speedBps: Long) { samples.add(done to total) } - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - if (stage == "stream" && items < itemsTotal) assertTrue(samples.last().first < samples.last().second) - } - } - assertNotNull(install(callback = callback)) - assertEquals(listOf(0L, bytes.size / 2L, bytes.size.toLong()), samples.map { it.first }) - assertTrue(samples.all { it.second == bytes.size.toLong() }) - } - - @Test fun oversizedSharedChunkStaysPinnedAcrossPrefetchedFilesWithoutChangingStages() { - bytes = ByteArray((EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 1024).toInt()) { (it % 251).toByte() } - manifest.uniqueChunks.single().apply { - windowSize = bytes.size - fileSize = bytes.size.toLong() - sha1 = sha(bytes) - } - manifest.files = (0..2).map { gameFile("part-$it.bin", it * 1024, 1024) } - val chunk = manifest.uniqueChunks.single() - val estimatedPeak = EpicDownloadManager.estimatePeakAdditionalBytes( - manifest.files, - mapOf(chunk.guidStr() to chunk), - emptyMap(), - true, - ) - assertEquals(EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 4 * 1024L, estimatedPeak) - val samples = mutableListOf() - val stages = mutableListOf() - val callback = object : EpicDownloadManager.Callback { - override fun onProgress(message: String, pct: Int) {} - override fun onBytes(done: Long, total: Long, speedBps: Long) { - assertEquals(3072L, total) - assertFalse(stages.contains("install")) - samples.add(done) - } - override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { - stages.add(stage) - if (stage == "stream" && items < itemsTotal) assertTrue(samples.last() < total) - } - } - assertNotNull(install(callback = callback)) - assertEquals(1, requests.get()) - assertEquals(listOf(0L, 1024L, 2048L, 3072L), samples) - assertEquals("install", stages.last()) - repeat(3) { assertArrayEquals(bytes.copyOfRange(it * 1024, (it + 1) * 1024), File(root, "part-$it.bin").readBytes()) } - } - - @Test fun hogwartsSizedSequentialPlanFitsAvailableSpaceWithoutFullChunkCache() { - val gib = 1024L * 1024 * 1024 - val file = EpicDownloadManager.FileInfo() - val chunks = LinkedHashMap() - repeat(72) { index -> - val chunk = EpicDownloadManager.ChunkInfo().apply { - guid = intArrayOf(index + 1, 0, 0, 0) - windowSize = gib.toInt() - } - chunks[chunk.guidStr()] = chunk - file.parts.add(EpicDownloadManager.ChunkPart().apply { - guid = chunk.guid - size = chunk.windowSize - }) - } - val peak = EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true) - assertEquals(73 * gib, peak) - assertTrue(peak < 96 * gib) - } - - @Test fun plannersReclaimEachOldFileOnlyAfterItsOwnVerifiedSwap() { - val chunks = manifest.uniqueChunks.associateBy { it.guidStr() } - val half = bytes.size / 2L - val old = mapOf("game.bin" to half, "game.bin.part" to 100 * half) - assertEquals(bytes.size + half, EpicDownloadManager.estimatePeakAdditionalBytes(manifest.files, chunks, emptyMap(), true, old)) - assertEquals(half, EpicDownloadManager.estimateStagedFilePeak(manifest.files, old)) - assertEquals(2 * half, EpicDownloadManager.estimateStagedFilePeak(manifest.files, emptyMap())) - assertEquals(bytes.size.toLong(), EpicDownloadManager.estimatePeakAdditionalBytes(manifest.files, chunks, emptyMap(), false, old)) - } - - private fun distinctFiles(count: Int) { - val template = manifest.uniqueChunks.single() - manifest.uniqueChunks = (1..count).map { id -> - EpicDownloadManager.ChunkInfo().apply { - guid = intArrayOf(id, 0, 0, 0) - windowSize = template.windowSize - fileSize = template.fileSize - sha1 = template.sha1 - } - } - manifest.files = manifest.uniqueChunks.mapIndexed { index, chunk -> - gameFile("file-$index.bin", 0, bytes.size).apply { parts.single().guid = chunk.guid } - } - } - - @Test fun prefetchOverlapsEightSinglePartFilesButNeverFetchesTheWholeManifest() { - distinctFiles(20) - requestBarrier = CountDownLatch(8) - val cancel = AtomicBoolean() - var sawWindow = false - assertNull(install(cancel, EpicDownloadManager.Callback { message, _ -> - if (message.startsWith("Downloading chunks (")) { - assertEquals(8, requests.get()) - val cacheBytes = File(root, ".chunks").listFiles()!!.sumOf { it.length() } - assertTrue(cacheBytes <= 8L * bytes.size) - sawWindow = true - cancel.set(true) - } - })) - assertTrue("An eight-request barrier cannot pass with a sequential downloader", sawWindow) - assertEquals(8, requests.get()) - assertTrue(root.walk().none { it.name.endsWith(".stage") || it.name.endsWith(".part") }) - } - - @Test fun parallelWindowSpaceThresholdIsCheckedBeforeStartingAnyRequest() { - distinctFiles(20) - val cache = temp.newFolder("parallel-scratch") - val peak = EpicDownloadManager.estimatePeakAdditionalBytes( - manifest.files, manifest.uniqueChunks.associateBy { it.guidStr() }, emptyMap(), false, - ) - assertEquals(8L * bytes.size, peak) - ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096 - 1, peak.toInt() / 4096 - 1) - assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache.path) } - assertEquals(0, requests.get()) - ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096, peak.toInt() / 4096) - requestBarrier = CountDownLatch(8) - assertNotNull(install(cache = cache.path)) - assertEquals(20, requests.get()) - manifest.files.forEach { assertArrayEquals(bytes, File(root, it.filename).readBytes()) } - } - - @Test fun cancellationDuringParallelNetworkReadsJoinsWorkersBeforeReturning() { - distinctFiles(20) - requestBarrier = CountDownLatch(8) - deliveryGate = CountDownLatch(1) - val cancel = AtomicBoolean() - val outcome = java.util.concurrent.atomic.AtomicReference() - val installer = Thread { - try { outcome.set(install(cancel)) } catch (e: Throwable) { outcome.set(e) } - }.apply { start() } - try { - assertTrue(requestBarrier!!.await(5, TimeUnit.SECONDS)) - cancel.set(true) - installer.join(10_000) - assertFalse("Installer must disconnect and join all download workers", installer.isAlive) - assertNull(outcome.get()) - val cache = File(root, ".chunks") - val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty() - deliveryGate!!.countDown() - Thread.sleep(150) - assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty()) - assertTrue(snapshot.keys.none { it.endsWith(".part") }) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - assertEquals(8, requests.get()) - } finally { - cancel.set(true) - deliveryGate!!.countDown() - installer.join(10_000) - } - } - - @Test fun failedParallelFetchJoinsOtherNetworkReadersBeforeReturning() { - distinctFiles(20) - requestBarrier = CountDownLatch(8) - deliveryGate = CountDownLatch(1) - rejectGuid.set(1) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertEquals(8, requests.get()) - val cache = File(root, ".chunks") - val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty() - deliveryGate!!.countDown() - Thread.sleep(150) - assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty()) - assertTrue(snapshot.keys.none { it.endsWith(".part") }) - assertTrue(root.walk().none { it.name.endsWith(".stage") }) - } - - @Test fun ownedOrphanStagingIsRecoveredBeforeTheSpaceProbeWithoutTouchingManifestFiles() { - val staging = EpicDownloadManager.recoverOwnedStaging(root, manifest.files) - File(staging, "native-123-0.stage").writeBytes(ByteArray(bytes.size)) - val legitimate = File(root, ".droiddeck-epic-old.stage").apply { writeText("keep") } - oldInstall() - val peak = 2 * bytes.size - ShadowStatFs.registerStats(root.path, 1024, peak / 4096, peak / 4096) - assertNotNull(install()) - assertEquals("keep", legitimate.readText()) - // The orphan was recovered, and a finished install removes the then-empty owned namespace. - assertFalse(staging.exists()) - } - - @Test fun stagingRecoveryRefusesUnownedAndManifestOwnedNamespaces() { - val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY).apply { mkdir() } - val untouched = File(staging, "legitimate.stage").apply { writeText("keep") } - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertEquals("keep", untouched.readText()) - File(staging, "owner").writeText("DroidDeck Epic staging v1\n") - manifest.files.first().filename = "${EpicDownloadManager.STAGING_DIRECTORY}/legitimate.stage" - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertEquals("keep", untouched.readText()) - assertEquals(0, requests.get()) - } - - @Test fun loadedNativeAbiFailureIsExplicitAndNeverFallsBackToJavaDownloads() { - val field = com.droiddeck.launcher.stores.StoresNative::class.java.getDeclaredField("loaded") - field.isAccessible = true - field.set(null, true) - try { - oldInstall() - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertTrue(error.message.orEmpty().contains("Native Epic install failed")) - assertEquals(0, requests.get()) - assertOldInstallIntact() - } finally { - com.droiddeck.launcher.stores.StoresNative.forget() - } - } - - @Test fun stagingRecoveryRejectsSymlinkNamespaceOwnerAndEntriesWithoutDeletingOutsideData() { - val outside = temp.newFolder("outside-staging") - val externalStage = File(outside, "keep.stage").apply { writeText("keep") } - val externalOwner = File(outside, "owner").apply { writeText("DroidDeck Epic staging v1\n") } - val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY) - try { - java.nio.file.Files.createSymbolicLink(staging.toPath(), outside.toPath()) - } catch (e: java.nio.file.FileSystemException) { - org.junit.Assume.assumeNoException("Host does not permit symbolic links", e) - } - try { - val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertTrue(error.message.orEmpty().contains("Unsafe Epic staging")) - assertEquals("keep", externalStage.readText()) - java.nio.file.Files.delete(staging.toPath()) - staging.mkdir() - java.nio.file.Files.createSymbolicLink(File(staging, "owner").toPath(), externalOwner.toPath()) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - java.nio.file.Files.delete(File(staging, "owner").toPath()) - File(staging, "owner").writeText("DroidDeck Epic staging v1\n") - val localStage = File(staging, "local.stage").apply { writeText("local") } - java.nio.file.Files.createSymbolicLink(File(staging, "linked.stage").toPath(), externalStage.toPath()) - assertThrows(EpicDownloadManager.InstallException::class.java) { install() } - assertEquals("local", localStage.readText()) - assertEquals("keep", externalStage.readText()) - assertEquals(0, requests.get()) - } finally { - if (java.nio.file.Files.isSymbolicLink(staging.toPath())) java.nio.file.Files.delete(staging.toPath()) - for (name in listOf("owner", "linked.stage")) { - val entry = File(staging, name).toPath() - if (java.nio.file.Files.isSymbolicLink(entry)) java.nio.file.Files.delete(entry) - } - } - } -} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicCredentialStoreTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicCredentialStoreTest.kt new file mode 100644 index 0000000000..d80b476a01 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicCredentialStoreTest.kt @@ -0,0 +1,86 @@ +package com.droiddeck.launcher.stores.epic + +import com.droiddeck.launcher.stores.CredentialCipher +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreAccounts +import org.json.JSONObject +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import java.security.SecureRandom +import javax.crypto.spec.SecretKeySpec + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicCredentialStoreTest { + private val app get() = RuntimeEnvironment.getApplication() + private val key = SecretKeySpec(ByteArray(32).also { SecureRandom().nextBytes(it) }, "AES") + + @Before fun setUp() { StoreAccounts.keys = CredentialCipher.KeyProvider { key }; StoreAccounts.keystoreFailed = false } + @After fun tearDown() { + StoreAccounts.clear(app, Store.EPIC) + StoreAccounts.keys = CredentialCipher.Keystore; StoreAccounts.keystoreFailed = false + } + + private val existing = JSONObject().put("access_token", "AT").put("refresh_token", "RT").put("account_id", "acc") + .put("display_name", "Someone").put("expires_at", System.currentTimeMillis() + 3_600_000L) + + @Test fun anExistingSignInIsRead() { + StoreAccounts.write(app, Store.EPIC, existing) + val c = EpicCredentialStore.load(app)!! + assertEquals("AT", c.accessToken) + assertEquals("RT", c.refreshToken) + assertEquals("acc", c.accountId) + assertEquals("Someone", c.displayName) + assertTrue(EpicCredentialStore.isLoggedIn(app)) + assertFalse(EpicCredentialStore.expired(app)) + assertEquals("AT", EpicCredentialStore.getValidAccessToken(app)) + assertEquals("Someone", StoreAccounts.signedInAs(app, Store.EPIC)) + } + + @Test fun aRefreshAnswerKeepsWhatItOmits() { + StoreAccounts.write(app, Store.EPIC, existing) + EpicCredentialStore.save(app, EpicCredentialStore.TokenResult("AT2", null, null, null, 5L)) + val c = EpicCredentialStore.load(app)!! + assertEquals("AT2", c.accessToken) + assertEquals("RT", c.refreshToken) + assertEquals("acc", c.accountId) + assertEquals("Someone", c.displayName) + assertTrue(EpicCredentialStore.expired(app)) + val stored = StoreAccounts.read(app, Store.EPIC)!! + for (k in listOf("access_token", "refresh_token", "account_id", "display_name", "expires_at")) assertTrue(k, stored.has(k)) + } + + @Test fun aNewSignInReplacesTheOldAccount() { + StoreAccounts.write(app, Store.EPIC, existing) + EpicCredentialStore.save(app, EpicCredentialStore.TokenResult("NEW", null, "acc2", "Other", 9L), merge = false) + val c = EpicCredentialStore.load(app)!! + assertNull(c.refreshToken) + assertEquals("acc2", c.accountId) + assertFalse(StoreAccounts.read(app, Store.EPIC)!!.has("refresh_token")) + } + + @Test fun signedOutAndUnusableAnswers() { + assertNull(EpicCredentialStore.load(app)) + assertNull(EpicCredentialStore.getValidAccessToken(app)) + assertFalse(EpicCredentialStore.expired(app)) + assertNull(EpicCredentialStore.TokenResult.from(JSONObject().put("error", "x"))) + assertNull(EpicCredentialStore.TokenResult.from(JSONObject().put("refresh_token", "x"))) + StoreAccounts.write(app, Store.EPIC, JSONObject().put("access_token", "").put("account_id", "a")) + assertNull(EpicCredentialStore.load(app)) + } + + @Test fun anExpiringTokenWithoutRefreshTokenIsReturnedAsIs() { + StoreAccounts.write(app, Store.EPIC, JSONObject().put("access_token", "OLD").put("expires_at", 1L)) + assertEquals("OLD", EpicCredentialStore.getValidAccessToken(app)) + assertTrue(EpicCredentialStore.expired(app)) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt deleted file mode 100644 index 14bc3d364e..0000000000 --- a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt +++ /dev/null @@ -1,190 +0,0 @@ -package com.droiddeck.launcher.stores.epic - -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertThrows -import org.junit.Assert.assertTrue -import org.junit.Test -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config - -@RunWith(RobolectricTestRunner::class) -@Config(sdk = [33]) -class EpicJsonManifestTest { - @Test - fun boundedInstallPeakReservesParallelIncomingChunksBeforeWriting() { - val first = chunk(1) - val second = chunk(2) - val file = file(part(1, 0, 100), part(2, 0, 100)) - val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) - - assertEquals(300L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true)) - assertEquals(200L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false)) - } - - @Test - fun sharedCachedChunkIsReclaimedOnlyAfterItsLastPart() { - val shared = chunk(1) - val files = listOf(file(part(1, 0, 60)), file(part(1, 60, 40))) - - assertEquals( - 100L, - EpicDownloadManager.estimatePeakAdditionalBytes( - files, mapOf(shared.guidStr() to shared), mapOf(shared.guidStr() to 100L), true, - ), - ) - assertEquals( - 200L, - EpicDownloadManager.estimatePeakAdditionalBytes( - files, mapOf(shared.guidStr() to shared), emptyMap(), true, - ), - ) - } - - @Test - fun sharedChunksWithinTheWindowStayPinnedWithinTheCacheCeiling() { - val first = chunk(1).apply { windowSize = 40 * 1024 * 1024 } - val second = chunk(2).apply { windowSize = 40 * 1024 * 1024 } - val files = listOf( - file(part(1, 0, 40 * 1024 * 1024)), - file(part(2, 0, 40 * 1024 * 1024)), - file(part(1, 0, 40 * 1024 * 1024)), - file(part(2, 0, 40 * 1024 * 1024)), - ) - val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) - - val gamePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true) - val cachePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false) - - assertEquals(200L * 1024 * 1024, gamePeak) - assertTrue(cachePeak <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 40L * 1024 * 1024) - } - - @Test fun scheduleDeduplicatesWindowAndEvictsOnlyUnpinnedReusableChunks() { - val size = 40L * 1024 * 1024 - val chunks = (1..12).map { chunk(it).apply { windowSize = size.toInt() } } - .associateBy { it.guidStr() } - val files = (1..12).map { file(part(it, 0, 1)) } + (1..12).map { file(part(it, 0, 1)) } - val schedule = EpicDownloadManager.PrefetchSchedule(files, chunks, emptyMap()) - assertEquals(chunks.keys.take(8), schedule.fill()) - assertEquals(emptyList(), schedule.fill()) - assertEquals(8 * size, schedule.bytes()) - assertEquals(emptyList(), schedule.consume()) - assertEquals(listOf(chunks.keys.elementAt(8)), schedule.fill()) - assertEquals(9 * size, schedule.bytes()) - assertEquals(listOf(chunks.keys.first()), schedule.consume()) - var refetchedFirst = false - repeat(22) { - val fetched = schedule.fill() - if (chunks.keys.first() in fetched) refetchedFirst = true - assertTrue(schedule.bytes() <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 8 * size) - val released = schedule.consume() - assertTrue(released.none { it in schedule.window() }) - } - assertTrue(refetchedFirst) - assertEquals(0L, schedule.bytes()) - assertEquals(9 * size, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false)) - } - - @Test fun cachedAndOldOutputBytesAreCreditedOnlyWhenTheScheduleReleasesThem() { - val chunks = (1..8).map { chunk(it) }.associateBy { it.guidStr() } - val files = (1..8).map { id -> file(part(id, 0, 100)).apply { filename = "file-$id" } } - val existing = chunks.keys.associateWith { 100L } - val old = files.associate { it.filename to 100L } - assertEquals(800L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false)) - assertEquals(0L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, false)) - assertEquals(900L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true, old)) - assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, true, old)) - } - - @Test - fun resumeCacheDropsCorruptAndUnneededEntriesButKeepsVerifiedChunks() { - val dir = java.io.File(System.getProperty("user.dir"), "build/epic-cache-test-${System.nanoTime()}") - assertTrue(dir.mkdirs()) - try { - val bytes = ByteArray(100) { (it % 251).toByte() } - val expectedSha = java.security.MessageDigest.getInstance("SHA-1").digest(bytes) - val valid = chunk(1).apply { sha1 = expectedSha } - val corrupt = chunk(2).apply { sha1 = expectedSha } - java.io.File(dir, valid.guidStr()).writeBytes(bytes) - java.io.File(dir, corrupt.guidStr()).writeBytes(ByteArray(100) { 1 }) - java.io.File(dir, "orphan").writeBytes(bytes) - - val retained = EpicDownloadManager.prepareBoundedCache(dir, listOf(valid, corrupt)) - - assertEquals(mapOf(valid.guidStr() to 100L), retained) - assertTrue(java.io.File(dir, valid.guidStr()).exists()) - assertFalse(java.io.File(dir, corrupt.guidStr()).exists()) - assertFalse(java.io.File(dir, "orphan").exists()) - } finally { - EpicDownloadManager.deleteDir(dir) - } - } - - @Test - fun referenceCountsIncludeRepeatedPartsSoCancelledFilesCanBeReplanned() { - val pending = listOf(file(part(1, 0, 20), part(1, 20, 20)), file(part(1, 40, 60))) - - assertEquals(3, EpicDownloadManager.chunkReferences(pending)["00000001000000000000000000000000"]) - } - - private fun chunk(id: Int) = EpicDownloadManager.ChunkInfo().apply { - guid = intArrayOf(id, 0, 0, 0) - windowSize = 100 - fileSize = 100 - } - - private fun part(id: Int, offset: Int, size: Int) = EpicDownloadManager.ChunkPart().apply { - guid = intArrayOf(id, 0, 0, 0) - this.offset = offset - this.size = size - } - - private fun file(vararg parts: EpicDownloadManager.ChunkPart) = EpicDownloadManager.FileInfo().apply { - filename = "test.bin" - this.parts.addAll(parts) - } - - @Test - fun blobNumbersAreDecimalBytesLeastSignificantFirst() { - assertEquals(1L, EpicDownloadManager.blobToNum("001000000000")) - assertEquals(0x100L, EpicDownloadManager.blobToNum("000001000000")) - assertEquals(0x0807_0605_0403_0201L, EpicDownloadManager.blobToNum("001002003004005006007008")) - assertEquals(-1L, EpicDownloadManager.blobToNum("255255255255255255255255")) - assertEquals(7L, EpicDownloadManager.blobToNum("7")) - assertEquals(0L, EpicDownloadManager.blobToNum("")) - } - - @Test - fun aBlobIsDigitsOnly() { - assertThrows(NumberFormatException::class.java) { EpicDownloadManager.blobToNum("0x1") } - assertThrows(NumberFormatException::class.java) { EpicDownloadManager.blobToNum("-05") } - } - - @Test - fun jsonManifestUsesBlobs() { - val json = """{ - "ManifestFileVersion": "006000000000", - "LaunchExeString": "Game\\Binaries\\Game.exe", - "ChunkHashList": { "0000000100000002000000030000000F": "001002003004005006007008" }, - "DataGroupList": { "0000000100000002000000030000000F": "007" }, - "ChunkFilesizeList": { "0000000100000002000000030000000F": "001002000000000000000000" }, - "FileManifestList": [ - { "Filename": "a.bin", - "FileChunkParts": [ { "Guid": "0000000100000002000000030000000F", "Offset": "001000000000", "Size": "000000016000" } ] } - ] - }""" - val m = EpicDownloadManager.parseManifest(json.toByteArray()) - assertNotNull(m) - assertEquals("ChunksV3", m.chunkDir) - assertEquals("Game/Binaries/Game.exe", m.launchExe) - val c = m.uniqueChunks.single() - assertEquals(0x0201L, c.fileSize) - assertEquals("ChunksV3/07/0807060504030201_0000000100000002000000030000000F.chunk", c.getPath(m.chunkDir)) - val part = m.files.single().parts.single() - assertEquals(1, part.offset) - assertEquals(1_048_576, part.size) - } -} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicLibraryTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicLibraryTest.kt new file mode 100644 index 0000000000..4eb819043a --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicLibraryTest.kt @@ -0,0 +1,95 @@ +package com.droiddeck.launcher.stores.epic + +import com.droiddeck.launcher.stores.Store +import org.json.JSONArray +import org.json.JSONObject +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicLibraryTest { + private val app get() = RuntimeEnvironment.getApplication() + + @After fun tearDown() { EpicPrefs.get(app).edit().clear().commit() } + + @Test fun cacheRoundTripsInTheStoredShape() { + val game = EpicGame("Fig", "ns", "cat", "Fig Game", artCover = "tall", artSquare = "wide", developer = "Dev", + description = "Text", installSize = 42L, canRunOffline = false, releaseDate = "2024-01-01", publisher = "Pub") + val text = EpicLibrary.toCacheJson(listOf(game)) + val o = JSONArray(text).getJSONObject(0) + for (key in listOf("appName", "namespace", "catalogItemId", "title", "artCover", "artSquare", "developer", + "description", "version", "installSize", "canRunOffline", "releaseDate", "publisher")) assertTrue(key, o.has(key)) + assertEquals(listOf(game), EpicLibrary.fromCacheJson(text)) + } + + @Test fun readingSkipsDedupsSortsAndCapsSizes() { + val text = JSONArray() + .put(JSONObject().put("appName", "b").put("title", "beta").put("installSize", (1L shl 41))) + .put(JSONObject().put("title", "no app name")) + .put(JSONObject().put("appName", "a").put("title", "Alpha")) + .put(JSONObject().put("appName", "b").put("title", "beta again")) + .toString() + val games = EpicLibrary.fromCacheJson(text) + assertEquals(listOf("a", "b"), games.map { it.appName }) + assertEquals(0L, games[1].installSize) + assertTrue(games[0].canRunOffline) + assertTrue(EpicLibrary.fromCacheJson("not json").isEmpty()) + } + + @Test fun catalogItemMapping() { + EpicPrefs.get(app).edit().putLong("size_Fig", 1234L).putString("release_Old", "2001").commit() + val item = EpicLibrary.toCatalogItem(app, EpicGame("Fig", "ns", "cat", "Fig & Co", artCover = "tall", description = "

Hi

", installSize = 9L)) + assertEquals(Store.EPIC, item.store) + assertEquals("Fig", item.id) + assertEquals("tall", item.imageUrl) + assertEquals("tall", item.tallImageUrl) + assertEquals(1234L, item.sizeBytes) + assertEquals("Hi", item.description) + assertTrue(item.owned) + assertEquals("https://store.epicgames.com/en-US/browse?q=Fig+%26+Co", item.storeUrl) + assertEquals(mapOf("namespace" to "ns", "catalogItemId" to "cat", "version" to ""), item.extra) + val old = EpicLibrary.toCatalogItem(app, EpicGame("Old", "ns", "c", "Old", artSquare = "wide", installSize = 9L)) + assertEquals("2001", old.releaseDate) + assertEquals(9L, old.sizeBytes) + assertEquals("wide", old.tallImageUrl) + } + + @Test fun dlcAreHiddenUnlessEverythingIsDlc() { + val base = EpicGame("base", "ns", "c1", "Base") + val dlc = EpicGame("dlc", "ns", "c2", "Add-on", isDlc = true) + assertEquals(listOf("base"), EpicLibrary.visible(listOf(dlc, base)).map { it.appName }) + assertEquals(listOf("dlc"), EpicLibrary.visible(listOf(dlc)).map { it.appName }) + } + + @Test fun syncedGamesKeepCachedValuesWhenDetailsAreMissing() { + val record = JSONObject().put("appName", "Fig").put("namespace", "ns").put("catalogItemId", "cat") + val before = EpicGame("Fig", "ns", "cat", "Cached Title", artCover = "c", version = "v1", installSize = 7L) + val kept = EpicLibrary.gameFrom(record, null, before) + assertEquals("Cached Title", kept.title) + assertEquals("v1", kept.version) + assertEquals(7L, kept.installSize) + val fresh = EpicLibrary.gameFrom(record, JSONObject().put("title", "New").put("dlc", true).put("canRunOffline", false), before) + assertEquals("New", fresh.title) + assertTrue(fresh.isDlc) + assertFalse(fresh.canRunOffline) + assertEquals("Fig", EpicLibrary.gameFrom(record, null, null).title) + } + + @Test fun signOutForgetsOnlyTheLibrary() { + EpicPrefs.get(app).edit().putString(EpicLibrary.KEY_CACHE, "[]").putLong(EpicLibrary.KEY_SYNCED_AT, 1L) + .putString("cloud_template_Fig", "{AppData}/x").commit() + EpicBackend.signOut(app) + val prefs = EpicPrefs.get(app) + assertFalse(prefs.contains(EpicLibrary.KEY_CACHE)) + assertFalse(prefs.contains(EpicLibrary.KEY_SYNCED_AT)) + assertTrue(prefs.contains("cloud_template_Fig")) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplateTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplateTest.kt new file mode 100644 index 0000000000..5ecc030d91 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicSaveTemplateTest.kt @@ -0,0 +1,30 @@ +package com.droiddeck.launcher.stores.epic + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class EpicSaveTemplateTest { + @Test fun rootsAreCaseInsensitive() { + val p = EpicSaveTemplate.parse("{appdata}\\Game\\Saved", "acc", "app")!! + assertEquals(EpicSaveTemplate.Root.LOCAL_APPDATA, p.root) + assertEquals("/Game/Saved", p.rest) + assertEquals(EpicSaveTemplate.Root.LOCAL_APPDATA, EpicSaveTemplate.parse("{LocalAppData}/x", "a", "b")!!.root) + assertEquals(EpicSaveTemplate.Root.ROAMING_APPDATA, EpicSaveTemplate.parse("{RoamingAppData}/x", "a", "b")!!.root) + assertEquals(EpicSaveTemplate.Root.DOCUMENTS, EpicSaveTemplate.parse("{UserDir}/x", "a", "b")!!.root) + assertEquals(EpicSaveTemplate.Root.SAVED_GAMES, EpicSaveTemplate.parse("{UserSavedGames}/x", "a", "b")!!.root) + assertEquals(EpicSaveTemplate.Root.USER_PROFILE, EpicSaveTemplate.parse("{UserProfile}/x", "a", "b")!!.root) + assertEquals(EpicSaveTemplate.Root.INSTALL_DIR, EpicSaveTemplate.parse("{InstallDir}/x", "a", "b")!!.root) + } + + @Test fun placeholdersAreFilledLiterally() { + val p = EpicSaveTemplate.parse("{UserSavedGames}/S/{epicid}/{APPNAME}/x", "a\$1", "B")!! + assertEquals("/S/a\$1/B/x", p.rest) + } + + @Test fun unknownOrMissingRootIsNull() { + assertNull(EpicSaveTemplate.parse("{Nope}/x", "a", "b")) + assertNull(EpicSaveTemplate.parse("Saves/x", "a", "b")) + assertNull(EpicSaveTemplate.parse("", "a", "b")) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicShelvesTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicShelvesTest.kt new file mode 100644 index 0000000000..3dddd02b58 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicShelvesTest.kt @@ -0,0 +1,58 @@ +package com.droiddeck.launcher.stores.epic + +import com.droiddeck.launcher.stores.CatalogItem +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreShelves +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicShelvesTest { + private fun item(id: String, items: String, mature: Boolean? = null, tags: List = emptyList()) = JSONObject() + .put("id", id).put("title", "T$id").put("image", "w").put("tall", "t").put("tags", "Action").put("free", false) + .put("hasPrice", true).put("final", "$5").put("orig", "$10").put("disc", 50).put("url", "u").put("dev", "D") + .put("pub", "P").put("rel", "2024").put("desc", "x") + .put("extra", JSONObject().put("namespace", "ns").put("items", items)) + .apply { if (mature != null) put("mature", mature) else put("allTags", JSONArray(tags)) } + + @Test fun nativeAnswerAndPersistedCopyReadTheSame() { + val answer = JSONObject().put("new", JSONArray().put(item("o1", "c1", tags = listOf("Mature")))) + .put("deals", JSONArray()).put("free", JSONArray()).put("trending", JSONArray().put(item("o2", "c2"))) + val shelves = EpicShelves.fromJson(answer.toString())!! + assertEquals(1, shelves.whatsNew.size) + assertTrue(shelves.whatsNew[0].mature) + assertEquals("x", shelves.whatsNew[0].description) + assertEquals(50, shelves.whatsNew[0].discountPercent) + val again = EpicShelves.fromJson(EpicShelves.toJson(shelves))!! + assertEquals(shelves.whatsNew, again.whatsNew) + assertEquals(shelves.trending, again.trending) + } + + @Test fun allEmptyIsAbsent() { + val empty = JSONObject().put("new", JSONArray()).put("deals", JSONArray()).put("free", JSONArray()).put("trending", JSONArray()) + assertNull(EpicShelves.fromJson(empty.toString())) + assertNull(EpicShelves.fromJson(null)) + } + + @Test fun ownedShelfEntriesBecomeTheLibraryItem() { + val shelf = EpicShelves.fromJson(JSONObject().put("free", JSONArray().put(item("offer", "x,c2", mature = false))).toString())!! + val owned = CatalogItem(Store.EPIC, "FigApp", "Fig", imageUrl = null, owned = true, developer = "", + publisher = "Lib Pub", extra = mapOf("namespace" to "ns", "catalogItemId" to "c2")) + val marked = EpicShelves.markOwned(shelf, listOf(owned)) + val entry = marked.free.single() + assertEquals("FigApp", entry.id) + assertTrue(entry.owned) + assertEquals("w", entry.imageUrl) + assertEquals("D", entry.developer) + assertEquals("Lib Pub", entry.publisher) + assertEquals(StoreShelves().isEmpty, EpicShelves.markOwned(StoreShelves(), listOf(owned)).isEmpty) + } +} diff --git a/docs/development/amazon-cleanroom.md b/docs/development/amazon-cleanroom.md new file mode 100644 index 0000000000..e907db5526 --- /dev/null +++ b/docs/development/amazon-cleanroom.md @@ -0,0 +1,175 @@ +# Amazon Games module: clean-room rewrite + +The Amazon Games integration (the Rust `amazon` module in `libdroiddeckstores.so` and the Kotlin +package `com.droiddeck.launcher.stores.amazon`) was rewritten so that it carries no code from the +earlier implementation. This page records how that was done and every judgement call the new code +makes. + +## Process + +1. **Spec, written separately.** A functional specification (six documents: summary and unknowns, + service protocol, manifest format, install behaviour, on-disk compatibility, and the app-facing + interface) was written from the old code by a separate pass. It describes behaviour and data + formats only, not code structure. +2. **Implementation from the spec only.** The implementer worked from those documents and from + code that is not Amazon-specific: + - read: the spec; the crate's `Cargo.toml`, `Cargo.lock`, `lib.rs`, `stores_jni.rs`, + `priority.rs`, `fetch_tuning.rs` and the shared `fetch_core.rs`; the clean-room Epic module + (Rust and Kotlin) as the structural template; `JNI.md`; the shared Kotlin store code (account + store, install records, download queue, login activity, catalog and state); the signatures and + call sites of `StoreSizes`' Amazon branch (to keep them compiling); build files and CI; + - not read: anything under `stores/amazon/` (Kotlin, Java, tests) or `src/amazon*` (Rust) of the + old tree, other branches or checkouts, or any Amazon client source. The old files were deleted + with `git rm` without being opened. +3. **Fresh code.** Every file has its own structure, names and comments, with a plain description + and `SPDX-License-Identifier: GPL-3.0-or-later` (the crate's licence). Where the spec left a fact + open, the choice was made from general knowledge of Amazon's public services or from + unauthenticated probes of the live endpoints (below), and is listed here. + +## Layout + +Rust, `app/src/main/rust/stores/src/amazon/`: + +| File | Role | +|---|---| +| `net.rs` | blocking HTTP (reqwest + rustls), the three User-Agent identities, hex / percent / base64url helpers, URL scrubbing for logs | +| `auth.rs` | per-attempt device serial, client id, PKCE verifier and challenge; sign-in URL; code from the return URL; registration, refresh, deregistration | +| `distribution.rs` | the distribution service call shape (`X-Amz-Target`, `x-amzn-token`, `Content-Encoding: amz-1.0`), refusal detection, the download lookup | +| `library.rs` | entitlement pages, field fallbacks, de-duplication, add-on filter, sort | +| `cdn.rs` | path insertion before the signed query, file URLs, the manifest GET | +| `codec.rs` | manifest framing (big-endian header length) and xz / LZMA-alone decompression (`lzma-rs`) | +| `proto.rs` | a strict protobuf wire reader (and test writers) | +| `manifest.rs` | packages → files, path normalisation and safety, duplicate handling | +| `install/` | `mod.rs` (the run), `check.rs` (completeness, stray `.tmp`, free space), `writer.rs` (streamed download, SHA-256, atomic placement), `tests.rs` (loopback end-to-end) | +| `launch.rs` | `fuel.json`, exe scoring, the FuelPump environment | +| `service.rs`, `ffi.rs` | named operations and the JNI boundary | + +Kotlin, `app/src/main/java/com/droiddeck/launcher/stores/amazon/`: `AmazonBackend` (the +`StoreBackend`, installs the login flow), `AmazonLogin` (the WebView flow), `AmazonCredentialStore` +(the account through `StoreAccounts`, single-flight refresh), `AmazonPrefs`, `AmazonLibrary` (sync, +cache, posters, publishing), `AmazonInstaller` (the queue job), `AmazonNative` (JNI), and two thin +wrappers kept for `StoreSizes`: `AmazonApiClient` (`getGameDownload`, `getBytes`, `appendPath`) and +`AmazonManifest` (`parse(bytes).totalInstallSize`). + +The JNI surface (`nativeConfigure`, `nativeCall`, `nativeFetch`, `nativeInspect`, `nativeInstall`, +`nativeCancel`, `nativeRelease`) is documented in `app/src/main/rust/stores/JNI.md`. The old +`nativeStart` symbol is gone. The Java xz library stays in `app/build.gradle`: commons-compress +needs it for the `.tar.xz` Windows components, so only Amazon stopped using it. + +## Compatibility kept exactly + +- Credentials (`stores/amazon/credentials.json` through `StoreAccounts`): the same keys + (`access_token`, `refresh_token`, `device_serial`, `client_id`, `expires_at`, `display_name`). A + refresh merges `access_token`, `device_serial`, `client_id`, `expires_at` into the stored record + and keeps every other key; a new sign-in writes the whole record. Registration and refresh keep + the launcher's device constants, so existing refresh tokens keep working. +- Preferences `stores.amazon`: `library_cache` (same element keys), `library_synced_at`, + `vcover_`, `size_`, `version_`; sign-out removes only the first two. +- Install folders, the sidecar keys (`extra.entitlementId`, `extra.sku`, no `cloud` key), the + launcher name `.droiddeck-launch.bat`, the `.tmp` naming and the two marker files. +- The catalog item's `extra` (`entitlementId`, `sku`, `version` = `""`) so the shared size cache and + the saved queue keep working. + +## User-Agents and what the probes showed + +Unauthenticated probes of the live services (2026-10-14): + +- **Distribution service** (`gaming.amazon.com`): `GetEntitlements`, `GetGameDownload` and + `GetLiveVersionIds` answered identically for `AGSLauncher/1.0.0`, `DroidDeck/…`, a curl UA, with + and without the `UserAgent` header. A bad token gives HTTP 401 ``; no + token gives HTTP 400 with `com.amazon.coral.service#NotAuthorizedException`. The probes cannot + show whether the UA matters for an authorised call, so the launcher's `User-Agent` and + `UserAgent` values are kept (Amazon's own client identifiers, one constant each in `net.rs`). +- **Identity API** (`api.amazon.com`): register, token and deregister gave the same validation + errors whatever the UA (Firefox, `DroidDeck/…`, none). The desktop Firefox string is kept for + these calls, the same as the WebView: it is a generic browser label, not a client name, and + existing refresh tokens are known to work with it. +- **Sign-in page** (`www.amazon.com/ap/signin`): every UA got Amazon's bot check from the probe + host, so nothing could be learned; the WebView keeps the desktop Firefox string (a sign-in page + that misbehaves would lock every user out). +- **CDN**: no URL is reachable without a token. Manifest and file requests now send + `DroidDeck/`, the only third-party label the old integration sent; this needs the + device test below. + +Refusal detection follows the probes: 401, 403, and 400 whose body names `NotAuthorizedException` +mean "sign in again". + +## Judgement calls + +Recommended fixes from the spec, adopted: + +1. **Complete = size and SHA-256.** One test for the space check and the download plan; a + same-size corrupt file is repaired. Files without a SHA-256 (another algorithm) are judged by + size. No cross-run "verified" cache: every present file is hashed on each run. +2. **A failed library page fails the sync** and keeps the old cache; a refused token is reported + with the shared "session expired: sign in again" line, also when a cache is shown. +3. **Single-flight refresh** (`AmazonCredentialStore`); a forced refresh after a failed library call + skips the refresh when another caller already replaced the token. +4. **Mark updating** (`beforeReplace` → `StoreInstalls.markUpdating`) once, before the first file + is put in place (new or replacing; the shared step only acts on an installed record). +5. **Manifest path safety**: absolute paths, any `:` (drive letters and stream names; Windows file + names cannot contain it), NUL, `.` / `..` and empty segments, and DroidDeck's own root files + (`.droiddeck-store.json` and its `.tmp`, `.droiddeck-launch.bat`, the two markers, compared + ignoring case) fail the install naming the path. A manifest that lists both `X` and `X.tmp` is + refused (the second would collide with the first's temporary file). The protobuf reader rejects + truncation, over-long varints, groups and unknown wire types, field number 0, wrong wire types + on the fields used, and a non-empty file without a hash or a SHA-256 that is not 32 bytes. +6. **Exe scoring on relative paths.** The Unreal-folder rule is tested against `"/" + relative + path` so a top-level `Binaries/Win64/…` still matches. The "first five letters" rule compares the + title with the file name **without** `.exe` (the extension's letters would otherwise count). An + empty title never matches the "path contains the title" rule. Ties: larger file, then the earlier + path. Only regular files are candidates; symlinks are not followed; depth is capped at 16. +7. **CDN User-Agent** is `DroidDeck/` (see above). +8. **Empty files** whose hash is absent or the SHA-256 of empty input are created locally; stray + `.tmp` siblings of manifest files are removed at run start (other `.tmp` files are untouched). +9. **Markers** `.amazon_download_in_progress` / `.amazon_download_complete` are still written. +10. **Duplicate paths**: the later entry wins with a log warning; files sharing one hash are fetched + once per path (no copy optimisation). + +Other choices: + +- **Install size** is the sum over distinct paths (equal to the spec's "duplicates included" sum + for every manifest without duplicate paths, which none have been seen to have); it is the + progress total, the disk size and `size_`. +- **Free space** is measured in Rust (`statvfs` on the folder or its nearest existing parent), + which is what the shared probe measures; unknown never blocks, a measured zero does. +- **Pre-download check** is reported in the queue's Manifest stage ("Checking files…" with byte + progress), like GOG and Epic: the Downloads UI orders stages Manifest → Download → Verify → + Install, so a Verify stage before Download would read backwards. +- **Concurrency**: one streamed GET per file through the shared fetch core in stream mode (the + speed tier's window clamped to 1–128, per-host cap `max(6, ceil(window / hosts))`, 120 s to the + response headers, the core's 30 s idle cap on the body, its five attempts with back-off and its + stall and 429 handling). A piece at offset 0 restarts the file (a retry); an out-of-order piece, + a body longer than the manifest size, a short body or a hash mismatch deletes the `.tmp` and is + retried; a write error, a refused `beforeReplace` or a failed rename fails the run. +- **Decompression** uses the `lzma-rs` crate (pure Rust; xz with any check type and LZMA-alone), + with a 256 MiB cap on the decompressed manifest. +- **Live version lookup** (`GetLiveVersionIds`) is not implemented; nothing used it. +- **DLC filter**: add-ons are hidden only when they also name a parent product (as the spec's sync + contract says), unless that would hide everything; the Rust call returns the filtered, sorted list + plus the unfiltered count. +- **Sign-in code**: the authorization code is read from the return URL's query (`%XX` decoded, `+` + kept literally; codes are URL-safe). The WebView flow passes the URL to Rust and never logs it. +- **Randomness** for the serial and verifier comes from `/dev/urandom`. +- **`fuel.json`** is found by exact name, else a case-insensitive match at the folder root; a UTF-8 + BOM is accepted. `Main.Command` is accepted after stripping leading `/` and `./`, and must be a + safe relative path naming a regular file. +- **Logs**: the Rust side strips the query string and fragment of every URL in a line before it + reaches `onLog`; the shared redaction runs after that as before. +- **No engine, no store**: without the native library, sign-in, sync and install report + "Amazon Games: the store engine is not available"; there is no Java fallback. + +## What needs device testing + +- An existing signed-in user stays signed in after the update: the library shows from cache at once, + and a forced sync refreshes the token with the stored refresh token and lists the games. +- A fresh sign-in through the WebView (registration, display name), and sign-out (deregistration + fired, library cache cleared). +- A library with many pages; add-ons hidden; posters. +- Install, pause, resume, cancel (folder removed); repair of an installed game (nothing downloaded + when intact; a corrupted file fetched again; "Resume install" offered when a repair is + interrupted); an SD-card install; out-of-space refusal. +- Every endpoint with the chosen User-Agents, the CDN with `DroidDeck/…` in particular. +- Launch of an existing install through its existing Steam shortcut (same shortcut, same prefix) + and of a fresh install (exe choice, `fuel.json` arguments, the five FuelPump variables in + `.droiddeck-launch.bat`); uninstall. diff --git a/docs/development/epic-cleanroom.md b/docs/development/epic-cleanroom.md new file mode 100644 index 0000000000..029c7c1e6c --- /dev/null +++ b/docs/development/epic-cleanroom.md @@ -0,0 +1,143 @@ +# Epic Games Store module: clean-room rewrite + +The Epic integration (the Rust `epic` module in `libdroiddeckstores.so` and the Kotlin package +`com.droiddeck.launcher.stores.epic`) was rewritten so that it carries no code from the earlier +implementation, whose Rust module stated derivation from third-party Epic clients. This page +records how that was done and every judgement call the new code makes. + +## Process + +1. **Spec, written separately.** A functional specification (six documents: summary and unknowns, + service protocol, manifest and chunk formats, install behaviour, on-disk compatibility, and the + app-facing interface) was written from the old code by a separate pass. It describes behaviour + and data formats only, not code structure. +2. **Implementation from the spec only.** The implementer worked from those documents and from + code that is not Epic-specific: + - read: the spec; the crate's `Cargo.toml`, `lib.rs`, `stores_jni.rs`, `priority.rs`, + `fetch_tuning.rs`, the shared `fetch_core.rs` and the GOG module (for crate, threading and JNI + conventions); `JNI.md`; the Kotlin code outside `stores/epic/` that calls Epic (to keep its API); + build files and CI; + - not read: anything under `stores/epic/` (Kotlin, Java, tests) or `src/epic*` (Rust) of the old + tree, the body of the old `EpicCloud` class and of `CloudSavePaths.epic`, other branches or + checkouts, or any Epic client source. The old files were deleted with `git rm` without being + opened. (A search over shared files incidentally printed two lines of the old `EpicCloud`, its + base URL and User-Agent constant; both values are in the spec and only the URL is used.) +3. **Fresh code.** Every file of the new module has its own structure, names and comments, with a + plain description and `SPDX-License-Identifier: GPL-3.0-or-later` (the crate's licence). Where + the spec left a fact open, the choice was made from general knowledge of Epic's public services + and is listed below. + +## Layout + +| Rust (`app/src/main/rust/stores/src/epic/`) | | +|---|---| +| `net.rs` | blocking HTTP helper, the one User-Agent, percent/base64 encoders | +| `account.rs` | sign-in with an authorization code, refresh, exchange codes, redirect-page parsing | +| `library.rs`, `catalog.rs` | owned items (all pages), catalog details and attributes | +| `build_info.rs` | live build lookup, CDN list, manifest download | +| `manifest.rs`, `chunk.rs` | binary and JSON manifests, chunk files | +| `locale.rs` | device language → install tags, tag selection | +| `install/` | the install engine: `staging`, `delta`, `space`, `cache`, `schedule`, `writer`, `mod` (orchestration), end-to-end `tests` against a loopback server | +| `shop.rs`, `achievements.rs`, `savesync.rs`, `launch.rs`, `timefmt.rs` | shelves, achievements, cloud saves, launch arguments, ISO dates | +| `service.rs`, `ffi.rs` | the JSON operations and the JNI entry points | + +| Kotlin (`stores/epic/`) | | +|---|---| +| `EpicBackend` | `StoreBackend` + `EpicLaunchSupport`; installs the login flow | +| `EpicLogin` | the `StoreLoginActivity.Flow` | +| `EpicCredentialStore`, `EpicPrefs` | account (shared Keystore store), `stores.epic` prefs | +| `EpicLibrary`, `EpicShelves` | library sync and cache, shop shelves | +| `EpicInstaller` | the download-queue job | +| `EpicApiClient`, `EpicDownloadManager`, `EpicInstallTags`, `EpicAchievements` | API kept for callers outside the package | +| `EpicSaveSync`, `EpicSaveTemplate` | cloud-save transfers for `EpicCloud`, template parsing for `CloudSavePaths.epic` | +| `EpicNative` | JNI wrapper (see `rust/stores/JNI.md`) | + +## Judgement calls + +Numbers in brackets refer to the spec's list of unknowns. + +**Fixed where clearly correct** + +1. OAuth form values (authorization code, refresh token) are percent-encoded [9]. +2. A library page that fails ends the sync with an error and keeps the cached library instead of + caching a partial one; an HTTP 401 is reported as "session expired: sign in again" rather than + an empty library [7]. + +**Chosen where the spec left it open** + +3. **User-Agent** [1]: every HTTP request (account, library, catalog, build info, CDNs, store + GraphQL, promotions, achievements, cloud saves) sends `DroidDeck/`, defined once in + `epic/net.rs` (`USER_AGENT_PRODUCT`). Whether every endpoint accepts it is untested; changing that + constant is the one place to adjust. The sign-in **WebView** keeps the launcher-style UA it used + before, because a sign-in page that rejects the UA would lock every user out; that string lives + in `epic/account.rs` (`WEBVIEW_USER_AGENT`). +4. **Shared fetch core** [14]: chunk downloads reuse the crate's store-agnostic `fetch_core` (also + used by GOG and Amazon; it came from Bannerlator's own crate and, unlike the old Epic module, + states no derivation from an Epic client). It already provides the adaptive window, CDN + rotation, retries with back-off, 429 and stall cooldowns the spec asks for. If its lineage is + ever in doubt, it is the one shared dependency to replace. +5. **Out-of-package Epic code** [14]: `EpicCloud` and `CloudSavePaths.epic` were rewritten to call + the new module. `StoreLaunch`'s exchange-code handling, `EpicOptions`, `WinCompSources.epic` and + `SessionPrefs.storesEpicMode` are shared code and were left as they are. +6. **Save-path template** parsing is Kotlin (`EpicSaveTemplate`), not Rust: the prefix walk it feeds + is shared Kotlin code and its existing unit test runs on the JVM without the native library. The + Epic walk matches folder names case- and punctuation-insensitively (`MyGame` finds `My-Game`), as + that test expects. +7. **One engine** [12, 13]: the hidden `storesEpicMode` switch is not read and there is no Java + fallback; without the native library the Epic store reports itself unavailable. When the full + space plan does not fit, the engine runs in a narrow mode (one file staged at a time, no chunk + cache), which needs only the net peak of the pending files. +8. **Schedule and cache**: the cache budget is enforced by the plan (eviction of the chunk needed + furthest in the future). At run time files are admitted up to 48 downloads ahead of the + furthest dispatched one, so the cache can briefly hold a few entries more than the plan's peak. + A chunk is first written into every admitted file that uses it; it touches the cache only when a + not-yet-admitted file still needs it. +9. **Space probing** uses `statvfs` (available blocks × fragment size, the same figure as Android's + `StatFs.getAvailableBytes`) from Rust; a failed probe is "unknown". A scratch directory on the same + filesystem as the game is treated as "beside the game". +10. **File hashing** is done in file order while parts land; parts that arrive early are read back + from the stage file. Files without a manifest hash are not hashed. +11. **Staging folder recovery**: an *empty* staging folder without its marker (a run stopped between + creating the folder and writing the marker) is adopted and the marker written; any other missing + or foreign marker fails the run without deleting. +12. **Unsafe manifest paths** (absolute, `..`, empty segments) fail the install. +13. **Encrypted manifests** (stored-as bit 1) are refused with an explicit message. +14. **Meta section prerequisite ids** are all read (no cap at 64), so the following fields stay aligned. +15. **Store GraphQL shelves** are requested with their arguments written into the query text + (country reduced to two letters first) instead of typed GraphQL variables, so the query does not + depend on the server's variable type names. +16. **Achievements**: when an account id is stored and the player-progress query gets no answer, the + result is "unreachable" rather than a list with everything locked. +17. **Token refresh** is single-flight (one refresh at a time; a waiting caller re-reads the stored + account first), so a rotated refresh token is never spent twice. A fresh sign-in replaces the + stored account instead of merging with it. +18. **Download progress** reported to the queue never moves backwards, although samples come from + several native threads. +19. **Network window** is the speed tier's value clamped to 1–128; the per-host cap is + `max(6, ceil(window / hosts))`. + +**Kept at parity (known weak spots)** + +- Chunk requests do not carry the CDN's signing query; only the manifest request does; the + `cloudflare.epicgamescdn.com` CDN stays excluded [2]. +- The manifest file name comes from the first manifest location and is requested from every CDN [3]. +- The manifest header SHA-1 and the chunk header GUID/hash are not checked (chunk size and chunk + SHA-1 are) [4]. +- Per-file flags, symlink targets and custom fields are ignored [5]; JSON manifests carry no hashes + as read, so their files are rewritten on every run [6]. +- The refresh token's own expiry is not tracked; a failed refresh still returns the stored token [8]. +- The cloud `hash` is compared as if it were an MD5 [10]. +- `AdditionalCommandLine` is one argument, frozen at install [11]. +- Shop search, an unused "wants an exchange code" helper and per-game uninstall work are not + ported; DLC stay hidden and are never installed separately; optional (non-language) install tags + are never selected; resume restarts a partially written file from zero [15–18]. +- Library details are fetched one game at a time, as before. + +## What needs device testing + +- Sign-in through the WebView, and that an **existing** saved account (sealed `credentials.json`) + stays signed in after the update. +- Every endpoint with the DroidDeck User-Agent: library, catalog, build info, CDN downloads, store + GraphQL and promotions, achievements, cloud saves. +- Library sync and shelves; install, pause/resume, cancel, update and repair; a card install with + the internal scratch cache; launch with an exchange code; cloud-save up/down. diff --git a/docs/development/stores.md b/docs/development/stores.md index c3c7223c21..4fb78f4da0 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -35,8 +35,8 @@ to manifest checks of the files still needing work (GOG MD5, Amazon SHA-256 wher Free-space display probes the nearest existing ancestor without creating the target folder; if the probe cannot determine space, it reports **Free space unknown** rather than blocking an install. `StoreInstallRoot` uses `-1` for unknown and preserves a measured `StatFs` zero as **0 B free**. -Fresh GOG/Amazon preflight and all three Java backend space checks reject known zero when bytes -remain. The read-only probe walks existing ancestors without creating directories; a failed +Fresh GOG/Amazon preflight and the backends' own space checks (GOG in Kotlin, Epic and Amazon in +the Rust engine) reject known zero when bytes remain. The read-only probe walks existing ancestors without creating directories; a failed `StatFs` probe can use a positive `File.usableSpace` fallback, but that API's ambiguous zero is unknown, not a measurement of a full disk. Preflight failures remain failed Downloads rows with Clear and Retry. Retry reruns preflight @@ -45,143 +45,80 @@ with a placeholder job. Failures are kept in the Downloads UI, notification and duplicated as transient failure toasts. Active queue persistence and ordinary backend pause/retry are unchanged. -Epic installs have four writers, all with the same per-file safety rules (below). The hidden -`storesEpicMode` string in the `session` preferences picks one: `adaptive` (default), `lookahead`, -`batch` or `java`. There is no UI for it; set it on a test device to compare the modes on the same -game. Every mode logs an `epic: summary mode=` line (files, bytes written, elapsed, average -MB/s). When `libdroiddeckstores.so` does not load, every mode runs the Java writer. Once it is -loaded, an engine failure (refused plan, ABI mismatch, network, hash or I/O failure, unsupported -atomic swap) fails the install; it never silently reruns as Java downloads. +Epic installs run in the Rust `epic` module of `libdroiddeckstores.so` (`epic/install/`), driven by +`EpicInstaller` through `EpicNative.install`. There is one engine; the old hidden `storesEpicMode` +switch is no longer read. Without the native library the Epic store cannot install (the facade +reports "the store engine is not available"). The design, and how it was written from a +functional spec only, is in [epic-cleanroom.md](epic-cleanroom.md). -**Adaptive (default)** - `EpicNative.stream`, `epic/stream.rs`. Measured on device: about -83 MB/s on a 71 GB install. The chunks go through the shared fetch core in the order their files -need them, with its full adaptivity (window ramp, per-host caps, in-flight byte budget, retries, -host rotation, stall handoff). A chunk only one file part uses is verified and written straight -into that file's staging file at the part's offset, in whatever order it arrives; it is never -cached, so each byte is written once. A chunk several parts use is written into every part whose -file the download has reached and kept, decompressed, in the cache until its last part is written. -The cache has a byte budget; the plan simulates the install in file order and, when the budget -would be exceeded, drops the cached chunk needed furthest ahead and downloads it again later. -Files are opened only once the download reaches them, so the set of half-written files stays -within the fetch window. All writes happen in the fetch core's process pool, whose completion -frees the in-flight byte budget: a slow card throttles the download instead of growing memory. -Each file's SHA-1 is computed as its parts are written (parts that arrived out of order are read -back once the bytes before them are in). Stage files go into the owned staging namespace below. -The run logs `stream-plan` (direct vs shared chunks, planned refetches, budget) and -`stream-summary mode=adaptive` (peak cache, peak staging, refetches, bytes written from the -download vs from the cache). +A run, in order: CDN list from the build information (`EpicApiClient.getManifestApiJson`) → +manifest from the first CDN that serves it → parse (binary or JSON) → files selected by the device +language's install tags (`EpicInstallTags`) → staging recovery → the delta pass (a selected file is +up to date when it has the manifest size and SHA-1; a file without a manifest hash is always +rewritten) → space plan → chunk cache preparation → fetch schedule → download and write → the +installed record (`StoreInstalls.complete`). Resume, update and repair are the same rerun. -On an update, a staged file that replaces an existing one sits beside its old copy until its swap, -so the engine paces replacements with an overlap budget: files are admitted in order, a replaced -file only while the planned sizes of admitted, not-yet-swapped replaced files fit the budget (one -is always admitted, so the run progresses), and a chunk is fetched only once its file is admitted. -Fresh files are never held back, and admission only delays files, never reorders them, so the -cache plan stays exact. The run logs `overlap_budget=` in `stream-plan` and -`peak_replace_overlap_bytes=` in `stream-summary`. With room to spare the budget is large and -updates run as wide as fresh installs; on a nearly full volume, replaced files are staged one -after another and the fetch pauses at each replaced-file boundary until the previous one is -swapped in. +- **Schedule** (`epic/install/schedule.rs`). The pending files' parts form one timeline. A chunk is + downloaded at its first use and feeds every later use in the same file directly. If it is used + again in a later file it is kept, decompressed and verified, in the chunk cache, which has a byte + budget; when keeping a chunk would exceed it, the cached chunk needed furthest in the future is + dropped and downloaded again when needed. Chunks a previous run left in the cache are reused, + soonest-needed first, as far as the budget allows. +- **Writing** (`epic/install/writer.rs`). Files are admitted strictly in order, a little ahead of + the download, and a download is dispatched only once the file of its first use is admitted, so + the number of half-written files stays small. Each admitted file is written into a unique + `epic---.stage` file in `/.droiddeck-epic-staging/`; the file SHA-1 is + computed in file order as parts land. A complete file is synced, size- and hash-checked, and + renamed over its destination. Right before the first rename of the run `StoreInstalls.markUpdating` + switches an installed record to `updating`; a refusal fails the run with nothing replaced. +- **Fetching** uses the shared `fetch_core` (adaptive window, CDN rotation, retries with back-off, + stall and 429 cooldowns) with the speed tier's window and decode workers. Every chunk is checked + (header, zlib stream end, exact size, SHA-1) before any byte reaches a file or the cache. +- **Space** (`epic/install/space.rs`). Free space is probed per volume (`statvfs`; unknown is never + "no space"). The plan: net peak of the pending files in order (each old copy credited at its own + swap), a cache budget of half the remaining space (at most 1 GiB, 256 MiB when unknown, 0 is + allowed), an overlap budget that limits how many bytes of files replacing existing data are + staged at once on an update (at least the largest replaced file), and a 64 MiB transient + allowance. A card install keeps its cache in `cacheDir/stores/epic//` when internal storage + can hold a useful cache and the game has no non-empty `.chunks` yet; otherwise the cache is + `/.chunks`. When the full plan does not fit, the run falls back to one file at a time with + no cache (needs only the net peak); when even that does not fit the install is refused before any + chunk request, naming the storage, the need, the free space and the shortfall. +- **Progress**: the bar counts pending-file bytes written (not compressed chunk bytes), at most every + 50 ms plus once per finished file. The run logs `epic: plan …` and `epic: summary mode=adaptive|narrow + files=… bytes_written=… avg_mbps=… fetches=… refetches=… peak_cache_mib=… hosts=…`. +- **Cancel / pause / failure**: the cancel flag is forwarded within 250 ms; the run stops its + requests, joins its threads, removes every unfinished stage file and `.part` cache entry, and only + then returns. Swapped files stay; unswapped destinations keep their old content; the chunk cache + stays for the next run. Success removes the cache and the staging folder (when only its marker is + left). -Its space plan (`EpicDownloadManager.planAdaptiveSpace`, unit-tested in `EpicAdaptivePlanTest`) -checks each volume for what lands on it, with the lookahead planner's rule (the need must not -exceed the free reading). The game volume: the net growth at its worst point (`settledNetPeak`, -the running Σ(new − old) with each old copy released at its swap), plus the overlap budget, plus -the cache when it is beside the game. The overlap budget is whatever the game volume has left -after the net growth and the cache, never less than the largest replaced file; when even that -does not fit, the plan hands over to the lookahead engine. The cache volume: the budget plus 64 MiB of -chunks in transit. The cache budget is half the cache volume's free space after those needs, at most -1 GiB (256 MiB when free space is unknown; 0 is allowed and means every reuse is a refetch). On a -card install the cache stays on internal scratch unless internal storage cannot hold at least -64 MiB of it; then the cache goes beside the game (still released chunk by chunk) instead of -failing. A plan that does not fit logs the short storage and the shortfall and hands over to the -lookahead engine, which narrows its window to the space there is (or fails its own preflight with -an explicit error). Running out of space in the middle of an adaptive run is not destructive: a -failed write fails the run, unfinished staging files are removed, the old files stay. +The staging folder is recognised by its `owner` marker (`DroidDeck Epic staging v1` + LF). Before +the delta pass the run checks that the folder is a real directory at the expected canonical path, +that the marker matches and that every other entry is a plain `.stage` file directly inside it; only +then are those orphans deleted. Anything unexpected fails the run without deleting. A manifest path +naming the staging folder, an absolute path or a `..` segment is refused. `.stage` / `.part` files +elsewhere in the game folder are never touched. -**Lookahead** - `EpicNative.installAdaptive`, `epic/adaptive.rs`; also the adaptive engine's -fallback when its space plan does not fit. The Java delta pass sends a versioned snapshot of -pending files, chunk metadata, verified cache lengths, old output lengths and both measured -free-space probes to Rust. The native planner and execution share one ordered moving-window -policy. Before any chunk request, Rust chooses the largest safe lookahead (128, 64, 32, 16, 8, 4, -2 or 1 part references) and reusable-cache budget (64 MiB, 16 MiB or zero) whose simulated -staged-file and cache peaks fit. Native device IDs also detect an explicitly named cache on the -game volume, so two separate paths cannot incorrectly turn one filesystem's requirement into two -independent checks. +Amazon installs run in the Rust `amazon` module (`amazon/install/`), driven by `AmazonInstaller` +through `AmazonNative.install`; sign-in, the library sync and the size lookup call the same module +through `AmazonNative.call`. The design, and how it was written from a functional spec only, is in +[amazon-cleanroom.md](amazon-cleanroom.md). -| Storage situation | Lookahead policy | -| --- | --- | -| Plenty of room | Up to 128 references ahead, up to 32 network requests, 64 MiB reusable data | -| Tight | Smaller lookahead and/or reusable budget; parallel fetching remains where space permits | -| Barely fits | One-reference lookahead, one request, no reusable retention if needed | -| Does not fit even the minimal schedule | Explicit preflight error before requests or replacement | -| Free space unknown | Conservative bounded lookahead (at most eight), not an invented zero | - -The lookahead fetch coordinator reuses `fetch_core`'s host scheduling, retries, adaptive network -window and decode pool. Each chunk is cached, then its parts are copied into staged files by a -writer that runs concurrently, waiting only for its next required chunk, through the native -assembler's streaming copy primitive. GUIDs in the lookahead are reserved once before submission, -at their full decoded size. Chunks are released at last use on **both** cache layouts; distant -reusable data is oldest-first bounded and may be refetched. At most the selected reusable budget -plus distinct lookahead decoded sizes is resident/reserved. New request admission re-probes both -volumes and can shrink unsent lookahead or retention if headroom disappears; already in-flight -reservations remain protected. If even existing reservations cannot fit, the run fails and joins -its workers rather than waiting indefinitely or deleting old game files. An install to SD retains -its internal scratch cache, so assembled output is the only game data written to the card. - -**Java** - without the native library, or with `storesEpicMode=java`. Eight Java workers prefetch -the next eight chunk-part references, across file boundaries, while one writer assembles and -verifies files in manifest order. Shared GUIDs within that window are fetched only once. Chunks -are deleted after their last use; other reusable chunks are retained up to 64 MiB. At most eight -additional decoded chunks (complete or in flight) coexist with that retained cache. The size -bound is 64 MiB plus the decoded sizes of the window's distinct GUIDs, not a fixed eight-MiB -allowance: manifests can supply larger chunk windows. Evicted distant shared chunks may be -downloaded again. No worker downloads outside the moving window; cancellation and failure -disconnect and join all workers before returning or removing the cache. Its summary line is -`mode=java-bounded`. - -**Batch** - the earlier native design, kept for A/B benchmarks: every chunk fetched into the -cache in parallel, then the files assembled from it (`EpicNative.run` + `EpicNative.assemble`). -It needs the cache and the files at once (about twice the game) and checks exactly that before -starting; it writes files in place, so an existing install is marked `updating` before assembly. -`epic/assemble.rs::build_assemble_plan` enables `scratch_cache` only for a nonempty cache path: -`run_assemble` then removes chunks at their last use; the default beside-game cache is removed at -the end (`default_cache_assembles_like_the_kotlin_loop_and_removes_the_cache_at_the_end` and the -scratch-cache assembly tests cover that distinction). - -The Epic download bar, measured rate and ETA (adaptive, lookahead and Java) count pending file bytes written, not unique -compressed chunks fetched. Shared chunks therefore cannot complete the bar before later files -are written, and an evicted chunk's refetch is reflected in the time spent doing that work. -Streaming stays in the Download stage until all files are written and verified, then enters -Install for registration; it does not bounce between stages for each file. - -For the lookahead and Java writers, the space check follows the manifest delta pass and uses the -same pure moving-window policy as execution, reserving all incoming decoded chunk sizes before writing parts and simulating -last-use releases and unpinned oldest-first evictions. Existing verified cache bytes count against -current free space and are credited only at their scheduled release. This avoids adding the entire -cache to the entire game without undercounting concurrent downloads. Each file is written to a -unique file in an owned staging namespace on its output volume (never the literal `.part`, which can -be a legitimate manifest file), size/hash verified, synced and atomically renamed over its output. -The old file remains until that swap succeeds. The planner credits its bytes only after that -file's complete staging write, not by subtracting all old outputs up front. Both volume checks -must accept this safe staging peak; insufficient space is rejected explicitly, with no destructive -delete-first fallback. An unsupported/failed atomic rename fails the install and retains the old -destination. Completed files and -verified cached chunks are accounted for on resume. On a card, game and cache requirements -are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed -or cancelled staging files are removed without deleting old outputs. A successful run removes the -cache. A resume within a large incomplete file restarts that file, not its individual parts. -All three staged writers (adaptive, lookahead, Java) use `/.droiddeck-epic-staging/`, whose `owner` marker identifies this -namespace. A successful install removes it when it holds nothing but the marker (same no-follow -and canonical-path checks); the next run recreates it. Before the delta pass and free-space probes, the manager removes orphan `.stage` -entries only from this owned namespace. A manifest naming that directory or any descendant -fails without deletion; an unowned directory or unexpected entry also fails conservatively. -The namespace must be a real directory at the expected canonical child of the install root. -Ownership markers and staging entries must be no-follow regular files; symlink directories, -markers and entries are rejected before recovery deletes anything. -Ordinary game files with `.stage` or `.part` suffixes elsewhere are never recovery targets. -Abrupt process termination is therefore recoverable on the next run without inferring ownership -from a loose filename prefix. Cancellation/failure waits for all native/Java fetch writers before -returning; completed verified cache chunks remain usable on resume. +An Amazon run, in order: download lookup for the entitlement (`GetGameDownload`: a CDN base URL with +a signed query, and the version id) → `manifest.proto` from that base (a 4-byte big-endian header +length, an opaque header, then an xz or LZMA-alone compressed protobuf) → parse (every file of every +package; paths refused when absolute, with a drive letter, `..`, empty segments, or naming +DroidDeck's own root files) → leftover `.tmp` siblings removed → the check of files on disk (a +file is complete when it has the manifest size and SHA-256; the same test drives the space check and +the download plan) → space check (`Not enough free space: need …, only … free`; unknown free space +never blocks) → empty files created locally → every other pending file fetched from +`files/` under the base, one streamed GET each through the shared fetch core (the speed +tier's window; at least half the CPU cores verifying and writing), hashed as it arrives, written to +`.tmp` and renamed over the destination after the size and hash checks → the installed record. +Before the first file is put in place the record is switched to `updating`, so an interrupted repair +is offered as "Resume install". There is no partial-file resume: a file is complete or fetched again +from its first byte. Files the manifest does not list are left alone. ## The sidecar @@ -266,6 +203,15 @@ An Epic game's launch choices are the Epic card in the Games tab's launch settin read the same thing - and "Resolve Epic sign-in", which opens Epic's account page behind its sign-in in Android's browser. +An Amazon game's exe and arguments come from its `fuel.json` (`Main.Command`, used when it names +an existing file; `Main.Args`, used even when it does not), else from a score over every `.exe` +below the folder (Unreal shipping binaries and `Binaries/Win64` first, crash handlers, setups, +launchers and tiny stubs last; the rules see paths relative to the install folder), else the shared +picker. Its launcher sets five FuelPump variables frozen at install time: `FUEL_DIR`, +`AMAZON_GAMES_SDK_PATH`, `AMAZON_GAMES_FUEL_ENTITLEMENT_ID`, `AMAZON_GAMES_FUEL_PRODUCT_SKU` and +`AMAZON_GAMES_FUEL_DISPLAY_NAME` (the account name, `Player` without one). No Amazon Games SDK files +are deployed into the prefix and `fuel.json` `PostInstall` steps are not run. + A game's web links reach Android: `droiddeck-store-launch` sets the prefix's `HKCU\Software\Wine\WineBrowser` `Browsers` to `/usr/local/bin/droiddeck-open-url`, which is also `xdg-open` first on the game's PATH, and hands http(s) addresses to the app, which opens them in @@ -306,7 +252,9 @@ the prefix's user folder appears, within a minute. Every path logs has no cloud saves. - **The services:** GOG `cloudstorage.gog.com/v1//` with a token issued to the game's own client (the client secret is kept at install; the Galaxy token is the fallback); Epic's - savesync data storage with signed read and write links. + savesync data storage with signed read and write links (`EpicCloud` → `EpicSaveSync` → the Rust + `epic/savesync.rs`; the template's root token and `{EpicId}` / `{AppName}` are read by + `EpicSaveTemplate`). - **Which launches:** `droiddeck-store-launch` recognises a store game by the launcher .bat Steam runs or, for a game without one (most GOG games), by its exe - the folders above it are searched for the sidecar. @@ -332,7 +280,9 @@ keeps the files; every store's install skips complete, verified files on the rer rerun. Cancel deletes the folder. `StoreDownloadService` holds a foreground notification while anything runs. The native engine (`libdroiddeckstores.so`, `StoresNative`, `GogNative`, `EpicNative`, `AmazonNative`; contract in `app/src/main/rust/stores/JNI.md`) is loaded on first use; without it -each manager runs its Java fetch loop. +the GOG manager runs its Java fetch loop, and the Epic and Amazon stores (all of whose service calls +and installs run in the Rust `epic` and `amazon` modules) are unavailable: sign-in, sync and install +report "the store engine is not available". ## Settings diff --git a/tools/release/check_android_libraries.py b/tools/release/check_android_libraries.py index 8f79e8844e..80b543e803 100644 --- a/tools/release/check_android_libraries.py +++ b/tools/release/check_android_libraries.py @@ -12,10 +12,10 @@ SYSTEM = set('libc.so libm.so libdl.so liblog.so libandroid.so libz.so libvulkan 'libOpenSLES.so libmediandk.so libcamera2ndk.so libsync.so libneuralnetworks.so'.split()) EXPORTS = ['Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion', 'Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart', - 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart', - 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble', - 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream', - 'Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart'] + 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeCall', + 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstall', + 'Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCall', + 'Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeInstall'] def check(apk, readelf, stores=True):