diff --git a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt index 0d4b827fee..9ed3ec8189 100644 --- a/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt +++ b/app/src/main/java/com/droiddeck/launcher/session/SessionPrefs.kt @@ -82,6 +82,15 @@ object SessionPrefs { prefs(context).edit().putString("gameStoresSpeedTier", tier).apply() } + /** + * Hidden A/B switch for the Epic installer: "adaptive" (default: native parallel fetch written + * straight into staged files), "lookahead" (the native ordered-lookahead engine that caches each + * chunk and copies it into staged files), "batch" (the older whole-cache native fetch, then + * native assembly; needs about twice the game's size free) or "java" (the bounded Java writer). + * No UI; set it in the `session` preferences for benchmarks. + */ + fun storesEpicMode(context: Context): String = prefs(context).getString("storesEpicMode", "adaptive") ?: "adaptive" + fun storesOpenTab(context: Context): String = prefs(context).getString("storesOpenTab", STORES_OPEN_LIBRARY)?.takeIf { it == STORES_OPEN_STORE } ?: STORES_OPEN_LIBRARY diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoreGameSidecar.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoreGameSidecar.kt index 1753003d50..25850f7479 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoreGameSidecar.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoreGameSidecar.kt @@ -68,6 +68,7 @@ class StoreGameSidecar( const val VERSION = 1 const val STATE_INSTALLED = "installed" const val STATE_INSTALLING = "installing" + const val STATE_UPDATING = "updating" fun file(folder: File): File = File(folder, FILE_NAME) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt index e122aa0573..d36f886784 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt @@ -9,12 +9,67 @@ import java.io.File import java.util.zip.CRC32 object StoreInstallRoot { + const val SPACE_UNKNOWN = -1L private const val INTERNAL = "root/Games/Stores" private const val ON_VOLUME = "Games" const val GUEST_EXTERNAL = "/mnt/droiddeck-stores" class Target(val root: File, val label: String, val removable: Boolean) { - val freeBytes: Long get() = try { StatFs((if (root.isDirectory) root else root.parentFile ?: root).path).availableBytes } catch (e: Exception) { 0L } + val freeBytes: Long get() = availableBytes(root) + } + + data class StorageCheck( + val enough: Boolean, + val requiredBytes: Long, + val availableBytes: Long, + val errorMessage: String? = null + ) + + @JvmStatic + fun availableBytes(file: File): Long = probeAvailableBytes(file, { StatFs(it.path).availableBytes }, { it.usableSpace }) + + internal fun probeAvailableBytes(file: File, statBytes: (File) -> Long, usableBytes: (File) -> Long): Long { + var dir: File? = if (file.isDirectory) file else file.parentFile + if (dir == null) dir = file + while (dir != null && !dir.exists()) dir = dir.parentFile + val existing = dir ?: return SPACE_UNKNOWN + return try { + statBytes(existing).coerceAtLeast(SPACE_UNKNOWN) + } catch (e: Exception) { + StoresState.logLine("storage: StatFs failed for store install target (${e.javaClass.simpleName})") + try { + // File's zero also means an unavailable measurement; only StatFs establishes zero. + usableBytes(existing).takeIf { it > 0L } ?: SPACE_UNKNOWN + } catch (fallback: Exception) { + StoresState.logLine("storage: usable-space fallback failed for store install target (${fallback.javaClass.simpleName})") + SPACE_UNKNOWN + } + } + } + + @JvmStatic + fun insufficientSpace(required: Long, available: Long): Boolean = required > 0L && available >= 0L && available < required + + /** + * The catalog size is the known minimum for an install. Epic's actual chunk-cache requirement + * is determined from its manifest before chunk downloads begin. + */ + fun estimatedBytesNeeded(item: CatalogItem): Long = StoreSizes.size(item).coerceAtLeast(0L) + + /** Pre-install storage check to detect insufficient space before starting. */ + fun checkStorage(context: Context, item: CatalogItem, targetRoot: File): StorageCheck { + val size = estimatedBytesNeeded(item) + val existing = existingFolder(context, item.store, item.id, item.title) + val freeTarget = availableBytes(existing ?: targetRoot) + // Backends account for completed files on resume; the catalog cannot describe remaining work. + // Comparing the full catalog size to remaining free space would reject valid resumes. + if (item.store == Store.EPIC || existing != null) return StorageCheck(enough = true, requiredBytes = size, availableBytes = freeTarget) + if (size <= 0L) return StorageCheck(enough = true, requiredBytes = 0L, availableBytes = freeTarget) + if (insufficientSpace(size, freeTarget)) { + val msg = context.getString(com.droiddeck.launcher.R.string.stores_storage_insufficient, formatBytes(size), formatBytes(freeTarget)) + return StorageCheck(enough = false, requiredBytes = size, availableBytes = freeTarget, errorMessage = msg) + } + return StorageCheck(enough = true, requiredBytes = size, availableBytes = freeTarget) } fun internalRoot(context: Context): File = File(LinuxRuntime.rootDir(context), INTERNAL) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstalls.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstalls.kt index 4616c51e60..0e629a6e7e 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstalls.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstalls.kt @@ -7,15 +7,35 @@ import com.droiddeck.launcher.frontend.AddedGames import com.droiddeck.launcher.frontend.LibraryCache import com.droiddeck.launcher.frontend.Library import java.io.File +import java.nio.file.Files +import java.nio.file.StandardCopyOption object StoreInstalls { private const val TAG = "StoreInstalls" fun begin(folder: File, store: Store, id: String, title: String, cover: String?, hero: String?) { - if (StoreGameSidecar.read(folder)?.isInstalled == true) return + val before = StoreGameSidecar.read(folder) + if (before?.isInstalled == true || before?.state == StoreGameSidecar.STATE_UPDATING) return StoreGameSidecar(store, id, title, exe = "", cover = cover, hero = hero, state = StoreGameSidecar.STATE_INSTALLING).write(folder) } + /** Epic commits verified files individually, not a whole-game transaction. Keep updates resumable. */ + @JvmStatic + fun markUpdating(folder: File) { + val before = StoreGameSidecar.read(folder) ?: return + if (!before.isInstalled) return + val stage = File.createTempFile(".droiddeck-update-", ".stage", folder) + try { + java.io.FileOutputStream(stage).use { + it.write(before.copy(state = StoreGameSidecar.STATE_UPDATING, installVersion = "").toJson().toString(2).toByteArray(Charsets.UTF_8)) + it.fd.sync() + } + Files.move(stage.toPath(), StoreGameSidecar.file(folder).toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING) + } finally { + stage.delete() + } + } + fun complete(context: Context, folder: File, sidecar: StoreGameSidecar, onStep: ((Int, Int) -> Unit)? = null) { val app = context.applicationContext onStep?.invoke(0, STEPS) @@ -66,10 +86,11 @@ object StoreInstalls { fun discard(context: Context, folder: File, caches: List) { val app = context.applicationContext Thread({ - val keep = StoreGameSidecar.read(folder)?.isInstalled == true + val sidecar = StoreGameSidecar.read(folder) + val keep = sidecar?.isInstalled == true || sidecar?.state == StoreGameSidecar.STATE_UPDATING caches.forEach { deleteTree(it) } if (!keep) deleteTree(folder) - StoresState.logLine(if (keep) "removed the partial update of ${folder.name}" else "removed ${folder.name}") + StoresState.logLine(if (keep) "removed update caches for ${folder.name}; verified file swaps are retained" else "removed ${folder.name}") StoresState.notifyLibraryChanged(app) }, "store-cancel-clean").start() } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt index 5551535d3f..74af04fb80 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoresState.kt @@ -170,7 +170,25 @@ object StoresState { fun install(context: Context, item: CatalogItem, root: java.io.File? = null) { val app = context.applicationContext - val target = root ?: StoreInstallRoot.installRoot(app) + val folder = StoreInstallRoot.folderFor(app, item.store, item.id, item.title, root ?: StoreInstallRoot.installRoot(app)) + val target = folder.parentFile!!.parentFile!! + val check = StoreInstallRoot.checkStorage(app, item, target) + if (!check.enough) { + val errorMsg = check.errorMessage ?: app.getString(com.droiddeck.launcher.R.string.stores_storage_insufficient, formatBytes(check.requiredBytes), formatBytes(check.availableBytes)) + val entry = com.droiddeck.launcher.stores.download.DownloadEntry( + item.store, + item.id, + item.title, + cover = item.imageUrl, + bytesTotal = item.sizeBytes, + location = StoreInstallRoot.labelFor(app, folder) + ) + com.droiddeck.launcher.stores.download.SavedQueue.remember(app, item, target) + com.droiddeck.launcher.stores.download.DownloadQueue.recordFailed(app, entry, errorMsg) { + install(app, item, target) + } + return + } com.droiddeck.launcher.stores.download.SavedQueue.remember(app, item, target) backends[item.store]?.install(app, item, target) ?: logLine("${item.store.label}: this build cannot install yet") } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java index 2eaaa31a29..6145f0250b 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/amazon/AmazonDownloadManager.java @@ -69,9 +69,10 @@ public final class AmazonDownloadManager { if (manifestBytes == null) throw new InstallException("The manifest could not be downloaded"); AmazonManifest.ParsedManifest manifest = AmazonManifest.parse(manifestBytes); cb.onLog("amazon: " + manifest.allFiles.size() + " files, " + fmt(manifest.totalInstallSize)); - long usable = installDir.getUsableSpace(); - if (manifest.totalInstallSize > 0 && usable > 0 && manifest.totalInstallSize > usable) - throw new InstallException("Not enough free space: need " + fmt(manifest.totalInstallSize) + ", only " + fmt(usable) + " free"); + long remainingBytes = remainingInstallBytes(manifest.allFiles, installDir); + long usable = com.droiddeck.launcher.stores.StoreInstallRoot.availableBytes(installDir); + if (com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(remainingBytes, usable)) + throw new InstallException("Not enough free space: need " + fmt(remainingBytes) + ", only " + fmt(usable) + " free"); cb.onSizes(manifest.totalInstallSize, manifest.totalInstallSize); cb.onBytes(0, manifest.totalInstallSize, 0); if (cancel.get()) return null; @@ -175,7 +176,7 @@ public final class AmazonDownloadManager { AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps, Callback cb, AtomicBoolean cancel) { File destFile = new File(installDir, file.unixPath()); File tmpFile = new File(installDir, file.unixPath() + ".tmp"); - if (destFile.exists() && destFile.length() == file.size) { totalDownloaded.addAndGet(file.size); return true; } + if (fileComplete(file, destFile)) { totalDownloaded.addAndGet(file.size); return true; } String fileUrl = AmazonApiClient.appendPath(baseUrl, "files/" + file.hashHex()); for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { destFile.getParentFile().mkdirs(); @@ -245,6 +246,21 @@ public final class AmazonDownloadManager { public static boolean isInstalled(File installDir) { return new File(installDir, COMPLETE_MARKER).exists(); } + static long remainingInstallBytes(List files, File installDir) { + long remaining = 0; + for (AmazonManifest.ManifestFile file : files) { + if (!fileComplete(file, new File(installDir, file.unixPath()))) remaining += file.size; + } + return remaining; + } + + private static boolean fileComplete(AmazonManifest.ManifestFile file, File output) { + if (!output.isFile() || output.length() != file.size) return false; + if (file.hashAlgorithm != 0 || file.hashBytes.length == 0) return true; + try { return Arrays.equals(sha256(output), file.hashBytes); } + catch (Exception e) { return false; } + } + private static byte[] sha256(File file) throws Exception { MessageDigest digest = MessageDigest.getInstance("SHA-256"); try (FileInputStream fis = new FileInputStream(file)) { diff --git a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt index 93621aa9ff..e2549a3be6 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/download/DownloadQueue.kt @@ -88,7 +88,7 @@ object DownloadQueue { } } - private class Item(var entry: DownloadEntry, val factory: () -> DownloadJob, var job: DownloadJob? = null, val cancelled: AtomicBoolean = AtomicBoolean(false), var pauseRequested: Boolean = false) { + private class Item(var entry: DownloadEntry, val factory: (() -> DownloadJob)? = null, val retry: (() -> Unit)? = null, var job: DownloadJob? = null, val cancelled: AtomicBoolean = AtomicBoolean(false), var pauseRequested: Boolean = false) { var speedAt = 0L var speedBytes = 0L var speedEwma = 0.0 @@ -137,6 +137,32 @@ object DownloadQueue { advance(app) } + /** Immediately records a failed download (e.g. from pre-flight storage check). */ + fun recordFailed(context: Context, entry: DownloadEntry, error: String, retry: (() -> Unit)? = null) { + val app = context.applicationContext + val now = System.currentTimeMillis() + synchronized(lock) { + if (items[entry.key]?.entry?.isActive == true) return + items.remove(entry.key) + heldPaused.remove(entry.key) + items[entry.key] = Item( + entry.copy( + state = DownloadState.FAILED, + error = error, + speedBps = 0, + etaSeconds = -1, + startedAt = now, + finishedAt = now + ), + retry = retry + ) + renumber() + publishLocked() + } + StoresState.logLine("${entry.store.label}: \"${entry.name}\" failed: $error") + StoreDownloadService.ended(app, entry.copy(state = DownloadState.FAILED, error = error), installed = false, gamesKey = null) + } + fun pause(key: String) { val item = synchronized(lock) { items[key] } ?: return synchronized(lock) { @@ -151,14 +177,21 @@ object DownloadQueue { fun resume(context: Context, key: String) { val app = context.applicationContext + val retry: (() -> Unit)? synchronized(lock) { val item = items[key] ?: return if (item.entry.state != DownloadState.PAUSED && item.entry.state != DownloadState.FAILED) return + retry = item.retry + if (retry == null && item.factory == null) return items.remove(key) - items[key] = Item(item.entry.copy(state = DownloadState.QUEUED, speedBps = 0, etaSeconds = -1, error = null, startedAt = System.currentTimeMillis()), item.factory) + if (retry == null) { + items[key] = Item(item.entry.copy(state = DownloadState.QUEUED, speedBps = 0, etaSeconds = -1, error = null, startedAt = System.currentTimeMillis()), item.factory) + } renumber() publishLocked() } + // A preflight retry rechecks storage and enqueues the real backend, never a running placeholder. + if (retry != null) { retry(); return } StoresState.logLine("resumed \"${entry(key)?.name}\"") advance(app) } @@ -182,7 +215,7 @@ object DownloadQueue { item.cancelled.set(true) item.job?.cancel(deleteFiles = true) } else { - Thread({ runCatching { item.factory().cancel(deleteFiles = true) } }, "store-dl-cancel").start() + Thread({ runCatching { item.factory?.invoke()?.cancel(deleteFiles = true) } }, "store-dl-cancel").start() } StoresState.logLine("cancelled \"${item.entry.name}\"") StoreDownloadService.finish(app, key) @@ -194,7 +227,7 @@ object DownloadQueue { val item = synchronized(lock) { items[key] } ?: return if (item.entry.state == DownloadState.RUNNING || item.entry.state == DownloadState.QUEUED) return if (item.entry.state == DownloadState.FAILED || item.entry.state == DownloadState.PAUSED) { - Thread({ runCatching { item.factory().cancel(deleteFiles = true) } }, "store-dl-clear").start() + Thread({ runCatching { item.factory?.invoke()?.cancel(deleteFiles = true) } }, "store-dl-clear").start() } synchronized(lock) { items.remove(key); publishLocked() } StoreDownloadService.finish(context.applicationContext, key) @@ -276,7 +309,7 @@ object DownloadQueue { var result: String? = null var error: String? = null try { - val job = item.factory() + val job = checkNotNull(item.factory).invoke() synchronized(lock) { item.job = job } StoresState.logLine("${item.entry.store.label}: fetching manifest for \"${item.entry.name}\"") result = job.run(handle) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/download/StoreDownloadService.kt b/app/src/main/java/com/droiddeck/launcher/stores/download/StoreDownloadService.kt index 791dd06b11..a32bceeb69 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/download/StoreDownloadService.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/download/StoreDownloadService.kt @@ -239,12 +239,16 @@ class StoreDownloadService : Service() { val app = context.applicationContext channels(app) val code = entry.key.hashCode() - val n = Notification.Builder(app, DONE_CHANNEL) + val builder = Notification.Builder(app, DONE_CHANNEL) .setSmallIcon(if (installed) android.R.drawable.stat_sys_download_done else android.R.drawable.stat_notify_error) .setContentTitle(app.getString(if (installed) R.string.stores_notification_installed else R.string.stores_notification_failed, entry.name)) .setAutoCancel(true) .setContentIntent(if (installed && gamesKey != null) openPage(app, gamesKey, code + 10) else openDownloads(app, code + 11)) - .build() + if (!installed && !entry.error.isNullOrBlank()) { + builder.setContentText(entry.error) + builder.style = Notification.BigTextStyle().bigText(entry.error) + } + val n = builder.build() runCatching { app.getSystemService(NotificationManager::class.java)?.notify(ID + 1 + (code and 0xFFFF), n) } } } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt index e631cf78b7..5603ba7424 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicBackend.kt @@ -136,6 +136,8 @@ object EpicBackend : StoreBackend, EpicLaunchSupport { EpicLibrary.rememberSize(app, item.id, diskBytes) } override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + // Assembly overlaps chunk fetching; only the final install event advances the UI. + if (stage == "stream") return handle.stage(when (stage) { "check" -> DownloadStage.MANIFEST; "verify" -> DownloadStage.VERIFY; else -> DownloadStage.INSTALL }) handle.stageProgress(done, total, items, itemsTotal, bytes = stage == "install") } diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java index 958e9b1587..88307fd25e 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java @@ -4,13 +4,15 @@ import android.content.Context; import android.util.Log; import com.droiddeck.launcher.stores.StoreLog; +import com.droiddeck.launcher.stores.StoreInstallRoot; +import com.droiddeck.launcher.stores.StoreInstalls; +import com.droiddeck.launcher.stores.StoresNative; import com.droiddeck.launcher.stores.download.StoreDownloadTier; import org.json.JSONArray; import org.json.JSONObject; import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -21,6 +23,10 @@ import java.net.URL; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Iterator; @@ -29,11 +35,13 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.zip.Inflater; @@ -41,6 +49,39 @@ public final class EpicDownloadManager { private static final String TAG = "EpicDownload"; private static final String UA = EpicAuthClient.USER_AGENT; + /** Keep reusable chunk data bounded even when a manifest shares chunks across distant files. */ + static final long MAX_RETAINED_CHUNK_CACHE_BYTES = 64L * 1024 * 1024; + static final int PREFETCH_PARTS = 8; + static final String STAGING_DIRECTORY = ".droiddeck-epic-staging"; + private static final String STAGING_OWNER = "DroidDeck Epic staging v1\n"; + /** Most the adaptive engine caches of shared chunks; more only saves the occasional refetch. */ + static final long ADAPTIVE_CACHE_MAX_BYTES = 1024L * 1024 * 1024; + /** Its budget when free space cannot be read. */ + static final long ADAPTIVE_CACHE_DEFAULT_BYTES = 256L * 1024 * 1024; + /** + * Below this, internal scratch is too small to be worth a cache of its own: the shared chunks + * then go beside the game (still released at their last use) rather than failing the install. + */ + static final long ADAPTIVE_CACHE_MIN_SCRATCH_BYTES = MAX_RETAINED_CHUNK_CACHE_BYTES; + /** Chunks being decoded, written or released at any moment, beyond the budget itself. */ + static final long ADAPTIVE_TRANSIENT_BYTES = 64L * 1024 * 1024; + + static final String MODE_ADAPTIVE = "adaptive"; + static final String MODE_LOOKAHEAD = "lookahead"; + static final String MODE_BATCH = "batch"; + static final String MODE_JAVA = "java"; + + /** Which writer {@link #install} asks for and how wide the native fetch may go. */ + static final class NativeOptions { + final String mode; + final int maxWorkers, processWorkers; + final String tierId; + NativeOptions(String mode, int maxWorkers, int processWorkers, String tierId) { + this.mode = mode; this.maxWorkers = maxWorkers; this.processWorkers = processWorkers; this.tierId = tierId; + } + /** The default: the adaptive engine at the Fast tier's window. */ + static NativeOptions defaults() { return new NativeOptions(MODE_ADAPTIVE, 32, 2, "fast"); } + } public interface Callback { void onProgress(String message, int pct); @@ -93,6 +134,8 @@ public final class EpicDownloadManager { public String prereqPath = ""; public boolean prereqRead = false; public List cdnUrls = new ArrayList<>(); + /** The manifest as downloaded, for the native engines (which parse it themselves); null = Java only. */ + byte[] rawBytes = null; } public static final class Result { @@ -125,13 +168,45 @@ public final class EpicDownloadManager { Manifest manifest = parseManifest(manifestBytes); if (manifest == null) throw new InstallException("The manifest could not be parsed"); manifest.cdnUrls = cdnUrls; + manifest.rawBytes = manifestBytes; cb.onLog("epic: chunkDir=" + manifest.chunkDir + " chunks=" + manifest.uniqueChunks.size() + " files=" + manifest.files.size() + " exe=" + manifest.launchExe); + StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); + String mode = com.droiddeck.launcher.session.SessionPrefs.INSTANCE.storesEpicMode(ctx); + NativeOptions options = new NativeOptions(mode, Math.max(1, Math.min(128, tier.getNetworkWindow())), Math.max(2, tier.getProcessWorkers()), tier.getId()); + return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, options); + } catch (InstallException e) { + throw e; + } catch (Exception e) { + Log.w(TAG, "install failed", e); + throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "")); + } + } + static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, + List installTags, AtomicBoolean cancel, Callback cb) throws InstallException { + return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, NativeOptions.defaults()); + } + + static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, + List installTags, AtomicBoolean cancel, Callback cb, + NativeOptions nativeOptions) throws InstallException { + Result result = installSelected(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb, nativeOptions); + // A finished install leaves no staging; the owned namespace is recreated by the next run. + if (result != null) releaseOwnedStaging(new File(installDirPath)); + return result; + } + + private static Result installSelected(Manifest manifest, String installDirPath, String chunkCacheDirPath, + List installTags, AtomicBoolean cancel, Callback cb, + NativeOptions nativeOptions) throws InstallException { + try { + List cdnUrls = manifest.cdnUrls; File installDir = new File(installDirPath); installDir.mkdirs(); final String cachePath = chunkCacheDirPath == null ? "" : chunkCacheDirPath; final File chunkCacheDir = cachePath.isEmpty() ? new File(installDir, ".chunks") : new File(cachePath); chunkCacheDir.mkdirs(); + File stagingDir = recoverOwnedStaging(installDir, manifest.files); List selected = resolveInstallFiles(manifest, installTags); cb.onLog("epic: " + selected.size() + "/" + manifest.files.size() + " files for tags " + (installTags == null ? "(all)" : installTags.toString())); @@ -160,111 +235,159 @@ public final class EpicDownloadManager { return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); } List needed = uniqueChunksForFiles(manifest, pending); - long cacheMissing = 0, pendingBytes = 0; - for (ChunkInfo c : needed) if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += Math.max(c.windowSize, 1); - for (FileInfo f : pending) pendingBytes += f.fileSize(); - if (cachePath.isEmpty()) { - long free = installDir.getUsableSpace(); - if (free > 0 && cacheMissing + pendingBytes > free) throw new InstallException("Not enough free space: need " + fmt(cacheMissing + pendingBytes) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(free) + " free"); - } else { - long freeCache = chunkCacheDir.getUsableSpace(), freeGame = installDir.getUsableSpace(); - if (freeCache > 0 && cacheMissing > freeCache) throw new InstallException("Not enough internal space for the download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free"); - if (freeGame > 0 && pendingBytes > freeGame) throw new InstallException("Not enough free space: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free"); + Map chunksByGuid = new LinkedHashMap<>(); + for (ChunkInfo c : needed) chunksByGuid.put(c.guidStr(), c); + if (!chunkCacheDir.isDirectory() && !chunkCacheDir.mkdirs()) throw new InstallException("Could not create Epic chunk cache"); + long pendingBytes = 0; + Map oldFileBytes = new LinkedHashMap<>(); + for (FileInfo f : pending) { + pendingBytes += f.fileSize(); + File output = new File(installDir, f.filename.replace("\\", "/")); + if (output.isFile()) oldFileBytes.put(f.filename, output.length()); } - cb.onLog("epic: cache " + (cachePath.isEmpty() ? "beside the game" : "scratch") + ", " + fmt(cacheMissing) + " to fetch for " + fmt(pendingBytes) + " of files"); - long totalBytes = 0; - for (ChunkInfo c : needed) totalBytes += Math.max(c.fileSize, 1); - final long fTotalBytes = totalBytes; - final int totalChunks = needed.size(); - final AtomicLong completedBytes = new AtomicLong(0); - final AtomicInteger completedCount = new AtomicInteger(0); - final AtomicInteger failCount = new AtomicInteger(0); - final AtomicLong lastSpeedMs = new AtomicLong(System.currentTimeMillis()); - final AtomicLong lastSpeedBytes = new AtomicLong(0); - final AtomicLong speedBps = new AtomicLong(0); - cb.onSizes(fTotalBytes, planned); - cb.onBytes(0, fTotalBytes, 0); - cb.onLog("epic: " + fmt(fTotalBytes) + " in " + totalChunks + " chunks"); - - boolean javaPool = true; - final int[] pendingIdx = pendingIndices(manifest, pending); - boolean engine = com.droiddeck.launcher.stores.StoresNative.INSTANCE.getAvailable() && pendingIdx != null; - if (engine) { - NativeOutcome r = runNativePool(ctx, manifestBytes, pendingIdx, needed, fTotalBytes, installDirPath, cachePath, cdnUrls, cancel, cb, - completedBytes, completedCount, lastSpeedMs, lastSpeedBytes, speedBps); - if (r.started) { - javaPool = false; - if (r.cancelled) return null; - if (!r.success) { cb.onLog("epic: engine failed: " + r.error); failCount.incrementAndGet(); } - } else cb.onLog("epic: engine not started (" + r.error + "); using the built-in pool"); + NativeOptions options = nativeOptions != null ? nativeOptions : NativeOptions.defaults(); + String mode = options.mode; + if (!MODE_LOOKAHEAD.equals(mode) && !MODE_BATCH.equals(mode) && !MODE_JAVA.equals(mode)) mode = MODE_ADAPTIVE; + boolean nativeAvailable = StoresNative.INSTANCE.ensureLoaded(); + if (!nativeAvailable && !MODE_JAVA.equals(mode)) { + cb.onLog("epic: native engine unavailable; mode=java-bounded"); + mode = MODE_JAVA; + } else if (MODE_JAVA.equals(mode)) { + cb.onLog("epic: mode=java-bounded chosen by storesEpicMode"); } - if (javaPool) { - cb.onLog("epic: engine=built-in (8 threads)"); - ExecutorService pool = Executors.newFixedThreadPool(8, com.droiddeck.launcher.stores.download.DownloadQueue.workerFactory("epic-dl")); - for (ChunkInfo chunk : needed) { - final ChunkInfo fc = chunk; - pool.submit(() -> { - if (cancel.get()) return; - File cached = new File(chunkCacheDir, fc.guidStr()); - if (!cached.exists() && !downloadChunkStreaming(fc, manifest.chunkDir, cdnUrls, cached)) { - cb.onLog("FAIL chunk=" + fc.guidStr()); - failCount.incrementAndGet(); - return; - } - long done = completedBytes.addAndGet(Math.max(fc.fileSize, 1)); - int cnt = completedCount.incrementAndGet(); - sampleSpeed(done, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading chunks (" + cnt + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (done * 80L / Math.max(1, fTotalBytes))); - cb.onBytes(done, fTotalBytes, speedBps.get()); - }); - } - pool.shutdown(); - try { - while (!pool.awaitTermination(250, TimeUnit.MILLISECONDS)) { - if (cancel.get()) { pool.shutdownNow(); pool.awaitTermination(5, TimeUnit.SECONDS); return null; } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - pool.shutdownNow(); - throw new InstallException("interrupted"); - } + if (MODE_ADAPTIVE.equals(mode)) { + NativeAttempt attempt = installAdaptive(manifest, pending, pendingBytes, oldFileBytes, planned, installDir, cachePath, chunkCacheDir, stagingDir, options, cancel, cb); + if (attempt.handled) return attempt.result; + // Its space plan did not fit: the lookahead engine sizes its window to what is free. + mode = MODE_LOOKAHEAD; + } else if (MODE_BATCH.equals(mode)) { + return installBatch(manifest, pending, needed, pendingBytes, planned, installDir, cachePath, chunkCacheDir, options, cancel, cb); + } + boolean lookahead = MODE_LOOKAHEAD.equals(mode); + Map cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed); + boolean cacheOnInstallVolume = cachePath.isEmpty(); + long peakGameAdditional = lookahead ? 0 : cacheOnInstallVolume + ? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true, oldFileBytes) + : estimateStagedFilePeak(pending, oldFileBytes); + long peakCacheAdditional = lookahead || cacheOnInstallVolume + ? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false); + long freeGame = StoreInstallRoot.availableBytes(installDir); + // The lookahead engine plans its own window against both free-space readings. + if (!lookahead && cacheOnInstallVolume) { + if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free"); + } else if (!lookahead) { + long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir); + if (StoreInstallRoot.insufficientSpace(peakCacheAdditional, freeCache)) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free"); + if (StoreInstallRoot.insufficientSpace(peakGameAdditional, freeGame)) throw new InstallException("Not enough free space for safe staged Epic replacement: need " + fmt(peakGameAdditional) + ", only " + fmt(freeGame) + " free"); } if (cancel.get()) return null; - if (failCount.get() > 0) throw new InstallException(failCount.get() + " chunk(s) failed to download"); - - if (engine) { - NativeOutcome r = runNativeAssembly(manifestBytes, pendingIdx, pending.size(), installDirPath, cachePath, cancel, cb); - if (r.started) { - if (r.cancelled) return null; - if (!r.success) throw new InstallException(r.error.isEmpty() ? "assembly failed" : r.error); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } - cb.onLog("epic: assembler not started (" + r.error + "); writing the files here"); - } - int totalFiles = pending.size(), doneFiles = 0; - long writtenBytes = 0, writeTotal = 0; - for (FileInfo f : pending) writeTotal += f.fileSize(); - for (FileInfo file : pending) { - if (cancel.get()) return null; - String relPath = file.filename.replace("\\", "/"); - File outFile = new File(installDir, relPath); - File parent = outFile.getParentFile(); - if (parent != null) parent.mkdirs(); - cb.onProgress("Writing: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), 80 + (int) (doneFiles * 20L / totalFiles)); - try (FileOutputStream fos = new FileOutputStream(outFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) { - for (ChunkPart part : file.parts) { - File cachedChunk = new File(chunkCacheDir, part.guidStr()); - if (!cachedChunk.exists()) throw new InstallException("Missing chunk " + part.guidStr() + " for " + relPath); - byte[] data = readFile(cachedChunk); - bos.write(data, part.offset, part.size); + if (lookahead) { + JSONObject request = nativeRequest(manifest, pending, chunksByGuid, cachedChunkBytes, + oldFileBytes, installDir, chunkCacheDir, stagingDir, cacheOnInstallVolume); + request.put("free_game", StoreInstallRoot.availableBytes(installDir)); + request.put("free_cache", StoreInstallRoot.availableBytes(chunkCacheDir)); + final long nativeTotal = pendingBytes; + final long nativeStarted = System.nanoTime(); + cb.onSizes(nativeTotal, planned); + cb.onBytes(0, nativeTotal, 0); + EpicNative.Result outcome = EpicNative.installAdaptive(request.toString(), cancel, new EpicNative.InstallListener() { + @Override public void onPlan(int chunks, long bytes, String plan) { + cb.onLog("epic: mode=lookahead engine=lookahead-rust " + plan); } + @Override public void onProgress(long done, long total, int files, int filesTotal) { + long elapsedMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000); + cb.onBytes(done, total, done * 1000 / elapsedMs); + cb.onProgress("Downloading chunks…", (int) (done * 80L / Math.max(1, total))); + cb.onStage("stream", done, total, files, filesTotal); + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onComplete(boolean success, String error, long bytes) {} + @Override public boolean beforeSwap() { + try { + if (cancel.get()) return false; + StoreInstalls.markUpdating(installDir); + return !cancel.get(); + } catch (Exception e) { + cb.onLog("epic: cannot mark updating: " + e.getMessage()); + return false; + } + } + }); + if (outcome.cancelled || cancel.get()) return null; + if (!outcome.success) throw new InstallException("Native Epic install failed: " + outcome.error); + long lookaheadMs = Math.max(1, (System.nanoTime() - nativeStarted) / 1_000_000); + cb.onLog("epic: summary mode=lookahead files=" + pending.size() + " bytes_written=" + nativeTotal + " elapsed_ms=" + lookaheadMs + + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", nativeTotal * 1000.0 / lookaheadMs / 1048576.0)); + cb.onStage("install", nativeTotal, nativeTotal, pending.size(), pending.size()); + cb.onProgress("Complete", 100); + return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); + } + cb.onLog("epic: cache " + (cacheOnInstallVolume ? "beside the game" : "scratch") + ", peak game additional=" + fmt(peakGameAdditional) + (cacheOnInstallVolume ? "" : ", peak cache additional=" + fmt(peakCacheAdditional)) + " for " + fmt(pendingBytes) + " of files"); + final long fTotalBytes = pendingBytes; + final int totalChunks = needed.size(); + cb.onSizes(fTotalBytes, planned); + cb.onBytes(0, fTotalBytes, 0); + cb.onLog("epic: " + fmt(fTotalBytes) + " to write using " + totalChunks + " unique chunks"); + cb.onLog("epic: engine=bounded-java (8-worker prefetch; native availability does not change cache semantics)"); + long completedBytes = 0, writtenBytes = 0; + long transferStarted = System.nanoTime(); + int completedCount = 0, doneFiles = 0, totalFiles = pending.size(); + Set reservedPaths = new LinkedHashSet<>(); + for (FileInfo file : manifest.files) reservedPaths.add(new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath()); + boolean updateRecorded = false; + try (ChunkPrefetch prefetch = new ChunkPrefetch(pending, chunksByGuid, cachedChunkBytes, + chunkCacheDir, manifest.chunkDir, cdnUrls, cancel)) { + for (FileInfo file : pending) { + if (cancel.get()) return null; + String relPath = file.filename.replace("\\", "/"); + File outFile = new File(installDir, relPath); + cb.onProgress("Downloading chunks: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), (int) (completedBytes * 80L / Math.max(1, fTotalBytes))); + MessageDigest fileSha = expectedDigest(file.sha1); + File partFile = createStagingFile(outFile, reservedPaths, stagingDir); + try { + try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) { + for (ChunkPart part : file.parts) { + if (cancel.get()) throw new IOException("cancelled"); + String guid = part.guidStr(); + File cachedChunk = prefetch.await(guid); + completedCount++; + cb.onProgress("Downloading chunks (part " + completedCount + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes))); + copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath); + completedBytes += part.size & 0xFFFFFFFFL; + long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); + cb.onBytes(completedBytes, fTotalBytes, completedBytes * 1000 / elapsedMs); + prefetch.consume(); + } + bos.flush(); + fos.getFD().sync(); + } + if (cancel.get()) return null; + if (partFile.length() != file.fileSize()) throw new InstallException("Size mismatch for " + relPath); + if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) { + throw new InstallException("SHA-1 mismatch for " + relPath); + } + // Once a swap can mix versions, the old installed record is no longer truthful. + if (!updateRecorded) { + StoreInstalls.markUpdating(installDir); + updateRecorded = true; + } + if (cancel.get()) return null; + Files.move(partFile.toPath(), outFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + if (cancel.get()) return null; + throw new InstallException("Could not assemble or atomically replace " + relPath + ": " + e.getMessage()); + } finally { + partFile.delete(); + } + doneFiles++; + writtenBytes += file.fileSize(); + cb.onStage("stream", writtenBytes, pendingBytes, doneFiles, totalFiles); } - doneFiles++; - writtenBytes += file.fileSize(); - cb.onStage("install", writtenBytes, writeTotal, doneFiles, totalFiles); } deleteDir(chunkCacheDir); + long boundedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); + cb.onLog("epic: summary mode=java-bounded files=" + doneFiles + " bytes_written=" + writtenBytes + " elapsed_ms=" + boundedMs + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", writtenBytes * 1000.0 / boundedMs / 1048576.0)); + cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles); cb.onProgress("Complete", 100); return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); } catch (InstallException e) { @@ -275,9 +398,17 @@ public final class EpicDownloadManager { } } - private static final class NativeOutcome { boolean started, success, cancelled; String error = ""; } + /** What a native attempt did: {@code handled} = the install ended there ({@code result} null when cancelled). */ + private static final class NativeAttempt { + final boolean handled; + final Result result; + private NativeAttempt(boolean handled, Result result) { this.handled = handled; this.result = result; } + static final NativeAttempt FALLBACK = new NativeAttempt(false, null); + static NativeAttempt done(Result result) { return new NativeAttempt(true, result); } + } - private static int[] pendingIndices(Manifest manifest, List pending) { + /** The pending files as indices into the manifest's file list, what the native runs take; null when one is not in it. */ + static int[] pendingIndices(Manifest manifest, List pending) { java.util.IdentityHashMap index = new java.util.IdentityHashMap<>(); for (int i = 0; i < manifest.files.size(); i++) index.put(manifest.files.get(i), i); int[] pendingIdx = new int[pending.size()]; @@ -289,58 +420,696 @@ public final class EpicDownloadManager { return pendingIdx; } - private static NativeOutcome runNativePool(Context ctx, byte[] manifestBytes, int[] pendingIdx, List needed, long totalBytes, - String installDirPath, String cachePath, List cdnUrls, AtomicBoolean cancel, Callback cb, - AtomicLong completedBytes, AtomicInteger completedCount, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - NativeOutcome r = new NativeOutcome(); - try { - String[] prefixes = new String[cdnUrls.size()]; - for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir; - final int totalChunks = needed.size(); - StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); - int workers = Math.max(1, Math.min(128, tier.getNetworkWindow())); - int process = Math.max(2, tier.getProcessWorkers()); - cb.onLog("epic: engine=native workers=" + workers + " process_workers=" + process + " tier=" + tier.getId() + (cachePath.isEmpty() ? "" : " cache=scratch")); - EpicNative.Result res = EpicNative.run(manifestBytes, installDirPath, cachePath, prefixes, pendingIdx, totalChunks, totalBytes, "", workers, process, cancel, - new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); } - @Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) { - completedBytes.set(bytesDone); - completedCount.set(chunksDone); - sampleSpeed(bytesDone, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (bytesDone * 80L / Math.max(1, totalBytes))); - cb.onBytes(bytesDone, totalBytes, speedBps.get()); - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytesCredited) {} - }); - r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error; - } catch (Throwable t) { - r.started = false; - r.error = t.getClass().getSimpleName() + ": " + t.getMessage(); - } - return r; + private static String[] cdnPrefixes(List cdnUrls) { + String[] prefixes = new String[cdnUrls.size()]; + for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir; + return prefixes; } - private static NativeOutcome runNativeAssembly(byte[] manifestBytes, int[] pendingIdx, int totalFiles, String installDirPath, String cachePath, AtomicBoolean cancel, Callback cb) { - NativeOutcome r = new NativeOutcome(); + /** + * Where the adaptive engine's shared-chunk cache goes and how big it may get, and what each + * volume must hold. A card install keeps the cache on internal scratch (each byte hits the + * card once); when internal storage cannot hold a useful cache, the cache goes beside the game + * instead of failing. {@code error} names the storage that is short, and by how much. + */ + static final class AdaptiveSpacePlan { + boolean cacheBesideGame; + long cacheBudget; + /** + * Most planned bytes of replaced files the engine may stage at once (each sits beside its + * old copy until its swap); -1 = nothing is replaced, no limit. + */ + long overlapBudget = -1; + long gameNeed; + long scratchNeed; + String error; + } + + /** The part of free space the shared cache may use, after {@code need}: half the headroom, capped. */ + static long adaptiveCacheBudget(long free, long need) { + if (free < 0) return ADAPTIVE_CACHE_DEFAULT_BYTES; + long headroom = free - need - ADAPTIVE_TRANSIENT_BYTES; + if (headroom <= 0) return 0; + return Math.min(ADAPTIVE_CACHE_MAX_BYTES, headroom / 2); + } + + /** + * The game volume's net growth at its worst point when files are swapped in order and each + * replaced file's old copy is released at its swap: the running Σ(new − old) after each file. + * What a staged file costs while it sits beside its old copy is the overlap budget's part. + */ + static long settledNetPeak(List files, Map oldFileBytes) { + long running = 0, peak = 0; + for (FileInfo file : files) { + Long old = oldFileBytes.get(file.filename); + running += file.fileSize() - (old == null ? 0 : old); + peak = Math.max(peak, running); + } + return peak; + } + + /** The largest pending file whose destination has data (the engine's smallest overlap budget). */ + static long largestReplacedFile(List files, Map oldFileBytes) { + long largest = 0; + for (FileInfo file : files) { + Long old = oldFileBytes.get(file.filename); + if (old != null && old > 0) largest = Math.max(largest, file.fileSize()); + } + return largest; + } + + /** + * The cache placement and budget as before; for an update, the replacement-overlap budget is + * what the game volume has left after the net growth and the cache, never below the largest + * replaced file (one at a time is the minimum that still progresses). The check is the + * lookahead planner's: the need must not exceed the free reading, no extra margin. + */ + static AdaptiveSpacePlan planAdaptiveSpace(long netPeak, long largestReplaced, long freeGame, boolean scratchRequested, long freeScratch) { + AdaptiveSpacePlan plan = new AdaptiveSpacePlan(); + long minimum = netPeak + largestReplaced; + long scratchBudget = scratchRequested ? adaptiveCacheBudget(freeScratch, 0) : 0; + plan.cacheBesideGame = !scratchRequested || (freeScratch >= 0 && scratchBudget < ADAPTIVE_CACHE_MIN_SCRATCH_BYTES); + long cacheOnGame; + if (plan.cacheBesideGame) { + plan.cacheBudget = adaptiveCacheBudget(freeGame, minimum); + cacheOnGame = plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; + plan.scratchNeed = 0; + } else { + plan.cacheBudget = scratchBudget; + cacheOnGame = 0; + plan.scratchNeed = plan.cacheBudget + ADAPTIVE_TRANSIENT_BYTES; + } + long overlap = 0; + if (largestReplaced > 0) { + overlap = freeGame < 0 ? largestReplaced : Math.max(largestReplaced, freeGame - netPeak - cacheOnGame); + plan.overlapBudget = overlap; + } + plan.gameNeed = netPeak + overlap + cacheOnGame; + if (StoreInstallRoot.insufficientSpace(plan.gameNeed, freeGame)) { + plan.error = "Not enough free space on the game's storage: need " + fmt(plan.gameNeed) + + (plan.cacheBesideGame && scratchRequested ? " (download cache moved beside the game: internal storage is full)" : "") + + ", only " + fmt(freeGame) + " free (short by " + fmt(plan.gameNeed - freeGame) + ")"; + } else if (!plan.cacheBesideGame && StoreInstallRoot.insufficientSpace(plan.scratchNeed, freeScratch)) { + plan.error = "Not enough internal space for the download cache: need " + fmt(plan.scratchNeed) + ", only " + fmt(freeScratch) + + " free (short by " + fmt(plan.scratchNeed - freeScratch) + ")"; + } + return plan; + } + + /** + * The adaptive native install; the library is loaded. Hands back {@code FALLBACK} (nothing + * fetched or written) only when its space plan does not fit, so the caller can run the + * lookahead engine, which narrows its window to the free space. Any other failure is final. + */ + private static NativeAttempt installAdaptive(Manifest manifest, List pending, long pendingBytes, Map oldFileBytes, + long planned, File installDir, String cachePath, File chunkCacheDir, File stagingDir, + NativeOptions options, AtomicBoolean cancel, Callback cb) throws Exception { + int[] pendingIdx = pendingIndices(manifest, pending); + if (manifest.rawBytes == null || pendingIdx == null) { + throw new InstallException("Native Epic install failed: the manifest is not available to the native engine"); + } + long netPeak = settledNetPeak(pending, oldFileBytes); + long largestReplaced = largestReplacedFile(pending, oldFileBytes); + long freeGame = StoreInstallRoot.availableBytes(installDir); + boolean scratchRequested = !cachePath.isEmpty(); + long freeScratch = scratchRequested ? StoreInstallRoot.availableBytes(chunkCacheDir) : -1; + AdaptiveSpacePlan space = planAdaptiveSpace(netPeak, largestReplaced, freeGame, scratchRequested, freeScratch); + if (space.error != null) { + cb.onLog("epic: adaptive space plan does not fit (" + space.error + "); mode=lookahead"); + return NativeAttempt.FALLBACK; + } + String engineCache = space.cacheBesideGame ? "" : cachePath; + File engineCacheDir = engineCache.isEmpty() ? new File(installDir, ".chunks") : chunkCacheDir; + if (space.cacheBesideGame && scratchRequested) { + cb.onLog("epic: internal storage cannot hold the shared-chunk cache (" + fmt(freeScratch) + " free); cache beside the game"); + } + if (cancel.get()) return NativeAttempt.done(null); + cb.onLog("epic: mode=adaptive tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers + + " cache=" + (engineCache.isEmpty() ? "beside the game" : "scratch") + " budget=" + fmt(space.cacheBudget) + + " overlap_budget=" + (space.overlapBudget < 0 ? "none" : fmt(space.overlapBudget)) + + " game need=" + fmt(space.gameNeed) + " of " + fmt(freeGame) + (space.cacheBesideGame ? "" : ", scratch need=" + fmt(space.scratchNeed) + " of " + fmt(freeScratch))); + cb.onSizes(pendingBytes, planned); + cb.onBytes(0, pendingBytes, 0); + final int totalFiles = pending.size(); + final File fInstallDir = installDir; + final Object lock = new Object(); + final long started = System.nanoTime(); + final long[] state = new long[] {0, 0, System.currentTimeMillis(), 0, 0}; // bytes, files, speed ms, speed bytes, speed bps + EpicNative.Result res = EpicNative.stream(manifest.rawBytes, installDir.getPath(), engineCache, stagingDir.getCanonicalPath(), cdnPrefixes(manifest.cdnUrls), pendingIdx, pendingBytes, + space.cacheBudget, space.overlapBudget, "", options.maxWorkers, options.processWorkers, cancel, new EpicNative.StreamListener() { + @Override public void onStreamProgress(long bytesWritten, long bytesTotal, int filesDone, int filesTotal, long filesBytes) { + synchronized (lock) { + if (bytesWritten < state[0] && filesDone <= state[1]) return; // a late, older sample + state[0] = Math.max(state[0], bytesWritten); + long nowMs = System.currentTimeMillis(); + if (nowMs - state[2] >= 500) { + state[4] = (state[0] - state[3]) * 1000L / Math.max(1, nowMs - state[2]); + state[2] = nowMs; + state[3] = state[0]; + } + cb.onBytes(state[0], pendingBytes, state[4]); + cb.onProgress("Downloading chunks (" + filesDone + "/" + totalFiles + " files)" + speedSuffix(state[4]), + (int) (state[0] * 80L / Math.max(1, pendingBytes))); + if (filesDone > state[1]) { + state[1] = filesDone; + cb.onStage("stream", filesBytes, pendingBytes, filesDone, totalFiles); + } + } + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public boolean onBeforeReplace() { + // Once a swap can mix versions, the old installed record is no longer truthful. + try { + StoreInstalls.markUpdating(fInstallDir); + return true; + } catch (Exception e) { + cb.onLog("epic: could not mark the install as updating: " + e.getMessage()); + return false; + } + } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (!res.started) throw new InstallException("Native Epic install failed: " + res.error); + long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); + if (res.cancelled) return NativeAttempt.done(null); + if (!res.success) throw new InstallException("Native Epic install failed: " + res.error); + deleteDir(engineCacheDir); + deleteDir(chunkCacheDir); + cb.onLog("epic: summary mode=adaptive files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs + + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); + cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); + cb.onProgress("Complete", 100); + return NativeAttempt.done(new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest)); + } + + /** + * The older native batch install, kept for A/B benchmarks: every chunk fetched into the cache + * in parallel, then the files assembled from it. It needs the whole cache and the files at + * once (about twice the game), and it writes files in place rather than staged, so an + * existing install is marked updating before anything is fetched. + */ + private static Result installBatch(Manifest manifest, List pending, List needed, long pendingBytes, + long planned, File installDir, String cachePath, File chunkCacheDir, NativeOptions options, + AtomicBoolean cancel, Callback cb) throws InstallException { + int[] pendingIdx = pendingIndices(manifest, pending); + if (manifest.rawBytes == null || pendingIdx == null) { + throw new InstallException("Native Epic install failed: the manifest is not available to the native engine"); + } + long cacheMissing = 0, totalBytes = 0; + for (ChunkInfo c : needed) { + if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += chunkWindowSize(c); + totalBytes += Math.max(c.fileSize, 1); + } + long freeGame = StoreInstallRoot.availableBytes(installDir); + if (cachePath.isEmpty()) { + long need = cacheMissing + pendingBytes; + if (StoreInstallRoot.insufficientSpace(need, freeGame)) throw new InstallException("Not enough free space for the batch Epic download: need " + fmt(need) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(freeGame) + " free (short by " + fmt(need - freeGame) + ")"); + } else { + long freeCache = StoreInstallRoot.availableBytes(chunkCacheDir); + if (StoreInstallRoot.insufficientSpace(cacheMissing, freeCache)) throw new InstallException("Not enough internal space for the batch download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free (short by " + fmt(cacheMissing - freeCache) + ")"); + if (StoreInstallRoot.insufficientSpace(pendingBytes, freeGame)) throw new InstallException("Not enough free space on the game's storage: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free (short by " + fmt(pendingBytes - freeGame) + ")"); + } + if (cancel.get()) return null; + final long fTotal = totalBytes; + final int totalChunks = needed.size(); + final int totalFiles = pending.size(); + cb.onLog("epic: mode=batch tier=" + options.tierId + " workers=" + options.maxWorkers + " process_workers=" + options.processWorkers + (cachePath.isEmpty() ? "" : " cache=scratch") + " cache_missing=" + fmt(cacheMissing)); + cb.onSizes(pendingBytes, planned); + final AtomicLong lastMs = new AtomicLong(System.currentTimeMillis()), lastBytes = new AtomicLong(), speed = new AtomicLong(); + final long started = System.nanoTime(); + EpicNative.Result fetch = EpicNative.run(manifest.rawBytes, installDir.getPath(), cachePath, cdnPrefixes(manifest.cdnUrls), pendingIdx, totalChunks, fTotal, "", + options.maxWorkers, options.processWorkers, cancel, new EpicNative.Listener() { + @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); } + @Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) { + sampleSpeed(bytesDone, lastMs, lastBytes, speed); + cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speed.get()), (int) (bytesDone * 80L / Math.max(1, fTotal))); + cb.onBytes(bytesDone, fTotal, speed.get()); + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (!fetch.started) throw new InstallException("Native Epic install failed: " + fetch.error); + if (fetch.cancelled) return null; + if (!fetch.success) throw new InstallException("Epic batch download failed: " + fetch.error); + // Assembly overwrites files in place: the old installed record stops being truthful now. + try { StoreInstalls.markUpdating(installDir); } catch (Exception e) { throw new InstallException("Could not mark the install as updating: " + e.getMessage()); } + EpicNative.Result asm = EpicNative.assemble(manifest.rawBytes, installDir.getPath(), cachePath, pendingIdx, cancel, new EpicNative.Listener() { + @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {} + @Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) { + cb.onStage("stream", bytesDone, bytesTotal, done, Math.max(total, totalFiles)); + cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles)))); + } + @Override public void onLog(String line) { cb.onLog(line); } + @Override public void onComplete(boolean success, String error, long bytes) {} + }); + if (asm.cancelled) return null; + if (!asm.started || !asm.success) throw new InstallException(asm.error == null || asm.error.isEmpty() ? "Epic batch assembly failed" : asm.error); + long elapsedMs = Math.max(1, (System.nanoTime() - started) / 1_000_000); + cb.onLog("epic: summary mode=batch files=" + totalFiles + " bytes_written=" + pendingBytes + " elapsed_ms=" + elapsedMs + + " avg_MBps=" + String.format(java.util.Locale.ROOT, "%.2f", pendingBytes * 1000.0 / elapsedMs / 1048576.0)); + deleteDir(chunkCacheDir); + cb.onStage("install", pendingBytes, pendingBytes, totalFiles, totalFiles); + cb.onProgress("Complete", 100); + return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); + } + + private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { + long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); + if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { + long prevB = lastSpeedBytes.getAndSet(done); + long dt = nowMs - prevMs; + if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); + } + } + + private static String speedSuffix(long bps) { + if (bps <= 0) return ""; + return " " + (bps >= 1048576 ? String.format(java.util.Locale.ROOT, "%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); + } + + static File recoverOwnedStaging(File installDir, List manifestFiles) throws IOException { + File staging = new File(installDir, STAGING_DIRECTORY); + Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY); + if (!staging.getCanonicalFile().toPath().equals(expected) || + (Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS) && + !Files.isDirectory(staging.toPath(), LinkOption.NOFOLLOW_LINKS))) { + throw new IOException("Unsafe Epic staging namespace; refusing to traverse it"); + } + String namespace = expected.toString(); + for (FileInfo file : manifestFiles) { + String path = new File(installDir, file.filename.replace("\\", "/")).getCanonicalPath(); + if (path.equals(namespace) || path.startsWith(namespace + File.separator)) { + throw new IOException("Manifest conflicts with Epic staging namespace"); + } + } + File owner = new File(staging, "owner"); + if (Files.exists(staging.toPath(), LinkOption.NOFOLLOW_LINKS)) { + if (!Files.isRegularFile(owner.toPath(), LinkOption.NOFOLLOW_LINKS) || + !owner.getCanonicalFile().toPath().equals(expected.resolve("owner")) || + owner.length() != STAGING_OWNER.length() || + !STAGING_OWNER.equals(new String(Files.readAllBytes(owner.toPath()), StandardCharsets.UTF_8))) { + throw new IOException("Unowned Epic staging namespace; refusing to delete it"); + } + File[] files = staging.listFiles(); + if (files == null) throw new IOException("Could not inspect Epic staging"); + for (File file : files) { + if (file.getName().equals("owner")) continue; + if (!Files.isRegularFile(file.toPath(), LinkOption.NOFOLLOW_LINKS) || + !file.getCanonicalFile().toPath().equals(expected.resolve(file.getName())) || + !file.getName().endsWith(".stage")) throw new IOException("Unexpected Epic staging entry"); + } + for (File file : files) { + if (file.getName().equals("owner")) continue; + if (!file.delete()) throw new IOException("Could not recover orphan Epic staging"); + } + } else { + if (!staging.mkdir()) throw new IOException("Could not create Epic staging namespace"); + try (FileOutputStream out = new FileOutputStream(owner)) { + out.write(STAGING_OWNER.getBytes(StandardCharsets.UTF_8)); + out.getFD().sync(); + } + } + return staging; + } + + /** + * Remove the owned staging namespace when it holds nothing but its own marker. Same no-follow + * and canonical-path checks as {@link #recoverOwnedStaging}; anything unexpected is left alone. + */ + static void releaseOwnedStaging(File installDir) { try { - cb.onLog("epic: assembler=native files=" + totalFiles + (cachePath.isEmpty() ? "" : " cache=scratch")); - EpicNative.Result res = EpicNative.assemble(manifestBytes, installDirPath, cachePath, pendingIdx, cancel, new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {} - @Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) { - cb.onStage("install", bytesDone, bytesTotal, done, Math.max(total, totalFiles)); - cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles)))); + File staging = new File(installDir, STAGING_DIRECTORY); + if (!Files.isDirectory(staging.toPath(), LinkOption.NOFOLLOW_LINKS)) return; + Path expected = installDir.getCanonicalFile().toPath().resolve(STAGING_DIRECTORY); + if (!staging.getCanonicalFile().toPath().equals(expected)) return; + String[] names = staging.list(); + if (names == null || names.length != 1 || !names[0].equals("owner")) return; + File owner = new File(staging, "owner"); + if (!Files.isRegularFile(owner.toPath(), LinkOption.NOFOLLOW_LINKS) + || owner.length() != STAGING_OWNER.length() + || !STAGING_OWNER.equals(new String(Files.readAllBytes(owner.toPath()), StandardCharsets.UTF_8))) return; + if (owner.delete()) staging.delete(); + } catch (IOException e) { + Log.w(TAG, "staging namespace kept: " + e.getMessage()); + } + } + + private static JSONObject nativeRequest(Manifest manifest, List files, + Map chunks, Map cached, Map old, + File install, File cache, File staging, boolean sameVolume) throws Exception { + JSONObject request = new JSONObject(); + request.put("schema_version", 1); + request.put("install_dir", install.getCanonicalPath()); + request.put("cache_dir", cache.getCanonicalPath()); + request.put("staging_dir", staging.getCanonicalPath()); + request.put("same_volume", sameVolume); + request.put("chunk_dir", manifest.chunkDir); + request.put("max_workers", 32); + JSONArray hosts = new JSONArray(); + for (CdnUrl cdn : manifest.cdnUrls) hosts.put(cdn.baseUrl + cdn.cloudDir); + request.put("hosts", hosts); + JSONArray chunkList = new JSONArray(); + for (ChunkInfo chunk : chunks.values()) { + JSONObject item = new JSONObject(); + item.put("guid", chunk.guidStr()); + item.put("path", chunk.getPath(manifest.chunkDir)); + item.put("size", chunkWindowSize(chunk)); + item.put("exact_size", chunk.windowSize > 0); + item.put("wire_size", Math.max(1, chunk.fileSize)); + item.put("sha1", hexDigest(chunk.sha1)); + item.put("cached", cached.containsKey(chunk.guidStr()) ? cached.get(chunk.guidStr()) : -1); + chunkList.put(item); + } + request.put("chunks", chunkList); + JSONArray fileList = new JSONArray(); + for (FileInfo file : files) { + JSONObject item = new JSONObject(); + item.put("path", file.filename.replace("\\", "/")); + item.put("sha1", hexDigest(file.sha1)); + item.put("old_size", old.containsKey(file.filename) ? old.get(file.filename) : 0); + JSONArray parts = new JSONArray(); + for (ChunkPart part : file.parts) { + parts.put(new JSONObject().put("guid", part.guidStr()).put("offset", part.offset & 0xFFFFFFFFL).put("size", part.size & 0xFFFFFFFFL)); + } + item.put("parts", parts); + fileList.put(item); + } + request.put("files", fileList); + return request; + } + + private static String hexDigest(byte[] digest) { + if (digest == null || digest.length != 20) return ""; + StringBuilder out = new StringBuilder(); + for (byte b : digest) out.append(String.format("%02x", b & 255)); + return out.toString(); + } + + private static File createStagingFile(File output, Set reservedPaths, File staging) throws IOException { + File parent = output.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) throw new IOException("Could not create output directory"); + // Owned staging is on the output volume, never a manifest's literal ".part". + while (true) { + File stage = File.createTempFile("file-", ".stage", staging); + if (!reservedPaths.contains(stage.getCanonicalPath())) return stage; + if (!stage.delete()) throw new IOException("Could not clear staging collision"); + } + } + + static long estimateStagedFilePeak(List files, Map oldFileBytes) { + long delta = 0, peak = 0; + for (FileInfo file : files) { + delta += file.fileSize(); + peak = Math.max(peak, delta); + delta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0; + } + return peak; + } + + /** + * Discard abandoned/orphaned cache entries and retain only verified chunks this install uses. + * A bounded run consumes each chunk at its last part reference, so stale chunks from an older + * all-at-once run must not silently occupy the space needed by the game files. + */ + static Map prepareBoundedCache(File cacheDir, List needed) throws InstallException { + Map byGuid = new LinkedHashMap<>(); + for (ChunkInfo chunk : needed) byGuid.put(chunk.guidStr(), chunk); + Map valid = new LinkedHashMap<>(); + File[] entries = cacheDir.listFiles(); + if (entries == null) throw new InstallException("Could not inspect Epic chunk cache"); + for (File entry : entries) { + ChunkInfo chunk = byGuid.get(entry.getName()); + if (chunk == null || !isValidCachedChunk(chunk, entry)) { + deleteDir(entry); + if (entry.exists()) throw new InstallException("Could not clear stale Epic cache entry " + entry.getName()); + continue; + } + valid.put(entry.getName(), entry.length()); + } + trimCacheToBound(cacheDir, valid); + return valid; + } + + /** Evict oldest reusable chunks until retained cache data is under the fixed disk ceiling. */ + private static void trimCacheToBound(File cacheDir, Map cacheSizes) throws InstallException { + for (String guid : evictOldestReusableChunks(cacheSizes).keySet()) { + File chunk = new File(cacheDir, guid); + if (chunk.exists() && !chunk.delete() && chunk.exists()) { + throw new InstallException("Could not release Epic cache chunk " + guid); + } + } + } + + /** + * The writer and space planner use the same oldest-first retained-cache policy. Entries leave + * this map as soon as their last reference is consumed; an oversized incoming chunk is + * therefore allowed to coexist with the retained cache until its current part is written. + */ + private static Map evictOldestReusableChunks(Map cacheSizes) { + long retained = 0; + for (long bytes : cacheSizes.values()) retained += bytes; + Map evicted = new LinkedHashMap<>(); + java.util.Iterator> iterator = cacheSizes.entrySet().iterator(); + while (retained > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) { + Map.Entry entry = iterator.next(); + retained -= entry.getValue(); + evicted.put(entry.getKey(), entry.getValue()); + iterator.remove(); + } + return evicted; + } + + private static boolean consumeChunkReference(Map refs, String guid) { + Integer count = refs.get(guid); + if (count == null) return false; + if (count <= 1) { + refs.remove(guid); + return true; + } + refs.put(guid, count - 1); + return false; + } + + /** + * Pure disk schedule shared by execution and preflight. The next eight part references pin + * at most eight GUIDs (including across file boundaries); other reusable data is oldest-first + * capped at 64 MiB. Incoming downloads reserve their entire decoded size before submission. + */ + static final class PrefetchSchedule { + final List parts = new ArrayList<>(); + final Map resident; + final Map refs; + final Map chunks; + int cursor; + + PrefetchSchedule(List files, Map chunks, Map cached) { + for (FileInfo file : files) for (ChunkPart part : file.parts) parts.add(part.guidStr()); + this.chunks = chunks; + resident = new LinkedHashMap<>(cached); + refs = chunkReferences(files); + } + + Set window() { + return new LinkedHashSet<>(parts.subList(cursor, Math.min(parts.size(), cursor + PREFETCH_PARTS))); + } + + List fill() { + List fetch = new ArrayList<>(); + for (String guid : window()) if (!resident.containsKey(guid)) { + ChunkInfo chunk = chunks.get(guid); + if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid); + resident.put(guid, chunkWindowSize(chunk)); + fetch.add(guid); + } + return fetch; + } + + List consume() { + List release = new ArrayList<>(); + String guid = parts.get(cursor++); + if (consumeChunkReference(refs, guid)) { + resident.remove(guid); + release.add(guid); + } + Set pinned = window(); + long reusable = 0; + for (Map.Entry entry : resident.entrySet()) { + if (!pinned.contains(entry.getKey())) reusable += entry.getValue(); + } + Iterator> it = resident.entrySet().iterator(); + while (reusable > MAX_RETAINED_CHUNK_CACHE_BYTES && it.hasNext()) { + Map.Entry entry = it.next(); + if (pinned.contains(entry.getKey())) continue; + reusable -= entry.getValue(); + release.add(entry.getKey()); + it.remove(); + } + return release; + } + + long bytes() { + long bytes = 0; + for (long size : resident.values()) bytes += size; + return bytes; + } + } + + private static final class ChunkPrefetch implements AutoCloseable { + final PrefetchSchedule schedule; + final File cacheDir; + final AtomicBoolean cancel; + final AtomicBoolean stopped = new AtomicBoolean(); + final ExecutorService workers = Executors.newFixedThreadPool(PREFETCH_PARTS); + final Map> jobs = new LinkedHashMap<>(); + final Set connections = ConcurrentHashMap.newKeySet(); + final String chunkDir; + final List cdns; + + ChunkPrefetch(List files, Map chunks, Map cached, + File cacheDir, String chunkDir, List cdns, AtomicBoolean cancel) { + schedule = new PrefetchSchedule(files, chunks, cached); + this.cacheDir = cacheDir; + this.chunkDir = chunkDir; + this.cdns = cdns; + this.cancel = cancel; + } + + File await(String guid) throws Exception { + for (String fetch : schedule.fill()) { + jobs.put(fetch, workers.submit(() -> downloadChunkStreaming(schedule.chunks.get(fetch), + chunkDir, cdns, new File(cacheDir, fetch), cancel, stopped, connections))); + } + Future job = jobs.get(guid); + if (job != null) while (true) { + if (cancel.get()) throw new IOException("cancelled"); + try { + if (!job.get(100, TimeUnit.MILLISECONDS)) { + if (cancel.get()) throw new IOException("cancelled"); + throw new InstallException("Failed to download Epic chunk " + guid); + } + break; + } catch (TimeoutException ignored) { } + } + return new File(cacheDir, guid); + } + + void consume() throws InstallException { + for (String guid : schedule.consume()) { + // Only the writer advances the schedule, after closing its chunk reader. + jobs.remove(guid); + File chunk = new File(cacheDir, guid); + if (chunk.exists() && !chunk.delete() && chunk.exists()) { + throw new InstallException("Could not release Epic cache chunk " + guid); } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytes) {} - }); - r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error; - } catch (Throwable t) { - r.started = false; - r.error = t.getClass().getSimpleName() + ": " + t.getMessage(); + } + } + + @Override public void close() { + stopped.set(true); + for (HttpURLConnection connection : connections) connection.disconnect(); + workers.shutdownNow(); + boolean interrupted = false; + while (true) { + try { + if (workers.awaitTermination(100, TimeUnit.MILLISECONDS)) break; + for (HttpURLConnection connection : connections) connection.disconnect(); + } catch (InterruptedException e) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } + } + + static boolean isValidCachedChunk(ChunkInfo chunk, File file) { + if (!file.isFile()) return false; + if (chunk.windowSize > 0 && file.length() != chunk.windowSize) return false; + byte[] expected = chunk.sha1; + if (expected == null || expected.length != 20) return true; + boolean allZero = true; + for (byte b : expected) if (b != 0) { allZero = false; break; } + if (allZero) return true; + try (FileInputStream in = new FileInputStream(file)) { + MessageDigest sha = MessageDigest.getInstance("SHA-1"); + byte[] buffer = new byte[64 * 1024]; + int count; + while ((count = in.read(buffer)) != -1) sha.update(buffer, 0, count); + return MessageDigest.isEqual(sha.digest(), expected); + } catch (Exception e) { + return false; + } + } + + private static MessageDigest expectedDigest(byte[] expected) throws InstallException { + if (expected == null || expected.length != 20) return null; + for (byte b : expected) if (b != 0) { + try { return MessageDigest.getInstance("SHA-1"); } + catch (java.security.NoSuchAlgorithmException e) { throw new InstallException("SHA-1 verification unavailable"); } + } + return null; + } + + static Map chunkReferences(List files) { + Map refs = new LinkedHashMap<>(); + for (FileInfo file : files) for (ChunkPart part : file.parts) { + String guid = part.guidStr(); + refs.put(guid, refs.containsKey(guid) ? refs.get(guid) + 1 : 1); + } + return refs; + } + + /** + * Simulates reserved parallel prefetch → append part → last-reference eviction order. The returned + * additional-space peak is measured against the current free-space reading: existing verified + * chunks are already reflected in that reading and count as reclaimed only when actually freed. + * Old output bytes are reclaimed after the complete new file is staged and verified, never + * before its parts are written. The same schedule is used for scratch-cache game-volume checks. + */ + static long estimatePeakAdditionalBytes(List files, Map chunksByGuid, + Map cachedChunkBytes, boolean includeFileWrites) { + return estimatePeakAdditionalBytes(files, chunksByGuid, cachedChunkBytes, includeFileWrites, new LinkedHashMap<>()); + } + + static long estimatePeakAdditionalBytes(List files, Map chunksByGuid, + Map cachedChunkBytes, boolean includeFileWrites, + Map oldFileBytes) { + PrefetchSchedule schedule = new PrefetchSchedule(files, chunksByGuid, cachedChunkBytes); + long initialCache = schedule.bytes(); + long fileDelta = 0, peak = 0; + for (FileInfo file : files) { + for (ChunkPart part : file.parts) { + schedule.fill(); + peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache); + if (includeFileWrites) { + fileDelta += part.size & 0xFFFFFFFFL; + peak = Math.max(peak, fileDelta + schedule.bytes() - initialCache); + } + schedule.consume(); + } + // The verified staging file coexists with the old output until its atomic swap. + if (includeFileWrites) fileDelta -= oldFileBytes.containsKey(file.filename) ? oldFileBytes.get(file.filename) : 0; + } + return Math.max(0, peak); + } + + private static void copyChunkPart(File chunkFile, ChunkPart part, BufferedOutputStream out, + MessageDigest fileSha, AtomicBoolean cancel, String relPath) throws IOException { + long offset = part.offset & 0xFFFFFFFFL; + long remaining = part.size & 0xFFFFFFFFL; + long length = chunkFile.length(); + if (offset > length || remaining > length - offset) { + throw new IOException("Chunk " + part.guidStr() + " is too short for " + relPath); + } + try (FileInputStream in = new FileInputStream(chunkFile)) { + in.getChannel().position(offset); + byte[] buffer = new byte[64 * 1024]; + while (remaining > 0) { + if (cancel.get()) throw new IOException("cancelled"); + int count = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (count < 0) throw new IOException("Unexpected end of chunk " + part.guidStr()); + out.write(buffer, 0, count); + if (fileSha != null) fileSha.update(buffer, 0, count); + remaining -= count; + } } - return r; } @@ -605,14 +1374,19 @@ public final class EpicDownloadManager { } } - private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List cdnUrls, File outFile) { + private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List cdnUrls, + File outFile, AtomicBoolean cancel, AtomicBoolean stopped, + Set connections) { String chunkPath = chunk.getPath(chunkDir); File tmp = new File(outFile.getPath() + ".part"); tmp.delete(); for (CdnUrl cdn : cdnUrls) { + if (cancel.get() || stopped.get()) return false; HttpURLConnection conn = null; try { conn = (HttpURLConnection) new URL(cdn.baseUrl + cdn.cloudDir + "/" + chunkPath).openConnection(); + connections.add(conn); + if (cancel.get() || stopped.get()) return false; conn.setConnectTimeout(30000); conn.setReadTimeout(60000); conn.setRequestProperty("User-Agent", UA); @@ -635,12 +1409,14 @@ public final class EpicDownloadManager { int storedAs = hdr.get() & 0xFF; if (headerSize > 41) skipFully(in, headerSize - 41); byte[] iobuf = new byte[131072]; + long outputBytes = 0; if ((storedAs & 1) != 0) { Inflater inflater = new Inflater(); byte[] obuf = new byte[131072]; int remaining = compressedSize; try { while (remaining > 0 && !inflater.finished()) { + if (cancel.get() || stopped.get()) break; if (inflater.needsInput()) { int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); if (n <= 0) break; @@ -648,31 +1424,56 @@ public final class EpicDownloadManager { inflater.setInput(iobuf, 0, n); } int out = inflater.inflate(obuf); - if (out > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); } + if (out > 0) { + outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); + fos.write(obuf, 0, out); + if (sha != null) sha.update(obuf, 0, out); + } + if (out == 0 && !inflater.needsInput() && !inflater.finished()) { + throw new IOException("invalid compressed Epic chunk"); + } } int out; - while ((out = inflater.inflate(obuf)) > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); } + while ((out = inflater.inflate(obuf)) > 0) { + if (cancel.get() || stopped.get()) break; + outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); + fos.write(obuf, 0, out); + if (sha != null) sha.update(obuf, 0, out); + } + if (!cancel.get() && !stopped.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk"); } finally { inflater.end(); } } else { int remaining = compressedSize; while (remaining > 0) { + if (cancel.get() || stopped.get()) break; int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); if (n <= 0) break; + outputBytes = checkedChunkWriteSize(chunk, outputBytes, n); fos.write(iobuf, 0, n); if (sha != null) sha.update(iobuf, 0, n); remaining -= n; } + if (!cancel.get() && !stopped.get() && remaining != 0) throw new IOException("truncated Epic chunk"); } } + if (cancel.get() || stopped.get()) { tmp.delete(); conn.disconnect(); return false; } + if (chunk.windowSize > 0 && tmp.length() != chunk.windowSize) { tmp.delete(); conn.disconnect(); continue; } if (sha != null && !MessageDigest.isEqual(sha.digest(), chunk.sha1)) { tmp.delete(); conn.disconnect(); continue; } if (!tmp.renameTo(outFile)) { tmp.delete(); conn.disconnect(); continue; } conn.disconnect(); return true; } catch (Exception e) { + Log.w(TAG, "chunk " + chunk.guidStr() + " failed: " + e.getMessage()); tmp.delete(); if (conn != null) conn.disconnect(); + } finally { + if (conn != null) { + connections.remove(conn); + conn.disconnect(); + } + tmp.delete(); } } tmp.delete(); @@ -680,6 +1481,17 @@ public final class EpicDownloadManager { return false; } + private static long checkedChunkWriteSize(ChunkInfo chunk, long written, int count) throws IOException { + long next = written + count; + if (next > chunkWindowSize(chunk)) throw new IOException("chunk exceeds expected window size"); + return next; + } + + private static long chunkWindowSize(ChunkInfo chunk) { + // Legacy JSON manifests omit window sizes; their chunks use 1 MiB windows. + return chunk.windowSize > 0 ? chunk.windowSize : 1024L * 1024; + } + private static void readFully(InputStream in, byte[] buf) throws IOException { int off = 0; while (off < buf.length) { int n = in.read(buf, off, buf.length - off); if (n < 0) throw new IOException("stream ended"); off += n; } @@ -768,15 +1580,6 @@ public final class EpicDownloadManager { } - static byte[] readFile(File f) throws IOException { - try (FileInputStream fis = new FileInputStream(f)) { - byte[] data = new byte[(int) f.length()]; - int off = 0; - while (off < data.length) { int r = fis.read(data, off, data.length - off); if (r < 0) break; off += r; } - return data; - } - } - static void deleteDir(File dir) { if (!dir.exists()) return; File[] files = dir.listFiles(); @@ -784,20 +1587,6 @@ public final class EpicDownloadManager { dir.delete(); } - private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); - if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { - long prevB = lastSpeedBytes.getAndSet(done); - long dt = nowMs - prevMs; - if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); - } - } - - private static String speedSuffix(long bps) { - if (bps <= 0) return ""; - return " " + (bps >= 1048576 ? String.format("%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); - } - private static String fmt(long bytes) { if (bytes >= 1073741824L) return String.format("%.1f GB", bytes / 1073741824.0); if (bytes >= 1048576L) return String.format("%.1f MB", bytes / 1048576.0); diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt index 28510a8f17..41b6637c29 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt @@ -17,6 +17,30 @@ object EpicNative { fun onComplete(success: Boolean, error: String, bytes: Long) } + interface InstallListener : Listener { + /** Synchronous approval before the first verified atomic replacement. False aborts. */ + fun beforeSwap(): Boolean + } + + /** One native budgeted fetch/assembly run. A loaded-engine failure is never a Java retry. */ + @JvmStatic + fun installAdaptive(request: String, cancel: AtomicBoolean, listener: InstallListener): Result = + drive("nativeInstallAdaptive", cancel, listener, strict = true) { inner -> + nativeInstallAdaptive(request, inner) + } + + /** Callbacks from a [stream] run; every method is called on a native thread. */ + interface StreamListener { + /** File bytes written so far (staging included), and the files verified and swapped in so far. */ + fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) + /** Engine log line (already in logcat under `EpicNative`). */ + fun onLog(line: String) + /** Once, before the first file replaces its destination; false (or a throw) fails the run with nothing replaced. */ + fun onBeforeReplace(): Boolean + /** Terminal; `bytes` = file bytes written. */ + fun onComplete(success: Boolean, error: String, bytes: Long) + } + class Result( @JvmField val started: Boolean, @JvmField val success: Boolean, @JvmField val cancelled: Boolean, @JvmField val error: String, @JvmField val bytes: Long, @JvmField val done: Int, @JvmField val total: Int, @@ -36,13 +60,69 @@ object EpicNative { fun assemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, cancel: AtomicBoolean?, listener: Listener): Result = drive("nativeAssemble", cancel, listener) { inner -> nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, inner) } - private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, start: (Listener) -> Long): Result { + /** + * The adaptive install of the files [pendingFileIdx] (indices into the manifest's file list), + * with at most [cacheBudget] bytes of shared chunks cached and at most [overlapBudget] planned + * bytes of replaced files staged at once (`-1` = no limit), staged in [stagingDir] (the + * manager's owned staging directory). `started == false` means nothing was fetched or written. + */ + @JvmStatic + fun stream( + manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, + cacheBudget: Long, overlapBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, cancel: AtomicBoolean?, listener: StreamListener, + ): Result { + if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) + val latch = CountDownLatch(1) + val completion = AtomicReference(null) + val doneRef = AtomicReference(0) + val totalRef = AtomicReference(pendingFileIdx.size) + val inner = object : StreamListener { + override fun onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long) { + doneRef.set(filesDone); totalRef.set(filesTotal) + listener.onStreamProgress(bytesWritten, bytesTotal, filesDone, filesTotal, filesBytes) + } + override fun onLog(line: String) = listener.onLog(line) + override fun onBeforeReplace(): Boolean = runCatching { listener.onBeforeReplace() }.getOrDefault(false) + override fun onComplete(success: Boolean, error: String, bytes: Long) { + completion.set(Completion(success, error, bytes)) + latch.countDown() + runCatching { listener.onComplete(success, error, bytes) } + } + } + val handle: Long = try { + nativeStream(manifest, installDir, chunkCacheDir, stagingDir, cdnPrefixes, pendingFileIdx, expectedBytes, cacheBudget, overlapBudget, caBundlePath, maxWorkers, processWorkers, inner) + } catch (t: Throwable) { + Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}") + return Result(false, false, false, "nativeStream: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0) + } + if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) + var cancelSent = false + try { + while (!latch.await(250, TimeUnit.MILLISECONDS)) { + if (!cancelSent && cancel != null && cancel.get()) { + nativeCancel(handle) + cancelSent = true + } + } + } finally { + if (completion.get() == null) nativeCancel(handle) + nativeRelease(handle) + } + val c = completion.get() + val cancelled = cancelSent || (c != null && !c.success && c.error == "cancelled") + val success = !cancelled && c != null && c.success + val error = when { cancelled -> "cancelled"; c == null -> "no completion"; else -> c.error } + return Result(true, success, cancelled, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get()) + } + + private fun drive(what: String, cancel: AtomicBoolean?, listener: Listener, strict: Boolean = false, start: (Listener) -> Long): Result { if (!StoresNative.ensureLoaded()) return Result(false, false, false, "engine not built", 0L, 0, 0) val latch = CountDownLatch(1) val completion = AtomicReference(null) val doneRef = AtomicReference(0) val totalRef = AtomicReference(0) - val inner = object : Listener { + val inner = object : InstallListener { + override fun beforeSwap(): Boolean = (listener as? InstallListener)?.beforeSwap() ?: false override fun onPlan(chunksTotal: Int, bytesTotal: Long, chunkDir: String) { totalRef.set(chunksTotal); listener.onPlan(chunksTotal, bytesTotal, chunkDir) } override fun onProgress(bytesDone: Long, bytesTotal: Long, done: Int, total: Int) { doneRef.set(done); totalRef.set(total); listener.onProgress(bytesDone, bytesTotal, done, total) } override fun onLog(line: String) = listener.onLog(line) @@ -55,21 +135,26 @@ object EpicNative { val handle: Long = try { start(inner) } catch (t: Throwable) { - Log.w(TAG, "engine unavailable: ${t.javaClass.simpleName}: ${t.message}") - return Result(false, false, false, "$what: ${t.javaClass.simpleName}", 0L, 0, 0) + Log.w(TAG, "${if (strict) "engine start failed (no fallback)" else "engine unavailable"}: ${t.javaClass.simpleName}: ${t.message}") + return Result(strict, false, false, "$what: ${t.javaClass.simpleName}: ${t.message}", 0L, 0, 0) } - if (handle == 0L) return Result(false, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) + if (handle == 0L) return Result(strict, false, false, completion.get()?.error ?: "not started", 0L, 0, totalRef.get()) var cancelSent = false try { while (!latch.await(250, TimeUnit.MILLISECONDS)) { if (cancel != null && cancel.get()) { nativeCancel(handle) cancelSent = true - latch.await(5, TimeUnit.SECONDS) + if (strict) { + while (!latch.await(250, TimeUnit.MILLISECONDS)) { } + } else { + latch.await(5, TimeUnit.SECONDS) + } break } } } finally { + if (completion.get() == null) nativeCancel(handle) nativeRelease(handle) } val c = completion.get() @@ -78,6 +163,9 @@ object EpicNative { return Result(true, success, cancelSent, error, c?.bytes ?: 0L, doneRef.get(), totalRef.get()) } + @JvmStatic + private external fun nativeInstallAdaptive(request: String, listener: Listener): Long + @JvmStatic private external fun nativeStart( manifest: ByteArray, installDir: String, chunkCacheDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedChunks: Int, expectedBytes: Long, @@ -87,6 +175,12 @@ object EpicNative { @JvmStatic private external fun nativeAssemble(manifest: ByteArray, installDir: String, chunkCacheDir: String, fileIdx: IntArray, listener: Listener): Long + @JvmStatic + private external fun nativeStream( + manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, + cacheBudget: Long, overlapBudget: Long, caBundlePath: String, maxWorkers: Int, processWorkers: Int, listener: StreamListener, + ): Long + @JvmStatic private external fun nativeCancel(handle: Long) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java index 0bbbaa238a..b61af1e96c 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/gog/GogDownloadManager.java @@ -152,11 +152,12 @@ public final class GogDownloadManager { installPath.mkdirs(); long plannedBytes = 0; for (DepotFile df : files) plannedBytes += df.totalSize; - if (plannedBytes > 0) { - long usable = installPath.getUsableSpace(); - if (usable > 0 && plannedBytes > usable) { + long remainingBytes = remainingInstallBytes(files, installPath); + if (remainingBytes > 0) { + long usable = com.droiddeck.launcher.stores.StoreInstallRoot.availableBytes(installPath); + if (com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(remainingBytes, usable)) { out.hardStop = true; - out.error = "Not enough free space: need " + formatBytes(plannedBytes) + ", only " + formatBytes(usable) + " free"; + out.error = "Not enough free space: need " + formatBytes(remainingBytes) + ", only " + formatBytes(usable) + " free"; return out; } } @@ -946,7 +947,16 @@ public final class GogDownloadManager { for (File f : files) if (f.isDirectory()) collectExeRecursive(f, out); } - private static final class DepotFile { + static long remainingInstallBytes(List files, File installPath) { + long remaining = 0; + for (DepotFile file : files) { + File output = new File(installPath, file.relativePath); + if (!fileVerified(output, file.totalSize, file.md5)) remaining += file.totalSize; + } + return remaining; + } + + static final class DepotFile { final String relativePath; final List chunks = new ArrayList<>(); String md5; diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt index 1a3480f950..125e4f1082 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresDetail.kt @@ -51,7 +51,9 @@ internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: Fro val id = key.substringAfter(':') val item = (StoresState.library[store].orEmpty() + StoresState.shelves[store]?.all.orEmpty()).firstOrNull { it.id == id } val installed = StoresState.installedGame(store, id) - val download = StoresState.download("${store.id}:$id")?.takeIf { it.isActive } + val rawDownload = StoresState.download("${store.id}:$id") + val download = rawDownload?.takeIf { it.isActive } + val failedDownload = rawDownload?.takeIf { it.state == DownloadState.FAILED } val title = item?.title ?: installed?.sidecar?.title ?: id Rise(0) { Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp), modifier = Modifier.padding(bottom = 12.dp)) { @@ -73,12 +75,24 @@ internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: Fro val state = when { installed != null -> stringResource(R.string.stores_eyebrow_installed) + " · " + com.droiddeck.launcher.stores.StoreInstallRoot.labelFor(ctx, installed.folder) download != null && download.location.isNotBlank() -> stringResource(R.string.stores_eyebrow_library) + " · " + download.location + failedDownload != null && failedDownload.location.isNotBlank() -> stringResource(R.string.stores_eyebrow_library) + " · " + failedDownload.location item?.owned == true -> stringResource(R.string.stores_eyebrow_library) else -> stringResource(R.string.stores_eyebrow_store) } val size = (item?.sizeBytes ?: 0L).takeIf { it > 0 }?.let { " · " + formatBytes(it) } ?: "" Text((state + size).uppercase(), fontSize = 12.sp, fontWeight = FontWeight.SemiBold, letterSpacing = 1.2.sp, color = pal.signal) Text(title, fontSize = if (narrow) 24.sp else 30.sp, lineHeight = if (narrow) 27.sp else 33.sp, fontWeight = FontWeight.Black, color = colors.onBackground, maxLines = 2, overflow = TextOverflow.Ellipsis) + if (!failedDownload?.error.isNullOrBlank()) { + Text( + failedDownload!!.error!!, + fontSize = 13.sp, + lineHeight = 17.sp, + fontWeight = FontWeight.Medium, + color = colors.error, + maxLines = 3, + overflow = TextOverflow.Ellipsis + ) + } Actions { when { installed != null -> { @@ -95,6 +109,16 @@ internal fun StoreGameDetail(store: Store, key: String, s: FrontEndState, a: Fro else PrimaryButton(stringResource(R.string.stores_dl_resume), main = true) { com.droiddeck.launcher.stores.download.DownloadQueue.resume(ctx, download.key) } ConfirmButton(stringResource(R.string.stores_dl_cancel), stringResource(R.string.stores_dl_cancel_confirm)) { com.droiddeck.launcher.stores.download.DownloadQueue.cancel(ctx, download.key) } } + failedDownload != null -> { + ActionChip(stringResource(R.string.stores_dl_failed), ok = false) + PrimaryButton(stringResource(R.string.stores_dl_resume), main = true) { + if (item != null) StoresState.requestInstall(ctx, item) + else com.droiddeck.launcher.stores.download.DownloadQueue.retry(ctx, failedDownload.key) + } + ConfirmButton(stringResource(R.string.stores_dl_clear), stringResource(R.string.stores_dl_cancel_confirm), compact = true) { + com.droiddeck.launcher.stores.download.DownloadQueue.clear(ctx, failedDownload.key) + } + } item?.owned == true && StoresState.isUnfinished(item) -> { InstallButton(stringResource(R.string.stores_resume_install)) { StoresState.requestInstall(ctx, item) } ConfirmButton(stringResource(R.string.stores_dl_clear), stringResource(R.string.stores_dl_cancel_confirm), compact = true) { StoresState.clearUnfinished(ctx, item) } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt index d332ee0e55..d9f2d1417d 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresDownloads.kt @@ -305,6 +305,7 @@ private fun Modifier.timeRim(left: () -> Float, color: Color): Modifier = drawWi @Composable private fun DownloadCard(d: DownloadEntry, s: FrontEndState, a: FrontEndActions) { val ctx = LocalContext.current + val colors = MaterialTheme.colorScheme val pal = LocalPalette.current val narrow = LocalNarrowPane.current val ink = Color.White @@ -326,7 +327,11 @@ private fun DownloadCard(d: DownloadEntry, s: FrontEndState, a: FrontEndActions) val meta = when (d.state) { DownloadState.INSTALLED -> stringResource(R.string.stores_installed_chip) DownloadState.CANCELLED -> stringResource(R.string.stores_dl_cancelled) - DownloadState.FAILED -> if (d.bytesDone > 0) stringResource(R.string.stores_dl_failed_kept, formatBytes(d.bytesDone)) else stringResource(R.string.stores_dl_failed) + DownloadState.FAILED -> { + if (!d.error.isNullOrBlank()) stringResource(R.string.stores_dl_failed_reason, d.error) + else if (d.bytesDone > 0) stringResource(R.string.stores_dl_failed_kept, formatBytes(d.bytesDone)) + else stringResource(R.string.stores_dl_failed) + } DownloadState.PAUSED -> stringResource(R.string.stores_dl_paused) DownloadState.QUEUED -> if (d.queuePosition > 0) stringResource(R.string.stores_dl_queued_at, d.queuePosition) else stringResource(R.string.stores_dl_queued) DownloadState.RUNNING -> { @@ -338,7 +343,8 @@ private fun DownloadCard(d: DownloadEntry, s: FrontEndState, a: FrontEndActions) } } } - Text(meta, fontSize = 12.sp, color = dim, maxLines = 2, overflow = TextOverflow.Ellipsis, modifier = Modifier.weight(1f, fill = false)) + val metaColor = if (d.state == DownloadState.FAILED) colors.error else dim + Text(meta, fontSize = 12.sp, color = metaColor, maxLines = 2, overflow = TextOverflow.Ellipsis, modifier = Modifier.weight(1f, fill = false)) if (d.state == DownloadState.RUNNING && (d.stage == DownloadStage.DOWNLOAD || d.stage == DownloadStage.INSTALL) && d.speedBps > 0) { Text(formatSpeed(d.speedBps), fontSize = 12.sp, color = dim, maxLines = 1) if (d.etaSeconds >= 0) Text(eta(d.etaSeconds), fontSize = 12.sp, color = dim, maxLines = 1) @@ -355,7 +361,10 @@ private fun DownloadCard(d: DownloadEntry, s: FrontEndState, a: FrontEndActions) RoundAction(Icons.Filled.PlayArrow, stringResource(R.string.stores_dl_resume), primary = true) { DownloadQueue.resume(ctx, d.key) } CancelAction { DownloadQueue.cancel(ctx, d.key) } } - DownloadState.FAILED -> RoundAction(Icons.Filled.Refresh, stringResource(R.string.stores_dl_resume), primary = true) { DownloadQueue.retry(ctx, d.key) } + DownloadState.FAILED -> { + RoundAction(Icons.Filled.Refresh, stringResource(R.string.stores_dl_resume), primary = true) { DownloadQueue.retry(ctx, d.key) } + CancelAction { DownloadQueue.clear(ctx, d.key) } + } DownloadState.INSTALLED -> Box(Modifier.bloomIn(true, 420)) { RoundAction(Icons.Filled.PlayArrow, stringResource(R.string.stores_play), primary = true) { launchStoreGame(ctx, d.store, d.id, s, a) } } @@ -411,6 +420,7 @@ private fun StageBar(d: DownloadEntry) { val x = i * (segW + gap) drawRoundRect(colors.surfaceVariant, Offset(x, cy - h / 2f), Size(segW, h), CornerRadius(h / 2f)) val base = when { + d.state == DownloadState.FAILED -> colors.error done[i] -> pal.good actives[i] && paused -> pausedColor.copy(alpha = breath) actives[i] && stages[i] == DownloadStage.VERIFY -> verifyColor diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresInstallDialog.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresInstallDialog.kt index fb7f2d4ee8..a8392465d4 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresInstallDialog.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresInstallDialog.kt @@ -67,10 +67,21 @@ internal fun InstallWhereDialog(item: CatalogItem, onDismiss: () -> Unit) { Rise(1) { Column(verticalArrangement = Arrangement.spacedBy(8.dp), modifier = Modifier.fillMaxWidth()) { targets.forEachIndexed { i, t -> + val needed = com.droiddeck.launcher.stores.StoreInstallRoot.estimatedBytesNeeded(item) + val freeBytes = t.freeBytes + val isLow = com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(needed, freeBytes) + val freeText = if (freeBytes < 0L) { + stringResource(R.string.stores_target_free_unknown) + } else if (isLow) { + stringResource(R.string.stores_storage_low, formatBytes(freeBytes), formatBytes(needed)) + } else { + stringResource(R.string.stores_target_free, formatBytes(freeBytes)) + } TargetRow( label = if (t.removable) t.label else stringResource(R.string.stores_target_internal), - free = stringResource(R.string.stores_target_free, formatBytes(t.freeBytes)), + free = freeText, removable = t.removable, focus = focus[i], + warning = isLow, ) { SessionPrefs.setStoresInstallTarget(ctx, t.root.absolutePath) StoresState.install(ctx, item, t.root) @@ -83,7 +94,7 @@ internal fun InstallWhereDialog(item: CatalogItem, onDismiss: () -> Unit) { } @Composable -private fun TargetRow(label: String, free: String, removable: Boolean, focus: FocusRequester, onPick: () -> Unit) { +private fun TargetRow(label: String, free: String, removable: Boolean, focus: FocusRequester, warning: Boolean = false, onPick: () -> Unit) { val colors = MaterialTheme.colorScheme val pal = LocalPalette.current val src = remember { MutableInteractionSource() } @@ -99,6 +110,6 @@ private fun TargetRow(label: String, free: String, removable: Boolean, focus: Fo ) { Icon(if (removable) Icons.Outlined.SdCard else Icons.Outlined.Smartphone, null, tint = if (hot) pal.signal else colors.onBackground, modifier = Modifier.size(22.dp)) Text(label, fontSize = 14.sp, fontWeight = FontWeight.SemiBold, color = colors.onBackground, maxLines = 1, overflow = TextOverflow.Ellipsis, modifier = Modifier.weight(1f)) - Text(free, fontSize = 12.sp, color = colors.onSurfaceVariant, maxLines = 1) + Text(free, fontSize = 12.sp, color = if (warning) colors.error else colors.onSurfaceVariant, maxLines = 1) } } diff --git a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt index a070e01581..c0471d5605 100644 --- a/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt +++ b/app/src/main/java/com/droiddeck/launcher/ui/StoresPage.kt @@ -438,10 +438,20 @@ private fun InstallWhere(item: CatalogItem) { StepTitle(title) StepList { targets.forEachIndexed { i, t -> + val needed = com.droiddeck.launcher.stores.StoreInstallRoot.estimatedBytesNeeded(item) + val freeBytes = t.freeBytes + val isLow = com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(needed, freeBytes) + val trailingText = if (freeBytes < 0L) { + stringResource(R.string.stores_target_free_unknown) + } else if (isLow) { + stringResource(R.string.stores_storage_low, formatBytes(freeBytes), formatBytes(needed)) + } else { + stringResource(R.string.stores_target_free, formatBytes(freeBytes)) + } StepChoice( label = if (t.removable) t.label else internalLabel, icon = if (t.removable) androidx.compose.material.icons.Icons.Outlined.SdCard else androidx.compose.material.icons.Icons.Outlined.Smartphone, - trailing = stringResource(R.string.stores_target_free, formatBytes(t.freeBytes)), + trailing = trailingText, enabled = open, modifier = Modifier.fillMaxWidth().stepItem(1 + i).then(if (i == defaultIndex) Modifier.focusRequester(first) else Modifier), ) { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index cc33a651ff..457ef53e6f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2084,6 +2084,9 @@ Install %1$s: Internal %1$s free + Free space unknown + Not enough free space: need %1$s, only %2$s free + Insufficient space (%1$s free, need %2$s) Store downloads Shows game download progress and keeps downloads running in the background Downloading diff --git a/app/src/main/rust/stores/JNI.md b/app/src/main/rust/stores/JNI.md index 9f1b385504..9cd314dfd3 100644 --- a/app/src/main/rust/stores/JNI.md +++ b/app/src/main/rust/stores/JNI.md @@ -112,7 +112,45 @@ On-disk protocol (what a resumed run, by either engine, relies on): a file is as ## `com.droiddeck.launcher.stores.epic.EpicNative` -Two runs make up the native half of an Epic install: +The manager's default Epic install is the **stream** run (`nativeStream`, mode `adaptive`, +below): the chunks are fetched in parallel and written straight into staged files as they arrive; +on device it measured about 83 MB/s on a 71 GB install. The **lookahead** install +(`nativeInstallAdaptive`, mode `lookahead`) is its fallback when the stream engine's space plan +does not fit, and is selectable directly, as is the legacy batch pair (`nativeStart` + +`nativeAssemble`, mode `batch`), through the hidden `storesEpicMode` preference. + +### Lookahead install (snapshot schema v1) + +`nativeInstallAdaptive(request: String, listener: EpicNative.Listener): Long` +exports `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive`. +Its exact Kotlin descriptor is +`(Ljava/lang/String;Lcom/droiddeck/launcher/stores/epic/EpicNative$Listener;)J`. +The listener must also implement `EpicNative.InstallListener.beforeSwap(): Boolean` (`()Z`). +Rust invokes it synchronously after staging/hash/sync and before the first atomic replacement; +false or a Java exception aborts without replacing that output. Java marks the sidecar updating +there. Later verified swaps preserve mixed-update/resume semantics. + +The UTF-8 JSON request contains: + +- `schema_version: 1`, absolute `install_dir`, `cache_dir`, owned `staging_dir`, + `same_volume`, signed `free_game`/`free_cache` (`-1` unknown), `max_workers`, CDN `hosts`. +- `chunks`: distinct `guid`, relative CDN `path`, decoded `size`, `exact_size`, `wire_size`, + SHA-1 hex `sha1` (empty/all-zero means absent), `cached` verified length (`-1` absent). +- `files`: pending manifest-order relative `path`, `sha1`, current `old_size`, ordered `parts` + containing `guid`, unsigned `offset` and `size`. + +Parse, owned-namespace validation and adaptive storage planning happen synchronously. Errors +return zero after `onComplete(false, error, 0)` and are **terminal**, not fallback signals. +`onPlan` reports the selected lookahead/retention/peaks; `onProgress` counts staged file bytes +and verified swapped file counts, not fetched wire bytes. Fetch and staged assembly overlap +in one native run, using the same pure reservation/release policy as the peak planner. +Cancellation/release joins native workers; no worker survives the terminal callback/handle +release. Only `StoresNative.ensureLoaded() == false` selects bounded parallel Java; the same rule +holds for `nativeStream`. + +### Legacy batch APIs (mode `batch`, for A/B runs) + +Two explicit calls provide the legacy batch interface: 1. **Fetch** (`nativeStart`): fills the chunk cache with verified, decompressed chunks for the files the manager says are pending. The Kotlin manager still fetches and parses the manifest @@ -224,6 +262,57 @@ Bannerlator's `BlEpicDownload.run(...)` wraps `nativeStart` in a blocking call t `onComplete`, then `nativeRelease` in `finally`. The same wrapper shape fits `nativeAssemble` (cancel is honoured between files; a multi-GB file finishes before the run stops). +### `nativeStream` — the adaptive install + +| Method | JNI signature | Symbol | +|---|---|---| +| `nativeStream(...)`: Long | `([BLjava/lang/String;Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;[IJJJLjava/lang/String;IILcom/droiddeck/launcher/stores/epic/EpicNative$StreamListener;)J` | `Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream` | + +```kotlin +@JvmStatic private external fun nativeStream( + manifest: ByteArray, installDir: String, chunkCacheDir: String, stagingDir: String, + cdnPrefixes: Array, pendingFileIdx: IntArray, expectedBytes: Long, cacheBudget: Long, + overlapBudget: Long, caBundlePath: String, + maxWorkers: Int, processWorkers: Int, listener: StreamListener, +): Long +``` + +Parameters as `nativeStart`, plus `stagingDir` (the manager's owned +`/.droiddeck-epic-staging`, which must exist; `""` = stage beside each file), and +except `expectedBytes` (the manager's `Σ fileSize` of the pending +files, `-1` = no cross-check; no chunk count) and `cacheBudget` (the most bytes of shared chunks +the cache may hold; the manager sizes it from free space, `0` = never cache, refetch instead) and +`overlapBudget` (the most planned bytes of *replaced* files - destination already holds data - +staged at once, since each sits beside its old copy until its swap; one is always allowed so the +run progresses; `-1` = no limit). Files are admitted in order and a chunk is dispatched only once +the file of its first use is admitted; fresh files are never held back. +`chunkCacheDir` is `""` (`/.chunks`) or a scratch directory; the manager falls back to +`""` for a card install when internal storage cannot hold the budget. + +What the run does (`epic/stream.rs`): a chunk one file part uses is decompressed, verified and +written at the part's offset into a unique `stream-*.stage` file in `stagingDir`; a +chunk several parts use is also kept in the cache, decompressed, as `` (a later copy after +an eviction as `.`) and deleted after its last covered part. The plan evicts by furthest +next use when the budget would be exceeded and fetches the evicted chunk again later. Each file, +once complete: synced, size and SHA-1 checked, `onBeforeReplace()` once for the whole run, then +renamed over its destination. Cancel or failure removes every unfinished stage file before +`onComplete`, leaves the destinations and the cache; success removes the cache directory. + +Planning runs on the calling thread and also prepares the cache (keeps verified chunks this +install needs, deletes everything else, as the Java writer does). A refusal returns `0` after a +synchronous `onComplete(false, reason, 0)`; nothing was fetched or written, and the manager fails +the install with that reason (it does not switch engines once the library is loaded). + +| `StreamListener` method | JNI signature | When | +|---|---|---| +| `onStreamProgress(bytesWritten: Long, bytesTotal: Long, filesDone: Int, filesTotal: Int, filesBytes: Long)` | `(JJIIJ)V` | after written parts (at most every 50 ms) and after every finished file; `filesBytes` = sizes of the files swapped in | +| `onLog(line: String)` | `(Ljava/lang/String;)V` | engine lines: `stream-plan …`, the fetch core's lines, `stream-summary mode=adaptive …` (peak cache, peak staging, refetches, direct vs cached bytes) | +| `onBeforeReplace(): Boolean` | `()Z` | once, before the first file replaces its destination; `false` or a throw fails the run with nothing replaced | +| `onComplete(success: Boolean, error: String, bytes: Long)` | `(ZLjava/lang/String;J)V` | terminal, exactly once; `bytes` = file bytes written; `error` = `"cancelled"` on cancel | + +Unlike the batch runs, the Kotlin wrapper (`EpicNative.stream`) waits for `onComplete` after a +cancel without a time limit: the stage-file cleanup must be finished before the manager returns. + ## `com.droiddeck.launcher.stores.amazon.AmazonNative` The file-fetch pool of an Amazon Games install: one streamed HTTP GET per manifest file, SHA-256 diff --git a/app/src/main/rust/stores/src/epic.rs b/app/src/main/rust/stores/src/epic.rs index 41ac19767b..9e131391b2 100644 --- a/app/src/main/rust/stores/src/epic.rs +++ b/app/src/main/rust/stores/src/epic.rs @@ -1,11 +1,13 @@ //! Derived from GameNative's Epic manager (itself derived from Legendary); both are GPL-3.0 and this crate carries the same licence (see Cargo.toml). pub mod assemble; +pub mod adaptive; pub mod chunk; pub mod driver; pub mod jni; pub mod manifest; pub mod plan; +pub mod stream; pub const USER_AGENT: &str = "UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit"; diff --git a/app/src/main/rust/stores/src/epic/adaptive.rs b/app/src/main/rust/stores/src/epic/adaptive.rs new file mode 100644 index 0000000000..7113723eaa --- /dev/null +++ b/app/src/main/rust/stores/src/epic/adaptive.rs @@ -0,0 +1,1585 @@ +//! Budgeted native install: ordered lookahead, shared peak schedule, fetch-core overlap and +//! verified per-file atomic swaps. The JNI snapshot is the Java delta pass's exact pending set. + +use crate::fetch_core::{run_fetch, FetchItem, FetchOptions, FetchSink, SinkError}; +use serde_json::Value; +use sha1::{Digest, Sha1}; +use std::collections::{HashMap, HashSet}; +use std::fs::{self, OpenOptions}; +use std::io::{BufWriter, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Condvar, Mutex}; +use std::time::Duration; + +const MIB: u64 = 1024 * 1024; +const RETAINED: u64 = 64 * MIB; + +#[derive(Clone, Debug)] +pub struct Chunk { + guid: String, + path: String, + size: u64, + wire: u64, + sha: Option<[u8; 20]>, + cached: Option, + exact: bool, +} +#[derive(Clone, Debug)] +pub struct Part { + chunk: usize, + offset: u64, + size: u64, +} +#[derive(Clone, Debug)] +pub struct Output { + path: String, + sha: Option<[u8; 20]>, + old: u64, + parts: Vec, +} +#[derive(Clone, Debug)] +pub struct Request { + pub install: PathBuf, + pub cache: PathBuf, + pub staging: PathBuf, + pub same_volume: bool, + pub free_game: Option, + pub free_cache: Option, + pub chunks: Vec, + pub files: Vec, + pub hosts: Vec, + pub max_workers: usize, +} + +fn uint(v: &Value, key: &str) -> Result { + v[key].as_u64().ok_or_else(|| format!("invalid {key}")) +} +fn text(v: &Value, key: &str) -> Result { + v[key] + .as_str() + .map(str::to_string) + .ok_or_else(|| format!("invalid {key}")) +} +fn digest(v: &Value) -> Result, String> { + let s = v.as_str().ok_or("invalid sha1")?; + if s.is_empty() { + return Ok(None); + } + if s.len() != 40 || !s.is_ascii() { + return Err("invalid sha1".into()); + } + let mut out = [0; 20]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(&s[i * 2..i * 2 + 2], 16).map_err(|_| "invalid sha1")?; + } + Ok(out.iter().any(|b| *b != 0).then_some(out)) +} +fn relative(s: &str) -> Result<(), String> { + if s.is_empty() + || s.contains('\\') + || s.contains(':') + || Path::new(s) + .components() + .any(|c| !matches!(c, Component::Normal(_))) + { + return Err(format!("unsafe manifest path {s}")); + } + Ok(()) +} +impl Request { + pub fn parse(json: &str) -> Result { + let v: Value = serde_json::from_str(json).map_err(|e| e.to_string())?; + if uint(&v, "schema_version")? != 1 { + return Err("unsupported adaptive request schema".into()); + } + let mut chunks = Vec::new(); + let mut by_guid = HashMap::new(); + for c in v["chunks"].as_array().ok_or("missing chunks")? { + let guid = text(c, "guid")?; + if guid.len() != 32 + || !guid.bytes().all(|b| b.is_ascii_hexdigit()) + || by_guid.contains_key(&guid) + { + return Err("invalid or duplicate chunk GUID".into()); + } + let path = text(c, "path")?; + relative(&path)?; + by_guid.insert(guid.clone(), chunks.len()); + chunks.push(Chunk { + guid, + path, + size: uint(c, "size")?, + wire: uint(c, "wire_size")?, + sha: digest(&c["sha1"])?, + cached: c["cached"].as_u64(), + exact: c["exact_size"].as_bool().ok_or("missing exact_size")?, + }); + } + let mut files = Vec::new(); + for f in v["files"].as_array().ok_or("missing files")? { + let path = text(f, "path")?; + relative(&path)?; + if path.split('/').next() == Some(".droiddeck-epic-staging") { + return Err("manifest conflicts with owned staging".into()); + } + let mut parts = Vec::new(); + for p in f["parts"].as_array().ok_or("missing parts")? { + let chunk = *by_guid + .get(&text(p, "guid")?) + .ok_or("missing chunk metadata")?; + let offset = uint(p, "offset")?; + let size = uint(p, "size")?; + if offset + .checked_add(size) + .filter(|n| *n <= chunks[chunk].size) + .is_none() + { + return Err("part exceeds decoded window".into()); + } + parts.push(Part { + chunk, + offset, + size, + }); + } + files.push(Output { + path, + parts, + sha: digest(&f["sha1"])?, + old: uint(f, "old_size")?, + }); + } + let install = PathBuf::from(text(&v, "install_dir")?); + let staging = PathBuf::from(text(&v, "staging_dir")?); + Self::validate_staging(&install, &staging)?; + let hosts = v["hosts"] + .as_array() + .ok_or("missing hosts")? + .iter() + .map(|h| { + h.as_str() + .map(str::to_string) + .ok_or_else(|| "invalid host".to_string()) + }) + .collect::, _>>()?; + if hosts.is_empty() { + return Err("no Epic CDN".into()); + } + let cache = PathBuf::from(text(&v, "cache_dir")?); + let declared_same = v["same_volume"].as_bool().ok_or("missing same_volume")?; + let same_volume = declared_same || Self::same_filesystem(&install, &cache); + let game_probe = v["free_game"].as_u64(); + let cache_probe = v["free_cache"].as_u64(); + let free_game = if same_volume { + match (game_probe, cache_probe) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } + } else { + game_probe + }; + Ok(Self { + install, + staging, + cache, + chunks, + files, + hosts, + same_volume, + free_game, + free_cache: cache_probe, + max_workers: uint(&v, "max_workers")?.clamp(1, 32) as usize, + }) + } + + fn validate_staging(install: &Path, staging: &Path) -> Result<(), String> { + let expected = fs::canonicalize(install) + .map_err(|e| e.to_string())? + .join(".droiddeck-epic-staging"); + let metadata = fs::symlink_metadata(staging).map_err(|e| e.to_string())?; + if !metadata.file_type().is_dir() + || fs::canonicalize(staging).map_err(|e| e.to_string())? != expected + { + return Err("unsafe staging directory; refusing to traverse it".into()); + } + let owner = staging.join("owner"); + let metadata = fs::symlink_metadata(&owner).map_err(|e| e.to_string())?; + if !metadata.file_type().is_file() + || metadata.len() != 26 + || fs::canonicalize(&owner).map_err(|e| e.to_string())? != expected.join("owner") + || fs::read(&owner).map_err(|e| e.to_string())? != b"DroidDeck Epic staging v1\n" + { + return Err("unowned staging directory".into()); + } + for entry in fs::read_dir(staging).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + if name == "owner" { + continue; + } + if !entry.file_type().map_err(|e| e.to_string())?.is_file() + || !name.to_string_lossy().ends_with(".stage") + || fs::canonicalize(entry.path()).map_err(|e| e.to_string())? + != expected.join(&name) + { + return Err("unsafe staging entry".into()); + } + } + Ok(()) + } + + #[cfg(unix)] + fn same_filesystem(a: &Path, b: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + match (fs::metadata(a), fs::metadata(b)) { + (Ok(a), Ok(b)) => a.dev() == b.dev(), + _ => false, + } + } + #[cfg(not(unix))] + fn same_filesystem(_a: &Path, _b: &Path) -> bool { + false + } + fn total(&self) -> u64 { + self.files + .iter() + .flat_map(|f| &f.parts) + .map(|p| p.size) + .sum() + } +} + +/// Resident includes full-size reservations for in-flight writes; only ready chunks can be read. +#[derive(Clone, Debug)] +struct Schedule { + order: Vec, + cursor: usize, + refs: Vec, + resident: Vec>, + oldest: Vec, + lookahead: usize, + retained: u64, +} +impl Schedule { + fn new(req: &Request, lookahead: usize, retained: u64) -> Self { + let order: Vec<_> = req + .files + .iter() + .flat_map(|f| f.parts.iter().map(|p| p.chunk)) + .collect(); + let mut refs = vec![0; req.chunks.len()]; + for &c in &order { + refs[c] += 1; + } + let resident: Vec<_> = req.chunks.iter().map(|c| c.cached).collect(); + let oldest = (0..resident.len()) + .filter(|&c| resident[c].is_some()) + .collect(); + Self { + order, + cursor: 0, + refs, + resident, + oldest, + lookahead, + retained, + } + } + fn window(&self) -> HashSet { + self.order[self.cursor..self.order.len().min(self.cursor + self.lookahead)] + .iter() + .copied() + .collect() + } + fn fill(&mut self, req: &Request) -> Vec { + let mut fetch = Vec::new(); + for &c in &self.order[self.cursor..self.order.len().min(self.cursor + self.lookahead)] { + if self.resident[c].is_none() { + self.resident[c] = Some(req.chunks[c].size); + self.oldest.push(c); + fetch.push(c); + } + } + fetch + } + fn consume(&mut self) -> Vec { + let c = self.order[self.cursor]; + self.cursor += 1; + self.refs[c] -= 1; + let mut release = Vec::new(); + if self.refs[c] == 0 { + self.resident[c] = None; + release.push(c); + } + let pinned = self.window(); + let mut reusable: u64 = self + .oldest + .iter() + .filter(|c| !pinned.contains(c)) + .map(|&c| self.resident[c].unwrap_or(0)) + .sum(); + for &c in &self.oldest { + if reusable <= self.retained { + break; + } + if pinned.contains(&c) { + continue; + } + if let Some(bytes) = self.resident[c].take() { + reusable -= bytes; + release.push(c); + } + } + self.oldest.retain(|&c| self.resident[c].is_some()); + release + } + fn bytes(&self) -> u64 { + self.oldest + .iter() + .map(|&c| self.resident[c].unwrap_or(0)) + .sum() + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Budget { + pub lookahead: usize, + pub retained: u64, + pub game_peak: u64, + pub cache_peak: u64, +} +fn peaks(req: &Request, lookahead: usize, retained: u64) -> (u64, u64) { + let mut s = Schedule::new(req, lookahead, retained); + let initial = s.bytes() as i128; + let mut game = 0i128; + let mut game_peak = 0i128; + let mut cache_peak = 0i128; + for file in &req.files { + for part in &file.parts { + s.fill(req); + let cache = s.bytes() as i128 - initial; + cache_peak = cache_peak.max(cache); + game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); + game += part.size as i128; + game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); + s.consume(); + } + game -= file.old as i128; + } + (game_peak.max(0) as u64, cache_peak.max(0) as u64) +} +fn fits(needed: u64, free: Option) -> bool { + free.is_none_or(|n| needed <= n) +} +pub fn choose_budget(req: &Request) -> Result { + // Unknown probes deliberately choose a conservative bounded mode, never "0 free". + let unknown = req.free_game.is_none() || (!req.same_volume && req.free_cache.is_none()); + for lookahead in [128, 64, 32, 16, 8, 4, 2, 1] { + if unknown && lookahead > 8 { + continue; + } + for retained in [RETAINED, 16 * MIB, 0] { + let (game_peak, cache_peak) = peaks(req, lookahead, retained); + if fits(game_peak, req.free_game) + && (req.same_volume || fits(cache_peak, req.free_cache)) + { + return Ok(Budget { + lookahead, + retained, + game_peak, + cache_peak, + }); + } + } + } + let (game, cache) = peaks(req, 1, 0); + Err(format!("Not enough free space for safe staged Epic replacement: minimum game additional={game}, cache additional={cache}; free game={:?}, cache={:?}", req.free_game, req.free_cache)) +} + +fn remaining_peaks( + req: &Request, + schedule: &Schedule, + fi: usize, + pi: usize, + disk_cache: u64, +) -> (u64, u64) { + let mut s = schedule.clone(); + let mut game = 0i128; + let mut game_peak = 0i128; + let mut cache_peak = 0i128; + let outstanding = s.bytes() as i128 - disk_cache as i128; + cache_peak = cache_peak.max(outstanding); + if req.same_volume { + game_peak = game_peak.max(outstanding); + } + for (index, file) in req.files.iter().enumerate().skip(fi) { + for part in file.parts.iter().skip(if index == fi { pi } else { 0 }) { + s.fill(req); + let cache = s.bytes() as i128 - disk_cache as i128; + cache_peak = cache_peak.max(cache); + game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); + game += part.size as i128; + game_peak = game_peak.max(game + if req.same_volume { cache } else { 0 }); + s.consume(); + } + game -= file.old as i128; + } + (game_peak.max(0) as u64, cache_peak.max(0) as u64) +} + +#[cfg(unix)] +fn available(path: &Path) -> Option { + use std::os::unix::ffi::OsStrExt; + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?; + let mut stat = std::mem::MaybeUninit::::uninit(); + if unsafe { libc::statvfs(path.as_ptr(), stat.as_mut_ptr()) } != 0 { + return None; + } + let stat = unsafe { stat.assume_init() }; + Some((stat.f_bavail as u64).saturating_mul(stat.f_frsize as u64)) +} +#[cfg(not(unix))] +fn available(_path: &Path) -> Option { + None +} + +/// Re-probe at admission boundaries. Already dispatched reservations cannot be withdrawn. +/// Shrink only unsent lookahead/reusable budget; never wait forever for unrelated space use. +fn live_admit( + req: &Request, + state: &mut State, + fi: usize, + pi: usize, + stage_remaining: u64, +) -> Result<(), String> { + let disk_cache = state + .schedule + .oldest + .iter() + .map(|&index| { + if state.ready[index] { + return state.disk_sizes[index]; + } + if !state.writing[index] { + return 0; + } + let c = &req.chunks[index]; + fs::metadata(req.cache.join(&c.guid)) + .map(|m| m.len()) + .unwrap_or(0) + + fs::metadata(super::chunk::part_path(&req.cache.join(&c.guid))) + .map(|m| m.len()) + .unwrap_or(0) + }) + .sum(); + let free_game = available(&req.install); + let free_cache = available(&req.cache); + admit_with_probes( + req, + state, + fi, + pi, + stage_remaining, + disk_cache, + free_game, + free_cache, + ) +} + +fn admit_with_probes( + req: &Request, + state: &mut State, + fi: usize, + pi: usize, + stage_remaining: u64, + disk_cache: u64, + free_game: Option, + free_cache: Option, +) -> Result<(), String> { + loop { + let incoming: u64 = state + .schedule + .window() + .iter() + .filter(|&&c| state.schedule.resident[c].is_none()) + .map(|&c| req.chunks[c].size) + .sum(); + let cache_now = state + .schedule + .bytes() + .saturating_add(incoming) + .saturating_sub(disk_cache); + let game_now = stage_remaining.saturating_add(if req.same_volume { cache_now } else { 0 }); + if fits(game_now, free_game) && (req.same_volume || fits(cache_now, free_cache)) { + return Ok(()); + } + // Full remaining simulation is only needed when live admission headroom is exhausted. + let (game, cache) = remaining_peaks(req, &state.schedule, fi, pi, disk_cache); + if fits(game, free_game) && (req.same_volume || fits(cache, free_cache)) { + return Ok(()); + } + if state.schedule.oldest.iter().any(|&c| !state.ready[c]) { + return Err("Not enough free space for already reserved Epic downloads".into()); + } + if state.schedule.retained > 0 { + state.schedule.retained = 0; + continue; + } + if state.schedule.lookahead > 1 { + state.schedule.lookahead /= 2; + continue; + } + return Err(format!("Not enough free space during Epic install: need additional game={game} cache={cache}, free game={free_game:?} cache={free_cache:?}")); + } +} + +struct State { + schedule: Schedule, + ready: Vec, + writing: Vec, + disk_sizes: Vec, + pending: Vec, + error: Option, + done: bool, +} +struct Shared { + state: Mutex, + changed: Condvar, + stop: AtomicBool, +} +struct Sink<'a> { + req: &'a Request, + shared: &'a Shared, +} +impl FetchSink for Sink<'_> { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + if self.shared.stop.load(Ordering::Acquire) { + return Err(SinkError::Fatal("cancelled".into())); + } + let c = item.id as usize; + let chunk = &self.req.chunks[c]; + self.shared.state.lock().unwrap().writing[c] = true; + let written = super::chunk::write_verified_chunk_bounded( + &body, + chunk.sha.as_ref(), + &self.req.cache.join(&chunk.guid), + chunk.size, + chunk.exact, + ); + let mut state = self.shared.state.lock().unwrap(); + state.writing[c] = false; + let written = written.map_err(SinkError::Retry)?; + state.ready[c] = true; + state.disk_sizes[c] = written; + self.shared.changed.notify_all(); + Ok(chunk.wire) + } +} + +/// JNI owns this entire blocking run. All scoped workers join before completion is signalled. +pub fn run( + req: &Request, + budget: Budget, + cancel: &AtomicBool, + before_swap: &(dyn Fn() -> Result<(), String> + Sync), + progress: &(dyn Fn(u64, u64, u64, u64) + Sync), + log: &(dyn Fn(&str) + Sync), +) -> Result { + let shared = Shared { + state: Mutex::new(State { + schedule: Schedule::new(req, budget.lookahead, budget.retained), + ready: req.chunks.iter().map(|c| c.cached.is_some()).collect(), + writing: vec![false; req.chunks.len()], + disk_sizes: req.chunks.iter().map(|c| c.cached.unwrap_or(0)).collect(), + pending: Vec::new(), + error: None, + done: false, + }), + changed: Condvar::new(), + stop: AtomicBool::new(false), + }; + log(&format!( + "adaptive-native lookahead={} retained={} game_peak={} cache_peak={} workers={}", + budget.lookahead, + budget.retained, + budget.game_peak, + budget.cache_peak, + req.max_workers.min(budget.lookahead) + )); + let result = std::thread::scope(|scope| { + scope.spawn(|| { + crate::priority::background(); + let sink = Sink { + req, + shared: &shared, + }; + loop { + let batch = { + let mut state = shared.state.lock().unwrap(); + while state.pending.is_empty() + && !state.done + && !shared.stop.load(Ordering::Acquire) + { + state = shared + .changed + .wait_timeout(state, Duration::from_millis(100)) + .unwrap() + .0; + } + if state.done || shared.stop.load(Ordering::Acquire) { + break; + } + std::mem::take(&mut state.pending) + }; + let items = batch + .into_iter() + .map(|c| FetchItem { + id: c as u64, + urls: req + .hosts + .iter() + .map(|h| format!("{h}/{}", req.chunks[c].path)) + .collect(), + reserve: req.chunks[c].wire, + range: None, + }) + .collect(); + let opts = FetchOptions { + max_workers: req.max_workers.min(budget.lookahead), + per_host_cap: super::plan::per_host_cap( + req.max_workers.min(budget.lookahead), + req.hosts.len(), + ), + timeout: Duration::from_secs(60), + process_workers: 2, + headers: vec![("User-Agent".into(), super::USER_AGENT.into())], + label: "epic adaptive".into(), + ..Default::default() + }; + let fetched = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + run_fetch( + items, + &req.hosts, + &opts, + &sink, + &shared.stop, + &|_, _| {}, + log, + ) + })); + let outcome = match fetched { + Ok(outcome) => outcome, + Err(_) => { + let mut state = shared.state.lock().unwrap(); + state.error = Some("native fetch panicked".into()); + shared.stop.store(true, Ordering::Release); + shared.changed.notify_all(); + break; + } + }; + if !outcome.ok() { + let mut state = shared.state.lock().unwrap(); + state.error = Some(outcome.error.unwrap_or_else(|| "cancelled".into())); + shared.stop.store(true, Ordering::Release); + shared.changed.notify_all(); + break; + } + } + }); + // Cancel forwarding is independent of a writer blocked on disk I/O. + scope.spawn(|| { + while !shared.stop.load(Ordering::Acquire) { + if cancel.load(Ordering::Acquire) { + shared.stop.store(true, Ordering::Release); + shared.changed.notify_all(); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + }); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + assemble(req, &shared, cancel, before_swap, progress) + })) + .unwrap_or_else(|_| Err("native assembly panicked".into())); + shared.stop.store(true, Ordering::Release); + { + let mut state = shared.state.lock().unwrap(); + state.done = true; + shared.changed.notify_all(); + } + result + }); + // No .part writer survives this point; preserve verified resident chunks for resume. + let resident = shared.state.lock().unwrap().schedule.oldest.clone(); + for index in resident { + let _ = fs::remove_file(super::chunk::part_path( + &req.cache.join(&req.chunks[index].guid), + )); + } + if result.is_ok() { + fs::remove_dir_all(&req.cache).map_err(|e| format!("clear cache: {e}"))?; + } + result +} + +fn await_chunk( + req: &Request, + shared: &Shared, + cancel: &AtomicBool, + c: usize, + fi: usize, + pi: usize, + stage_remaining: u64, +) -> Result<(), String> { + let mut state = shared.state.lock().unwrap(); + if cancel.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + live_admit(req, &mut state, fi, pi, stage_remaining)?; + let fetch = state.schedule.fill(req); + state.pending.extend(fetch); + shared.changed.notify_all(); + while !state.ready[c] { + if cancel.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + if let Some(error) = &state.error { + return Err(error.clone()); + } + if shared.stop.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + state = shared + .changed + .wait_timeout(state, Duration::from_millis(100)) + .unwrap() + .0; + } + Ok(()) +} + +fn assemble( + req: &Request, + shared: &Shared, + cancel: &AtomicBool, + before_swap: &(dyn Fn() -> Result<(), String> + Sync), + progress: &(dyn Fn(u64, u64, u64, u64) + Sync), +) -> Result { + let mut done = 0; + let mut marked = false; + let total = req.total(); + for (fi, file) in req.files.iter().enumerate() { + Request::validate_staging(&req.install, &req.staging)?; + if cancel.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + let output = req.install.join(&file.path); + fs::create_dir_all(output.parent().ok_or("missing output parent")?) + .map_err(|e| e.to_string())?; + let stage = req + .staging + .join(format!("native-{}-{fi}.stage", std::process::id())); + let result = (|| { + let stage_file = OpenOptions::new() + .write(true) + .create_new(true) + .open(&stage) + .map_err(|e| e.to_string())?; + let mut out = BufWriter::with_capacity(64 * 1024, stage_file); + let mut sha = Sha1::new(); + let mut written = 0; + let file_total: u64 = file.parts.iter().map(|p| p.size).sum(); + for (pi, part) in file.parts.iter().enumerate() { + // A live free-space probe must include already buffered staging bytes. + out.flush().map_err(|e| e.to_string())?; + await_chunk( + req, + shared, + cancel, + part.chunk, + fi, + pi, + file_total - written, + )?; + written += super::assemble::copy_chunk_part( + &req.cache.join(&req.chunks[part.chunk].guid), + part.offset, + part.size, + &mut out, + &mut sha, + cancel, + )?; + let mut state = shared.state.lock().unwrap(); + for c in state.schedule.consume() { + if !state.ready[c] { + return Err("attempt to release in-flight chunk".into()); + } + fs::remove_file(req.cache.join(&req.chunks[c].guid)) + .map_err(|e| e.to_string())?; + state.ready[c] = false; + } + done += part.size; + progress(done, total, fi as u64, req.files.len() as u64); + } + out.flush().map_err(|e| e.to_string())?; + out.get_ref().sync_all().map_err(|e| e.to_string())?; + drop(out); + if written != file_total { + return Err("file size mismatch".into()); + } + if file + .sha + .is_some_and(|expected| sha.finalize().as_slice() != expected) + { + return Err(format!("SHA-1 mismatch for {}", file.path)); + } + if cancel.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + if !marked { + before_swap()?; + marked = true; + } + if cancel.load(Ordering::Acquire) { + return Err("cancelled".into()); + } + // Android/Linux rename replaces atomically on the same output volume. Never delete first. + fs::rename(&stage, &output) + .map_err(|e| format!("atomic replacement {}: {e}", file.path))?; + progress(done, total, fi as u64 + 1, req.files.len() as u64); + Ok(()) + })(); + let _ = fs::remove_file(&stage); + result?; + } + Ok(done) +} + +#[cfg(test)] +mod tests { + use super::super::chunk::test_support::{build_chunk_body, sha1_of}; + use super::*; + use std::io::Read; + use std::net::TcpListener; + use std::sync::atomic::{AtomicU64, AtomicUsize}; + use std::sync::{Arc, Barrier}; + use std::thread; + use std::time::{Instant, SystemTime, UNIX_EPOCH}; + + static SEQUENCE: AtomicU64 = AtomicU64::new(0); + struct Fixture { + root: PathBuf, + req: Request, + } + impl Fixture { + fn new(count: usize, size: usize) -> Self { + let root = PathBuf::from("target").join(format!( + "epic-adaptive-{}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(), + SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let install = root.join("game"); + let cache = root.join("cache"); + let staging = install.join(".droiddeck-epic-staging"); + fs::create_dir_all(&staging).unwrap(); + fs::create_dir_all(&cache).unwrap(); + fs::write(staging.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); + let data = vec![42; size]; + let sha = sha1_of(&data); + let chunks: Vec<_> = (0..count) + .map(|c| Chunk { + guid: format!("{c:032X}"), + path: format!("chunks/{c}"), + size: size as u64, + wire: size as u64 + 41, + sha: Some(sha), + cached: None, + exact: true, + }) + .collect(); + let files = (0..count) + .map(|c| Output { + path: format!("file-{c}.bin"), + sha: Some(sha), + old: 0, + parts: vec![Part { + chunk: c, + offset: 0, + size: size as u64, + }], + }) + .collect(); + Self { + root, + req: Request { + install, + cache, + staging, + chunks, + files, + same_volume: false, + free_game: Some(u64::MAX), + free_cache: Some(u64::MAX), + hosts: vec!["http://127.0.0.1:1".into()], + max_workers: 8, + }, + } + } + } + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } + } + + struct Server { + address: String, + requests: Arc, + active: Arc, + stop: Arc, + worker: Option>, + } + impl Server { + fn start(data: Vec, barrier_count: usize, stalled: bool) -> Self { + Self::start_mode(data, barrier_count, stalled.then_some(usize::MAX)) + } + + fn start_mode(data: Vec, barrier_count: usize, stall_chunk: Option) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + listener.set_nonblocking(true).unwrap(); + let requests = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let requests2 = requests.clone(); + let active2 = active.clone(); + let stop2 = stop.clone(); + let worker = thread::spawn(move || { + let body = Arc::new(build_chunk_body(&data, false, 0, None)); + let barrier = Arc::new(Barrier::new(barrier_count.max(1))); + let mut handlers = Vec::new(); + while !stop2.load(Ordering::Acquire) { + let (mut socket, _) = match listener.accept() { + Ok(s) => s, + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + continue; + } + Err(e) => panic!("{e}"), + }; + let body = body.clone(); + let requests = requests2.clone(); + let active = active2.clone(); + let stop = stop2.clone(); + let barrier = barrier.clone(); + handlers.push(thread::spawn(move || { + active.fetch_add(1, Ordering::Relaxed); + let _ = socket.set_read_timeout(Some(Duration::from_secs(5))); + let mut request = Vec::new(); + let mut byte = [0]; + while socket.read(&mut byte).unwrap_or(0) > 0 { + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + let n = requests.fetch_add(1, Ordering::Relaxed); + if n < barrier_count { + barrier.wait(); + } + let _ = write!( + socket, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let stalled = stall_chunk.is_some_and(|c| { + c == usize::MAX + || String::from_utf8_lossy(&request) + .contains(&format!("/chunks/{c} ")) + }); + if stalled { + // Cancellation must close the native client's connection without data. + let _ = socket.set_read_timeout(Some(Duration::from_millis(100))); + loop { + if stop.load(Ordering::Acquire) { + break; + } + match socket.read(&mut byte) { + Ok(0) => break, + Ok(_) => {} + Err(e) + if e.kind() == std::io::ErrorKind::WouldBlock + || e.kind() == std::io::ErrorKind::TimedOut => {} + Err(_) => break, + } + } + } else { + let _ = socket.write_all(&body); + } + active.fetch_sub(1, Ordering::Relaxed); + })); + } + for handler in handlers { + handler.join().unwrap(); + } + }); + Self { + address, + requests, + active, + stop, + worker: Some(worker), + } + } + } + impl Drop for Server { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(worker) = self.worker.take() { + worker.join().unwrap(); + } + } + } + + #[test] + fn situation_table_selects_large_tight_minimal_and_rejects_no_fit() { + let mut f = Fixture::new(300, MIB as usize); + assert_eq!(choose_budget(&f.req).unwrap().lookahead, 128); + f.req.free_cache = Some(8 * MIB); + assert_eq!(choose_budget(&f.req).unwrap().lookahead, 8); + f.req.free_cache = Some(MIB); + assert_eq!(choose_budget(&f.req).unwrap().lookahead, 1); + f.req.free_cache = Some(MIB - 1); + assert!(choose_budget(&f.req) + .unwrap_err() + .contains("Not enough free space")); + f.req.free_cache = None; + assert_eq!(choose_budget(&f.req).unwrap().lookahead, 8); + f.req.free_game = Some(0); + assert!(choose_budget(&f.req).is_err()); + } + + #[test] + fn planner_counts_all_incoming_old_outputs_and_existing_cache_release() { + let mut f = Fixture::new(20, 100); + f.req.same_volume = true; + for file in &mut f.req.files { + file.old = 100; + } + assert_eq!(peaks(&f.req, 8, 0), (900, 800)); + for chunk in &mut f.req.chunks { + chunk.cached = Some(100); + } + assert_eq!(peaks(&f.req, 8, 0), (100, 0)); + f.req.same_volume = false; + assert_eq!(peaks(&f.req, 8, 0), (100, 0)); + } + + #[test] + fn shared_references_deduplicate_and_distant_chunks_refetch_under_budget() { + let mut f = Fixture::new(12, (40 * MIB) as usize); + f.req.files.extend(f.req.files.clone()); + let mut schedule = Schedule::new(&f.req, 8, RETAINED); + assert_eq!(schedule.fill(&f.req), (0..8).collect::>()); + assert!(schedule.fill(&f.req).is_empty()); + assert!(schedule.consume().is_empty()); + assert_eq!(schedule.fill(&f.req), vec![8]); + assert_eq!(schedule.consume(), vec![0]); + let mut refetched = false; + for _ in 2..24 { + refetched |= schedule.fill(&f.req).contains(&0); + assert!(schedule.bytes() <= RETAINED + 8 * 40 * MIB); + let removed = schedule.consume(); + assert!(removed.iter().all(|c| !schedule.window().contains(c))); + } + assert!(refetched); + assert_eq!(schedule.bytes(), 0); + } + + #[test] + fn native_fetch_overlaps_across_files_and_never_fetches_outside_window() { + let mut f = Fixture::new(30, 4096); + let server = Server::start(vec![42; 4096], 8, false); + f.req.hosts = vec![server.address.clone()]; + f.req.free_cache = Some(8 * 4096); + let budget = choose_budget(&f.req).unwrap(); + assert_eq!(budget.lookahead, 8); + let cancel = AtomicBool::new(false); + let swapped = AtomicUsize::new(0); + let progress = |_, _, files, _| { + if files == 0 { + cancel.store(true, Ordering::Release); + } + }; + let result = run( + &f.req, + budget, + &cancel, + &|| { + swapped.fetch_add(1, Ordering::Relaxed); + Ok(()) + }, + &progress, + &|_| {}, + ); + assert!(result.is_err()); + assert_eq!(server.requests.load(Ordering::Relaxed), 8); + assert_eq!(swapped.load(Ordering::Relaxed), 0); + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + assert!(fs::read_dir(&f.req.cache).unwrap().all(|e| !e + .unwrap() + .path() + .to_string_lossy() + .ends_with(".part"))); + } + + #[test] + fn native_success_stages_verifies_swaps_and_reclaims_both_cache_layouts() { + for same_volume in [false, true] { + let mut f = Fixture::new(20, 4096); + let server = Server::start(vec![42; 4096], 8, false); + f.req.hosts = vec![server.address.clone()]; + f.req.same_volume = same_volume; + if same_volume { + f.req.cache = f.req.install.join(".chunks"); + fs::create_dir_all(&f.req.cache).unwrap(); + } + for file in &mut f.req.files { + fs::write(f.req.install.join(&file.path), b"old").unwrap(); + file.old = 3; + } + let swaps = AtomicUsize::new(0); + let before = || { + assert_eq!(fs::read(f.req.install.join("file-0.bin")).unwrap(), b"old"); + swaps.fetch_add(1, Ordering::Relaxed); + Ok(()) + }; + assert_eq!( + run( + &f.req, + choose_budget(&f.req).unwrap(), + &AtomicBool::new(false), + &before, + &|_, _, _, _| {}, + &|_| {} + ) + .unwrap(), + 20 * 4096 + ); + assert_eq!(swaps.load(Ordering::Relaxed), 1); + assert!(!f.req.cache.exists()); + for file in &f.req.files { + assert_eq!( + fs::read(f.req.install.join(&file.path)).unwrap(), + vec![42; 4096] + ); + } + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + } + } + + #[test] + fn native_file_hash_failure_and_updating_failure_keep_old_output() { + for updating_failure in [false, true] { + let mut f = Fixture::new(1, 4096); + let server = Server::start(vec![42; 4096], 1, false); + f.req.hosts = vec![server.address.clone()]; + fs::write(f.req.install.join("file-0.bin"), b"old").unwrap(); + if !updating_failure { + f.req.files[0].sha = Some([1; 20]); + } + let before = || Err("updating record failed".into()); + let result = run( + &f.req, + choose_budget(&f.req).unwrap(), + &AtomicBool::new(false), + &before, + &|_, _, _, _| {}, + &|_| {}, + ); + assert!(result.is_err()); + assert_eq!(fs::read(f.req.install.join("file-0.bin")).unwrap(), b"old"); + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + } + } + + #[test] + fn native_cancellation_closes_stalled_parallel_requests_and_joins_before_return() { + let mut f = Fixture::new(30, 4096); + let server = Server::start(vec![42; 4096], 8, true); + f.req.hosts = vec![server.address.clone()]; + f.req.free_cache = Some(8 * 4096); + let cancel = AtomicBool::new(false); + thread::scope(|scope| { + scope.spawn(|| { + let until = Instant::now() + Duration::from_secs(5); + while server.requests.load(Ordering::Relaxed) < 8 && Instant::now() < until { + thread::sleep(Duration::from_millis(10)); + } + cancel.store(true, Ordering::Release); + }); + let started = Instant::now(); + assert!(run( + &f.req, + choose_budget(&f.req).unwrap(), + &cancel, + &|| Ok(()), + &|_, _, _, _| {}, + &|_| {} + ) + .is_err()); + assert!(started.elapsed() < Duration::from_secs(7)); + }); + let until = Instant::now() + Duration::from_secs(2); + while server.active.load(Ordering::Relaxed) > 0 && Instant::now() < until { + thread::sleep(Duration::from_millis(10)); + } + assert_eq!(server.active.load(Ordering::Relaxed), 0); + assert_eq!(server.requests.load(Ordering::Relaxed), 8); + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + assert_eq!(fs::read_dir(&f.req.cache).unwrap().count(), 0); + } + + #[test] + fn native_bounded_decode_never_publishes_oversized_or_truncated_chunks() { + let f = Fixture::new(1, 10); + let path = f.req.cache.join("chunk"); + let body = build_chunk_body(&[42; 11], true, 0, None); + assert!( + super::super::chunk::write_verified_chunk_bounded(&body, None, &path, 10, true) + .is_err() + ); + assert!(!path.exists()); + assert!(!super::super::chunk::part_path(&path).exists()); + let body = build_chunk_body(&[42; 10], false, 0, Some(11)); + assert!( + super::super::chunk::write_verified_chunk_bounded(&body, None, &path, 10, true) + .is_err() + ); + let data = vec![42; 512 * 1024]; + let mut compressed = build_chunk_body(&data, true, 0, None); + assert_eq!( + super::super::chunk::write_verified_chunk_bounded( + &compressed, + Some(&sha1_of(&data)), + &path, + data.len() as u64, + true + ) + .unwrap(), + data.len() as u64 + ); + fs::remove_file(&path).unwrap(); + compressed.pop(); + // A matching shorter payload header leaves the missing zlib stream end to detect failure. + let short = (compressed.len() - 41) as i32; + compressed[12..16].copy_from_slice(&short.to_le_bytes()); + assert!(super::super::chunk::write_verified_chunk_bounded( + &compressed, + None, + &path, + data.len() as u64, + false + ) + .is_err()); + assert!(!path.exists()); + } + + #[test] + fn live_admission_shrinks_unsent_work_but_never_withdraws_inflight_reservations() { + let f = Fixture::new(30, 4096); + let mut state = State { + schedule: Schedule::new(&f.req, 128, RETAINED), + ready: vec![false; 30], + writing: vec![false; 30], + disk_sizes: vec![0; 30], + pending: vec![], + error: None, + done: false, + }; + admit_with_probes( + &f.req, + &mut state, + 0, + 0, + 4096, + 0, + Some(u64::MAX), + Some(8 * 4096), + ) + .unwrap(); + assert_eq!(state.schedule.lookahead, 8); + assert_eq!(state.schedule.retained, 0); + assert_eq!(state.schedule.fill(&f.req).len(), 8); + assert!(admit_with_probes( + &f.req, + &mut state, + 0, + 0, + 4096, + 0, + Some(u64::MAX), + Some(4096) + ) + .is_err()); + assert_eq!(state.schedule.lookahead, 8); + assert!(state + .schedule + .oldest + .iter() + .all(|&c| state.schedule.resident[c].is_some())); + } + + #[test] + fn snapshot_schema_rejects_unowned_staging_bad_paths_and_bad_version() { + let f = Fixture::new(1, 10); + let mut snapshot = serde_json::json!({ + "schema_version": 1, + "install_dir": f.req.install, "cache_dir": f.req.cache, "staging_dir": f.req.staging, + "same_volume": false, "free_game": -1, "free_cache": 0, "max_workers": 32, + "hosts": ["http://127.0.0.1"], + "chunks": [{ "guid": "00000000000000000000000000000000", "path": "chunks/00/chunk", + "size": 10, "wire_size": 51, "sha1": "", "cached": -1, "exact_size": true }], + "files": [{ "path": "legitimate.part", "old_size": 0, "sha1": "", + "parts": [{ "guid": "00000000000000000000000000000000", "offset": 0, "size": 10 }] }] + }); + let parsed = Request::parse(&snapshot.to_string()).unwrap(); + assert_eq!( + parsed.free_game, + if parsed.same_volume { Some(0) } else { None } + ); + assert_eq!(parsed.free_cache, Some(0)); + assert_eq!(parsed.chunks[0].cached, None); + assert!(choose_budget(&parsed).is_err()); + snapshot["schema_version"] = 2.into(); + assert!(Request::parse(&snapshot.to_string()).is_err()); + snapshot["schema_version"] = 1.into(); + snapshot["files"][0]["path"] = "../outside".into(); + assert!(Request::parse(&snapshot.to_string()).is_err()); + snapshot["files"][0]["path"] = ".droiddeck-epic-staging/owner".into(); + assert!(Request::parse(&snapshot.to_string()).is_err()); + snapshot["files"][0]["path"] = "legitimate.part".into(); + fs::remove_file(f.req.staging.join("owner")).unwrap(); + assert!(Request::parse(&snapshot.to_string()).is_err()); + } + + #[test] + fn native_shared_chunk_survives_cancelled_file_boundary_and_is_reused_on_resume() { + let mut f = Fixture::new(3, 4096); + let server = Server::start(vec![42; 4096], 1, false); + f.req.hosts = vec![server.address.clone()]; + for file in &mut f.req.files { + file.parts[0].chunk = 0; + } + let cancel = AtomicBool::new(false); + let before = AtomicUsize::new(0); + let progress = |_, _, files, _| { + if files == 1 { + cancel.store(true, Ordering::Release); + } + }; + assert!(run( + &f.req, + choose_budget(&f.req).unwrap(), + &cancel, + &|| { + before.fetch_add(1, Ordering::Relaxed); + Ok(()) + }, + &progress, + &|_| {} + ) + .is_err()); + assert_eq!(before.load(Ordering::Relaxed), 1); + assert_eq!(server.requests.load(Ordering::Relaxed), 1); + assert_eq!( + fs::read(f.req.install.join("file-0.bin")).unwrap(), + vec![42; 4096] + ); + assert!(!f.req.install.join("file-1.bin").exists()); + assert!(f.req.cache.join(&f.req.chunks[0].guid).is_file()); + f.req.files.remove(0); + f.req.chunks[0].cached = Some(4096); + assert_eq!( + run( + &f.req, + choose_budget(&f.req).unwrap(), + &AtomicBool::new(false), + &|| Ok(()), + &|_, _, _, _| {}, + &|_| {} + ) + .unwrap(), + 8192 + ); + assert_eq!(server.requests.load(Ordering::Relaxed), 1); + assert!(!f.req.cache.exists()); + } + + #[test] + fn native_later_hash_failure_preserves_verified_swap_and_unreplaced_old_file() { + let mut f = Fixture::new(2, 4096); + let server = Server::start(vec![42; 4096], 2, false); + f.req.hosts = vec![server.address.clone()]; + for file in &mut f.req.files { + fs::write(f.req.install.join(&file.path), b"old").unwrap(); + file.old = 3; + } + f.req.files[1].sha = Some([1; 20]); + let marked = AtomicUsize::new(0); + let error = run( + &f.req, + choose_budget(&f.req).unwrap(), + &AtomicBool::new(false), + &|| { + marked.fetch_add(1, Ordering::Relaxed); + Ok(()) + }, + &|_, _, _, _| {}, + &|_| {}, + ) + .unwrap_err(); + assert!(error.contains("SHA-1 mismatch")); + assert_eq!(marked.load(Ordering::Relaxed), 1); + assert_eq!( + fs::read(f.req.install.join("file-0.bin")).unwrap(), + vec![42; 4096] + ); + assert_eq!(fs::read(f.req.install.join("file-1.bin")).unwrap(), b"old"); + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + } + + #[test] + fn actual_native_file_and_cache_growth_never_exceeds_the_shared_planned_peak() { + let mut f = Fixture::new(20, 4096); + let server = Server::start(vec![42; 4096], 8, false); + f.req.hosts = vec![server.address.clone()]; + f.req.same_volume = true; + f.req.free_game = Some(21 * 4096); + for file in &mut f.req.files { + file.old = 4096; + fs::write(f.req.install.join(&file.path), vec![1; 4096]).unwrap(); + } + let budget = choose_budget(&f.req).unwrap(); + let observed = AtomicU64::new(0); + let progress = |_, _, _, _| { + let game: u64 = f + .req + .files + .iter() + .map(|file| fs::metadata(f.req.install.join(&file.path)).unwrap().len()) + .sum(); + let stage: u64 = fs::read_dir(&f.req.staging) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name() != "owner") + .map(|e| e.metadata().map(|m| m.len()).unwrap_or(0)) + .sum(); + let cache: u64 = fs::read_dir(&f.req.cache) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.metadata().map(|m| m.len()).unwrap_or(0)) + .sum(); + let additional = (game + stage + cache).saturating_sub(20 * 4096); + observed.fetch_max(additional, Ordering::Relaxed); + assert!( + additional <= budget.game_peak, + "{additional} > planned {}", + budget.game_peak + ); + }; + assert_eq!( + run( + &f.req, + budget, + &AtomicBool::new(false), + &|| Ok(()), + &progress, + &|_| {} + ) + .unwrap(), + 20 * 4096 + ); + assert!(observed.load(Ordering::Relaxed) > 8 * 4096); + } + + #[test] + fn native_assembly_swaps_a_verified_file_while_a_future_chunk_is_still_fetching() { + let mut f = Fixture::new(20, 4096); + let server = Server::start_mode(vec![42; 4096], 8, Some(7)); + f.req.hosts = vec![server.address.clone()]; + f.req.free_cache = Some(8 * 4096); + let cancel = AtomicBool::new(false); + let overlap = AtomicBool::new(false); + let progress = |_, _, files, _| { + if files == 1 { + assert!(server.active.load(Ordering::Relaxed) > 0); + assert_eq!( + fs::read(f.req.install.join("file-0.bin")).unwrap(), + vec![42; 4096] + ); + overlap.store(true, Ordering::Release); + cancel.store(true, Ordering::Release); + } + }; + assert!(run( + &f.req, + choose_budget(&f.req).unwrap(), + &cancel, + &|| Ok(()), + &progress, + &|_| {} + ) + .is_err()); + assert!(overlap.load(Ordering::Acquire)); + assert_eq!(server.requests.load(Ordering::Relaxed), 8); + assert_eq!(fs::read_dir(&f.req.staging).unwrap().count(), 1); + } + + #[cfg(unix)] + #[test] + fn staging_validation_rejects_symlink_namespace_owner_and_entries() { + use std::os::unix::fs::symlink; + let f = Fixture::new(1, 4096); + let outside = f.root.join("outside"); + fs::create_dir(&outside).unwrap(); + fs::write(outside.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); + fs::write(outside.join("keep.stage"), b"keep").unwrap(); + fs::remove_dir_all(&f.req.staging).unwrap(); + symlink(fs::canonicalize(&outside).unwrap(), &f.req.staging).unwrap(); + assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); + assert!(run( + &f.req, + choose_budget(&f.req).unwrap(), + &AtomicBool::new(false), + &|| Ok(()), + &|_, _, _, _| {}, + &|_| {} + ) + .is_err()); + assert_eq!(fs::read(outside.join("keep.stage")).unwrap(), b"keep"); + fs::remove_file(&f.req.staging).unwrap(); + fs::create_dir(&f.req.staging).unwrap(); + symlink( + fs::canonicalize(outside.join("owner")).unwrap(), + f.req.staging.join("owner"), + ) + .unwrap(); + assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); + fs::remove_file(f.req.staging.join("owner")).unwrap(); + fs::write(f.req.staging.join("owner"), b"DroidDeck Epic staging v1\n").unwrap(); + symlink( + fs::canonicalize(outside.join("keep.stage")).unwrap(), + f.req.staging.join("linked.stage"), + ) + .unwrap(); + assert!(Request::validate_staging(&f.req.install, &f.req.staging).is_err()); + assert_eq!(fs::read(outside.join("keep.stage")).unwrap(), b"keep"); + fs::remove_file(f.req.staging.join("linked.stage")).unwrap(); + } +} diff --git a/app/src/main/rust/stores/src/epic/assemble.rs b/app/src/main/rust/stores/src/epic/assemble.rs index 6881640b8d..8094d64fce 100644 --- a/app/src/main/rust/stores/src/epic/assemble.rs +++ b/app/src/main/rust/stores/src/epic/assemble.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::fs::{self, File}; -use std::io::{BufWriter, Write}; +use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -10,6 +10,33 @@ use sha1::{Digest, Sha1}; use super::manifest::{parse_manifest, FileInfo, Manifest}; use super::plan::resolve_cache_dir; +/// Streaming assembly primitive used by the budgeted installer; never holds a decoded chunk +/// in memory and closes the reader before the caller releases the cache reservation. +pub fn copy_chunk_part( + cache_path: &Path, offset: u64, size: u64, output: &mut impl Write, + sha: &mut Sha1, cancel: &AtomicBool, +) -> Result { + let mut input = File::open(cache_path).map_err(|e| format!("read chunk: {e}"))?; + let length = input.metadata().map_err(|e| e.to_string())?.len(); + if offset.checked_add(size).filter(|end| *end <= length).is_none() { + return Err("chunk part exceeds cached data".into()); + } + input.seek(SeekFrom::Start(offset)).map_err(|e| e.to_string())?; + let mut left = size; + let mut buffer = [0u8; 64 * 1024]; + while left > 0 { + if cancel.load(Ordering::Acquire) { return Err("cancelled".into()); } + let take = left.min(buffer.len() as u64) as usize; + let count = input.read(&mut buffer[..take]) + .map_err(|e| e.to_string())?; + if count == 0 { return Err("short cached chunk".into()); } + output.write_all(&buffer[..count]).map_err(|e| e.to_string())?; + sha.update(&buffer[..count]); + left -= count as u64; + } + Ok(size) +} + pub struct AssembleRequest { pub manifest_bytes: Vec, pub install_dir: String, diff --git a/app/src/main/rust/stores/src/epic/chunk.rs b/app/src/main/rust/stores/src/epic/chunk.rs index 3e0e96c499..e07c26c8ca 100644 --- a/app/src/main/rust/stores/src/epic/chunk.rs +++ b/app/src/main/rust/stores/src/epic/chunk.rs @@ -71,10 +71,14 @@ struct HashingWriter { inner: W, sha: Option, written: u64, + limit: u64, } impl Write for HashingWriter { fn write(&mut self, buf: &[u8]) -> io::Result { + if self.written.saturating_add(buf.len() as u64) > self.limit { + return Err(io::Error::other("chunk exceeds reserved decoded window")); + } let n = self.inner.write(buf)?; if let Some(sha) = self.sha.as_mut() { sha.update(&buf[..n]); @@ -102,10 +106,21 @@ pub fn write_verified_chunk( let tmp = part_path(final_path); let _ = fs::remove_file(&tmp); - let result = write_part(body, expected_sha1, &tmp, final_path); + let result = write_part(body, expected_sha1, &tmp, final_path, u64::MAX, false); if result.is_err() { let _ = fs::remove_file(&tmp); } + + result +} + +pub fn write_verified_chunk_bounded( + body: &[u8], expected_sha1: Option<&[u8; 20]>, final_path: &Path, + decoded_limit: u64, exact_size: bool, +) -> Result { + let tmp = part_path(final_path); + let result = write_part(body, expected_sha1, &tmp, final_path, decoded_limit, exact_size); + if result.is_err() { let _ = fs::remove_file(tmp); } result } @@ -114,20 +129,43 @@ fn write_part( expected_sha1: Option<&[u8; 20]>, tmp: &Path, final_path: &Path, + limit: u64, + exact_size: bool, ) -> Result { let hdr = parse_chunk_header(body)?; let payload = chunk_payload(body, &hdr)?; + if limit != u64::MAX && (hdr.compressed_size < 0 || payload.len() != hdr.compressed_size as usize) { + return Err("truncated chunk body".into()); + } let file = File::create(tmp).map_err(|e| format!("open {}: {e}", tmp.display()))?; let mut out = HashingWriter { inner: BufWriter::with_capacity(128 * 1024, file), sha: expected_sha1.map(|_| Sha1::new()), written: 0, + limit, }; if hdr.is_compressed() { - let mut dec = flate2::read::ZlibDecoder::new(payload); - io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?; + if limit == u64::MAX { + let mut dec = flate2::read::ZlibDecoder::new(payload); + io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?; + } else { + let mut dec = flate2::Decompress::new(true); + let mut buffer = [0u8; 128 * 1024]; + loop { + let input = dec.total_in() as usize; + let before = dec.total_out(); + let status = dec.decompress(&payload[input..], &mut buffer, flate2::FlushDecompress::None) + .map_err(|e| format!("inflate: {e}"))?; + let count = (dec.total_out() - before) as usize; + out.write_all(&buffer[..count]).map_err(|e| format!("inflate write: {e}"))?; + if status == flate2::Status::StreamEnd { break; } + if count == 0 && dec.total_in() as usize == input { + return Err("truncated compressed Epic chunk".into()); + } + } + } } else { out.write_all(payload).map_err(|e| format!("write: {e}"))?; } @@ -135,6 +173,9 @@ fn write_part( let written = out.written; let digest = out.sha.take().map(|s| s.finalize()); drop(out); + if exact_size && written != limit { + return Err("chunk decoded size mismatch".into()); + } if let (Some(expected), Some(actual)) = (expected_sha1, digest) { if actual.as_slice() != &expected[..] { @@ -146,6 +187,80 @@ fn write_part( Ok(written) } +/// The window a chunk decompresses to: the manifest's, or 1 MiB for a legacy JSON manifest that +/// names none (`EpicDownloadManager.chunkWindowSize`). The streaming installer also plans its +/// shared-chunk cache with this size. +pub fn chunk_window_bytes(window_size: i32) -> u64 { + if window_size > 0 { + window_size as u64 + } else { + 1024 * 1024 + } +} + +/// One downloaded body → the verified, decompressed chunk in memory, for the streaming installer +/// (`stream.rs`), which writes most chunks straight into the files that use them and never puts +/// them in a cache. The checks are the bounded Java writer's (`downloadChunkStreaming`), which is +/// stricter than [`write_verified_chunk`]: a stored payload shorter than `compressedSize` is +/// truncated, a zlib stream must reach its end, the output may not grow past the chunk's window +/// and must fill it exactly when the manifest names one, and the SHA-1 must match when the +/// manifest has one. `Err` = "try another CDN", exactly like a cache write that failed. +pub fn decode_verified_chunk( + body: &[u8], + window_size: i32, + expected_sha1: Option<&[u8; 20]>, +) -> Result, String> { + let hdr = parse_chunk_header(body)?; + let start = hdr.payload_start(); + if start > body.len() { + return Err("Stream ended during skip".to_string()); + } + let want = if hdr.compressed_size <= 0 { + 0 + } else { + hdr.compressed_size as usize + }; + let avail = body.len() - start; + let cap = chunk_window_bytes(window_size) as usize; + let out = if hdr.is_compressed() { + // Java stops reading once the inflater finishes, so a body may carry less than + // `compressedSize` as long as the stream itself is complete. + let payload = &body[start..start + want.min(avail)]; + let mut out: Vec = Vec::with_capacity(cap + 1); + let mut inflater = flate2::Decompress::new(true); + let status = inflater + .decompress_vec(payload, &mut out, flate2::FlushDecompress::Finish) + .map_err(|e| format!("inflate: {e}"))?; + if out.len() > cap { + return Err("chunk exceeds expected window size".to_string()); + } + if status != flate2::Status::StreamEnd { + return Err("truncated compressed Epic chunk".to_string()); + } + out + } else { + if avail < want { + return Err("truncated Epic chunk".to_string()); + } + if want > cap { + return Err("chunk exceeds expected window size".to_string()); + } + body[start..start + want].to_vec() + }; + if window_size > 0 && out.len() != window_size as usize { + return Err(format!( + "chunk is {} bytes, window is {window_size}", + out.len() + )); + } + if let Some(expected) = expected_sha1 { + if Sha1::digest(&out).as_slice() != &expected[..] { + return Err("Chunk SHA-1 mismatch (streaming)".to_string()); + } + } + Ok(out) +} + #[cfg(test)] pub(crate) mod test_support { use super::*; @@ -326,6 +441,29 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn decode_in_memory_applies_the_bounded_writer_checks() { + let data = sample_data(); + let window = data.len() as i32; + let sha = sha1_of(&data); + let zipped = build_chunk_body(&data, true, 2, None); + assert_eq!(decode_verified_chunk(&zipped, window, Some(&sha)).unwrap(), data); + let stored = build_chunk_body(&data, false, 0, None); + assert_eq!(decode_verified_chunk(&stored, window, Some(&sha)).unwrap(), data); + // No window and no hash (legacy JSON manifest): accepted up to 1 MiB. + assert_eq!(decode_verified_chunk(&zipped, 0, None).unwrap(), data); + // Window mismatch, SHA-1 mismatch, truncation, oversize. + assert!(decode_verified_chunk(&zipped, window + 1, None).is_err()); + let err = decode_verified_chunk(&zipped, window, Some(&[7; 20])).unwrap_err(); + assert!(err.contains("SHA-1 mismatch"), "{err}"); + let cut = zipped.len() - 500; + assert!(decode_verified_chunk(&zipped[..cut], window, None).is_err()); + assert!(decode_verified_chunk(&stored[..stored.len() - 1], 0, None) + .unwrap_err() + .contains("truncated")); + assert!(decode_verified_chunk(&zipped, window - 1, None).is_err()); + } + #[test] fn stale_part_file_is_cleared_before_the_attempt() { let dir = temp_dir("stale"); diff --git a/app/src/main/rust/stores/src/epic/jni.rs b/app/src/main/rust/stores/src/epic/jni.rs index 1c3c2a01e1..29edbe4b7b 100644 --- a/app/src/main/rust/stores/src/epic/jni.rs +++ b/app/src/main/rust/stores/src/epic/jni.rs @@ -9,6 +9,7 @@ use jni::{JNIEnv, JavaVM}; use super::assemble::{build_assemble_plan, run_assemble, AssembleRequest}; use super::driver::{build_plan, run_plan, EpicRequest}; +use super::stream::{build_stream_plan, run_stream, Hooks, StreamProgress, StreamRequest}; #[cfg(target_os = "android")] #[link(name = "log")] @@ -39,6 +40,7 @@ fn android_log(message: &str) { pub struct EpicDownloadHandle { cancel: Arc, + worker: Option>, } fn clear_pending_exception(env: &mut JNIEnv) { @@ -289,12 +291,11 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native let cancel = Arc::new(AtomicBool::new(false)); let thread_cancel = Arc::clone(&cancel); - let handle = Box::new(EpicDownloadHandle { cancel }); - - thread::spawn(move || { + let worker = thread::spawn(move || { crate::priority::background(); run_on_thread(vm, listener, plan, req, thread_cancel); }); + let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); Box::into_raw(handle) as jlong } @@ -393,9 +394,7 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native let cancel = Arc::new(AtomicBool::new(false)); let thread_cancel = Arc::clone(&cancel); - let handle = Box::new(EpicDownloadHandle { cancel }); - - thread::spawn(move || { + let worker = thread::spawn(move || { crate::priority::background(); let progress = |bytes_done: u64, bytes_total: u64, files_done: u64, files_total: u64| { let Ok(mut env) = vm.attach_current_thread_as_daemon() else { @@ -422,6 +421,194 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native outcome.bytes_written, ); }); + let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); + + Box::into_raw(handle) as jlong +} + +fn call_on_stream_progress(env: &mut JNIEnv, listener: &JObject, p: &StreamProgress) { + if listener.is_null() { + return; + } + let _ = env.call_method( + listener, + "onStreamProgress", + "(JJIIJ)V", + &[ + JValue::Long(p.bytes_written as jlong), + JValue::Long(p.bytes_total as jlong), + JValue::Int(p.files_done.min(i32::MAX as u64) as jint), + JValue::Int(p.files_total.min(i32::MAX as u64) as jint), + JValue::Long(p.files_bytes as jlong), + ], + ); + clear_pending_exception(env); +} + +/// `boolean onBeforeReplace()`: the Java side marks an existing install `updating`. A thrown +/// exception or `false` is a refusal. +fn call_on_before_replace(env: &mut JNIEnv, listener: &JObject) -> Result<(), String> { + if listener.is_null() { + return Ok(()); + } + let result = env + .call_method(listener, "onBeforeReplace", "()Z", &[]) + .and_then(|v| v.z()); + let threw = env.exception_check().unwrap_or(false); + clear_pending_exception(env); + match result { + Ok(true) if !threw => Ok(()), + _ => Err("could not mark the install as updating".to_string()), + } +} + +/// `EpicNative.nativeStream(...)`: the adaptive install (see `stream.rs`). Parse, plan and cache +/// preparation run on the calling thread, so a refusal returns 0 (after a synchronous +/// `onComplete(false, reason, 0)`) before anything is fetched or written, and the manager runs +/// the bounded Java writer instead. The run itself is on a new thread; `onComplete` fires after +/// every unfinished staging file of a cancelled or failed run is gone. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream( + mut env: JNIEnv, + _class: JClass, + manifest: JByteArray, + install_dir: JString, + chunk_cache_dir: JString, + staging_dir: JString, + cdn_prefixes: JObjectArray, + pending_file_idx: JIntArray, + expected_bytes: jlong, + cache_budget: jlong, + overlap_budget: jlong, + ca_bundle_path: JString, + max_workers: jint, + process_workers: jint, + listener: JObject, +) -> jlong { + if listener.is_null() { + android_log("nativeStream: null listener"); + return 0; + } + let Ok(manifest_bytes) = env.convert_byte_array(manifest) else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "manifest bytes unreadable", 0); + return 0; + }; + let install_dir = jstring_to_string(&mut env, &install_dir).unwrap_or_default(); + let chunk_cache_dir = jstring_to_string(&mut env, &chunk_cache_dir).unwrap_or_default(); + let staging_dir = jstring_to_string(&mut env, &staging_dir).unwrap_or_default(); + let cdn_prefixes = string_array_to_vec(&mut env, &cdn_prefixes); + let pending_file_indices: Vec = int_array_to_vec(&env, &pending_file_idx) + .into_iter() + .map(|i| i.max(0) as usize) + .collect(); + let ca_bundle_path = jstring_to_string(&mut env, &ca_bundle_path).unwrap_or_default(); + let max_workers = max_workers.max(1) as usize; + let process_workers = process_workers.max(1) as usize; + let label = { + let base = std::path::Path::new(&install_dir) + .file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_default(); + format!("epic app={base}") + }; + log_both( + &mut env, + &listener, + &format!( + "engine=rust-stream label=\"{label}\" install_dir={install_dir} cache_dir={} cdns={} pending_files={} cache_budget={} workers={max_workers} process_workers={process_workers} manifest_bytes={}", + if chunk_cache_dir.is_empty() { "/.chunks" } else { chunk_cache_dir.as_str() }, + cdn_prefixes.len(), + pending_file_indices.len(), + cache_budget.max(0), + manifest_bytes.len() + ), + ); + + let req = StreamRequest { + manifest_bytes, + install_dir, + chunk_cache_dir, + staging_dir, + cdn_prefixes, + pending_file_indices, + expected_bytes: if expected_bytes >= 0 { + Some(expected_bytes as u64) + } else { + None + }, + cache_budget: cache_budget.max(0) as u64, + // Negative = no limit (nothing the manager knows of is being replaced). + replace_overlap_budget: if overlap_budget < 0 { + u64::MAX + } else { + overlap_budget as u64 + }, + ca_bundle_path, + max_workers, + process_workers, + label, + }; + let plan = match build_stream_plan(&req) { + Ok(plan) => plan, + Err(err) => { + log_both(&mut env, &listener, &format!("not started: {err}")); + call_on_complete(&mut env, &listener, false, &err, 0); + return 0; + } + }; + let Ok(vm) = env.get_java_vm() else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); + return 0; + }; + let Ok(listener) = env.new_global_ref(&listener) else { + clear_pending_exception(&mut env); + call_on_complete(&mut env, &listener, false, "listener ref failed", 0); + return 0; + }; + + let cancel = Arc::new(AtomicBool::new(false)); + let thread_cancel = Arc::clone(&cancel); + let worker = thread::spawn(move || { + crate::priority::background(); + let progress = |p: &StreamProgress| { + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_stream_progress(&mut env, listener.as_obj(), p); + }; + let before_replace = || { + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return Err("JavaVM attach failed".to_string()); + }; + call_on_before_replace(&mut env, listener.as_obj()) + }; + let log = |line: &str| { + android_log(line); + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_log(&mut env, listener.as_obj(), line); + }; + let hooks = Hooks { + progress: &progress, + before_replace: &before_replace, + log: &log, + }; + let outcome = run_stream(&plan, &req, thread_cancel.as_ref(), &hooks); + let Ok(mut env) = vm.attach_current_thread_as_daemon() else { + return; + }; + call_on_complete( + &mut env, + listener.as_obj(), + outcome.success, + &outcome.error, + outcome.bytes_written, + ); + }); + let handle = Box::new(EpicDownloadHandle { cancel, worker: Some(worker) }); Box::into_raw(handle) as jlong } @@ -449,5 +636,73 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native if handle == 0 { return; } - drop(unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) }); + let mut handle = unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) }; + if let Some(worker) = handle.worker.take() { let _ = worker.join(); } +} + +/// ABI v1 snapshot: selected chunks/files, verified cache lengths, old output lengths and both +/// measured free-space probes. Parse/budget failures are terminal, never fallback requests. +#[no_mangle] +pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeInstallAdaptive( + mut env: JNIEnv, _class: JClass, request: JString, listener: JObject, +) -> jlong { + if listener.is_null() { return 0; } + let parsed = jstring_to_string(&mut env, &request).ok_or("unreadable adaptive request".to_string()) + .and_then(|s| super::adaptive::Request::parse(&s)) + .and_then(|req| super::adaptive::choose_budget(&req).map(|budget| (req, budget))); + let (req, budget) = match parsed { + Ok(plan) => plan, + Err(error) => { + call_on_complete(&mut env, &listener, false, &error, 0); + return 0; + } + }; + call_on_plan(&mut env, &listener, req.chunks.len() as u64, 0, + &format!("lookahead={} retained={} game_peak={} cache_peak={}", + budget.lookahead, budget.retained, budget.game_peak, budget.cache_peak)); + let Ok(vm) = env.get_java_vm() else { + call_on_complete(&mut env, &listener, false, "JavaVM unavailable", 0); + return 0; + }; + let Ok(listener) = env.new_global_ref(&listener) else { + call_on_complete(&mut env, &listener, false, "listener ref failed", 0); + return 0; + }; + let cancel = Arc::new(AtomicBool::new(false)); + let thread_cancel = cancel.clone(); + let worker = thread::spawn(move || { + crate::priority::background(); + let progress = |done, total, files, files_total| { + if let Ok(mut env) = vm.attach_current_thread_as_daemon() { + call_on_progress(&mut env, listener.as_obj(), done, total, files, files_total); + } + }; + let before_swap = || -> Result<(), String> { + let mut env = vm.attach_current_thread_as_daemon().map_err(|e| e.to_string())?; + match env.call_method(listener.as_obj(), "beforeSwap", "()Z", &[]) { + Ok(value) => if value.z().unwrap_or(false) { Ok(()) } else { Err("Could not mark installed record updating before replacement".into()) }, + _ => { + clear_pending_exception(&mut env); + Err("Could not mark installed record updating before replacement".into()) + } + } + }; + let log = |line: &str| { + android_log(line); + if let Ok(mut env) = vm.attach_current_thread_as_daemon() { + call_on_log(&mut env, listener.as_obj(), line); + } + }; + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| + super::adaptive::run(&req, budget, &thread_cancel, &before_swap, &progress, &log))) + .unwrap_or_else(|_| Err("Native Epic install panicked".into())); + let (success, error, bytes) = match outcome { + Ok(bytes) => (true, String::new(), bytes), + Err(error) => (false, error, 0), + }; + if let Ok(mut env) = vm.attach_current_thread_as_daemon() { + call_on_complete(&mut env, listener.as_obj(), success, &error, bytes); + } + }); + Box::into_raw(Box::new(EpicDownloadHandle { cancel, worker: Some(worker) })) as jlong } diff --git a/app/src/main/rust/stores/src/epic/stream.rs b/app/src/main/rust/stores/src/epic/stream.rs new file mode 100644 index 0000000000..5489801ddc --- /dev/null +++ b/app/src/main/rust/stores/src/epic/stream.rs @@ -0,0 +1,1956 @@ +//! The adaptive Epic install: chunks fetched in parallel and written straight into the files. +//! +//! The batch engine ([`super::driver`] + [`super::assemble`]) fetched every chunk into a cache +//! first and assembled the files afterwards: fast, but the cache and the game existed side by side +//! (twice the game's size on disk) and every byte was written twice. The bounded Java writer fixed +//! the space by fetching one chunk at a time, at the cost of the parallel fetch. This module keeps +//! the fetch core's parallel, adaptive download (window ramp, per-host caps, the in-flight byte +//! budget, retries, host rotation, stall handoff — none of it changes) and keeps the bounded +//! writer's safety rules, by deciding up front what each downloaded chunk is for: +//! +//! - **A chunk exactly one file part uses** is decompressed, verified and written straight into +//! that file's staging file at the part's offset (a positioned write, so chunks may arrive in +//! any order). It never touches a cache. +//! - **A chunk several parts use** is written into every part whose file is already open, and the +//! decompressed copy is kept in the cache (`/`, the same verified form and name the +//! batch engine and the Java writer use, so a resume on any engine reuses it) until its last +//! covered part is written; then it is deleted. Decompressed, not raw: a later part is then a +//! plain read and write with no second inflate or hash, and the file is exactly what a resume +//! already knows how to check. +//! - The cache has a **byte budget** (from the Java planner, sized from free space). The plan +//! simulates the whole install in file order and, when the budget would be exceeded, evicts the +//! cached chunk whose next use is furthest away (Belady's rule, the fewest refetches for that +//! budget). An evicted chunk is downloaded again for its later uses, as its own fetch item. +//! +//! **Files open in order.** The fetch items are queued in the order of their first use, so the +//! fetch core works through the game front to back. A file is opened (its staging file created) +//! only once the download has reached it: every file before the first chunk still to arrive, plus +//! the file of every chunk that has arrived. That keeps the set of half-written files to what the +//! fetch window spans, and it is also why the cache never holds more than the plan allowed: a +//! cached chunk is written into every newly reached file at once, so what stays in the cache is +//! exactly what the simulation kept at that point of the file order. +//! +//! **Updates are paced by a replacement-overlap budget.** A staged file that replaces an existing +//! one sits beside the old copy until its swap, so every replaced file open at once costs its full +//! new size in extra space (on a fresh install a staged file simply becomes the file). Files are +//! admitted in order; a replaced file is admitted only while the planned sizes of admitted, +//! not-yet-swapped replaced files stay within the budget the manager computed from free space, +//! and one is always admitted when none is open, so the install always progresses. The fetch core +//! dispatches a chunk only once the file of its first use is admitted ([`run_fetch_gated`]). +//! Admission never changes the order files are reached in, only when, so the cache simulation +//! below (which works in file order) stays exact: the cache holds at any moment what the plan holds +//! at that point of the order. Nothing waits on a gated file: every chunk an admitted file needs is +//! first used at or before that file, so it was dispatched before the gate closed. +//! +//! **A slow disk throttles the download by itself.** Every write happens in the fetch core's +//! process pool, inside [`FetchSink::process`]; the core frees a body's share of its in-flight +//! byte budget only when `process` returns, so while the disk is behind, the budget fills and no +//! new request goes out. Memory is that budget plus one decompressed chunk per pool thread. +//! +//! **Each file is replaced safely**, with the bounded Java writer's rules: written into a unique +//! `.stage` file in the manager's owned staging directory (`/.droiddeck-epic-staging`, +//! whose leftovers the manager recovers before the next run; a sibling `.droiddeck-epic-*.stage` +//! when no directory is given), never a manifest path, so a game file legitimately named `x.part` +//! is safe. Its SHA-1 computed as the bytes go out (parts written out of order are +//! read back once the bytes before them are in, while they are still in the page cache), then +//! synced, its size and hash checked, the install marked `updating` once (the Java callback, +//! before the first replacement), and the stage renamed over the destination. On cancel or failure +//! every unfinished stage is removed and the old files are untouched; verified files that were +//! already swapped stay, and the next run's delta pass skips them. The cache is left as it stands +//! for that run, and removed when the install completes. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs::{self, File, OpenOptions}; +use std::io::ErrorKind; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::Mutex; +use std::time::Instant; + +use sha1::{Digest, Sha1}; + +use crate::fetch_core::{run_fetch_gated, FetchItem, FetchOptions, FetchSink, SinkError}; + +use super::assemble::relative_path; +use super::chunk::{chunk_window_bytes, decode_verified_chunk, part_path}; +use super::driver::REQUEST_TIMEOUT; +use super::manifest::{parse_manifest, ChunkInfo, Manifest}; +use super::plan::{chunk_url, distinct_prefixes, per_host_cap, resolve_cache_dir}; + +/// "No further use" in the schedule's next-use table. +const NONE: usize = usize::MAX; + +/// Progress callbacks are rate-limited to one per this many milliseconds (a finished file always +/// reports): a large game has hundreds of thousands of parts, each a JNI call otherwise. +const PROGRESS_EVERY_MS: u64 = 50; + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// The schedule: which download serves which file part +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// One download of a chunk (or one cached copy left by an earlier run) and the file parts it +/// serves. Positions index the install's part sequence: every pending file's parts, file by file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScheduledEvent { + /// Index into `manifest.unique_chunks`. + pub chunk: usize, + /// The part positions this copy serves, in order. + pub uses: Vec, + /// Already in the cache when the run started; not fetched. + pub preloaded: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Schedule { + pub events: Vec, + /// Position → the event that serves it. + pub use_event: Vec, + /// Cached chunks left by an earlier run that this run will not use (no pending part needs + /// them, or keeping them would exceed the budget): deleted before anything is fetched. + pub discarded: Vec, + /// The most the cache holds at any point of the file order. + pub peak_cache_bytes: u64, +} + +/// Plans the downloads for the part sequence `chunk_of_use` (the chunk each part position reads) +/// with at most `budget` bytes of cached chunks (`chunk_bytes[c]` each). A chunk with one use is +/// a plain download. A chunk with more uses stays cached from one use to the next; when the cache +/// would exceed the budget, the cached chunk needed furthest in the future (possibly the one just +/// downloaded) is dropped, and its next use becomes a new download. `preloaded` chunks start in +/// the cache, the soonest needed first while they fit. +pub fn schedule( + chunk_of_use: &[usize], + chunk_bytes: &[u64], + preloaded: &[usize], + budget: u64, +) -> Schedule { + let n = chunk_of_use.len(); + let mut next_use = vec![NONE; n]; + let mut first_use: HashMap = HashMap::new(); + for p in (0..n).rev() { + let c = chunk_of_use[p]; + next_use[p] = first_use.get(&c).copied().unwrap_or(NONE); + first_use.insert(c, p); + } + + let mut out = Schedule { + use_event: vec![NONE; n], + ..Schedule::default() + }; + // Cached chunk → (its event, the position it is next needed at), and the same keyed by next + // use so the furthest one is the last entry. + let mut resident: HashMap = HashMap::new(); + let mut by_next: BTreeSet<(usize, usize)> = BTreeSet::new(); + let mut bytes: u64 = 0; + + let mut pre: Vec<(usize, usize)> = Vec::new(); + let mut seen_pre: HashSet = HashSet::new(); + for &c in preloaded { + if !seen_pre.insert(c) { + continue; + } + match first_use.get(&c) { + Some(&p) => pre.push((p, c)), + None => out.discarded.push(c), + } + } + pre.sort_unstable(); + for (p, c) in pre { + let size = chunk_bytes[c]; + if bytes + size <= budget { + let e = out.events.len(); + out.events.push(ScheduledEvent { + chunk: c, + uses: Vec::new(), + preloaded: true, + }); + resident.insert(c, (e, p)); + by_next.insert((p, c)); + bytes += size; + } else { + out.discarded.push(c); + } + } + out.peak_cache_bytes = bytes; + + for p in 0..n { + let c = chunk_of_use[p]; + let next = next_use[p]; + if let Some((e, _)) = resident.remove(&c) { + by_next.remove(&(p, c)); + out.events[e].uses.push(p); + out.use_event[p] = e; + if next == NONE { + bytes -= chunk_bytes[c]; + } else { + resident.insert(c, (e, next)); + by_next.insert((next, c)); + } + continue; + } + let e = out.events.len(); + out.events.push(ScheduledEvent { + chunk: c, + uses: vec![p], + preloaded: false, + }); + out.use_event[p] = e; + if next == NONE { + continue; + } + resident.insert(c, (e, next)); + by_next.insert((next, c)); + bytes += chunk_bytes[c]; + while bytes > budget { + let Some(&(far, victim)) = by_next.iter().next_back() else { + break; + }; + by_next.remove(&(far, victim)); + resident.remove(&victim); + bytes -= chunk_bytes[victim]; + } + out.peak_cache_bytes = out.peak_cache_bytes.max(bytes); + } + out +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// The plan +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// Everything the JNI layer hands over for one run. +pub struct StreamRequest { + pub manifest_bytes: Vec, + pub install_dir: String, + /// `""` = `/.chunks`; a path = scratch cache (internal storage for a card install). + pub chunk_cache_dir: String, + /// The manager's owned staging directory on the install volume; `""` = stage beside each file. + pub staging_dir: String, + pub cdn_prefixes: Vec, + /// Indices into `manifest.files` of the files still to write (Java's delta pass), in order. + pub pending_file_indices: Vec, + /// Java's `Σ fileSize` over the pending files, for a cross-check (`None` = skip). + pub expected_bytes: Option, + /// Most bytes of shared chunks the cache may hold. + pub cache_budget: u64, + /// Most planned bytes of replaced files (destination already there) staged at once; one is + /// always allowed. Fresh files are not limited by it. + pub replace_overlap_budget: u64, + pub ca_bundle_path: String, + pub max_workers: usize, + pub process_workers: usize, + pub label: String, +} + +#[derive(Debug)] +pub struct PlanPart { + /// Index into `manifest.unique_chunks`. + pub chunk: usize, + pub chunk_offset: usize, + pub file_offset: u64, + pub size: usize, +} + +#[derive(Debug)] +pub struct PlanFile { + pub rel_path: String, + pub size: u64, + /// The manifest's SHA-1 when it has one (all-zero = none, as the Java delta pass treats it). + pub sha1: Option<[u8; 20]>, + pub parts: Vec, + /// Position of the file's first part in the install's part sequence. + pub first_use: usize, + /// The destination exists with data: staging this file costs space until its swap. + pub replaces: bool, +} + +#[derive(Debug)] +pub struct StreamPlan { + pub manifest: Manifest, + pub install_dir: PathBuf, + pub cache_dir: PathBuf, + pub scratch_cache: bool, + pub files: Vec, + /// Position → (file, part index in that file). + pub use_file: Vec, + pub use_part: Vec, + pub schedule: Schedule, + /// The events that are fetched, in fetch order; a fetch item's id indexes this. + pub fetch_events: Vec, + /// Event → the file of its first use (0 for a preloaded copy nothing ends up using). + pub event_first_file: Vec, + /// Event → it is its chunk's first copy, cached as `` (later copies get `.`, + /// so a copy being deleted can never remove the next one). + pub event_first_copy: Vec, + /// Every manifest file's destination: a staging name may never be one of them. + pub reserved: HashSet, + /// Where staging files go; `None` = beside each destination. + pub staging_dir: Option, + pub bytes_total: u64, + pub budget: u64, + pub overlap_budget: u64, + pub hosts: Vec, + pub direct_chunks: usize, + pub shared_chunks: usize, + pub refetches: usize, +} + +/// A cached chunk left by an earlier run is reused only when it is what the manifest says: +/// the window size when there is one, the SHA-1 when there is one (`isValidCachedChunk`). +pub fn cached_chunk_is_valid(path: &Path, chunk: &ChunkInfo) -> bool { + let Ok(meta) = fs::metadata(path) else { + return false; + }; + if !meta.is_file() { + return false; + } + if chunk.window_size > 0 && meta.len() != chunk.window_size as u64 { + return false; + } + let Some(expected) = chunk.verifiable_sha1() else { + return true; + }; + match fs::read(path) { + Ok(data) => Sha1::digest(&data).as_slice() == &expected[..], + Err(_) => false, + } +} + +/// Parse, check, clean the cache and schedule. `Err` = nothing was fetched or written (stale cache +/// entries may have been removed, as the Java writer's own preparation does); the manager then +/// runs the bounded Java writer instead. +pub fn build_stream_plan(req: &StreamRequest) -> Result { + if req.install_dir.is_empty() { + return Err("stream: empty install dir".to_string()); + } + let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("stream: {e}"))?; + let by_guid = manifest.chunk_index_by_guid(); + + let mut files: Vec = Vec::with_capacity(req.pending_file_indices.len()); + let mut use_file: Vec = Vec::new(); + let mut use_part: Vec = Vec::new(); + let mut chunk_of_use: Vec = Vec::new(); + let mut bytes_total: u64 = 0; + for &fi in &req.pending_file_indices { + let Some(f) = manifest.files.get(fi) else { + return Err(format!( + "stream: pending file index {fi} out of range ({} files)", + manifest.files.len() + )); + }; + let rel_path = relative_path(f); + let first_use = chunk_of_use.len(); + let mut parts = Vec::with_capacity(f.parts.len()); + let mut file_offset: u64 = 0; + for (k, part) in f.parts.iter().enumerate() { + let guid = part.guid_str(); + let Some(&ci) = by_guid.get(&guid) else { + return Err(format!("Missing chunk metadata {guid} for {rel_path}")); + }; + let size = part.size as u32 as usize; + let chunk_offset = usize::try_from(part.offset) + .map_err(|_| format!("Chunk {guid} is too short for {rel_path}"))?; + let window = chunk_window_bytes(manifest.unique_chunks[ci].window_size); + if (chunk_offset as u64).saturating_add(size as u64) > window { + return Err(format!("Chunk {guid} is too short for {rel_path}")); + } + parts.push(PlanPart { + chunk: ci, + chunk_offset, + file_offset, + size, + }); + file_offset += size as u64; + use_file.push(files.len()); + use_part.push(k); + chunk_of_use.push(ci); + } + bytes_total += file_offset; + let replaces = fs::metadata(Path::new(&req.install_dir).join(&rel_path)) + .map(|m| m.is_file() && m.len() > 0) + .unwrap_or(false); + files.push(PlanFile { + replaces, + rel_path, + size: file_offset, + sha1: f.sha1.filter(|h| h.iter().any(|&b| b != 0)), + parts, + first_use, + }); + } + if let Some(expected) = req.expected_bytes { + if expected != bytes_total { + return Err(format!( + "stream: byte total mismatch java={expected} rust={bytes_total}" + )); + } + } + let hosts = distinct_prefixes(&req.cdn_prefixes); + if hosts.is_empty() { + return Err("stream: no CDN prefixes".to_string()); + } + + let install_dir = PathBuf::from(&req.install_dir); + let staging_dir = if req.staging_dir.is_empty() { + None + } else { + let dir = PathBuf::from(&req.staging_dir); + if !dir.is_dir() { + return Err(format!("stream: staging dir {} missing", dir.display())); + } + Some(dir) + }; + let cache_dir = resolve_cache_dir(&req.install_dir, &req.chunk_cache_dir); + fs::create_dir_all(&cache_dir) + .map_err(|e| format!("stream: mkdirs {}: {e}", cache_dir.display()))?; + let reserved: HashSet = manifest + .files + .iter() + .map(|f| install_dir.join(relative_path(f))) + .collect(); + + // The cache keeps only verified chunks this install still needs, as the Java writer's + // `prepareBoundedCache` does: anything else (a stale chunk, a `.part`, a later copy from a + // cancelled run) would only take space the files need. + let needed: HashSet = chunk_of_use.iter().copied().collect(); + let mut needed_by_name: HashMap = HashMap::new(); + for &ci in &needed { + needed_by_name.insert(manifest.unique_chunks[ci].guid_str(), ci); + } + let entries = fs::read_dir(&cache_dir) + .map_err(|e| format!("stream: list {}: {e}", cache_dir.display()))?; + let mut preloaded: Vec = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + if let Some(&ci) = needed_by_name.get(&name) { + if cached_chunk_is_valid(&path, &manifest.unique_chunks[ci]) { + preloaded.push(ci); + continue; + } + } + remove_entry(&path); + } + + let chunk_bytes: Vec = manifest + .unique_chunks + .iter() + .map(|c| chunk_window_bytes(c.window_size)) + .collect(); + let schedule = schedule(&chunk_of_use, &chunk_bytes, &preloaded, req.cache_budget); + for &ci in &schedule.discarded { + remove_entry(&cache_dir.join(manifest.unique_chunks[ci].guid_str())); + } + + let mut fetch_events = Vec::new(); + let mut event_first_file = Vec::with_capacity(schedule.events.len()); + let mut event_first_copy = Vec::with_capacity(schedule.events.len()); + let mut copies: HashMap = HashMap::new(); + for (e, ev) in schedule.events.iter().enumerate() { + event_first_file.push(ev.uses.first().map(|&p| use_file[p]).unwrap_or(0)); + let count = copies.entry(ev.chunk).or_insert(0); + event_first_copy.push(*count == 0); + *count += 1; + if !ev.preloaded { + fetch_events.push(e); + } + } + let mut uses_per_chunk: HashMap = HashMap::new(); + for &c in &chunk_of_use { + *uses_per_chunk.entry(c).or_insert(0) += 1; + } + let direct_chunks = uses_per_chunk.values().filter(|&&n| n == 1).count(); + let shared_chunks = uses_per_chunk.len() - direct_chunks; + let refetches: usize = copies.values().map(|&n| n - 1).sum(); + + Ok(StreamPlan { + manifest, + install_dir, + cache_dir, + scratch_cache: !req.chunk_cache_dir.is_empty(), + files, + use_file, + use_part, + schedule, + fetch_events, + event_first_file, + event_first_copy, + reserved, + staging_dir, + bytes_total, + budget: req.cache_budget, + overlap_budget: req.replace_overlap_budget, + hosts, + direct_chunks, + shared_chunks, + refetches, + }) +} + +fn remove_entry(path: &Path) { + if path.is_dir() { + let _ = fs::remove_dir_all(path); + } else { + let _ = fs::remove_file(path); + } +} + +// ───────────────────────────────────────────────────────────────────────────────────────────── +// The run +// ───────────────────────────────────────────────────────────────────────────────────────────── + +/// What the Java side sees of a run's progress. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct StreamProgress { + /// File bytes written so far (into staging files). + pub bytes_written: u64, + pub bytes_total: u64, + pub files_done: u64, + pub files_total: u64, + /// Sizes of the files verified and swapped into place. + pub files_bytes: u64, +} + +/// The run's callbacks. All may be called from any engine thread. +pub struct Hooks<'a> { + pub progress: &'a (dyn Fn(&StreamProgress) + Sync), + /// Called once, before the first file replaces its destination: the Java side marks an + /// existing install `updating`. `Err` fails the run and nothing is replaced. + pub before_replace: &'a (dyn Fn() -> Result<(), String> + Sync), + pub log: &'a (dyn Fn(&str) + Sync), +} + +#[derive(Clone, Debug, Default)] +pub struct StreamOutcome { + pub success: bool, + pub cancelled: bool, + pub error: String, + pub bytes_written: u64, + pub files_done: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum EventState { + /// Not downloaded yet. + Pending, + /// Downloaded, its cache copy being written; uses are only claimed by the downloading thread. + Caching, + /// In the cache: newly reached files read it from there. + Cached, + /// Every use is claimed; no cache copy (left). + Done, +} + +struct Shared { + state: Vec, + /// Event → uses not written yet. + remaining: Vec, + /// Event → bytes its cache copy takes. + cached_size: Vec, + /// Position → a thread has taken this part to write. + claimed: Vec, + /// File → parts not written yet. + parts_left: Vec, + /// Files `[0, active)` have been reached and may be written. + active: usize, + /// One past the highest first-use file of a downloaded event. + reached: usize, + /// `fetch_events[..next_fetch]` have all been downloaded. + next_fetch: usize, + /// Files `[0, admitted)` passed the replacement-overlap budget; `active` never passes it. + admitted: usize, + /// Planned bytes of admitted replaced files not swapped in yet. + overlap_open: u64, + peak_overlap: u64, + cache_bytes: u64, + peak_cache: u64, +} + +#[derive(Default)] +struct FileWriter { + staging: Option, + file: Option, + sha: Option, + /// Parts `[0, hashed)` are in the SHA-1. + hashed: usize, + done: Vec, + /// Bytes written into the staging file. + written: u64, +} + +#[derive(Default)] +struct Stats { + written: AtomicU64, + direct: AtomicU64, + from_cache: AtomicU64, + cache_stored: AtomicU64, + staging_now: AtomicU64, + staging_peak: AtomicU64, + files_done: AtomicU64, + files_bytes: AtomicU64, + fetched: AtomicU64, + last_progress_ms: AtomicU64, +} + +/// Parts to write from cache copies (event → positions), and part-less files to finish. +#[derive(Default)] +struct Work { + from_cache: HashMap>, + empty_files: Vec, +} + +/// Unique part of every staging name this process creates. +static STAGE_SEQ: AtomicU64 = AtomicU64::new(0); + +/// The run's state; it is the fetch core's sink. +pub struct Engine<'a> { + plan: &'a StreamPlan, + cancel: &'a AtomicBool, + hooks: &'a Hooks<'a>, + shared: Mutex, + writers: Vec>, + replace_gate: Mutex>>, + stats: Stats, + started: Instant, + /// `Shared::admitted`, readable by the fetch driver without the lock. + admitted: AtomicUsize, +} + +fn fatal_write(rel: &str, err: impl std::fmt::Display) -> SinkError { + SinkError::Fatal(format!("Could not assemble or atomically replace {rel}: {err}")) +} + +impl<'a> Engine<'a> { + pub fn new(plan: &'a StreamPlan, cancel: &'a AtomicBool, hooks: &'a Hooks<'a>) -> Self { + let events = &plan.schedule.events; + let mut state = Vec::with_capacity(events.len()); + let mut cached_size = vec![0u64; events.len()]; + let mut cache_bytes = 0u64; + for (e, ev) in events.iter().enumerate() { + if !ev.preloaded { + state.push(EventState::Pending); + continue; + } + let path = cache_path(plan, e); + if ev.uses.is_empty() { + // Evicted before its first use in the plan: it only takes space. + let _ = fs::remove_file(&path); + state.push(EventState::Done); + continue; + } + let size = fs::metadata(&path) + .map(|m| m.len()) + .unwrap_or(plan_chunk_bytes(plan, ev.chunk)); + cached_size[e] = size; + cache_bytes += size; + state.push(EventState::Cached); + } + Engine { + plan, + cancel, + hooks, + shared: Mutex::new(Shared { + state, + remaining: events.iter().map(|ev| ev.uses.len() as u32).collect(), + cached_size, + claimed: vec![false; plan.use_file.len()], + parts_left: plan.files.iter().map(|f| f.parts.len() as u32).collect(), + active: 0, + reached: 0, + next_fetch: 0, + admitted: 0, + overlap_open: 0, + peak_overlap: 0, + cache_bytes, + peak_cache: cache_bytes, + }), + writers: plan.files.iter().map(|_| Mutex::new(FileWriter::default())).collect(), + replace_gate: Mutex::new(None), + stats: Stats::default(), + started: Instant::now(), + admitted: AtomicUsize::new(0), + } + } + + /// The fetch driver's gate: fetch item `id` may go out once its first file is admitted. + pub fn may_dispatch(&self, id: usize) -> bool { + let p = self.plan; + match p.fetch_events.get(id) { + Some(&e) => p.event_first_file[e] < self.admitted.load(Ordering::Acquire), + None => true, + } + } + + /// The most planned bytes of replaced files that were staged at once. + pub fn peak_replace_overlap(&self) -> u64 { + self.shared.lock().map(|st| st.peak_overlap).unwrap_or_default() + } + + /// Keep reaching files until nothing new can be written: a file that finishes can admit the + /// next replaced file, whose parts may already be in the cache. Iterative, so a long run of + /// cache-only files never deepens the stack. + fn settle(&self) -> Result<(), SinkError> { + loop { + let mut work = Work::default(); + { + let mut st = self.shared.lock().expect("stream state poisoned"); + self.advance(&mut st, &mut work); + } + if work.from_cache.is_empty() && work.empty_files.is_empty() { + return Ok(()); + } + self.run_work(work)?; + } + } + + /// Before the first fetch: the files before the first chunk to download can be written from + /// the cache an earlier run left (or are empty). + pub fn start(&self) -> Result<(), SinkError> { + self.settle() + } + + /// Reach every file that can be written now: all files before the first chunk still to + /// download, and every file up to the furthest first use of a chunk that has arrived. Parts + /// of newly reached files whose chunk is in the cache are claimed for writing. + fn advance(&self, st: &mut Shared, work: &mut Work) { + let p = self.plan; + while st.admitted < p.files.len() { + let file = &p.files[st.admitted]; + if file.replaces { + if st.overlap_open > 0 && st.overlap_open + file.size > p.overlap_budget { + break; + } + st.overlap_open += file.size; + st.peak_overlap = st.peak_overlap.max(st.overlap_open); + } + st.admitted += 1; + } + self.admitted.store(st.admitted, Ordering::Release); + while st.next_fetch < p.fetch_events.len() + && st.state[p.fetch_events[st.next_fetch]] != EventState::Pending + { + st.next_fetch += 1; + } + let frontier = match p.fetch_events.get(st.next_fetch) { + Some(&e) => p.event_first_file[e], + None => p.files.len(), + }; + let target = st.active.max(frontier).max(st.reached).min(st.admitted); + for f in st.active..target { + let file = &p.files[f]; + if file.parts.is_empty() { + work.empty_files.push(f); + continue; + } + for k in 0..file.parts.len() { + let pos = file.first_use + k; + let e = p.schedule.use_event[pos]; + if st.state[e] == EventState::Cached && !st.claimed[pos] { + st.claimed[pos] = true; + work.from_cache.entry(e).or_default().push(pos); + } + } + } + st.active = target; + } + + /// One downloaded chunk body (fetch item `id`). + fn on_body(&self, id: usize, body: Vec) -> Result { + let p = self.plan; + let Some(&e) = p.fetch_events.get(id) else { + return Err(SinkError::Fatal(format!("item id {id} out of range"))); + }; + let ev = &p.schedule.events[e]; + let chunk = &p.manifest.unique_chunks[ev.chunk]; + let data = decode_verified_chunk(&body, chunk.window_size, chunk.verifiable_sha1()) + .map_err(|reason| SinkError::Retry(format!("{} {reason}", chunk.guid_str())))?; + drop(body); + self.stats.fetched.fetch_add(1, Ordering::Relaxed); + + let mut work = Work::default(); + let mut now: Vec = Vec::new(); + let caching; + { + let mut st = self.shared.lock().expect("stream state poisoned"); + if st.state[e] != EventState::Pending { + return Ok(chunk.credit_bytes()); + } + // Provisional: until the cache copy exists, nobody else may claim this chunk's uses. + st.state[e] = EventState::Caching; + st.reached = st.reached.max(p.event_first_file[e] + 1); + self.advance(&mut st, &mut work); + for &pos in &ev.uses { + if p.use_file[pos] < st.active && !st.claimed[pos] { + st.claimed[pos] = true; + now.push(pos); + } + } + caching = now.len() < ev.uses.len(); + if !caching { + st.state[e] = EventState::Done; + } + } + self.write_from_memory(&data, &now)?; + self.run_work(work)?; + if caching { + let path = cache_path(p, e); + write_cache_file(&path, &data).map_err(|err| { + SinkError::Fatal(format!("cache {}: {err}", chunk.guid_str())) + })?; + self.stats + .cache_stored + .fetch_add(data.len() as u64, Ordering::Relaxed); + let mut more = Vec::new(); + { + let mut st = self.shared.lock().expect("stream state poisoned"); + st.state[e] = EventState::Cached; + st.cached_size[e] = data.len() as u64; + st.cache_bytes += data.len() as u64; + st.peak_cache = st.peak_cache.max(st.cache_bytes); + // Files reached while the copy was being written. + for &pos in &ev.uses { + if p.use_file[pos] < st.active && !st.claimed[pos] { + st.claimed[pos] = true; + more.push(pos); + } + } + } + self.write_from_memory(&data, &more)?; + } + self.settle()?; + Ok(chunk.credit_bytes()) + } + + fn write_from_memory(&self, data: &[u8], positions: &[usize]) -> Result<(), SinkError> { + for &pos in positions { + let written = self.write_use(pos, data)?; + self.stats.direct.fetch_add(written, Ordering::Relaxed); + self.complete_use(pos)?; + } + Ok(()) + } + + fn run_work(&self, work: Work) -> Result<(), SinkError> { + let mut groups: Vec<(usize, Vec)> = work.from_cache.into_iter().collect(); + groups.sort_unstable(); + for (e, positions) in groups { + let ev = &self.plan.schedule.events[e]; + let guid = self.plan.manifest.unique_chunks[ev.chunk].guid_str(); + let data = fs::read(cache_path(self.plan, e)) + .map_err(|err| SinkError::Fatal(format!("read cached chunk {guid}: {err}")))?; + for pos in positions { + let written = self.write_use(pos, &data)?; + self.stats.from_cache.fetch_add(written, Ordering::Relaxed); + self.complete_use(pos)?; + } + } + for f in work.empty_files { + self.finalize(f)?; + } + Ok(()) + } + + /// Write the part at `pos` from its chunk's bytes into its file's staging file. + fn write_use(&self, pos: usize, chunk_data: &[u8]) -> Result { + let p = self.plan; + let f = p.use_file[pos]; + let k = p.use_part[pos]; + let file = &p.files[f]; + let part = &file.parts[k]; + let guid = p.manifest.unique_chunks[part.chunk].guid_str(); + let end = part + .chunk_offset + .checked_add(part.size) + .filter(|&end| end <= chunk_data.len()) + .ok_or_else(|| { + SinkError::Fatal(format!("Chunk {guid} is too short for {}", file.rel_path)) + })?; + let slice = &chunk_data[part.chunk_offset..end]; + { + let mut guard = self.writers[f].lock().expect("writer poisoned"); + if guard.file.is_none() { + self.open_staging(f, &mut guard)?; + } + let FileWriter { + file: out, + sha, + hashed, + done, + written, + .. + } = &mut *guard; + let out = out.as_ref().expect("staging open"); + out.write_all_at(slice, part.file_offset) + .map_err(|err| fatal_write(&file.rel_path, err))?; + done[k] = true; + *written += slice.len() as u64; + // SHA-1 in file order: the part just written comes from memory, parts that arrived + // earlier out of order are read back from the staging file. + if let Some(sha) = sha.as_mut() { + while *hashed < file.parts.len() && done[*hashed] { + let h = *hashed; + if h == k { + sha.update(slice); + } else { + let q = &file.parts[h]; + let mut buf = vec![0u8; q.size]; + out.read_exact_at(&mut buf, q.file_offset) + .map_err(|err| fatal_write(&file.rel_path, err))?; + sha.update(&buf); + } + *hashed += 1; + } + } + } + let len = slice.len() as u64; + let staged = self.stats.staging_now.fetch_add(len, Ordering::Relaxed) + len; + self.stats.staging_peak.fetch_max(staged, Ordering::Relaxed); + self.stats.written.fetch_add(len, Ordering::Relaxed); + self.report(false); + Ok(len) + } + + /// A part is written: release its chunk copy after its last use, finish its file after its + /// last part. + fn complete_use(&self, pos: usize) -> Result<(), SinkError> { + let p = self.plan; + let e = p.schedule.use_event[pos]; + let f = p.use_file[pos]; + let (release, finish) = { + let mut st = self.shared.lock().expect("stream state poisoned"); + st.remaining[e] = st.remaining[e].saturating_sub(1); + let release = st.remaining[e] == 0 && st.state[e] == EventState::Cached; + if release { + st.state[e] = EventState::Done; + st.cache_bytes = st.cache_bytes.saturating_sub(st.cached_size[e]); + } + st.parts_left[f] = st.parts_left[f].saturating_sub(1); + (release, st.parts_left[f] == 0) + }; + if release { + let _ = fs::remove_file(cache_path(p, e)); + } + if finish { + self.finalize(f)?; + } + Ok(()) + } + + /// A unique `.stage` file in the owned staging directory (or beside the destination), never a + /// manifest path (`createStagingFile`). The destination's directory is created either way. + fn open_staging(&self, f: usize, w: &mut FileWriter) -> Result<(), SinkError> { + let file = &self.plan.files[f]; + let dest = self.plan.install_dir.join(&file.rel_path); + let parent = dest + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| self.plan.install_dir.clone()); + fs::create_dir_all(&parent).map_err(|err| fatal_write(&file.rel_path, err))?; + let (stage_dir, prefix) = match &self.plan.staging_dir { + Some(dir) => (dir.clone(), "stream-"), + None => (parent.clone(), ".droiddeck-epic-"), + }; + for _ in 0..64 { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos()) + .unwrap_or(0); + let seq = STAGE_SEQ.fetch_add(1, Ordering::Relaxed); + let path = stage_dir.join(format!( + "{prefix}{}-{seq}-{nanos}.stage", + std::process::id() + )); + if self.plan.reserved.contains(&path) { + continue; + } + match OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + { + Ok(out) => { + w.staging = Some(path); + w.file = Some(out); + w.sha = file.sha1.map(|_| Sha1::new()); + w.hashed = 0; + w.done = vec![false; file.parts.len()]; + w.written = 0; + return Ok(()); + } + Err(err) if err.kind() == ErrorKind::AlreadyExists => continue, + Err(err) => return Err(fatal_write(&file.rel_path, err)), + } + } + Err(fatal_write(&file.rel_path, "no free staging name")) + } + + /// All parts are in: sync, check size and SHA-1, mark the install updating (once), rename + /// over the destination. On any failure the staging file goes and the destination stays. + fn finalize(&self, f: usize) -> Result<(), SinkError> { + let mut guard = self.writers[f].lock().expect("writer poisoned"); + if guard.file.is_none() { + self.open_staging(f, &mut guard)?; + } + let result = self.finish_file(f, &mut guard); + let staged = guard.written; + if result.is_err() { + guard.file = None; + if let Some(path) = guard.staging.take() { + let _ = fs::remove_file(&path); + } + } + drop(guard); + self.stats + .staging_now + .fetch_sub(staged.min(self.stats.staging_now.load(Ordering::Relaxed)), Ordering::Relaxed); + if result.is_ok() { + let size = self.plan.files[f].size; + if self.plan.files[f].replaces { + let mut st = self.shared.lock().expect("stream state poisoned"); + st.overlap_open = st.overlap_open.saturating_sub(size); + } + self.stats.files_done.fetch_add(1, Ordering::Relaxed); + self.stats.files_bytes.fetch_add(size, Ordering::Relaxed); + self.report(true); + } + result + } + + fn finish_file(&self, f: usize, w: &mut FileWriter) -> Result<(), SinkError> { + let file = &self.plan.files[f]; + let rel = file.rel_path.as_str(); + let out = w.file.take().expect("staging open"); + out.sync_all().map_err(|err| fatal_write(rel, err))?; + if self.cancel.load(Ordering::Relaxed) { + return Err(SinkError::Fatal("cancelled".to_string())); + } + let len = out.metadata().map_err(|err| fatal_write(rel, err))?.len(); + drop(out); + if len != file.size { + return Err(SinkError::Fatal(format!("Size mismatch for {rel}"))); + } + if let (Some(expected), Some(sha)) = (file.sha1.as_ref(), w.sha.take()) { + if w.hashed != file.parts.len() || sha.finalize().as_slice() != &expected[..] { + return Err(SinkError::Fatal(format!("SHA-1 mismatch for {rel}"))); + } + } + // Once a swap can mix versions, the old installed record is no longer truthful. + self.mark_updating().map_err(|err| fatal_write(rel, err))?; + if self.cancel.load(Ordering::Relaxed) { + return Err(SinkError::Fatal("cancelled".to_string())); + } + let staging = w.staging.clone().expect("staging path"); + let dest = self.plan.install_dir.join(&file.rel_path); + fs::rename(&staging, &dest).map_err(|err| fatal_write(rel, err))?; + w.staging = None; + Ok(()) + } + + fn mark_updating(&self) -> Result<(), String> { + let mut gate = self.replace_gate.lock().expect("replace gate poisoned"); + if let Some(result) = gate.as_ref() { + return result.clone(); + } + let result = (self.hooks.before_replace)(); + *gate = Some(result.clone()); + result + } + + fn report(&self, force: bool) { + let now_ms = self.started.elapsed().as_millis() as u64; + let last = self.stats.last_progress_ms.load(Ordering::Relaxed); + if !force { + if now_ms.saturating_sub(last) < PROGRESS_EVERY_MS { + return; + } + if self + .stats + .last_progress_ms + .compare_exchange(last, now_ms, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return; + } + } else { + self.stats.last_progress_ms.store(now_ms, Ordering::Relaxed); + } + (self.hooks.progress)(&self.snapshot()); + } + + fn snapshot(&self) -> StreamProgress { + StreamProgress { + bytes_written: self.stats.written.load(Ordering::Relaxed), + bytes_total: self.plan.bytes_total, + files_done: self.stats.files_done.load(Ordering::Relaxed), + files_total: self.plan.files.len() as u64, + files_bytes: self.stats.files_bytes.load(Ordering::Relaxed), + } + } + + /// Cancel or failure: every unfinished staging file goes; the destinations are untouched and + /// the cache stays for the next run. Call only when no other thread is writing. + pub fn discard_staging(&self) -> usize { + let mut removed = 0; + for writer in &self.writers { + let mut guard = writer.lock().unwrap_or_else(|e| e.into_inner()); + guard.file = None; + if let Some(path) = guard.staging.take() { + if fs::remove_file(&path).is_ok() { + removed += 1; + } + } + } + removed + } + + /// Files not yet swapped into place. + pub fn unfinished_files(&self) -> u64 { + self.plan.files.len() as u64 - self.stats.files_done.load(Ordering::Relaxed) + } + + pub fn summary(&self) -> String { + let secs = self.started.elapsed().as_secs_f64().max(0.001); + let written = self.stats.written.load(Ordering::Relaxed); + let (peak_cache, peak_overlap) = self + .shared + .lock() + .map(|st| (st.peak_cache, st.peak_overlap)) + .unwrap_or_default(); + format!( + "stream-summary mode=adaptive files={}/{} bytes_written={written} direct_bytes={} from_cache_bytes={} cache_stored_bytes={} peak_cache_bytes={peak_cache} budget={} plan_peak_cache_bytes={} peak_staging_bytes={} peak_replace_overlap_bytes={peak_overlap} overlap_budget={} fetched_chunks={} refetches={} elapsed={secs:.1}s avg_MBps={:.2}", + self.stats.files_done.load(Ordering::Relaxed), + self.plan.files.len(), + self.stats.direct.load(Ordering::Relaxed), + self.stats.from_cache.load(Ordering::Relaxed), + self.stats.cache_stored.load(Ordering::Relaxed), + self.plan.budget, + self.plan.schedule.peak_cache_bytes, + self.stats.staging_peak.load(Ordering::Relaxed), + self.plan.overlap_budget, + self.stats.fetched.load(Ordering::Relaxed), + self.plan.refetches, + written as f64 / secs / (1024.0 * 1024.0), + ) + } +} + +impl FetchSink for Engine<'_> { + fn process(&self, item: &FetchItem, body: Vec) -> Result { + self.on_body(item.id as usize, body) + } +} + +fn plan_chunk_bytes(plan: &StreamPlan, chunk: usize) -> u64 { + chunk_window_bytes(plan.manifest.unique_chunks[chunk].window_size) +} + +/// Where event `e`'s copy lives in the cache. +fn cache_path(plan: &StreamPlan, e: usize) -> PathBuf { + let guid = plan.manifest.unique_chunks[plan.schedule.events[e].chunk].guid_str(); + if plan.event_first_copy[e] { + plan.cache_dir.join(guid) + } else { + plan.cache_dir.join(format!("{guid}.{e}")) + } +} + +/// `.part` + rename, so a cache file under its final name is always complete. +fn write_cache_file(path: &Path, data: &[u8]) -> Result<(), String> { + let tmp = part_path(path); + let _ = fs::remove_file(&tmp); + let result = fs::write(&tmp, data) + .map_err(|e| format!("write {}: {e}", tmp.display())) + .and_then(|()| fs::rename(&tmp, path).map_err(|e| format!("rename: {e}"))); + if result.is_err() { + let _ = fs::remove_file(&tmp); + } + result +} + +/// Run the plan to the end: the fetch core downloads the chunks in order, the engine writes them. +pub fn run_stream( + plan: &StreamPlan, + req: &StreamRequest, + cancel: &AtomicBool, + hooks: &Hooks<'_>, +) -> StreamOutcome { + let log = hooks.log; + let host_cap = per_host_cap(req.max_workers, plan.hosts.len()); + log(&format!( + "stream-plan chunk_dir={} cache_dir={} scratch={} {} files={} bytes={} uses={} chunks_direct={} chunks_shared={} fetches={} preloaded={} refetches={} budget={} plan_peak_cache={} replaced_files={} overlap_budget={} hosts={} workers={} per_host_cap={host_cap} process_workers={}", + plan.manifest.chunk_dir, + plan.cache_dir.display(), + plan.scratch_cache, + crate::priority::log_field(), + plan.files.len(), + plan.bytes_total, + plan.use_file.len(), + plan.direct_chunks, + plan.shared_chunks, + plan.fetch_events.len(), + plan.schedule.events.len() - plan.fetch_events.len(), + plan.refetches, + plan.budget, + plan.schedule.peak_cache_bytes, + plan.files.iter().filter(|f| f.replaces).count(), + plan.overlap_budget, + plan.hosts.len(), + req.max_workers, + req.process_workers + )); + for (i, h) in plan.hosts.iter().enumerate() { + log(&format!("host[{i}]={h}")); + } + + let engine = Engine::new(plan, cancel, hooks); + let mut outcome = StreamOutcome::default(); + let mut error: Option = match engine.start() { + Ok(()) => None, + Err(SinkError::Fatal(m)) | Err(SinkError::Retry(m)) => Some(m), + }; + + if error.is_none() && !cancel.load(Ordering::Relaxed) && !plan.fetch_events.is_empty() { + let items: Vec = plan + .fetch_events + .iter() + .enumerate() + .map(|(id, &e)| { + let chunk = &plan.manifest.unique_chunks[plan.schedule.events[e].chunk]; + FetchItem { + id: id as u64, + urls: plan + .hosts + .iter() + .map(|h| chunk_url(h, &plan.manifest.chunk_dir, chunk)) + .collect(), + reserve: if chunk.file_size > 0 { + chunk.file_size as u64 + } else { + 0 + }, + range: None, + } + }) + .collect(); + let opts = FetchOptions { + max_workers: req.max_workers.max(1), + per_host_cap: host_cap, + timeout: REQUEST_TIMEOUT, + headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())], + ca_bundle_path: req.ca_bundle_path.clone(), + process_workers: req.process_workers.max(1), + label: req.label.clone(), + ..FetchOptions::default() + }; + // Progress is the engine's (file bytes written); the core's own credit only feeds its + // `fetch-end` line. + let no_progress = |_: u64, _: u64| {}; + let gate = |id: usize| engine.may_dispatch(id); + let fetched = run_fetch_gated( + items, + &plan.hosts, + &opts, + &engine, + cancel, + &no_progress, + log, + &gate, + ); + error = fetched.error; + } + + let progress = engine.snapshot(); + outcome.bytes_written = progress.bytes_written; + outcome.files_done = progress.files_done; + if cancel.load(Ordering::Relaxed) { + let removed = engine.discard_staging(); + outcome.cancelled = true; + outcome.error = "cancelled".to_string(); + log(&engine.summary()); + log(&format!( + "cancelled ({}/{} files, {removed} staging files removed)", + progress.files_done, progress.files_total + )); + return outcome; + } + if error.is_none() && engine.unfinished_files() > 0 { + error = Some(format!( + "stream ended with {} files unfinished", + engine.unfinished_files() + )); + } + if let Some(err) = error { + let removed = engine.discard_staging(); + outcome.error = err; + log(&engine.summary()); + log(&format!( + "FAIL {} ({}/{} files ok, {removed} staging files removed)", + outcome.error, progress.files_done, progress.files_total + )); + return outcome; + } + outcome.success = true; + log(&engine.summary()); + match fs::remove_dir_all(&plan.cache_dir) { + Ok(()) => log(&format!("cache removed {}", plan.cache_dir.display())), + Err(e) => log(&format!("cache not removed {}: {e}", plan.cache_dir.display())), + } + outcome +} + +#[cfg(test)] +mod tests { + use super::super::chunk::test_support::{build_chunk_body, sha1_of, temp_dir}; + use super::super::manifest::test_support::{build_manifest, TestChunk, TestFile}; + use super::*; + + // ── schedule ── + + #[test] + fn single_use_chunks_are_plain_downloads_and_shared_ones_stay_cached() { + // Chunk 0 is used at positions 0 and 2, chunk 1 once, chunk 2 once. + let s = schedule(&[0, 1, 0, 2], &[10, 10, 10], &[], 100); + assert_eq!(s.events.len(), 3); + assert_eq!(s.events[0].uses, vec![0, 2]); + assert_eq!(s.events[1].uses, vec![1]); + assert_eq!(s.events[2].uses, vec![3]); + assert_eq!(s.use_event, vec![0, 1, 0, 2]); + assert_eq!(s.peak_cache_bytes, 10, "only the shared chunk is ever cached"); + assert!(s.discarded.is_empty()); + } + + #[test] + fn a_chunk_bigger_than_the_budget_is_downloaded_for_every_use() { + let s = schedule(&[0, 1, 0, 0], &[10, 1], &[], 5); + let chunk0: Vec<&ScheduledEvent> = s.events.iter().filter(|e| e.chunk == 0).collect(); + assert_eq!(chunk0.len(), 3, "never cached → one download per use"); + assert!(chunk0.iter().all(|e| e.uses.len() == 1)); + assert_eq!(s.peak_cache_bytes, 0); + } + + #[test] + fn eviction_drops_the_chunk_needed_furthest_away() { + // Budget fits two chunks. At position 2 chunk 2 arrives (next use 6) while 0 (next 3) + // and 1 (next 4) are cached: chunk 2 itself is the furthest and is not kept. + let uses = [0, 1, 2, 0, 1, 9, 2]; + let s = schedule(&uses, &[4, 4, 4, 0, 0, 0, 0, 0, 0, 1], &[], 8); + let copies = |c: usize| s.events.iter().filter(|e| e.chunk == c).count(); + assert_eq!(copies(0), 1); + assert_eq!(copies(1), 1); + assert_eq!(copies(2), 2, "chunk 2 downloaded again for its later use"); + assert!(s.peak_cache_bytes <= 8); + // Same sequence with room for all three: no refetch. + let s = schedule(&uses, &[4, 4, 4, 0, 0, 0, 0, 0, 0, 1], &[], 12); + assert_eq!(s.events.iter().filter(|e| e.chunk == 2).count(), 1); + assert_eq!(s.peak_cache_bytes, 12); + } + + #[test] + fn preloaded_chunks_serve_their_uses_until_the_budget_says_otherwise() { + // Chunk 0 cached by an earlier run, chunk 1 too but nothing needs it, chunk 2 cached but + // over budget. + let s = schedule(&[0, 2, 0], &[4, 4, 4], &[0, 1, 2], 4); + assert!(s.events[0].preloaded); + assert_eq!(s.events[0].chunk, 0); + assert_eq!(s.events[0].uses, vec![0, 2]); + let mut discarded = s.discarded.clone(); + discarded.sort_unstable(); + assert_eq!(discarded, vec![1, 2]); + let fetched: Vec<&ScheduledEvent> = s.events.iter().filter(|e| !e.preloaded).collect(); + assert_eq!(fetched.len(), 1); + assert_eq!(fetched[0].chunk, 2); + } + + // ── engine (no network: bodies are handed to the sink directly) ── + + const A: [u32; 4] = [1, 0, 0, 0]; + const B: [u32; 4] = [2, 0, 0, 0]; + const C: [u32; 4] = [3, 0, 0, 0]; + + fn data(seed: u32, len: usize) -> Vec { + (0..len as u32).map(|i| ((i * 7 + seed) % 251) as u8).collect() + } + + struct Fixture { + dir: PathBuf, + install: PathBuf, + chunks: Vec<([u32; 4], Vec)>, + files: Vec<(String, Vec<([u32; 4], i32, i32)>)>, + } + + impl Fixture { + /// Three files: `a.bin` = A[0..100] + B[0..50]; `dir/b.bin` = A[100..300] (A is shared); + /// `c.bin` = C whole. + fn new(tag: &str) -> Self { + let dir = temp_dir(tag); + let install = dir.join("game"); + fs::create_dir_all(&install).unwrap(); + Fixture { + dir, + install, + chunks: vec![(A, data(1, 300)), (B, data(2, 50)), (C, data(3, 80))], + files: vec![ + ("a.bin".to_string(), vec![(A, 0, 100), (B, 0, 50)]), + ("dir\\b.bin".to_string(), vec![(A, 100, 200)]), + ("c.bin".to_string(), vec![(C, 0, 80)]), + ], + } + } + + fn chunk(&self, guid: [u32; 4]) -> &Vec { + &self.chunks.iter().find(|(g, _)| *g == guid).unwrap().1 + } + + fn expected(&self, file: usize) -> Vec { + let mut out = Vec::new(); + for (g, off, size) in &self.files[file].1 { + out.extend_from_slice(&self.chunk(*g)[*off as usize..(*off + *size) as usize]); + } + out + } + + fn manifest(&self) -> Vec { + let chunks: Vec = self + .chunks + .iter() + .map(|(g, d)| TestChunk { + guid: *g, + hash: g[0] as u64, + sha1: sha1_of(d), + group: 1, + window: d.len() as i32, + file_size: d.len() as u64, + }) + .collect(); + let files: Vec = (0..self.files.len()) + .map(|i| TestFile { + name: self.files[i].0.clone(), + sha1: sha1_of(&self.expected(i)), + tags: vec![], + parts: self.files[i].1.clone(), + }) + .collect(); + build_manifest(&chunks, &files, 21, true) + } + + fn request(&self, pending: Vec, budget: u64) -> StreamRequest { + StreamRequest { + manifest_bytes: self.manifest(), + install_dir: self.install.to_str().unwrap().to_string(), + chunk_cache_dir: self.dir.join("cache").to_str().unwrap().to_string(), + staging_dir: String::new(), + cdn_prefixes: vec!["http://127.0.0.1:9/Builds/x".to_string()], + pending_file_indices: pending, + expected_bytes: None, + cache_budget: budget, + replace_overlap_budget: u64::MAX, + ca_bundle_path: String::new(), + max_workers: 4, + process_workers: 2, + label: "epic test".to_string(), + } + } + + fn body_for(&self, plan: &StreamPlan, id: usize) -> Vec { + let ev = &plan.schedule.events[plan.fetch_events[id]]; + let guid = plan.manifest.unique_chunks[ev.chunk].guid; + build_chunk_body(self.chunk(guid), true, 0, None) + } + + fn stages(&self) -> Vec { + let mut out = Vec::new(); + let mut dirs = vec![self.install.clone()]; + while let Some(d) = dirs.pop() { + for entry in fs::read_dir(&d).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + dirs.push(path); + } else if path.to_string_lossy().ends_with(".stage") { + out.push(path); + } + } + } + out + } + } + + impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } + } + + struct Recorder { + marks: Mutex, + last: Mutex, + fail_mark: bool, + } + + impl Recorder { + fn new() -> Self { + Recorder { + marks: Mutex::new(0), + last: Mutex::new(StreamProgress::default()), + fail_mark: false, + } + } + } + + fn with_engine( + plan: &StreamPlan, + cancel: &AtomicBool, + rec: &Recorder, + body: impl FnOnce(&Engine) -> R, + ) -> R { + let progress = |p: &StreamProgress| *rec.last.lock().unwrap() = *p; + let before = || { + *rec.marks.lock().unwrap() += 1; + if rec.fail_mark { + Err("mark failed".to_string()) + } else { + Ok(()) + } + }; + let log = |_: &str| {}; + let hooks = Hooks { + progress: &progress, + before_replace: &before, + log: &log, + }; + let engine = Engine::new(plan, cancel, &hooks); + body(&engine) + } + + #[test] + fn plan_counts_direct_and_shared_chunks_and_orders_fetches_by_first_use() { + let fx = Fixture::new("stream-plan"); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); + assert_eq!(plan.direct_chunks, 2, "B and C"); + assert_eq!(plan.shared_chunks, 1, "A"); + assert_eq!(plan.fetch_events.len(), 3); + assert_eq!(plan.refetches, 0); + assert_eq!(plan.bytes_total, 150 + 200 + 80); + let order: Vec<[u32; 4]> = plan + .fetch_events + .iter() + .map(|&e| plan.manifest.unique_chunks[plan.schedule.events[e].chunk].guid) + .collect(); + assert_eq!(order, vec![A, B, C]); + let mut req = fx.request(vec![0, 1, 2], 1 << 20); + req.expected_bytes = Some(1); + assert!(build_stream_plan(&req).unwrap_err().contains("byte total mismatch")); + } + + #[test] + fn out_of_order_arrival_writes_verified_files_and_empties_the_cache() { + let fx = Fixture::new("stream-ooo"); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + // C, then B, then A: the shared chunk last. + for id in (0..plan.fetch_events.len()).rev() { + engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); + } + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..3 { + let rel = fx.files[i].0.replace('\\', "/"); + assert_eq!(fs::read(fx.install.join(&rel)).unwrap(), fx.expected(i), "{rel}"); + } + assert!(fx.stages().is_empty()); + assert_eq!(*rec.marks.lock().unwrap(), 1, "marked updating once"); + let last = *rec.last.lock().unwrap(); + assert_eq!(last.files_done, 3); + assert_eq!(last.bytes_written, 430); + assert_eq!(last.files_bytes, 430); + let leftovers = fs::read_dir(&plan.cache_dir).unwrap().count(); + assert_eq!(leftovers, 0, "the shared chunk is released after its last use"); + } + + #[test] + fn a_shared_chunk_arriving_first_is_cached_until_its_later_file_is_reached() { + let fx = Fixture::new("stream-cache"); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); // A + let cached = plan.cache_dir.join(guid_to(A)); + assert!(cached.is_file(), "A waits in the cache for dir/b.bin"); + engine.process(&item(1), fx.body_for(&plan, 1)).unwrap(); // B: reaches b.bin + assert!(!cached.exists(), "released after its last use"); + engine.process(&item(2), fx.body_for(&plan, 2)).unwrap(); // C + }); + assert_eq!(fs::read(fx.install.join("dir/b.bin")).unwrap(), fx.expected(1)); + } + + #[test] + fn a_zero_budget_refetches_shared_chunks_and_never_caches() { + let fx = Fixture::new("stream-budget"); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 0)).unwrap(); + assert_eq!(plan.refetches, 1, "A downloaded once per file"); + assert_eq!(plan.fetch_events.len(), 4); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + for id in [3, 1, 0, 2] { + engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); + assert_eq!(fs::read_dir(&plan.cache_dir).unwrap().count(), 0); + } + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..3 { + let rel = fx.files[i].0.replace('\\', "/"); + assert_eq!(fs::read(fx.install.join(&rel)).unwrap(), fx.expected(i)); + } + } + + #[test] + fn resume_writes_only_pending_files_and_reuses_a_cached_chunk() { + let fx = Fixture::new("stream-resume"); + // An earlier run left A verified in the cache, plus a stale entry and a later copy. + let cache = fx.dir.join("cache"); + fs::create_dir_all(&cache).unwrap(); + fs::write(cache.join(guid_to(A)), fx.chunk(A)).unwrap(); + fs::write(cache.join("FFFFFFFF000000000000000000000000"), b"stale").unwrap(); + fs::write(cache.join(format!("{}.7", guid_to(A))), fx.chunk(A)).unwrap(); + // a.bin was verified by the delta pass; only dir/b.bin and c.bin are pending. + let plan = build_stream_plan(&fx.request(vec![1, 2], 1 << 20)).unwrap(); + assert_eq!(plan.fetch_events.len(), 1, "only C is downloaded"); + assert_eq!(fs::read_dir(&cache).unwrap().count(), 1, "stale entries removed"); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + assert!(fx.install.join("dir/b.bin").is_file(), "written from the cache at start"); + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); + assert_eq!(engine.unfinished_files(), 0); + }); + assert!(!fx.install.join("a.bin").exists()); + assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), fx.expected(2)); + } + + #[test] + fn cancel_keeps_old_files_removes_staging_and_keeps_the_cache() { + let fx = Fixture::new("stream-cancel"); + fs::write(fx.install.join("a.bin"), b"old a").unwrap(); + fs::write(fx.install.join("c.bin"), b"old c").unwrap(); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + // A arrives: a.bin half written, A cached for dir/b.bin. + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); + assert_eq!(fx.stages().len(), 1); + cancel.store(true, Ordering::Relaxed); + // B completes a.bin, but a cancelled run never swaps it in. + assert!(engine.process(&item(1), fx.body_for(&plan, 1)).is_err()); + assert!(engine.discard_staging() <= 2); + }); + assert!(fx.stages().is_empty()); + assert_eq!(fs::read(fx.install.join("a.bin")).unwrap(), b"old a"); + assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); + assert_eq!(*rec.marks.lock().unwrap(), 0, "nothing was replaced"); + assert!(plan.cache_dir.join(guid_to(A)).is_file(), "kept for the resume"); + } + + #[test] + fn a_hash_failure_keeps_the_old_file_and_marks_nothing() { + let fx = Fixture::new("stream-hash"); + fs::write(fx.install.join("c.bin"), b"old c").unwrap(); + let mut plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); + // The manifest promises another hash than the bytes the chunk yields. + plan.files[0].sha1 = Some([9; 20]); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + let err = with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + let res = engine.process(&item(0), fx.body_for(&plan, 0)); + engine.discard_staging(); + res + }); + match err { + Err(SinkError::Fatal(m)) => assert!(m.contains("SHA-1 mismatch for c.bin"), "{m}"), + other => panic!("expected a fatal hash failure, got {other:?}"), + } + assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); + assert!(fx.stages().is_empty()); + assert_eq!(*rec.marks.lock().unwrap(), 0); + } + + #[test] + fn a_corrupt_body_is_a_retry_that_changes_nothing() { + let fx = Fixture::new("stream-corrupt"); + let plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + let mut body = fx.body_for(&plan, 0); + let n = body.len(); + body[n - 3] ^= 0xFF; + assert!(matches!(engine.process(&item(0), body), Err(SinkError::Retry(_)))); + assert!(fx.stages().is_empty()); + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); + }); + assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), fx.expected(2)); + } + + #[test] + fn a_failed_mark_stops_before_any_replacement() { + let fx = Fixture::new("stream-mark"); + fs::write(fx.install.join("c.bin"), b"old c").unwrap(); + let plan = build_stream_plan(&fx.request(vec![2], 1 << 20)).unwrap(); + let cancel = AtomicBool::new(false); + let mut rec = Recorder::new(); + rec.fail_mark = true; + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + let err = engine.process(&item(0), fx.body_for(&plan, 0)).unwrap_err(); + assert!(matches!(err, SinkError::Fatal(ref m) if m.contains("atomically replace"))); + }); + assert_eq!(fs::read(fx.install.join("c.bin")).unwrap(), b"old c"); + assert!(fx.stages().is_empty()); + } + + #[test] + fn an_owned_staging_dir_holds_the_stages_and_is_left_empty() { + let fx = Fixture::new("stream-staging"); + let staging = fx.install.join(".droiddeck-epic-staging"); + fs::create_dir_all(&staging).unwrap(); + let mut req = fx.request(vec![0, 1, 2], 1 << 20); + req.staging_dir = staging.to_str().unwrap().to_string(); + let plan = build_stream_plan(&req).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); + let stages: Vec<_> = fs::read_dir(&staging).unwrap().flatten().collect(); + assert_eq!(stages.len(), 1, "a.bin is staged in the owned directory"); + assert!(stages[0].file_name().to_string_lossy().ends_with(".stage")); + for id in 1..plan.fetch_events.len() { + engine.process(&item(id), fx.body_for(&plan, id)).unwrap(); + } + }); + assert_eq!(fs::read_dir(&staging).unwrap().count(), 0); + assert_eq!(fs::read(fx.install.join("dir/b.bin")).unwrap(), fx.expected(1)); + let mut missing = fx.request(vec![0], 1 << 20); + missing.staging_dir = fx.dir.join("nope").to_str().unwrap().to_string(); + assert!(build_stream_plan(&missing).unwrap_err().contains("staging dir")); + } + + #[test] + fn a_manifest_file_named_like_a_stage_is_never_used_for_staging() { + let fx = Fixture::new("stream-reserved"); + let plan = build_stream_plan(&fx.request(vec![0, 1, 2], 1 << 20)).unwrap(); + assert!(plan.reserved.contains(&fx.install.join("dir/b.bin"))); + assert!(plan.reserved.contains(&fx.install.join("a.bin"))); + } + + // ── replacement-overlap gate ── + + /// `n` files of `len` bytes, one chunk each. + fn big_fixture(tag: &str, n: usize, len: usize) -> Fixture { + let dir = temp_dir(tag); + let install = dir.join("game"); + fs::create_dir_all(&install).unwrap(); + let guid = |i: usize| [0x100 + i as u32, 0, 0, 0]; + Fixture { + dir, + install, + chunks: (0..n).map(|i| (guid(i), data(i as u32 + 10, len))).collect(), + files: (0..n) + .map(|i| (format!("f{i}.bin"), vec![(guid(i), 0, len as i32)])) + .collect(), + } + } + + /// Drive the engine the way the fetch core does: items in order while the gate allows, + /// bodies handed back newest first (the most out-of-order the core can be). + fn drive_gated(fx: &Fixture, plan: &StreamPlan, engine: &Engine) -> Result<(), SinkError> { + engine.start()?; + let mut next = 0; + let mut in_flight: Vec = Vec::new(); + while next < plan.fetch_events.len() || !in_flight.is_empty() { + while next < plan.fetch_events.len() && engine.may_dispatch(next) { + in_flight.push(next); + next += 1; + } + let id = in_flight.pop().expect("gate closed with nothing in flight: deadlock"); + engine.process(&item(id), fx.body_for(plan, id))?; + } + Ok(()) + } + + fn write_old(fx: &Fixture, files: &[usize]) { + for &i in files { + fs::write(fx.install.join(&fx.files[i].0), b"old").unwrap(); + } + } + + #[test] + fn replaced_files_stay_within_the_overlap_budget_and_complete() { + let fx = big_fixture("stream-overlap", 6, 1000); + write_old(&fx, &[0, 1, 2, 3, 4, 5]); + let mut req = fx.request((0..6).collect(), 1 << 20); + req.replace_overlap_budget = 2500; + let plan = build_stream_plan(&req).unwrap(); + assert!(plan.files.iter().all(|f| f.replaces)); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + assert!(engine.may_dispatch(1), "two 1000-byte files fit 2500"); + assert!(!engine.may_dispatch(2), "a third does not"); + drive_gated(&fx, &plan, engine).unwrap(); + assert_eq!(engine.peak_replace_overlap(), 2000); + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..6 { + assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); + } + assert!(fx.stages().is_empty()); + } + + #[test] + fn a_budget_below_the_largest_file_still_progresses_one_at_a_time() { + let fx = big_fixture("stream-overlap-one", 4, 800); + write_old(&fx, &[0, 1, 2, 3]); + let mut req = fx.request((0..4).collect(), 1 << 20); + req.replace_overlap_budget = 1; + let plan = build_stream_plan(&req).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + assert!(engine.may_dispatch(0)); + assert!(!engine.may_dispatch(1)); + drive_gated(&fx, &plan, engine).unwrap(); + assert_eq!(engine.peak_replace_overlap(), 800); + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..4 { + assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); + } + } + + #[test] + fn a_fresh_install_is_never_gated() { + let fx = big_fixture("stream-overlap-fresh", 5, 600); + let mut req = fx.request((0..5).collect(), 1 << 20); + req.replace_overlap_budget = 1; + let plan = build_stream_plan(&req).unwrap(); + assert!(plan.files.iter().all(|f| !f.replaces)); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + assert!((0..5).all(|id| engine.may_dispatch(id))); + drive_gated(&fx, &plan, engine).unwrap(); + assert_eq!(engine.peak_replace_overlap(), 0); + }); + assert_eq!(fs::read(fx.install.join("f4.bin")).unwrap(), fx.expected(4)); + } + + #[test] + fn mixed_fresh_and_replaced_files_gate_only_the_replacements() { + let fx = big_fixture("stream-overlap-mixed", 6, 700); + write_old(&fx, &[1, 3, 5]); + let mut req = fx.request((0..6).collect(), 1 << 20); + req.replace_overlap_budget = 700; + let plan = build_stream_plan(&req).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + // f0 fresh, f1 replaced (admitted), f2 fresh, f3 replaced (waits for f1's swap). + assert!(engine.may_dispatch(2)); + assert!(!engine.may_dispatch(3)); + drive_gated(&fx, &plan, engine).unwrap(); + assert_eq!(engine.peak_replace_overlap(), 700); + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..6 { + assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); + } + } + + #[test] + fn a_shared_chunk_for_a_gated_file_waits_in_the_cache() { + // f0 and f2 are replaced and share chunk A; f1 is fresh. With room for one replaced file, + // f2 is admitted only after f0's swap, and A is cached until then. + let dir = temp_dir("stream-overlap-shared"); + let install = dir.join("game"); + fs::create_dir_all(&install).unwrap(); + let fx = Fixture { + dir, + install, + chunks: vec![(A, data(1, 300)), (B, data(2, 50))], + files: vec![ + ("f0.bin".to_string(), vec![(A, 0, 100)]), + ("f1.bin".to_string(), vec![(B, 0, 50)]), + ("f2.bin".to_string(), vec![(A, 100, 200)]), + ], + }; + write_old(&fx, &[0, 2]); + let mut req = fx.request(vec![0, 1, 2], 1 << 20); + req.replace_overlap_budget = 150; + let plan = build_stream_plan(&req).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + drive_gated(&fx, &plan, engine).unwrap(); + assert!(engine.peak_replace_overlap() <= 200); + assert_eq!(engine.unfinished_files(), 0); + }); + for i in 0..3 { + assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), fx.expected(i)); + } + assert_eq!(fs::read_dir(&plan.cache_dir).unwrap().count(), 0); + } + + #[test] + fn cancelling_a_gated_update_keeps_every_old_file() { + let fx = big_fixture("stream-overlap-cancel", 4, 900); + write_old(&fx, &[0, 1, 2, 3]); + let mut req = fx.request((0..4).collect(), 1 << 20); + req.replace_overlap_budget = 1000; + let plan = build_stream_plan(&req).unwrap(); + let cancel = AtomicBool::new(false); + let rec = Recorder::new(); + with_engine(&plan, &cancel, &rec, |engine| { + engine.start().unwrap(); + engine.process(&item(0), fx.body_for(&plan, 0)).unwrap(); // f0 swapped in + assert!(engine.may_dispatch(1)); + cancel.store(true, Ordering::Relaxed); + assert!(engine.process(&item(1), fx.body_for(&plan, 1)).is_err()); + engine.discard_staging(); + }); + assert_eq!(fs::read(fx.install.join("f0.bin")).unwrap(), fx.expected(0)); + for i in 1..4 { + assert_eq!(fs::read(fx.install.join(&fx.files[i].0)).unwrap(), b"old"); + } + assert!(fx.stages().is_empty()); + } + + fn item(id: usize) -> FetchItem { + FetchItem { + id: id as u64, + urls: vec!["http://127.0.0.1:9/x".to_string()], + reserve: 0, + range: None, + } + } + + fn guid_to(g: [u32; 4]) -> String { + super::super::manifest::guid_to_string(&g) + } +} diff --git a/app/src/main/rust/stores/src/fetch_core.rs b/app/src/main/rust/stores/src/fetch_core.rs index 9dc96faa6d..d0a5fa36c7 100644 --- a/app/src/main/rust/stores/src/fetch_core.rs +++ b/app/src/main/rust/stores/src/fetch_core.rs @@ -117,6 +117,25 @@ pub fn run_fetch( cancel: &AtomicBool, progress: &(dyn Fn(u64, u64) + Sync), log: &(dyn Fn(&str) + Sync), +) -> FetchOutcome { + run_fetch_gated(items, hosts, opts, sink, cancel, progress, log, &|_: usize| true) +} + +/// [`run_fetch`] with an admission gate: the next fresh item (by index) is only dispatched once +/// `gate(index)` says so. Items are still taken strictly in order, and retries are never held +/// back. While the gate is closed the driver idles as it does on a full window, so a gate that +/// opens as the sink makes progress (e.g. as files finish) paces the run without stalling the +/// requests already in flight. The adaptivity is unchanged. +#[allow(clippy::too_many_arguments)] +pub fn run_fetch_gated( + items: Vec, + hosts: &[String], + opts: &FetchOptions, + sink: &dyn FetchSink, + cancel: &AtomicBool, + progress: &(dyn Fn(u64, u64) + Sync), + log: &(dyn Fn(&str) + Sync), + gate: &(dyn Fn(usize) -> bool + Sync), ) -> FetchOutcome { let started = Instant::now(); let label = opts.label.as_str(); @@ -359,6 +378,7 @@ mode={}", Ok(rt) => { rt.block_on(run_driver(DriverCtx { items: items.as_slice(), + gate, hosts: hosts.as_slice(), opts, label, @@ -832,6 +852,7 @@ struct FetchDone { struct DriverCtx<'a> { items: &'a [FetchItem], + gate: &'a (dyn Fn(usize) -> bool + Sync), hosts: &'a [String], opts: &'a FetchOptions, label: &'a str, @@ -853,6 +874,7 @@ struct DriverCtx<'a> { async fn run_driver(ctx: DriverCtx<'_>) { let DriverCtx { items, + gate, hosts, opts, label, @@ -997,7 +1019,7 @@ async fn run_driver(ctx: DriverCtx<'_>) { let (idx, attempts) = if use_retry { let p = *retry.front().expect("retry nonempty"); (p.idx, p.attempts) - } else if next_item < items.len() { + } else if next_item < items.len() && gate(next_item) { (next_item, 0u32) } else { break; diff --git a/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt index 217bc7d5a3..de903c80f8 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt @@ -11,6 +11,7 @@ import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner import org.robolectric.RuntimeEnvironment import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowStatFs @RunWith(RobolectricTestRunner::class) @Config(sdk = [33]) @@ -55,4 +56,87 @@ class StoreInstallRootTest { assertTrue(dir.absolutePath.startsWith(app.cacheDir.absolutePath)) assertEquals("a_b_c", dir.name) } + + @Test fun estimatedBytesNeededUsesKnownInstallSizeForAllStores() { + val item = CatalogItem(Store.EPIC, "game1", "Game 1", imageUrl = null, owned = true, sizeBytes = 10_000_000L) + assertEquals(10_000_000L, StoreInstallRoot.estimatedBytesNeeded(item)) + + val gogItem = CatalogItem(Store.GOG, "game2", "Game 2", imageUrl = null, owned = true, sizeBytes = 10_000_000L) + assertEquals(10_000_000L, StoreInstallRoot.estimatedBytesNeeded(gogItem)) + } + + @Test fun checkStorageReportsEnoughWhenSpaceIsUnknownOrAvailable() { + val zeroItem = CatalogItem(Store.EPIC, "game1", "Game 1", imageUrl = null, owned = true, sizeBytes = 0L) + val check = StoreInstallRoot.checkStorage(app, zeroItem, StoreInstallRoot.internalRoot(app)) + assertTrue(check.enough) + } + + @Test fun availableBytesUsesNearestExistingAncestorWithoutCreatingDirectories() { + val existing = File(app.filesDir, "storage-probe").apply { mkdirs() } + val missing = File(existing, "not-created/another-level/game") + ShadowStatFs.registerStats(existing.path, 1024, 100, 100) + + assertTrue(StoreInstallRoot.availableBytes(missing) > 0L) + assertFalse(File(existing, "not-created").exists()) + } + + @Test fun epicDefersFullSizeChecksToItsManifestDeltaForResume() { + val item = CatalogItem(Store.EPIC, "large", "Large", imageUrl = null, owned = true, sizeBytes = Long.MAX_VALUE) + val check = StoreInstallRoot.checkStorage(app, item, StoreInstallRoot.internalRoot(app)) + assertTrue(check.enough) + assertEquals(Long.MAX_VALUE, check.requiredBytes) + } + + @Test fun gogAndAmazonResumeChecksUseTheExistingVolumeAndDeferRemainingWork() { + val card = File(app.filesDir, "resume-card").apply { mkdirs() } + SessionPrefs.setGameStorage(app, card.path, "Card") + for (store in listOf(Store.GOG, Store.AMAZON)) { + val item = CatalogItem(store, "resume-${store.id}", "Partial ${store.id}", imageUrl = null, sizeBytes = Long.MAX_VALUE) + val folder = File(StoreInstallRoot.storeDir(File(card, "Games"), store), StoreInstallRoot.folderName(item.title, item.id)).apply { mkdirs() } + StoreInstalls.begin(folder, store, item.id, item.title, null, null) + val check = StoreInstallRoot.checkStorage(app, item, StoreInstallRoot.internalRoot(app)) + assertTrue(check.enough) + assertEquals(StoreInstallRoot.availableBytes(folder), check.availableBytes) + assertEquals(folder, StoreInstallRoot.folderFor(app, store, item.id, item.title)) + } + } + + @Test fun freshGogAndAmazonStillRejectInsufficientCatalogSpace() { + StoreInstallRoot.internalRoot(app).mkdirs() + ShadowStatFs.registerStats(StoreInstallRoot.internalRoot(app).path, 1024, 100, 100) + for (store in listOf(Store.GOG, Store.AMAZON)) { + val item = CatalogItem(store, "too-large-${store.id}", "Too large ${store.id}", imageUrl = null, sizeBytes = Long.MAX_VALUE) + assertFalse(StoreInstallRoot.checkStorage(app, item, StoreInstallRoot.internalRoot(app)).enough) + } + } + + @Test fun measuredZeroRejectsFreshGogAndAmazonRatherThanBeingUnknown() { + val root = File(app.filesDir, "zero-space").apply { mkdirs() } + ShadowStatFs.registerStats(root.path, 1024, 0, 0) + val missing = File(root, "not-created/game") + assertEquals(0L, StoreInstallRoot.availableBytes(missing)) + assertFalse(File(root, "not-created").exists()) + for (store in listOf(Store.GOG, Store.AMAZON)) { + val item = CatalogItem(store, "zero-${store.id}", "Zero ${store.id}", imageUrl = null, sizeBytes = 1L) + val check = StoreInstallRoot.checkStorage(app, item, missing) + assertFalse(check.enough) + assertEquals(0L, check.availableBytes) + } + } + + @Test fun failedProbesAreUnknownAndDoNotCreateOrWriteToTargets() { + val root = File(app.filesDir, "read-only-probe").apply { mkdirs() } + val missing = File(root, "not-created/game") + val fail: (File) -> Long = { throw java.io.IOException("unavailable") } + assertEquals(-1L, StoreInstallRoot.probeAvailableBytes(missing, fail, fail)) + assertEquals(-1L, StoreInstallRoot.probeAvailableBytes(missing, fail, { 0L })) + assertEquals(42L, StoreInstallRoot.probeAvailableBytes(missing, fail, { 42L })) + assertEquals(0L, StoreInstallRoot.probeAvailableBytes(missing, { 0L }, fail)) + assertFalse(File(root, "not-created").exists()) + assertTrue(root.listFiles().orEmpty().isEmpty()) + assertFalse(StoreInstallRoot.insufficientSpace(1L, -1L)) + assertTrue(StoreInstallRoot.insufficientSpace(1L, 0L)) + assertFalse(StoreInstallRoot.insufficientSpace(1L, 1L)) + assertFalse(StoreInstallRoot.insufficientSpace(0L, 0L)) + } } diff --git a/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt new file mode 100644 index 0000000000..6382f9519a --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/amazon/AmazonResumeSpaceTest.kt @@ -0,0 +1,43 @@ +package com.droiddeck.launcher.stores.amazon + +import java.io.File +import java.security.MessageDigest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowStatFs +import com.droiddeck.launcher.stores.StoreInstallRoot + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class AmazonResumeSpaceTest { + @Test fun remainingWorkRejectsMeasuredZeroButNotUnknownSpace() { + val root = File(RuntimeEnvironment.getApplication().filesDir, "amazon-zero").apply { mkdirs() } + ShadowStatFs.registerStats(root.path, 1024, 0, 0) + val files = listOf(AmazonManifest.ManifestFile("missing", 4, 0, null)) + val needed = AmazonDownloadManager.remainingInstallBytes(files, root) + assertEquals(4L, needed) + assertTrue(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.availableBytes(root))) + assertFalse(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.SPACE_UNKNOWN)) + } + + @Test fun remainingSpaceExcludesVerifiedFilesButIncludesCorruptAndPartialFiles() { + val root = File(RuntimeEnvironment.getApplication().filesDir, "amazon-resume").apply { mkdirs() } + val data = byteArrayOf(1, 2, 3, 4) + val hash = MessageDigest.getInstance("SHA-256").digest(data) + val files = listOf("good", "corrupt", "partial", "missing").map { + AmazonManifest.ManifestFile(it, 4, 0, hash) + } + File(root, "good").writeBytes(data) + File(root, "corrupt").writeBytes(ByteArray(4)) + File(root, "partial").writeBytes(byteArrayOf(1)) + assertEquals(12L, AmazonDownloadManager.remainingInstallBytes(files, root)) + for (file in files) File(root, file.unixPath()).writeBytes(data) + assertEquals(0L, AmazonDownloadManager.remainingInstallBytes(files, root)) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/download/DownloadQueueRetryTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/download/DownloadQueueRetryTest.kt new file mode 100644 index 0000000000..879e8de91d --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/download/DownloadQueueRetryTest.kt @@ -0,0 +1,140 @@ +package com.droiddeck.launcher.stores.download + +import android.content.Context +import com.droiddeck.launcher.session.SessionPrefs +import com.droiddeck.launcher.stores.* +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import org.junit.After +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.json.JSONArray +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowStatFs + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class DownloadQueueRetryTest { + private val app get() = RuntimeEnvironment.getApplication() + + @After fun cleanRows() { DownloadQueue.dismissFinished(); StoreSizes.known.clear() } + + private fun awaitFailure(key: String, error: String) { + val until = System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + while (System.nanoTime() < until) { + val entry = DownloadQueue.entry(key) + if (entry?.state == DownloadState.FAILED && entry.error == error) return + Thread.sleep(10) + } + fail("Expected failed real job: ${DownloadQueue.entry(key)}") + } + + @Test fun storageRetryRechecksThenRunsTheRealBackendAndNormalRetryStillWorks() { + StoreInstallRoot.internalRoot(app).mkdirs() + ShadowStatFs.registerStats(StoreInstallRoot.internalRoot(app).path, 1024, 100, 100) + val item = CatalogItem(Store.GOG, "preflight-retry", "Preflight retry", imageUrl = null, sizeBytes = Long.MAX_VALUE) + val runs = AtomicInteger() + var lastTarget: File? = null + val previous = StoresState.backend(Store.GOG) + val backend = object : StoreBackend { + override val store = Store.GOG + override fun signIn(context: Context) {} + override fun signOut(context: Context) {} + override fun syncLibrary(context: Context, force: Boolean) {} + override fun loadShelves(context: Context, force: Boolean) {} + override fun uninstall(context: Context, game: InstalledStoreGame) {} + override fun install(context: Context, item: CatalogItem, root: File) { + lastTarget = root + DownloadQueue.enqueue(context, DownloadEntry(item.store, item.id, item.title)) { + object : DownloadQueue.DownloadJob { + override fun run(handle: DownloadQueue.JobHandle): String? { + throw IllegalStateException("backend-${runs.incrementAndGet()}") + } + override fun cancel(deleteFiles: Boolean) {} + } + } + } + } + StoresState.register(backend) + try { + StoresState.install(app, item) + assertEquals(DownloadState.FAILED, DownloadQueue.entry(item.key)?.state) + DownloadQueue.retry(app, item.key) + assertEquals(DownloadState.FAILED, DownloadQueue.entry(item.key)?.state) + assertEquals(0, runs.get()) + StoreSizes.known[item.key] = 1 + DownloadQueue.retry(app, item.key) + awaitFailure(item.key, "backend-1") + DownloadQueue.retry(app, item.key) + awaitFailure(item.key, "backend-2") + DownloadQueue.clear(app, item.key) + assertNull(DownloadQueue.entry(item.key)) + + // A card resume (including the restored paused path) must retain its real target. + val card = File(app.filesDir, "retry-card").apply { mkdirs() } + SessionPrefs.setGameStorage(app, card.path, "Card") + val target = File(card, "Games") + val folder = File(StoreInstallRoot.storeDir(target, item.store), StoreInstallRoot.folderName(item.title, item.id)).apply { mkdirs() } + StoreInstalls.begin(folder, item.store, item.id, item.title, null, null) + StoreSizes.known[item.key] = Long.MAX_VALUE + StoresState.unfinished = StoreInstallRoot.unfinished(app) + DownloadQueue.holdPaused(item.key) + StoresState.requestInstall(app, item) + assertEquals(target, lastTarget) + assertEquals(DownloadState.PAUSED, DownloadQueue.entry(item.key)?.state) + assertEquals(2, runs.get()) + val saved = File(app.filesDir, "store-queue.json") + val until = System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + var correctSavedRoot = false + while (!correctSavedRoot && System.nanoTime() < until) { + correctSavedRoot = runCatching { + val rows = JSONArray(saved.readText()) + (0 until rows.length()).any { i -> + val row = rows.getJSONObject(i) + row.getJSONObject("item").getString("id") == item.id && + row.getString("root") == target.absolutePath && row.getBoolean("paused") + } + }.getOrDefault(false) + if (!correctSavedRoot) Thread.sleep(10) + } + assertTrue("Paused saved queue must remember the existing card root", correctSavedRoot) + DownloadQueue.clear(app, item.key) + } finally { + if (previous != null) StoresState.register(previous) + SessionPrefs.setGameStorage(app, SessionPrefs.GAME_STORAGE_OFF, "") + StoresState.unfinished = emptyList() + } + } + + @Test fun preflightClearRemovesTheRowWithoutRunningItsRetry() { + val entry = DownloadEntry(Store.AMAZON, "clear-preflight", "Clear") + DownloadQueue.recordFailed(app, entry, "storage") { fail("Clear must not retry") } + DownloadQueue.clear(app, entry.key) + assertNull(DownloadQueue.entry(entry.key)) + } + + @Test fun heldPausedBackendRestoresAndResumesWithoutAPlaceholder() { + val entry = DownloadEntry(Store.GOG, "restored-paused", "Paused") + val ran = CountDownLatch(1) + DownloadQueue.holdPaused(entry.key) + DownloadQueue.enqueue(app, entry) { + object : DownloadQueue.DownloadJob { + override fun run(handle: DownloadQueue.JobHandle): String? { + ran.countDown() + throw IllegalStateException("restored") + } + override fun cancel(deleteFiles: Boolean) {} + } + } + assertEquals(DownloadState.PAUSED, DownloadQueue.entry(entry.key)?.state) + assertEquals(1L, ran.count) + DownloadQueue.resume(app, entry.key) + assertTrue(ran.await(5, TimeUnit.SECONDS)) + awaitFailure(entry.key, "restored") + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt new file mode 100644 index 0000000000..fdf46d0df5 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicAdaptivePlanTest.kt @@ -0,0 +1,107 @@ +package com.droiddeck.launcher.stores.epic + +import java.io.File +import org.junit.Assert.* +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicAdaptivePlanTest { + @get:Rule val temp = TemporaryFolder() + private val mib = 1024L * 1024 + private val gib = 1024 * mib + + private fun file(name: String, size: Long) = EpicDownloadManager.FileInfo().apply { + filename = name + var left = size + while (left > 0) { + val part = minOf(left, Int.MAX_VALUE.toLong()) + parts.add(EpicDownloadManager.ChunkPart().apply { this.size = part.toInt() }) + left -= part + } + } + + @Test fun netPeakCreditsEachOldCopyAtItsSwapAndLargestReplacedIgnoresFreshFiles() { + val files = listOf(file("a", 10 * gib), file("b", 2 * gib), file("c", 4 * gib)) + val old = mapOf("a" to 9 * gib, "c" to 5 * gib) + // Running Σ(new − old): +1, +3, +2 GiB. + assertEquals(3 * gib, EpicDownloadManager.settledNetPeak(files, old)) + assertEquals(10 * gib, EpicDownloadManager.largestReplacedFile(files, old)) + assertEquals(16 * gib, EpicDownloadManager.settledNetPeak(files, emptyMap())) + assertEquals(0L, EpicDownloadManager.largestReplacedFile(files, emptyMap())) + assertEquals(0L, EpicDownloadManager.largestReplacedFile(files, mapOf("b" to 0L))) + } + + @Test fun cacheBudgetIsHalfTheHeadroomCappedAndDefaultsWhenSpaceIsUnknown() { + assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_DEFAULT_BYTES, EpicDownloadManager.adaptiveCacheBudget(-1, 0)) + assertEquals(EpicDownloadManager.ADAPTIVE_CACHE_MAX_BYTES, EpicDownloadManager.adaptiveCacheBudget(100 * gib, 10 * gib)) + assertEquals(0L, EpicDownloadManager.adaptiveCacheBudget(gib, gib)) + val free = 1000 * mib + assertEquals((free - 400 * mib - EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES) / 2, EpicDownloadManager.adaptiveCacheBudget(free, 400 * mib)) + } + + @Test fun freshInstallsHaveNoOverlapLimit() { + val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 10 * gib) + assertNull(plan.error) + assertEquals(-1L, plan.overlapBudget) + assertEquals(20 * gib, plan.gameNeed) + } + + @Test fun updateOverlapBudgetIsTheLeftoverAfterNetGrowthAndCache() { + // Card install, cache on internal: the card's leftover after the net growth is the budget. + val roomy = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 20 * gib, true, 10 * gib) + assertNull(roomy.error) + assertFalse(roomy.cacheBesideGame) + assertEquals(19 * gib, roomy.overlapBudget) + assertEquals(20 * gib, roomy.gameNeed) + // Tight: floored at the largest replaced file, one at a time. + val tight = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 5 * gib, true, 10 * gib) + assertNull(tight.error) + assertEquals(4 * gib, tight.overlapBudget) + // Not even one replaced file fits: the shortfall is named. + val short = EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, 4 * gib, true, 10 * gib) + assertTrue(short.error!!.contains("game's storage")) + assertTrue(short.error!!.contains("short by 1.0 GB")) + // Unknown free space: one replaced file at a time. + assertEquals(4 * gib, EpicDownloadManager.planAdaptiveSpace(gib, 4 * gib, -1, true, -1).overlapBudget) + } + + @Test fun cacheBesideTheGameComesOutOfTheOverlapBudget() { + val plan = EpicDownloadManager.planAdaptiveSpace(gib, 2 * gib, 10 * gib, false, -1) + assertNull(plan.error) + assertTrue(plan.cacheBesideGame) + assertTrue(plan.cacheBudget > 0) + assertEquals(10 * gib - gib - plan.cacheBudget - EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.overlapBudget) + assertEquals(10 * gib, plan.gameNeed) + } + + @Test fun fullInternalStorageMovesTheCacheBesideTheGameInsteadOfFailing() { + val plan = EpicDownloadManager.planAdaptiveSpace(20 * gib, 0, 30 * gib, true, 100 * mib) + assertNull(plan.error) + assertTrue(plan.cacheBesideGame) + assertEquals(0L, plan.scratchNeed) + assertTrue(plan.cacheBudget > 0) + assertEquals(20 * gib + plan.cacheBudget + EpicDownloadManager.ADAPTIVE_TRANSIENT_BYTES, plan.gameNeed) + } + + @Test fun ownedStagingIsRemovedOnlyWhenItHoldsJustItsMarker() { + val root = temp.newFolder("game") + val staging = EpicDownloadManager.recoverOwnedStaging(root, emptyList()) + File(staging, "keep.stage").writeText("x") + EpicDownloadManager.releaseOwnedStaging(root) + assertTrue(File(staging, "owner").exists()) + File(staging, "keep.stage").delete() + EpicDownloadManager.releaseOwnedStaging(root) + assertFalse(staging.exists()) + // A directory without the marker is not ours. + staging.mkdir() + File(staging, "owner").writeText("someone else") + EpicDownloadManager.releaseOwnedStaging(root) + assertTrue(staging.exists()) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt new file mode 100644 index 0000000000..37dd7d849d --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt @@ -0,0 +1,646 @@ +package com.droiddeck.launcher.stores.epic + +import com.droiddeck.launcher.stores.Store +import com.droiddeck.launcher.stores.StoreGameSidecar +import com.droiddeck.launcher.stores.StoreInstalls +import java.io.File +import java.net.InetAddress +import java.net.ServerSocket +import java.net.SocketException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.net.Socket +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowStatFs + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicBoundedInstallTest { + @get:Rule val temp = TemporaryFolder() + private lateinit var server: ServerSocket + private lateinit var worker: Thread + private lateinit var manifest: EpicDownloadManager.Manifest + private lateinit var root: File + private val requests = AtomicInteger() + private val reject = AtomicBoolean() + private val rejectGuid = AtomicInteger() + private val handlers = Executors.newCachedThreadPool() + private val sockets = ConcurrentHashMap.newKeySet() + private var requestBarrier: CountDownLatch? = null + private var deliveryGate: CountDownLatch? = null + private var bytes = ByteArray(256 * 1024) { (it % 251).toByte() } + + @Before fun setUp() { + root = temp.newFolder("game") + ShadowStatFs.registerStats(root.path, 1_000_000_000, 1_000_000_000, 1_000_000_000) + server = ServerSocket(0, 8, InetAddress.getByName("127.0.0.1")) + worker = Thread { + while (!server.isClosed) { + val socket = try { server.accept() } catch (e: SocketException) { + if (server.isClosed) break else throw e + } + sockets.add(socket) + handlers.submit { + try { + socket.use { + it.soTimeout = 5000 + val reader = it.getInputStream().bufferedReader() + val request = reader.readLine() + while (!reader.readLine().isNullOrEmpty()) { } + requests.incrementAndGet() + requestBarrier?.let { barrier -> + barrier.countDown() + check(barrier.await(5, TimeUnit.SECONDS)) { "Requests did not overlap" } + } + val out = it.getOutputStream() + val rejectedGuid = rejectGuid.get() + if (reject.get() || (rejectedGuid != 0 && request.contains("_%08X".format(rejectedGuid)))) { + out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray()) + } else { + val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN) + .putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size) + .put(ByteArray(24)).put(0).array() + out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray()) + out.write(header) + out.flush() + deliveryGate?.await(5, TimeUnit.SECONDS) + out.write(bytes) + } + out.flush() + } + } catch (_: java.io.IOException) { + // Cancellation disconnects an owned request while this fixture serves it. + } finally { sockets.remove(socket) } + } + } + }.apply { start() } + val chunk = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(1, 0, 0, 0) + windowSize = bytes.size + fileSize = bytes.size.toLong() + sha1 = sha(bytes) + } + manifest = EpicDownloadManager.Manifest().apply { + chunkDir = "chunks" + uniqueChunks = arrayListOf(chunk) + cdnUrls = listOf(EpicDownloadManager.CdnUrl("http://127.0.0.1:${server.localPort}", "", "")) + files = arrayListOf(gameFile("game.bin", 0, bytes.size / 2), gameFile("game.bin.part", bytes.size / 2, bytes.size / 2)) + } + } + + @After fun tearDown() { + deliveryGate?.countDown() + server.close() + worker.join(6000) + sockets.forEach { it.close() } + handlers.shutdownNow() + assertTrue(handlers.awaitTermination(6, TimeUnit.SECONDS)) + assertFalse(worker.isAlive) + } + + private fun sha(data: ByteArray) = MessageDigest.getInstance("SHA-1").digest(data) + + private fun gameFile(name: String, offset: Int, size: Int) = EpicDownloadManager.FileInfo().apply { + filename = name + sha1 = sha(bytes.copyOfRange(offset, offset + size)) + parts.add(EpicDownloadManager.ChunkPart().apply { + guid = intArrayOf(1, 0, 0, 0) + this.offset = offset + this.size = size + }) + } + + private fun install(cancel: AtomicBoolean = AtomicBoolean(), callback: EpicDownloadManager.Callback = + EpicDownloadManager.Callback { _, _ -> }, cache: String = "") = + EpicDownloadManager.installManifest(manifest, root.path, cache, null, cancel, callback) + + private fun oldInstall() { + File(root, "game.bin").writeBytes(byteArrayOf(9, 8, 7)) + File(root, "game.bin.part").writeBytes(byteArrayOf(6, 5, 4)) + StoreGameSidecar(Store.EPIC, "game", "Game", "game.bin", installVersion = "old").write(root) + } + + private fun assertOldInstallIntact() { + assertArrayEquals(byteArrayOf(9, 8, 7), File(root, "game.bin").readBytes()) + assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) + assertEquals("old", StoreGameSidecar.read(root)?.installVersion) + assertTrue(StoreGameSidecar.read(root)?.isInstalled == true) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + } + + @Test fun networkFailureAfterAcceptedPreflightPreservesInstalledFilesAndRecord() { + oldInstall() + reject.set(true) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(requests.get() > 0) + assertOldInstallIntact() + } + + @Test fun hashFailureAfterAcceptedPreflightPreservesInstalledFilesAndRecord() { + oldInstall() + manifest.files.first().sha1 = ByteArray(20) { 1 } + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertOldInstallIntact() + } + + @Test fun cancellationWhileStagingPreservesInstalledFilesAndRecord() { + oldInstall() + val cancel = AtomicBoolean() + assertNull(install(cancel, EpicDownloadManager.Callback { message, _ -> + if (message.startsWith("Downloading chunks (")) cancel.set(true) + })) + assertOldInstallIntact() + } + + @Test fun laterFailureRetainsVerifiedSwapsAndMarksMixedUpdateResumable() { + oldInstall() + manifest.files.last().sha1 = ByteArray(20) { 1 } + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) + val record = StoreGameSidecar.read(root)!! + assertEquals(StoreGameSidecar.STATE_UPDATING, record.state) + assertFalse(record.isInstalled) + assertEquals("", record.installVersion) + StoreInstalls.begin(root, Store.EPIC, "game", "Game", null, null) + assertEquals("game.bin", StoreGameSidecar.read(root)?.exe) + assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) + } + + @Test fun laterNetworkFailurePreservesTheUnreplacedOldFile() { + oldInstall() + val secondChunk = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(2, 0, 0, 0) + windowSize = bytes.size + fileSize = bytes.size.toLong() + sha1 = sha(bytes) + } + manifest.uniqueChunks.add(secondChunk) + manifest.files.last().parts.single().guid = secondChunk.guid + // Prefetch may fail early, but replacement is still ordered by the writer. + rejectGuid.set(2) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) + assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + } + + @Test fun cancellationAfterFirstSwapLeavesAResumableMixedUpdate() { + oldInstall() + val cancel = AtomicBoolean() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) {} + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + if (stage == "stream" && items == 1) cancel.set(true) + } + } + assertNull(install(cancel, callback)) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(byteArrayOf(6, 5, 4), File(root, "game.bin.part").readBytes()) + assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) + // Registration, not an individual file swap, restores the installed state. + assertFalse(StoreGameSidecar.read(root)!!.isInstalled) + } + + @Test fun atomicRenameFailurePreservesDestinationAndCleansStagingWithoutDeleteFallback() { + val destination = File(root, "game.bin").apply { mkdir() } + File(destination, "old-content").writeText("keep") + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(error.message.orEmpty().contains("atomically replace")) + assertEquals("keep", File(destination, "old-content").readText()) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + } + + @Test fun safeStagingSpaceBoundaryRejectsWithoutReclaimingAllOldOutputs() { + // The first staged half-file must coexist with the shared chunk and BOTH old outputs. + val old = ByteArray(bytes.size / 2) { 9 } + File(root, "game.bin").writeBytes(old) + File(root, "game.bin.part").writeBytes(old) + val peak = bytes.size + old.size + ShadowStatFs.registerStats(root.path, 1024, peak / 4096 - 1, peak / 4096 - 1) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertEquals(0, requests.get()) + assertArrayEquals(old, File(root, "game.bin").readBytes()) + assertArrayEquals(old, File(root, "game.bin.part").readBytes()) + ShadowStatFs.registerStats(root.path, 1024, peak / 4096, peak / 4096) + assertNotNull(install()) + } + + @Test fun zeroSpaceRejectsPendingFilesWhileUnknownSpaceIsNotTreatedAsZero() { + oldInstall() + ShadowStatFs.registerStats(root.path, 1024, 0, 0) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertOldInstallIntact() + assertEquals(0, requests.get()) + assertTrue(com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(1, 0)) + assertFalse(com.droiddeck.launcher.stores.StoreInstallRoot.insufficientSpace(1, -1)) + } + + @Test fun zeroScratchSpaceRejectsBeforeFetchWithoutTouchingInstalledOutputs() { + oldInstall() + val cache = temp.newFolder("full-scratch") + ShadowStatFs.registerStats(cache.path, 1024, 0, 0) + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache.path) } + assertTrue(error.message.orEmpty().contains("Not enough internal space")) + assertOldInstallIntact() + assertEquals(0, requests.get()) + } + + @Test fun installsSharedChunksAndPreservesLegitimatePartFilename() { + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) + assertEquals(1, requests.get()) + assertFalse(File(root, ".chunks").exists()) + assertNotNull(install()) + assertEquals("Verified files must not be downloaded again", 1, requests.get()) + } + + private fun oversizedUpdate() { + val file = manifest.files[0] + val part = file.parts[0] + file.parts.clear() + repeat(100_000) { + file.parts.add(EpicDownloadManager.ChunkPart().apply { + guid = part.guid + size = Int.MAX_VALUE + }) + } + } + + @Test fun insufficientInstallSpacePreservesExistingFilesOnBothCacheLayouts() { + val oldGame = byteArrayOf(9, 8, 7) + val oldPart = byteArrayOf(6, 5, 4) + File(root, "game.bin").writeBytes(oldGame) + File(root, "game.bin.part").writeBytes(oldPart) + oversizedUpdate() + assertTrue(manifest.files[0].fileSize() > root.usableSpace) + + for (cache in listOf("", temp.newFolder("scratch-rejected").path)) { + if (cache.isNotEmpty()) ShadowStatFs.registerStats(cache, 1_000_000_000, 1_000_000_000, 1_000_000_000) + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache) } + assertTrue(error.message.orEmpty().contains("Not enough free space")) + assertArrayEquals(oldGame, File(root, "game.bin").readBytes()) + assertArrayEquals(oldPart, File(root, "game.bin.part").readBytes()) + } + assertEquals(0, requests.get()) + } + + @Test fun acceptedUpdateReplacesOutdatedFilesWithVerifiedOutputs() { + oldInstall() + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) + assertEquals(StoreGameSidecar.STATE_UPDATING, StoreGameSidecar.read(root)?.state) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + } + + @Test fun pauseKeepsCompletedFilesAndResumeReusesSharedCache() { + val cancel = AtomicBoolean() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) {} + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + if (stage == "stream" && items == 1) cancel.set(true) + } + } + assertNull(install(cancel, callback)) + assertTrue(File(root, "game.bin").exists()) + assertFalse(File(root, "game.bin.part").exists()) + assertEquals(1, requests.get()) + assertNotNull(install()) + assertEquals(1, requests.get()) + } + + @Test fun cancellationDuringChunkDeliveryRemovesIncompleteOutput() { + val cancel = AtomicBoolean() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) { + if (message.startsWith("Downloading chunks (")) cancel.set(true) + } + } + assertNull(install(cancel, callback)) + assertFalse(File(root, "game.bin").exists()) + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + } + + @Test fun fileHashFailureIsExplicitAndRetryRepairsIt() { + val expected = manifest.files[0].sha1 + manifest.files[0].sha1 = ByteArray(20) { 1 } + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(error.message.orEmpty().contains("SHA-1 mismatch")) + assertFalse(File(root, "game.bin").exists()) + manifest.files[0].sha1 = expected + assertNotNull(install()) + } + + @Test fun corruptCacheIsRefetchedAndScratchCacheIsRemovedOnSuccess() { + val cache = temp.newFolder("scratch") + ShadowStatFs.registerStats(cache.path, 1_000_000_000, 1_000_000_000, 1_000_000_000) + File(cache, manifest.uniqueChunks[0].guidStr()).writeBytes(ByteArray(bytes.size)) + assertNotNull(install(cache = cache.path)) + assertEquals(1, requests.get()) + assertFalse(cache.exists()) + } + + @Test fun networkFailureLeavesNoCompletedFileAndCanBeRetried() { + reject.set(true) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertFalse(File(root, "game.bin").exists()) + reject.set(false) + assertNotNull(install()) + } + + @Test fun boundedProgressNeverReentersDownloadAfterEnteringInstall() { + val events = mutableListOf() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) { + if (message.startsWith("Downloading chunks")) events.add("download") + if (message.startsWith("Writing")) events.add("install") + } + override fun onBytes(done: Long, total: Long, speedBps: Long) { events.add("download") } + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + if (stage == "install") { + assertEquals(total, done) + assertEquals(itemsTotal, items) + events.add("install") + } + } + } + assertNotNull(install(callback = callback)) + val firstInstall = events.indexOf("install") + assertTrue(firstInstall > 0) + assertFalse(events.drop(firstInstall).contains("download")) + } + + @Test fun sharedChunksDoNotCompleteProgressBeforeTheRemainingFilesAreWritten() { + val samples = mutableListOf>() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) {} + override fun onBytes(done: Long, total: Long, speedBps: Long) { samples.add(done to total) } + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + if (stage == "stream" && items < itemsTotal) assertTrue(samples.last().first < samples.last().second) + } + } + assertNotNull(install(callback = callback)) + assertEquals(listOf(0L, bytes.size / 2L, bytes.size.toLong()), samples.map { it.first }) + assertTrue(samples.all { it.second == bytes.size.toLong() }) + } + + @Test fun oversizedSharedChunkStaysPinnedAcrossPrefetchedFilesWithoutChangingStages() { + bytes = ByteArray((EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 1024).toInt()) { (it % 251).toByte() } + manifest.uniqueChunks.single().apply { + windowSize = bytes.size + fileSize = bytes.size.toLong() + sha1 = sha(bytes) + } + manifest.files = (0..2).map { gameFile("part-$it.bin", it * 1024, 1024) } + val chunk = manifest.uniqueChunks.single() + val estimatedPeak = EpicDownloadManager.estimatePeakAdditionalBytes( + manifest.files, + mapOf(chunk.guidStr() to chunk), + emptyMap(), + true, + ) + assertEquals(EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 4 * 1024L, estimatedPeak) + val samples = mutableListOf() + val stages = mutableListOf() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) {} + override fun onBytes(done: Long, total: Long, speedBps: Long) { + assertEquals(3072L, total) + assertFalse(stages.contains("install")) + samples.add(done) + } + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + stages.add(stage) + if (stage == "stream" && items < itemsTotal) assertTrue(samples.last() < total) + } + } + assertNotNull(install(callback = callback)) + assertEquals(1, requests.get()) + assertEquals(listOf(0L, 1024L, 2048L, 3072L), samples) + assertEquals("install", stages.last()) + repeat(3) { assertArrayEquals(bytes.copyOfRange(it * 1024, (it + 1) * 1024), File(root, "part-$it.bin").readBytes()) } + } + + @Test fun hogwartsSizedSequentialPlanFitsAvailableSpaceWithoutFullChunkCache() { + val gib = 1024L * 1024 * 1024 + val file = EpicDownloadManager.FileInfo() + val chunks = LinkedHashMap() + repeat(72) { index -> + val chunk = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(index + 1, 0, 0, 0) + windowSize = gib.toInt() + } + chunks[chunk.guidStr()] = chunk + file.parts.add(EpicDownloadManager.ChunkPart().apply { + guid = chunk.guid + size = chunk.windowSize + }) + } + val peak = EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true) + assertEquals(73 * gib, peak) + assertTrue(peak < 96 * gib) + } + + @Test fun plannersReclaimEachOldFileOnlyAfterItsOwnVerifiedSwap() { + val chunks = manifest.uniqueChunks.associateBy { it.guidStr() } + val half = bytes.size / 2L + val old = mapOf("game.bin" to half, "game.bin.part" to 100 * half) + assertEquals(bytes.size + half, EpicDownloadManager.estimatePeakAdditionalBytes(manifest.files, chunks, emptyMap(), true, old)) + assertEquals(half, EpicDownloadManager.estimateStagedFilePeak(manifest.files, old)) + assertEquals(2 * half, EpicDownloadManager.estimateStagedFilePeak(manifest.files, emptyMap())) + assertEquals(bytes.size.toLong(), EpicDownloadManager.estimatePeakAdditionalBytes(manifest.files, chunks, emptyMap(), false, old)) + } + + private fun distinctFiles(count: Int) { + val template = manifest.uniqueChunks.single() + manifest.uniqueChunks = (1..count).map { id -> + EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(id, 0, 0, 0) + windowSize = template.windowSize + fileSize = template.fileSize + sha1 = template.sha1 + } + } + manifest.files = manifest.uniqueChunks.mapIndexed { index, chunk -> + gameFile("file-$index.bin", 0, bytes.size).apply { parts.single().guid = chunk.guid } + } + } + + @Test fun prefetchOverlapsEightSinglePartFilesButNeverFetchesTheWholeManifest() { + distinctFiles(20) + requestBarrier = CountDownLatch(8) + val cancel = AtomicBoolean() + var sawWindow = false + assertNull(install(cancel, EpicDownloadManager.Callback { message, _ -> + if (message.startsWith("Downloading chunks (")) { + assertEquals(8, requests.get()) + val cacheBytes = File(root, ".chunks").listFiles()!!.sumOf { it.length() } + assertTrue(cacheBytes <= 8L * bytes.size) + sawWindow = true + cancel.set(true) + } + })) + assertTrue("An eight-request barrier cannot pass with a sequential downloader", sawWindow) + assertEquals(8, requests.get()) + assertTrue(root.walk().none { it.name.endsWith(".stage") || it.name.endsWith(".part") }) + } + + @Test fun parallelWindowSpaceThresholdIsCheckedBeforeStartingAnyRequest() { + distinctFiles(20) + val cache = temp.newFolder("parallel-scratch") + val peak = EpicDownloadManager.estimatePeakAdditionalBytes( + manifest.files, manifest.uniqueChunks.associateBy { it.guidStr() }, emptyMap(), false, + ) + assertEquals(8L * bytes.size, peak) + ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096 - 1, peak.toInt() / 4096 - 1) + assertThrows(EpicDownloadManager.InstallException::class.java) { install(cache = cache.path) } + assertEquals(0, requests.get()) + ShadowStatFs.registerStats(cache.path, 1024, peak.toInt() / 4096, peak.toInt() / 4096) + requestBarrier = CountDownLatch(8) + assertNotNull(install(cache = cache.path)) + assertEquals(20, requests.get()) + manifest.files.forEach { assertArrayEquals(bytes, File(root, it.filename).readBytes()) } + } + + @Test fun cancellationDuringParallelNetworkReadsJoinsWorkersBeforeReturning() { + distinctFiles(20) + requestBarrier = CountDownLatch(8) + deliveryGate = CountDownLatch(1) + val cancel = AtomicBoolean() + val outcome = java.util.concurrent.atomic.AtomicReference() + val installer = Thread { + try { outcome.set(install(cancel)) } catch (e: Throwable) { outcome.set(e) } + }.apply { start() } + try { + assertTrue(requestBarrier!!.await(5, TimeUnit.SECONDS)) + cancel.set(true) + installer.join(10_000) + assertFalse("Installer must disconnect and join all download workers", installer.isAlive) + assertNull(outcome.get()) + val cache = File(root, ".chunks") + val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty() + deliveryGate!!.countDown() + Thread.sleep(150) + assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty()) + assertTrue(snapshot.keys.none { it.endsWith(".part") }) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + assertEquals(8, requests.get()) + } finally { + cancel.set(true) + deliveryGate!!.countDown() + installer.join(10_000) + } + } + + @Test fun failedParallelFetchJoinsOtherNetworkReadersBeforeReturning() { + distinctFiles(20) + requestBarrier = CountDownLatch(8) + deliveryGate = CountDownLatch(1) + rejectGuid.set(1) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertEquals(8, requests.get()) + val cache = File(root, ".chunks") + val snapshot = cache.listFiles()?.associate { it.name to it.length() }.orEmpty() + deliveryGate!!.countDown() + Thread.sleep(150) + assertEquals(snapshot, cache.listFiles()?.associate { it.name to it.length() }.orEmpty()) + assertTrue(snapshot.keys.none { it.endsWith(".part") }) + assertTrue(root.walk().none { it.name.endsWith(".stage") }) + } + + @Test fun ownedOrphanStagingIsRecoveredBeforeTheSpaceProbeWithoutTouchingManifestFiles() { + val staging = EpicDownloadManager.recoverOwnedStaging(root, manifest.files) + File(staging, "native-123-0.stage").writeBytes(ByteArray(bytes.size)) + val legitimate = File(root, ".droiddeck-epic-old.stage").apply { writeText("keep") } + oldInstall() + val peak = 2 * bytes.size + ShadowStatFs.registerStats(root.path, 1024, peak / 4096, peak / 4096) + assertNotNull(install()) + assertEquals("keep", legitimate.readText()) + // The orphan was recovered, and a finished install removes the then-empty owned namespace. + assertFalse(staging.exists()) + } + + @Test fun stagingRecoveryRefusesUnownedAndManifestOwnedNamespaces() { + val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY).apply { mkdir() } + val untouched = File(staging, "legitimate.stage").apply { writeText("keep") } + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertEquals("keep", untouched.readText()) + File(staging, "owner").writeText("DroidDeck Epic staging v1\n") + manifest.files.first().filename = "${EpicDownloadManager.STAGING_DIRECTORY}/legitimate.stage" + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertEquals("keep", untouched.readText()) + assertEquals(0, requests.get()) + } + + @Test fun loadedNativeAbiFailureIsExplicitAndNeverFallsBackToJavaDownloads() { + val field = com.droiddeck.launcher.stores.StoresNative::class.java.getDeclaredField("loaded") + field.isAccessible = true + field.set(null, true) + try { + oldInstall() + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(error.message.orEmpty().contains("Native Epic install failed")) + assertEquals(0, requests.get()) + assertOldInstallIntact() + } finally { + com.droiddeck.launcher.stores.StoresNative.forget() + } + } + + @Test fun stagingRecoveryRejectsSymlinkNamespaceOwnerAndEntriesWithoutDeletingOutsideData() { + val outside = temp.newFolder("outside-staging") + val externalStage = File(outside, "keep.stage").apply { writeText("keep") } + val externalOwner = File(outside, "owner").apply { writeText("DroidDeck Epic staging v1\n") } + val staging = File(root, EpicDownloadManager.STAGING_DIRECTORY) + try { + java.nio.file.Files.createSymbolicLink(staging.toPath(), outside.toPath()) + } catch (e: java.nio.file.FileSystemException) { + org.junit.Assume.assumeNoException("Host does not permit symbolic links", e) + } + try { + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(error.message.orEmpty().contains("Unsafe Epic staging")) + assertEquals("keep", externalStage.readText()) + java.nio.file.Files.delete(staging.toPath()) + staging.mkdir() + java.nio.file.Files.createSymbolicLink(File(staging, "owner").toPath(), externalOwner.toPath()) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + java.nio.file.Files.delete(File(staging, "owner").toPath()) + File(staging, "owner").writeText("DroidDeck Epic staging v1\n") + val localStage = File(staging, "local.stage").apply { writeText("local") } + java.nio.file.Files.createSymbolicLink(File(staging, "linked.stage").toPath(), externalStage.toPath()) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertEquals("local", localStage.readText()) + assertEquals("keep", externalStage.readText()) + assertEquals(0, requests.get()) + } finally { + if (java.nio.file.Files.isSymbolicLink(staging.toPath())) java.nio.file.Files.delete(staging.toPath()) + for (name in listOf("owner", "linked.stage")) { + val entry = File(staging, name).toPath() + if (java.nio.file.Files.isSymbolicLink(entry)) java.nio.file.Files.delete(entry) + } + } + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt index 669e0adc49..14bc3d364e 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt @@ -1,8 +1,10 @@ package com.droiddeck.launcher.stores.epic import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner @@ -11,6 +13,140 @@ import org.robolectric.annotation.Config @RunWith(RobolectricTestRunner::class) @Config(sdk = [33]) class EpicJsonManifestTest { + @Test + fun boundedInstallPeakReservesParallelIncomingChunksBeforeWriting() { + val first = chunk(1) + val second = chunk(2) + val file = file(part(1, 0, 100), part(2, 0, 100)) + val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) + + assertEquals(300L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true)) + assertEquals(200L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false)) + } + + @Test + fun sharedCachedChunkIsReclaimedOnlyAfterItsLastPart() { + val shared = chunk(1) + val files = listOf(file(part(1, 0, 60)), file(part(1, 60, 40))) + + assertEquals( + 100L, + EpicDownloadManager.estimatePeakAdditionalBytes( + files, mapOf(shared.guidStr() to shared), mapOf(shared.guidStr() to 100L), true, + ), + ) + assertEquals( + 200L, + EpicDownloadManager.estimatePeakAdditionalBytes( + files, mapOf(shared.guidStr() to shared), emptyMap(), true, + ), + ) + } + + @Test + fun sharedChunksWithinTheWindowStayPinnedWithinTheCacheCeiling() { + val first = chunk(1).apply { windowSize = 40 * 1024 * 1024 } + val second = chunk(2).apply { windowSize = 40 * 1024 * 1024 } + val files = listOf( + file(part(1, 0, 40 * 1024 * 1024)), + file(part(2, 0, 40 * 1024 * 1024)), + file(part(1, 0, 40 * 1024 * 1024)), + file(part(2, 0, 40 * 1024 * 1024)), + ) + val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) + + val gamePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true) + val cachePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false) + + assertEquals(200L * 1024 * 1024, gamePeak) + assertTrue(cachePeak <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 40L * 1024 * 1024) + } + + @Test fun scheduleDeduplicatesWindowAndEvictsOnlyUnpinnedReusableChunks() { + val size = 40L * 1024 * 1024 + val chunks = (1..12).map { chunk(it).apply { windowSize = size.toInt() } } + .associateBy { it.guidStr() } + val files = (1..12).map { file(part(it, 0, 1)) } + (1..12).map { file(part(it, 0, 1)) } + val schedule = EpicDownloadManager.PrefetchSchedule(files, chunks, emptyMap()) + assertEquals(chunks.keys.take(8), schedule.fill()) + assertEquals(emptyList(), schedule.fill()) + assertEquals(8 * size, schedule.bytes()) + assertEquals(emptyList(), schedule.consume()) + assertEquals(listOf(chunks.keys.elementAt(8)), schedule.fill()) + assertEquals(9 * size, schedule.bytes()) + assertEquals(listOf(chunks.keys.first()), schedule.consume()) + var refetchedFirst = false + repeat(22) { + val fetched = schedule.fill() + if (chunks.keys.first() in fetched) refetchedFirst = true + assertTrue(schedule.bytes() <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 8 * size) + val released = schedule.consume() + assertTrue(released.none { it in schedule.window() }) + } + assertTrue(refetchedFirst) + assertEquals(0L, schedule.bytes()) + assertEquals(9 * size, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false)) + } + + @Test fun cachedAndOldOutputBytesAreCreditedOnlyWhenTheScheduleReleasesThem() { + val chunks = (1..8).map { chunk(it) }.associateBy { it.guidStr() } + val files = (1..8).map { id -> file(part(id, 0, 100)).apply { filename = "file-$id" } } + val existing = chunks.keys.associateWith { 100L } + val old = files.associate { it.filename to 100L } + assertEquals(800L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false)) + assertEquals(0L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, false)) + assertEquals(900L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true, old)) + assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, existing, true, old)) + } + + @Test + fun resumeCacheDropsCorruptAndUnneededEntriesButKeepsVerifiedChunks() { + val dir = java.io.File(System.getProperty("user.dir"), "build/epic-cache-test-${System.nanoTime()}") + assertTrue(dir.mkdirs()) + try { + val bytes = ByteArray(100) { (it % 251).toByte() } + val expectedSha = java.security.MessageDigest.getInstance("SHA-1").digest(bytes) + val valid = chunk(1).apply { sha1 = expectedSha } + val corrupt = chunk(2).apply { sha1 = expectedSha } + java.io.File(dir, valid.guidStr()).writeBytes(bytes) + java.io.File(dir, corrupt.guidStr()).writeBytes(ByteArray(100) { 1 }) + java.io.File(dir, "orphan").writeBytes(bytes) + + val retained = EpicDownloadManager.prepareBoundedCache(dir, listOf(valid, corrupt)) + + assertEquals(mapOf(valid.guidStr() to 100L), retained) + assertTrue(java.io.File(dir, valid.guidStr()).exists()) + assertFalse(java.io.File(dir, corrupt.guidStr()).exists()) + assertFalse(java.io.File(dir, "orphan").exists()) + } finally { + EpicDownloadManager.deleteDir(dir) + } + } + + @Test + fun referenceCountsIncludeRepeatedPartsSoCancelledFilesCanBeReplanned() { + val pending = listOf(file(part(1, 0, 20), part(1, 20, 20)), file(part(1, 40, 60))) + + assertEquals(3, EpicDownloadManager.chunkReferences(pending)["00000001000000000000000000000000"]) + } + + private fun chunk(id: Int) = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(id, 0, 0, 0) + windowSize = 100 + fileSize = 100 + } + + private fun part(id: Int, offset: Int, size: Int) = EpicDownloadManager.ChunkPart().apply { + guid = intArrayOf(id, 0, 0, 0) + this.offset = offset + this.size = size + } + + private fun file(vararg parts: EpicDownloadManager.ChunkPart) = EpicDownloadManager.FileInfo().apply { + filename = "test.bin" + this.parts.addAll(parts) + } + @Test fun blobNumbersAreDecimalBytesLeastSignificantFirst() { assertEquals(1L, EpicDownloadManager.blobToNum("001000000000")) diff --git a/app/src/test/java/com/droiddeck/launcher/stores/gog/GogResumeSpaceTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/gog/GogResumeSpaceTest.kt new file mode 100644 index 0000000000..4934b58980 --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/gog/GogResumeSpaceTest.kt @@ -0,0 +1,43 @@ +package com.droiddeck.launcher.stores.gog + +import java.io.File +import java.security.MessageDigest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowStatFs +import com.droiddeck.launcher.stores.StoreInstallRoot + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class GogResumeSpaceTest { + @Test fun remainingWorkRejectsMeasuredZeroButNotUnknownSpace() { + val root = File(RuntimeEnvironment.getApplication().filesDir, "gog-zero").apply { mkdirs() } + ShadowStatFs.registerStats(root.path, 1024, 0, 0) + val files = listOf(GogDownloadManager.DepotFile("missing").apply { totalSize = 4 }) + val needed = GogDownloadManager.remainingInstallBytes(files, root) + assertEquals(4L, needed) + assertTrue(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.availableBytes(root))) + assertFalse(StoreInstallRoot.insufficientSpace(needed, StoreInstallRoot.SPACE_UNKNOWN)) + } + + @Test fun remainingSpaceExcludesVerifiedFilesButIncludesCorruptAndPartialFiles() { + val root = File(RuntimeEnvironment.getApplication().filesDir, "gog-resume").apply { mkdirs() } + val data = byteArrayOf(1, 2, 3, 4) + val hash = MessageDigest.getInstance("MD5").digest(data).joinToString("") { "%02x".format(it) } + val files = listOf("good", "corrupt", "partial", "missing").map { + GogDownloadManager.DepotFile(it).apply { totalSize = 4; md5 = hash } + } + File(root, "good").writeBytes(data) + File(root, "corrupt").writeBytes(ByteArray(4)) + File(root, "partial").writeBytes(byteArrayOf(1)) + assertEquals(12L, GogDownloadManager.remainingInstallBytes(files, root)) + for (file in files) File(root, file.relativePath).writeBytes(data) + assertEquals(0L, GogDownloadManager.remainingInstallBytes(files, root)) + } +} diff --git a/docs/development/stores.md b/docs/development/stores.md index 79b1e2b73b..c3c7223c21 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -24,15 +24,164 @@ scanned (`StoreInstallRoot.roots`): internal, every card, and the Steam library' installs an earlier build put there. `` is `GOG`, `Epic` or `Amazon`; `` is the title with unsafe characters dropped, at most 60 characters (`StoreInstallRoot.folderName`), chosen once; a rerun (repair, update) lands on the folder whose sidecar carries the game's id, wherever it is. -An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`: the -`chunkCacheDir` of both native calls (`EpicNative.run` fetches into it, `EpicNative.assemble` -writes the files from it and drops each chunk after its last use, then the folder), and of the -manager's own loops when the engine is not there; `""` keeps the cache beside the game, as an -internal install has it. The scratch folder is removed on cancel-with-delete and on uninstall. -The cache holds whole ~1 MiB chunk windows, shared with files this device does not install, so it -is often larger than the game (Metalstorm: 9.1 GB of chunks for 4.4 GB of files); the free-space -check, made after the delta pass, counts the missing chunks and the missing files on their own -volumes. A successful run removes the cache. +An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`; +`""` keeps the cache beside the game, as an internal install has it. The scratch folder is +removed on cancel-with-delete and on uninstall. + +Preflight resolves an existing game's folder before selecting its target root, so a resume or +repair on a card checks and remembers that volume rather than internal storage. Fresh GOG and +Amazon installs can reject insufficient catalog-size space immediately; existing installs defer +to manifest checks of the files still needing work (GOG MD5, Amazon SHA-256 where supplied). +Free-space display probes the nearest existing ancestor without creating the target folder; if +the probe cannot determine space, it reports **Free space unknown** rather than blocking an install. +`StoreInstallRoot` uses `-1` for unknown and preserves a measured `StatFs` zero as **0 B free**. +Fresh GOG/Amazon preflight and all three Java backend space checks reject known zero when bytes +remain. The read-only probe walks existing ancestors without creating directories; a failed +`StatFs` probe can use a positive `File.usableSpace` fallback, but that API's ambiguous zero is +unknown, not a measurement of a full disk. +Preflight failures remain failed Downloads rows with Clear and Retry. Retry reruns preflight +outside the worker lifecycle and then enqueues the real backend; it never occupies a queue slot +with a placeholder job. Failures are kept in the Downloads UI, notification and logs rather than +duplicated as transient failure toasts. Active queue persistence and ordinary backend pause/retry +are unchanged. + +Epic installs have four writers, all with the same per-file safety rules (below). The hidden +`storesEpicMode` string in the `session` preferences picks one: `adaptive` (default), `lookahead`, +`batch` or `java`. There is no UI for it; set it on a test device to compare the modes on the same +game. Every mode logs an `epic: summary mode=<mode>` line (files, bytes written, elapsed, average +MB/s). When `libdroiddeckstores.so` does not load, every mode runs the Java writer. Once it is +loaded, an engine failure (refused plan, ABI mismatch, network, hash or I/O failure, unsupported +atomic swap) fails the install; it never silently reruns as Java downloads. + +**Adaptive (default)** - `EpicNative.stream`, `epic/stream.rs`. Measured on device: about +83 MB/s on a 71 GB install. The chunks go through the shared fetch core in the order their files +need them, with its full adaptivity (window ramp, per-host caps, in-flight byte budget, retries, +host rotation, stall handoff). A chunk only one file part uses is verified and written straight +into that file's staging file at the part's offset, in whatever order it arrives; it is never +cached, so each byte is written once. A chunk several parts use is written into every part whose +file the download has reached and kept, decompressed, in the cache until its last part is written. +The cache has a byte budget; the plan simulates the install in file order and, when the budget +would be exceeded, drops the cached chunk needed furthest ahead and downloads it again later. +Files are opened only once the download reaches them, so the set of half-written files stays +within the fetch window. All writes happen in the fetch core's process pool, whose completion +frees the in-flight byte budget: a slow card throttles the download instead of growing memory. +Each file's SHA-1 is computed as its parts are written (parts that arrived out of order are read +back once the bytes before them are in). Stage files go into the owned staging namespace below. +The run logs `stream-plan` (direct vs shared chunks, planned refetches, budget) and +`stream-summary mode=adaptive` (peak cache, peak staging, refetches, bytes written from the +download vs from the cache). + +On an update, a staged file that replaces an existing one sits beside its old copy until its swap, +so the engine paces replacements with an overlap budget: files are admitted in order, a replaced +file only while the planned sizes of admitted, not-yet-swapped replaced files fit the budget (one +is always admitted, so the run progresses), and a chunk is fetched only once its file is admitted. +Fresh files are never held back, and admission only delays files, never reorders them, so the +cache plan stays exact. The run logs `overlap_budget=` in `stream-plan` and +`peak_replace_overlap_bytes=` in `stream-summary`. With room to spare the budget is large and +updates run as wide as fresh installs; on a nearly full volume, replaced files are staged one +after another and the fetch pauses at each replaced-file boundary until the previous one is +swapped in. + +Its space plan (`EpicDownloadManager.planAdaptiveSpace`, unit-tested in `EpicAdaptivePlanTest`) +checks each volume for what lands on it, with the lookahead planner's rule (the need must not +exceed the free reading). The game volume: the net growth at its worst point (`settledNetPeak`, +the running Σ(new − old) with each old copy released at its swap), plus the overlap budget, plus +the cache when it is beside the game. The overlap budget is whatever the game volume has left +after the net growth and the cache, never less than the largest replaced file; when even that +does not fit, the plan hands over to the lookahead engine. The cache volume: the budget plus 64 MiB of +chunks in transit. The cache budget is half the cache volume's free space after those needs, at most +1 GiB (256 MiB when free space is unknown; 0 is allowed and means every reuse is a refetch). On a +card install the cache stays on internal scratch unless internal storage cannot hold at least +64 MiB of it; then the cache goes beside the game (still released chunk by chunk) instead of +failing. A plan that does not fit logs the short storage and the shortfall and hands over to the +lookahead engine, which narrows its window to the space there is (or fails its own preflight with +an explicit error). Running out of space in the middle of an adaptive run is not destructive: a +failed write fails the run, unfinished staging files are removed, the old files stay. + +**Lookahead** - `EpicNative.installAdaptive`, `epic/adaptive.rs`; also the adaptive engine's +fallback when its space plan does not fit. The Java delta pass sends a versioned snapshot of +pending files, chunk metadata, verified cache lengths, old output lengths and both measured +free-space probes to Rust. The native planner and execution share one ordered moving-window +policy. Before any chunk request, Rust chooses the largest safe lookahead (128, 64, 32, 16, 8, 4, +2 or 1 part references) and reusable-cache budget (64 MiB, 16 MiB or zero) whose simulated +staged-file and cache peaks fit. Native device IDs also detect an explicitly named cache on the +game volume, so two separate paths cannot incorrectly turn one filesystem's requirement into two +independent checks. + +| Storage situation | Lookahead policy | +| --- | --- | +| Plenty of room | Up to 128 references ahead, up to 32 network requests, 64 MiB reusable data | +| Tight | Smaller lookahead and/or reusable budget; parallel fetching remains where space permits | +| Barely fits | One-reference lookahead, one request, no reusable retention if needed | +| Does not fit even the minimal schedule | Explicit preflight error before requests or replacement | +| Free space unknown | Conservative bounded lookahead (at most eight), not an invented zero | + +The lookahead fetch coordinator reuses `fetch_core`'s host scheduling, retries, adaptive network +window and decode pool. Each chunk is cached, then its parts are copied into staged files by a +writer that runs concurrently, waiting only for its next required chunk, through the native +assembler's streaming copy primitive. GUIDs in the lookahead are reserved once before submission, +at their full decoded size. Chunks are released at last use on **both** cache layouts; distant +reusable data is oldest-first bounded and may be refetched. At most the selected reusable budget +plus distinct lookahead decoded sizes is resident/reserved. New request admission re-probes both +volumes and can shrink unsent lookahead or retention if headroom disappears; already in-flight +reservations remain protected. If even existing reservations cannot fit, the run fails and joins +its workers rather than waiting indefinitely or deleting old game files. An install to SD retains +its internal scratch cache, so assembled output is the only game data written to the card. + +**Java** - without the native library, or with `storesEpicMode=java`. Eight Java workers prefetch +the next eight chunk-part references, across file boundaries, while one writer assembles and +verifies files in manifest order. Shared GUIDs within that window are fetched only once. Chunks +are deleted after their last use; other reusable chunks are retained up to 64 MiB. At most eight +additional decoded chunks (complete or in flight) coexist with that retained cache. The size +bound is 64 MiB plus the decoded sizes of the window's distinct GUIDs, not a fixed eight-MiB +allowance: manifests can supply larger chunk windows. Evicted distant shared chunks may be +downloaded again. No worker downloads outside the moving window; cancellation and failure +disconnect and join all workers before returning or removing the cache. Its summary line is +`mode=java-bounded`. + +**Batch** - the earlier native design, kept for A/B benchmarks: every chunk fetched into the +cache in parallel, then the files assembled from it (`EpicNative.run` + `EpicNative.assemble`). +It needs the cache and the files at once (about twice the game) and checks exactly that before +starting; it writes files in place, so an existing install is marked `updating` before assembly. +`epic/assemble.rs::build_assemble_plan` enables `scratch_cache` only for a nonempty cache path: +`run_assemble` then removes chunks at their last use; the default beside-game cache is removed at +the end (`default_cache_assembles_like_the_kotlin_loop_and_removes_the_cache_at_the_end` and the +scratch-cache assembly tests cover that distinction). + +The Epic download bar, measured rate and ETA (adaptive, lookahead and Java) count pending file bytes written, not unique +compressed chunks fetched. Shared chunks therefore cannot complete the bar before later files +are written, and an evicted chunk's refetch is reflected in the time spent doing that work. +Streaming stays in the Download stage until all files are written and verified, then enters +Install for registration; it does not bounce between stages for each file. + +For the lookahead and Java writers, the space check follows the manifest delta pass and uses the +same pure moving-window policy as execution, reserving all incoming decoded chunk sizes before writing parts and simulating +last-use releases and unpinned oldest-first evictions. Existing verified cache bytes count against +current free space and are credited only at their scheduled release. This avoids adding the entire +cache to the entire game without undercounting concurrent downloads. Each file is written to a +unique file in an owned staging namespace on its output volume (never the literal `<filename>.part`, which can +be a legitimate manifest file), size/hash verified, synced and atomically renamed over its output. +The old file remains until that swap succeeds. The planner credits its bytes only after that +file's complete staging write, not by subtracting all old outputs up front. Both volume checks +must accept this safe staging peak; insufficient space is rejected explicitly, with no destructive +delete-first fallback. An unsupported/failed atomic rename fails the install and retains the old +destination. Completed files and +verified cached chunks are accounted for on resume. On a card, game and cache requirements +are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed +or cancelled staging files are removed without deleting old outputs. A successful run removes the +cache. A resume within a large incomplete file restarts that file, not its individual parts. +All three staged writers (adaptive, lookahead, Java) use `<installDir>/.droiddeck-epic-staging/`, whose `owner` marker identifies this +namespace. A successful install removes it when it holds nothing but the marker (same no-follow +and canonical-path checks); the next run recreates it. Before the delta pass and free-space probes, the manager removes orphan `.stage` +entries only from this owned namespace. A manifest naming that directory or any descendant +fails without deletion; an unowned directory or unexpected entry also fails conservatively. +The namespace must be a real directory at the expected canonical child of the install root. +Ownership markers and staging entries must be no-follow regular files; symlink directories, +markers and entries are rejected before recovery deletes anything. +Ordinary game files with `.stage` or `.part` suffixes elsewhere are never recovery targets. +Abrupt process termination is therefore recoverable on the next run without inferring ownership +from a loose filename prefix. Cancellation/failure waits for all native/Java fetch writers before +returning; completed verified cache chunks remain usable on resume. ## The sidecar @@ -59,7 +208,18 @@ store root whose sidecar says `installing`, or that has no sidecar at all, is an install: `AddedGames.scan` leaves it out (it is neither a game nor a Custom folder), and the Stores card and page offer **Resume install**, which reuses that folder - and an Epic `.chunks` beside it - instead of starting a fresh one (`StoreInstallRoot.existingFolder`). A finished install being -repaired keeps its finished sidecar. The launcher and the finished sidecar are written before the +repaired keeps its finished sidecar until Epic reaches the first verified file swap. Immediately +before that swap Epic atomically changes the record to `"state": "updating"`, preserves launch +choices and identifiers, and clears `installVersion`. Network/hash failures or cancellation before +that point leave the old installed record and files intact. A later failure can leave a mixture of +verified new files and untouched old files: replacement is atomic **per file**, not a transactional +whole-game update or rollback. The updating folder is excluded from installed scans and offered +for resume. Cancellation with deletion clears caches but retains that folder and its verified +swaps; retry preserves the updating record's metadata. If marking or swapping fails, no destructive +fallback is attempted (the conservative updating state can remain even when a swap fails). +Completion writes the installed record with the new version only after all selected files succeed. +An already registered live Steam shortcut is not revoked at each file swap; do not launch an +in-progress update. The launcher and the finished sidecar are written before the art and the Steam registration; a failure writing them fails the download with its message. `StoreGameSidecar.parse` refuses anything whose `exe` or `launcher` would point outside the folder. diff --git a/tools/release/check_android_libraries.py b/tools/release/check_android_libraries.py index 765e226acc..8f79e8844e 100644 --- a/tools/release/check_android_libraries.py +++ b/tools/release/check_android_libraries.py @@ -14,6 +14,7 @@ EXPORTS = ['Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion', 'Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart', 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStart', 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble', + 'Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeStream', 'Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart']