From 78b3f278164fcee8731dccc394152fe0f5957e86 Mon Sep 17 00:00:00 2001 From: Juan Marti Mercado Date: Tue, 13 Oct 2026 10:07:01 -0400 Subject: [PATCH] Bound Epic chunk cache during installation --- .../launcher/stores/StoreInstallRoot.kt | 3 + .../stores/epic/EpicDownloadManager.java | 479 +++++++++++------- .../launcher/stores/epic/EpicNative.kt | 11 +- .../launcher/stores/StoreInstallRootTest.kt | 7 + .../stores/epic/EpicBoundedInstallTest.kt | 178 +++++++ .../stores/epic/EpicJsonManifestTest.kt | 99 ++++ docs/development/stores.md | 28 +- 7 files changed, 595 insertions(+), 210 deletions(-) create mode 100644 app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt diff --git a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt index 09bf3c61f1..7d77c0330d 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/StoreInstallRoot.kt @@ -54,6 +54,9 @@ object StoreInstallRoot { fun checkStorage(context: Context, item: CatalogItem, targetRoot: File): StorageCheck { val size = estimatedBytesNeeded(item) val freeTarget = availableBytes(targetRoot) + // Epic's manifest delta accounts for completed files and reusable chunks on resume. + // Comparing the full catalog size to remaining free space would reject valid resumes. + if (item.store == Store.EPIC) return StorageCheck(enough = true, requiredBytes = size, availableBytes = freeTarget) if (size <= 0L) return StorageCheck(enough = true, requiredBytes = 0L, availableBytes = freeTarget) if (freeTarget in 1 until size) { val msg = context.getString(com.droiddeck.launcher.R.string.stores_storage_insufficient, formatBytes(size), formatBytes(freeTarget)) diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java index 09cc369e7f..21f1dc488c 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicDownloadManager.java @@ -4,13 +4,11 @@ import android.content.Context; import android.util.Log; import com.droiddeck.launcher.stores.StoreLog; -import com.droiddeck.launcher.stores.download.StoreDownloadTier; import org.json.JSONArray; import org.json.JSONObject; import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -29,20 +27,16 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.zip.Inflater; /** * The Epic install pipeline, ported from Bannerlator's EpicDownloadManager: the manifest API's CDN * list, the manifest binary (ChunksV4, or the older JSON form), the files narrowed to the base * set plus the device's language, a delta pass that skips files already on disk with the right - * size and SHA-1, the chunk fetch into `.chunks/` (the native engine when it is there, the - * Java pool otherwise), and the assembly of files from those chunks. + * size and SHA-1, then the bounded-cache chunk-to-file writer. The native batch engine remains + * available for other callers, but this install path intentionally uses the same serial writer + * whether or not native code is installed: a whole-game chunk cache can exceed available space. * * Chunk URLs carry no auth tokens; the subfolder is the decimal group number; the Cloudflare * mirror is skipped (it refuses chunks). Blocking; the download queue runs it on its own thread. @@ -51,6 +45,8 @@ public final class EpicDownloadManager { private static final String TAG = "EpicDownload"; private static final String UA = EpicAuthClient.USER_AGENT; + /** Keep reusable chunk data bounded even when a manifest shares chunks across distant files. */ + static final long MAX_RETAINED_CHUNK_CACHE_BYTES = 64L * 1024 * 1024; public interface Callback { void onProgress(String message, int pct); @@ -154,11 +150,23 @@ public final class EpicDownloadManager { if (manifest == null) throw new InstallException("The manifest could not be parsed"); manifest.cdnUrls = cdnUrls; cb.onLog("epic: chunkDir=" + manifest.chunkDir + " chunks=" + manifest.uniqueChunks.size() + " files=" + manifest.files.size() + " exe=" + manifest.launchExe); + return installManifest(manifest, installDirPath, chunkCacheDirPath, installTags, cancel, cb); + } catch (InstallException e) { + throw e; + } catch (Exception e) { + Log.w(TAG, "install failed", e); + throw new InstallException(e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "")); + } + } + static Result installManifest(Manifest manifest, String installDirPath, String chunkCacheDirPath, + List installTags, AtomicBoolean cancel, Callback cb) throws InstallException { + try { + List cdnUrls = manifest.cdnUrls; File installDir = new File(installDirPath); installDir.mkdirs(); // "" = the cache beside the game; a path = the scratch cache (the app's own storage when - // the game goes to a card). The engine and the Java loops read the same value. + // the game goes to a card). The bounded writer uses this location on every build. final String cachePath = chunkCacheDirPath == null ? "" : chunkCacheDirPath; final File chunkCacheDir = cachePath.isEmpty() ? new File(installDir, ".chunks") : new File(cachePath); chunkCacheDir.mkdirs(); @@ -191,125 +199,116 @@ public final class EpicDownloadManager { return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); } List needed = uniqueChunksForFiles(manifest, pending); - // The cache holds whole chunk windows (~1 MiB each), and a window is shared with files - // this device does not install (other tags, other builds' data): the cache is often - // larger than the game itself (Metalstorm: 9.1 GB of chunks for 4.4 GB of files). Both - // have to fit at once until assembly, so the check counts both, on their own volumes. - long cacheMissing = 0, pendingBytes = 0; - for (ChunkInfo c : needed) if (!new File(chunkCacheDir, c.guidStr()).isFile()) cacheMissing += Math.max(c.windowSize, 1); - for (FileInfo f : pending) pendingBytes += f.fileSize(); - if (cachePath.isEmpty()) { - long free = installDir.getUsableSpace(); - if (free > 0 && cacheMissing + pendingBytes > free) throw new InstallException("Not enough free space: need " + fmt(cacheMissing + pendingBytes) + " (" + fmt(pendingBytes) + " of game files + " + fmt(cacheMissing) + " of download cache), only " + fmt(free) + " free"); + Map chunksByGuid = new LinkedHashMap<>(); + for (ChunkInfo c : needed) chunksByGuid.put(c.guidStr(), c); + if (!chunkCacheDir.isDirectory() && !chunkCacheDir.mkdirs()) throw new InstallException("Could not create Epic chunk cache"); + Map cachedChunkBytes = prepareBoundedCache(chunkCacheDir, needed); + long pendingBytes = 0; + for (FileInfo f : pending) { + pendingBytes += f.fileSize(); + File output = new File(installDir, f.filename.replace("\\", "/")); + File parent = output.getParentFile(); + if (parent != null) parent.mkdirs(); + // This file failed the delta check, so it cannot be resumed as a valid install. + // Remove it before estimating/writing its replacement to avoid requiring space + // for both the corrupt file and its verified replacement. + if (output.isFile() && !output.delete() && output.exists()) throw new InstallException("Could not replace incomplete file " + output.getName()); + } + boolean cacheOnInstallVolume = cachePath.isEmpty(); + long peakGameAdditional = cacheOnInstallVolume + ? estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, true) + : pendingBytes; + long peakCacheAdditional = cacheOnInstallVolume + ? 0 : estimatePeakAdditionalBytes(pending, chunksByGuid, cachedChunkBytes, false); + long freeGame = installDir.getUsableSpace(); + if (cacheOnInstallVolume) { + if (freeGame > 0 && peakGameAdditional > freeGame) throw new InstallException("Not enough free space for the bounded Epic install: need up to " + fmt(peakGameAdditional) + " additional, only " + fmt(freeGame) + " free"); } else { - long freeCache = chunkCacheDir.getUsableSpace(), freeGame = installDir.getUsableSpace(); - if (freeCache > 0 && cacheMissing > freeCache) throw new InstallException("Not enough internal space for the download cache: need " + fmt(cacheMissing) + ", only " + fmt(freeCache) + " free"); + long freeCache = chunkCacheDir.getUsableSpace(); + if (freeCache > 0 && peakCacheAdditional > freeCache) throw new InstallException("Not enough internal space for the bounded Epic download cache: need up to " + fmt(peakCacheAdditional) + " additional, only " + fmt(freeCache) + " free"); if (freeGame > 0 && pendingBytes > freeGame) throw new InstallException("Not enough free space: need " + fmt(pendingBytes) + ", only " + fmt(freeGame) + " free"); } - cb.onLog("epic: cache " + (cachePath.isEmpty() ? "beside the game" : "scratch") + ", " + fmt(cacheMissing) + " to fetch for " + fmt(pendingBytes) + " of files"); + cb.onLog("epic: cache " + (cacheOnInstallVolume ? "beside the game" : "scratch") + ", peak game additional=" + fmt(peakGameAdditional) + (cacheOnInstallVolume ? "" : ", peak cache additional=" + fmt(peakCacheAdditional)) + " for " + fmt(pendingBytes) + " of files"); long totalBytes = 0; for (ChunkInfo c : needed) totalBytes += Math.max(c.fileSize, 1); final long fTotalBytes = totalBytes; final int totalChunks = needed.size(); - final AtomicLong completedBytes = new AtomicLong(0); - final AtomicInteger completedCount = new AtomicInteger(0); - final AtomicInteger failCount = new AtomicInteger(0); - final AtomicLong lastSpeedMs = new AtomicLong(System.currentTimeMillis()); - final AtomicLong lastSpeedBytes = new AtomicLong(0); - final AtomicLong speedBps = new AtomicLong(0); cb.onSizes(fTotalBytes, planned); cb.onBytes(0, fTotalBytes, 0); cb.onLog("epic: " + fmt(fTotalBytes) + " in " + totalChunks + " chunks"); - - boolean javaPool = true; - final int[] pendingIdx = pendingIndices(manifest, pending); - boolean engine = com.droiddeck.launcher.stores.StoresNative.INSTANCE.getAvailable() && pendingIdx != null; - String nativeError = null; - if (engine) { - NativeOutcome r = runNativePool(ctx, manifestBytes, pendingIdx, needed, fTotalBytes, installDirPath, cachePath, cdnUrls, cancel, cb, - completedBytes, completedCount, lastSpeedMs, lastSpeedBytes, speedBps); - if (r.started) { - javaPool = false; - if (r.cancelled) return null; - if (!r.success) { - cb.onLog("epic: engine failed: " + r.error); - nativeError = r.error; - failCount.incrementAndGet(); - } - } else cb.onLog("epic: engine not started (" + r.error + "); using the built-in pool"); - } - if (javaPool) { - cb.onLog("epic: engine=built-in (8 threads)"); - ExecutorService pool = Executors.newFixedThreadPool(8, com.droiddeck.launcher.stores.download.DownloadQueue.workerFactory("epic-dl")); - for (ChunkInfo chunk : needed) { - final ChunkInfo fc = chunk; - pool.submit(() -> { - if (cancel.get()) return; - File cached = new File(chunkCacheDir, fc.guidStr()); - if (!cached.exists() && !downloadChunkStreaming(fc, manifest.chunkDir, cdnUrls, cached)) { - cb.onLog("FAIL chunk=" + fc.guidStr()); - failCount.incrementAndGet(); - return; - } - long done = completedBytes.addAndGet(Math.max(fc.fileSize, 1)); - int cnt = completedCount.incrementAndGet(); - sampleSpeed(done, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading chunks (" + cnt + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (done * 80L / Math.max(1, fTotalBytes))); - cb.onBytes(done, fTotalBytes, speedBps.get()); - }); - } - pool.shutdown(); - try { - while (!pool.awaitTermination(250, TimeUnit.MILLISECONDS)) { - if (cancel.get()) { pool.shutdownNow(); pool.awaitTermination(5, TimeUnit.SECONDS); return null; } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - pool.shutdownNow(); - throw new InstallException("interrupted"); - } - } - if (cancel.get()) return null; - if (failCount.get() > 0) { - if (nativeError != null && !nativeError.isEmpty()) { - throw new InstallException(nativeError); - } - throw new InstallException(failCount.get() + " chunk(s) failed to download"); - } - - // The engine's assembler when it starts (it drops each chunk after its last use and the - // cache at the end); the loop below when it does not. - if (engine) { - NativeOutcome r = runNativeAssembly(manifestBytes, pendingIdx, pending.size(), installDirPath, cachePath, cancel, cb); - if (r.started) { - if (r.cancelled) return null; - if (!r.success) throw new InstallException(r.error.isEmpty() ? "assembly failed" : r.error); - cb.onProgress("Complete", 100); - return new Result(manifest.launchExe, manifest.buildVersion, planned).withPrereq(manifest); - } - cb.onLog("epic: assembler not started (" + r.error + "); writing the files here"); - } - int totalFiles = pending.size(), doneFiles = 0; - long writtenBytes = 0, writeTotal = 0; - for (FileInfo f : pending) writeTotal += f.fileSize(); + cb.onLog("epic: engine=bounded-java (shared path; native availability does not change cache semantics)"); + Map remainingRefs = chunkReferences(pending); + Set creditedChunks = new LinkedHashSet<>(); + long completedBytes = 0, writtenBytes = 0; + long fetchedBytes = 0; + long transferStarted = System.nanoTime(); + int completedCount = 0, doneFiles = 0, totalFiles = pending.size(); for (FileInfo file : pending) { if (cancel.get()) return null; String relPath = file.filename.replace("\\", "/"); File outFile = new File(installDir, relPath); - File parent = outFile.getParentFile(); - if (parent != null) parent.mkdirs(); cb.onProgress("Writing: " + (relPath.contains("/") ? relPath.substring(relPath.lastIndexOf('/') + 1) : relPath), 80 + (int) (doneFiles * 20L / totalFiles)); - try (FileOutputStream fos = new FileOutputStream(outFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) { + // Like the native assembler, write to the final path and remove it on failure. + // A sibling ".part" can itself be a legitimate file in the manifest. + File partFile = outFile; + MessageDigest fileSha = expectedDigest(file.sha1); + try (FileOutputStream fos = new FileOutputStream(partFile); BufferedOutputStream bos = new BufferedOutputStream(fos, 65536)) { for (ChunkPart part : file.parts) { - File cachedChunk = new File(chunkCacheDir, part.guidStr()); - if (!cachedChunk.exists()) throw new InstallException("Missing chunk " + part.guidStr() + " for " + relPath); - byte[] data = readFile(cachedChunk); - bos.write(data, part.offset, part.size); + if (cancel.get()) throw new IOException("cancelled"); + String guid = part.guidStr(); + ChunkInfo chunk = chunksByGuid.get(guid); + if (chunk == null) throw new InstallException("Missing chunk metadata " + guid + " for " + relPath); + File cachedChunk = new File(chunkCacheDir, guid); + if (!cachedChunk.isFile() || !isValidCachedChunk(chunk, cachedChunk)) { + if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) { + throw new InstallException("Could not clear invalid Epic chunk " + guid); + } + cachedChunkBytes.remove(guid); + cachedChunk = new File(chunkCacheDir, guid); + if (!downloadChunkStreaming(chunk, manifest.chunkDir, cdnUrls, cachedChunk, cancel)) { + if (cancel.get()) throw new IOException("cancelled"); + throw new InstallException("Failed to download Epic chunk " + guid); + } + cachedChunkBytes.put(guid, cachedChunk.length()); + fetchedBytes += Math.max(chunk.fileSize, 1); + } + if (creditedChunks.add(guid)) { + completedBytes += Math.max(chunk.fileSize, 1); + completedCount++; + cb.onProgress("Downloading chunks (" + completedCount + "/" + totalChunks + ")", (int) (completedBytes * 80L / Math.max(1, fTotalBytes))); + } + long elapsedMs = Math.max(1, (System.nanoTime() - transferStarted) / 1_000_000); + cb.onBytes(completedBytes, fTotalBytes, fetchedBytes * 1000 / elapsedMs); + copyChunkPart(cachedChunk, part, bos, fileSha, cancel, relPath); + Integer refs = remainingRefs.get(guid); + if (refs != null) { + if (refs <= 1) { + remainingRefs.remove(guid); + if (cachedChunk.exists() && !cachedChunk.delete() && cachedChunk.exists()) { + throw new InstallException("Could not release consumed Epic chunk " + guid); + } + cachedChunkBytes.remove(guid); + } else remainingRefs.put(guid, refs - 1); + } + trimCacheToBound(chunkCacheDir, cachedChunkBytes); } + bos.flush(); + } catch (IOException e) { + partFile.delete(); + if (cancel.get()) return null; + throw new InstallException("Could not assemble " + relPath + ": " + e.getMessage()); + } catch (InstallException e) { + partFile.delete(); + throw e; + } + if (cancel.get()) { partFile.delete(); return null; } + if (fileSha != null && !MessageDigest.isEqual(fileSha.digest(), file.sha1)) { + partFile.delete(); + throw new InstallException("SHA-1 mismatch for " + relPath); } doneFiles++; writtenBytes += file.fileSize(); - cb.onStage("install", writtenBytes, writeTotal, doneFiles, totalFiles); + cb.onStage("install", writtenBytes, pendingBytes, doneFiles, totalFiles); } deleteDir(chunkCacheDir); cb.onProgress("Complete", 100); @@ -322,75 +321,154 @@ public final class EpicDownloadManager { } } - private static final class NativeOutcome { boolean started, success, cancelled; String error = ""; } - - /** The pending files as indices into the manifest's file list, what both native runs take; null when one is not in it. */ - private static int[] pendingIndices(Manifest manifest, List pending) { - java.util.IdentityHashMap index = new java.util.IdentityHashMap<>(); - for (int i = 0; i < manifest.files.size(); i++) index.put(manifest.files.get(i), i); - int[] pendingIdx = new int[pending.size()]; - for (int i = 0; i < pending.size(); i++) { - Integer k = index.get(pending.get(i)); - if (k == null) return null; - pendingIdx[i] = k; + /** + * Discard abandoned/orphaned cache entries and retain only verified chunks this install uses. + * A bounded run consumes each chunk at its last part reference, so stale chunks from an older + * all-at-once run must not silently occupy the space needed by the game files. + */ + static Map prepareBoundedCache(File cacheDir, List needed) throws InstallException { + Map byGuid = new LinkedHashMap<>(); + for (ChunkInfo chunk : needed) byGuid.put(chunk.guidStr(), chunk); + Map valid = new LinkedHashMap<>(); + File[] entries = cacheDir.listFiles(); + if (entries == null) throw new InstallException("Could not inspect Epic chunk cache"); + for (File entry : entries) { + ChunkInfo chunk = byGuid.get(entry.getName()); + if (chunk == null || !isValidCachedChunk(chunk, entry)) { + deleteDir(entry); + if (entry.exists()) throw new InstallException("Could not clear stale Epic cache entry " + entry.getName()); + continue; + } + valid.put(entry.getName(), entry.length()); } - return pendingIdx; + trimCacheToBound(cacheDir, valid); + return valid; } - /** The native chunk pool on the same counters; never throws. */ - private static NativeOutcome runNativePool(Context ctx, byte[] manifestBytes, int[] pendingIdx, List needed, long totalBytes, - String installDirPath, String cachePath, List cdnUrls, AtomicBoolean cancel, Callback cb, - AtomicLong completedBytes, AtomicInteger completedCount, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - NativeOutcome r = new NativeOutcome(); - try { - String[] prefixes = new String[cdnUrls.size()]; - for (int i = 0; i < cdnUrls.size(); i++) prefixes[i] = cdnUrls.get(i).baseUrl + cdnUrls.get(i).cloudDir; - final int totalChunks = needed.size(); - StoreDownloadTier tier = StoreDownloadTier.Companion.current(ctx); - int workers = Math.max(1, Math.min(128, tier.getNetworkWindow())); - int process = Math.max(2, tier.getProcessWorkers()); - cb.onLog("epic: engine=native workers=" + workers + " process_workers=" + process + " tier=" + tier.getId() + (cachePath.isEmpty() ? "" : " cache=scratch")); - EpicNative.Result res = EpicNative.run(manifestBytes, installDirPath, cachePath, prefixes, pendingIdx, totalChunks, totalBytes, "", workers, process, cancel, - new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) { cb.onLog("epic: plan chunks=" + chunksTotal + " bytes=" + bytesTotal + " dir=" + chunkDir); } - @Override public void onProgress(long bytesDone, long bytesTotal, int chunksDone, int chunksTotal) { - completedBytes.set(bytesDone); - completedCount.set(chunksDone); - sampleSpeed(bytesDone, lastSpeedMs, lastSpeedBytes, speedBps); - cb.onProgress("Downloading chunks (" + chunksDone + "/" + totalChunks + ")" + speedSuffix(speedBps.get()), (int) (bytesDone * 80L / Math.max(1, totalBytes))); - cb.onBytes(bytesDone, totalBytes, speedBps.get()); - } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytesCredited) {} - }); - r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error; - } catch (Throwable t) { - r.started = false; - r.error = t.getClass().getSimpleName() + ": " + t.getMessage(); + /** Evict oldest reusable chunks until retained cache data is under the fixed disk ceiling. */ + private static void trimCacheToBound(File cacheDir, Map cacheSizes) throws InstallException { + long retained = 0; + for (long bytes : cacheSizes.values()) retained += bytes; + java.util.Iterator> iterator = cacheSizes.entrySet().iterator(); + while (retained > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) { + Map.Entry entry = iterator.next(); + File chunk = new File(cacheDir, entry.getKey()); + long bytes = entry.getValue(); + if (chunk.exists() && !chunk.delete() && chunk.exists()) { + throw new InstallException("Could not release Epic cache chunk " + entry.getKey()); + } + retained -= bytes; + iterator.remove(); } - return r; } - /** The native assembler: the pending files written from the cache, the cache removed on success; never throws. */ - private static NativeOutcome runNativeAssembly(byte[] manifestBytes, int[] pendingIdx, int totalFiles, String installDirPath, String cachePath, AtomicBoolean cancel, Callback cb) { - NativeOutcome r = new NativeOutcome(); - try { - cb.onLog("epic: assembler=native files=" + totalFiles + (cachePath.isEmpty() ? "" : " cache=scratch")); - EpicNative.Result res = EpicNative.assemble(manifestBytes, installDirPath, cachePath, pendingIdx, cancel, new EpicNative.Listener() { - @Override public void onPlan(int chunksTotal, long bytesTotal, String chunkDir) {} - @Override public void onProgress(long bytesDone, long bytesTotal, int done, int total) { - cb.onStage("install", bytesDone, bytesTotal, done, Math.max(total, totalFiles)); - cb.onProgress("Writing files (" + done + "/" + Math.max(total, totalFiles) + ")", 80 + (int) (done * 20L / Math.max(1, Math.max(total, totalFiles)))); + static boolean isValidCachedChunk(ChunkInfo chunk, File file) { + if (!file.isFile()) return false; + if (chunk.windowSize > 0 && file.length() != chunk.windowSize) return false; + byte[] expected = chunk.sha1; + if (expected == null || expected.length != 20) return true; + boolean allZero = true; + for (byte b : expected) if (b != 0) { allZero = false; break; } + if (allZero) return true; + try (FileInputStream in = new FileInputStream(file)) { + MessageDigest sha = MessageDigest.getInstance("SHA-1"); + byte[] buffer = new byte[64 * 1024]; + int count; + while ((count = in.read(buffer)) != -1) sha.update(buffer, 0, count); + return MessageDigest.isEqual(sha.digest(), expected); + } catch (Exception e) { + return false; + } + } + + private static MessageDigest expectedDigest(byte[] expected) throws InstallException { + if (expected == null || expected.length != 20) return null; + for (byte b : expected) if (b != 0) { + try { return MessageDigest.getInstance("SHA-1"); } + catch (java.security.NoSuchAlgorithmException e) { throw new InstallException("SHA-1 verification unavailable"); } + } + return null; + } + + static Map chunkReferences(List files) { + Map refs = new LinkedHashMap<>(); + for (FileInfo file : files) for (ChunkPart part : file.parts) { + String guid = part.guidStr(); + refs.put(guid, refs.containsKey(guid) ? refs.get(guid) + 1 : 1); + } + return refs; + } + + /** + * Simulates the serial fetch → append part → last-reference eviction order. The returned + * additional-space peak is measured against the current free-space reading: existing verified + * chunks are already reflected in that reading and count as reclaimed only when actually freed. + */ + static long estimatePeakAdditionalBytes(List files, Map chunksByGuid, + Map cachedChunkBytes, boolean includeFileWrites) { + Map refs = chunkReferences(files); + Map residentChunks = new LinkedHashMap<>(cachedChunkBytes); + long residentDelta = 0, peak = 0; + long cacheResident = 0; + for (long bytes : residentChunks.values()) cacheResident += bytes; + for (FileInfo file : files) { + for (ChunkPart part : file.parts) { + String guid = part.guidStr(); + ChunkInfo chunk = chunksByGuid.get(guid); + if (chunk == null) throw new IllegalArgumentException("Missing chunk metadata " + guid); + Long knownBytes = residentChunks.get(guid); + long chunkBytes = knownBytes != null ? knownBytes : chunkWindowSize(chunk); + if (knownBytes == null) { + residentChunks.put(guid, chunkBytes); + cacheResident += chunkBytes; + residentDelta += chunkBytes; + peak = Math.max(peak, residentDelta); } - @Override public void onLog(String line) { cb.onLog(line); } - @Override public void onComplete(boolean success, String error, long bytes) {} - }); - r.started = res.started; r.success = res.success; r.cancelled = res.cancelled; r.error = res.error == null ? "" : res.error; - } catch (Throwable t) { - r.started = false; - r.error = t.getClass().getSimpleName() + ": " + t.getMessage(); + if (includeFileWrites) { + residentDelta += part.size & 0xFFFFFFFFL; + peak = Math.max(peak, residentDelta); + } + int count = refs.get(guid); + if (count <= 1) { + refs.remove(guid); + Long removed = residentChunks.remove(guid); + if (removed != null) { cacheResident -= removed; residentDelta -= removed; } + } else refs.put(guid, count - 1); + java.util.Iterator> iterator = residentChunks.entrySet().iterator(); + while (cacheResident > MAX_RETAINED_CHUNK_CACHE_BYTES && iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (refs.containsKey(entry.getKey())) { + long removed = entry.getValue(); + cacheResident -= removed; + residentDelta -= removed; + iterator.remove(); + } + } + } + } + return Math.max(0, peak); + } + + private static void copyChunkPart(File chunkFile, ChunkPart part, BufferedOutputStream out, + MessageDigest fileSha, AtomicBoolean cancel, String relPath) throws IOException { + long offset = part.offset & 0xFFFFFFFFL; + long remaining = part.size & 0xFFFFFFFFL; + long length = chunkFile.length(); + if (offset > length || remaining > length - offset) { + throw new IOException("Chunk " + part.guidStr() + " is too short for " + relPath); + } + try (FileInputStream in = new FileInputStream(chunkFile)) { + in.getChannel().position(offset); + byte[] buffer = new byte[64 * 1024]; + while (remaining > 0) { + if (cancel.get()) throw new IOException("cancelled"); + int count = in.read(buffer, 0, (int) Math.min(buffer.length, remaining)); + if (count < 0) throw new IOException("Unexpected end of chunk " + part.guidStr()); + out.write(buffer, 0, count); + if (fileSha != null) fileSha.update(buffer, 0, count); + remaining -= count; + } } - return r; } // ---- CDN list and manifest download --------------------------------------------------------- @@ -667,11 +745,12 @@ public final class EpicDownloadManager { } /** One chunk streamed from the first CDN that serves it, SHA-1 checked, published by rename. */ - private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List cdnUrls, File outFile) { + private static boolean downloadChunkStreaming(ChunkInfo chunk, String chunkDir, List cdnUrls, File outFile, AtomicBoolean cancel) { String chunkPath = chunk.getPath(chunkDir); File tmp = new File(outFile.getPath() + ".part"); tmp.delete(); for (CdnUrl cdn : cdnUrls) { + if (cancel.get()) return false; HttpURLConnection conn = null; try { conn = (HttpURLConnection) new URL(cdn.baseUrl + cdn.cloudDir + "/" + chunkPath).openConnection(); @@ -697,12 +776,14 @@ public final class EpicDownloadManager { int storedAs = hdr.get() & 0xFF; if (headerSize > 41) skipFully(in, headerSize - 41); byte[] iobuf = new byte[131072]; + long outputBytes = 0; if ((storedAs & 1) != 0) { Inflater inflater = new Inflater(); byte[] obuf = new byte[131072]; int remaining = compressedSize; try { while (remaining > 0 && !inflater.finished()) { + if (cancel.get()) break; if (inflater.needsInput()) { int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); if (n <= 0) break; @@ -710,29 +791,47 @@ public final class EpicDownloadManager { inflater.setInput(iobuf, 0, n); } int out = inflater.inflate(obuf); - if (out > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); } + if (out > 0) { + outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); + fos.write(obuf, 0, out); + if (sha != null) sha.update(obuf, 0, out); + } + if (out == 0 && !inflater.needsInput() && !inflater.finished()) { + throw new IOException("invalid compressed Epic chunk"); + } } int out; - while ((out = inflater.inflate(obuf)) > 0) { fos.write(obuf, 0, out); if (sha != null) sha.update(obuf, 0, out); } + while ((out = inflater.inflate(obuf)) > 0) { + outputBytes = checkedChunkWriteSize(chunk, outputBytes, out); + fos.write(obuf, 0, out); + if (sha != null) sha.update(obuf, 0, out); + } + if (!cancel.get() && !inflater.finished()) throw new IOException("truncated compressed Epic chunk"); } finally { inflater.end(); } } else { int remaining = compressedSize; while (remaining > 0) { + if (cancel.get()) break; int n = in.read(iobuf, 0, Math.min(iobuf.length, remaining)); if (n <= 0) break; + outputBytes = checkedChunkWriteSize(chunk, outputBytes, n); fos.write(iobuf, 0, n); if (sha != null) sha.update(iobuf, 0, n); remaining -= n; } + if (!cancel.get() && remaining != 0) throw new IOException("truncated Epic chunk"); } } + if (cancel.get()) { tmp.delete(); conn.disconnect(); return false; } + if (chunk.windowSize > 0 && tmp.length() != chunk.windowSize) { tmp.delete(); conn.disconnect(); continue; } if (sha != null && !MessageDigest.isEqual(sha.digest(), chunk.sha1)) { tmp.delete(); conn.disconnect(); continue; } if (!tmp.renameTo(outFile)) { tmp.delete(); conn.disconnect(); continue; } conn.disconnect(); return true; } catch (Exception e) { + Log.w(TAG, "chunk " + chunk.guidStr() + " failed: " + e.getMessage()); tmp.delete(); if (conn != null) conn.disconnect(); } @@ -742,6 +841,17 @@ public final class EpicDownloadManager { return false; } + private static long checkedChunkWriteSize(ChunkInfo chunk, long written, int count) throws IOException { + long next = written + count; + if (next > chunkWindowSize(chunk)) throw new IOException("chunk exceeds expected window size"); + return next; + } + + private static long chunkWindowSize(ChunkInfo chunk) { + // Legacy JSON manifests omit window sizes; their chunks use 1 MiB windows. + return chunk.windowSize > 0 ? chunk.windowSize : 1024L * 1024; + } + private static void readFully(InputStream in, byte[] buf) throws IOException { int off = 0; while (off < buf.length) { int n = in.read(buf, off, buf.length - off); if (n < 0) throw new IOException("stream ended"); off += n; } @@ -835,15 +945,6 @@ public final class EpicDownloadManager { // ---- helpers -------------------------------------------------------------------------------- - static byte[] readFile(File f) throws IOException { - try (FileInputStream fis = new FileInputStream(f)) { - byte[] data = new byte[(int) f.length()]; - int off = 0; - while (off < data.length) { int r = fis.read(data, off, data.length - off); if (r < 0) break; off += r; } - return data; - } - } - static void deleteDir(File dir) { if (!dir.exists()) return; File[] files = dir.listFiles(); @@ -851,20 +952,6 @@ public final class EpicDownloadManager { dir.delete(); } - private static void sampleSpeed(long done, AtomicLong lastSpeedMs, AtomicLong lastSpeedBytes, AtomicLong speedBps) { - long nowMs = System.currentTimeMillis(), prevMs = lastSpeedMs.get(); - if (nowMs - prevMs >= 500 && lastSpeedMs.compareAndSet(prevMs, nowMs)) { - long prevB = lastSpeedBytes.getAndSet(done); - long dt = nowMs - prevMs; - if (dt > 0) speedBps.set((done - prevB) * 1000L / dt); - } - } - - private static String speedSuffix(long bps) { - if (bps <= 0) return ""; - return " " + (bps >= 1048576 ? String.format("%.1f MB/s", bps / 1048576.0) : (bps / 1024) + " KB/s"); - } - private static String fmt(long bytes) { if (bytes >= 1073741824L) return String.format("%.1f GB", bytes / 1073741824.0); if (bytes >= 1048576L) return String.format("%.1f MB", bytes / 1048576.0); diff --git a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt index 59849fd605..57f2681705 100644 --- a/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt +++ b/app/src/main/java/com/droiddeck/launcher/stores/epic/EpicNative.kt @@ -9,17 +9,18 @@ import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference /** - * The native half of an Epic install in `libdroiddeckstores.so`: [run] fills the chunk cache with + * The native batch APIs for Epic in `libdroiddeckstores.so`: [run] fills the chunk cache with * verified, decompressed chunks for the files the manager says are pending, [assemble] writes those - * files from the cache. Everything around them (manifest, install tags, delta pass, post-install) - * stays in the manager. + * files from the cache. The production installer intentionally uses one serial Java + * chunk-to-file path instead, so cache bounds and resume behavior do not depend on native + * availability. These APIs remain for callers that explicitly need batch fetch/assembly. * * The cache is `/.chunks` when `chunkCacheDir` is `""`, else the scratch folder given - * the app's own cache when the game installs to a card, where the assembler drops each chunk right * after its last use and the folder at the end. Both calls take the same value for one install. * - * Both block like the Java loops they replace and poll [AtomicBoolean] `cancel` every 250 ms: on - * cancel they flip the native flag, wait up to 5 s for the run to wind down, then return. + * Both block while polling [AtomicBoolean] `cancel` every 250 ms: on cancel they flip the native + * flag, wait up to 5 s for the run to wind down, then return. */ object EpicNative { private const val TAG = "EpicNative" diff --git a/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt index 10d8b0ea45..4248233e0f 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/StoreInstallRootTest.kt @@ -73,4 +73,11 @@ class StoreInstallRootTest { val check = StoreInstallRoot.checkStorage(app, zeroItem, StoreInstallRoot.internalRoot(app)) assertTrue(check.enough) } + + @Test fun epicDefersFullSizeChecksToItsManifestDeltaForResume() { + val item = CatalogItem(Store.EPIC, "large", "Large", imageUrl = null, owned = true, sizeBytes = Long.MAX_VALUE) + val check = StoreInstallRoot.checkStorage(app, item, StoreInstallRoot.internalRoot(app)) + assertTrue(check.enough) + assertEquals(Long.MAX_VALUE, check.requiredBytes) + } } diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt new file mode 100644 index 0000000000..686c6324fb --- /dev/null +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicBoundedInstallTest.kt @@ -0,0 +1,178 @@ +package com.droiddeck.launcher.stores.epic + +import java.io.File +import java.net.InetAddress +import java.net.ServerSocket +import java.net.SocketException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import org.junit.After +import org.junit.Assert.* +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [33]) +class EpicBoundedInstallTest { + @get:Rule val temp = TemporaryFolder() + private lateinit var server: ServerSocket + private lateinit var worker: Thread + private lateinit var manifest: EpicDownloadManager.Manifest + private lateinit var root: File + private val requests = AtomicInteger() + private val reject = AtomicBoolean() + private val bytes = ByteArray(256 * 1024) { (it % 251).toByte() } + + @Before fun setUp() { + root = temp.newFolder("game") + server = ServerSocket(0, 8, InetAddress.getByName("127.0.0.1")) + worker = Thread { + while (!server.isClosed) { + val socket = try { server.accept() } catch (e: SocketException) { + if (server.isClosed) break else throw e + } + socket.use { + it.soTimeout = 5000 + val reader = it.getInputStream().bufferedReader() + while (!reader.readLine().isNullOrEmpty()) { } + requests.incrementAndGet() + val out = it.getOutputStream() + if (reject.get()) { + out.write("HTTP/1.1 503 Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".toByteArray()) + } else { + val header = ByteBuffer.allocate(41).order(ByteOrder.LITTLE_ENDIAN) + .putInt(0xB1FE3AA2.toInt()).putInt(1).putInt(41).putInt(bytes.size) + .put(ByteArray(24)).put(0).array() + out.write("HTTP/1.1 200 OK\r\nContent-Length: ${header.size + bytes.size}\r\nConnection: close\r\n\r\n".toByteArray()) + out.write(header) + out.write(bytes) + } + out.flush() + } + } + }.apply { start() } + val chunk = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(1, 0, 0, 0) + windowSize = bytes.size + fileSize = bytes.size.toLong() + sha1 = sha(bytes) + } + manifest = EpicDownloadManager.Manifest().apply { + chunkDir = "chunks" + uniqueChunks = arrayListOf(chunk) + cdnUrls = listOf(EpicDownloadManager.CdnUrl("http://127.0.0.1:${server.localPort}", "", "")) + files = arrayListOf(gameFile("game.bin", 0, bytes.size / 2), gameFile("game.bin.part", bytes.size / 2, bytes.size / 2)) + } + } + + @After fun tearDown() { server.close(); worker.join(6000); assertFalse(worker.isAlive) } + + private fun sha(data: ByteArray) = MessageDigest.getInstance("SHA-1").digest(data) + + private fun gameFile(name: String, offset: Int, size: Int) = EpicDownloadManager.FileInfo().apply { + filename = name + sha1 = sha(bytes.copyOfRange(offset, offset + size)) + parts.add(EpicDownloadManager.ChunkPart().apply { + guid = intArrayOf(1, 0, 0, 0) + this.offset = offset + this.size = size + }) + } + + private fun install(cancel: AtomicBoolean = AtomicBoolean(), callback: EpicDownloadManager.Callback = + EpicDownloadManager.Callback { _, _ -> }, cache: String = "") = + EpicDownloadManager.installManifest(manifest, root.path, cache, null, cancel, callback) + + @Test fun installsSharedChunksAndPreservesLegitimatePartFilename() { + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + assertArrayEquals(bytes.copyOfRange(bytes.size / 2, bytes.size), File(root, "game.bin.part").readBytes()) + assertEquals(1, requests.get()) + assertFalse(File(root, ".chunks").exists()) + assertNotNull(install()) + assertEquals("Verified files must not be downloaded again", 1, requests.get()) + } + + @Test fun pauseKeepsCompletedFilesAndResumeReusesSharedCache() { + val cancel = AtomicBoolean() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) {} + override fun onStage(stage: String, done: Long, total: Long, items: Int, itemsTotal: Int) { + if (stage == "install" && items == 1) cancel.set(true) + } + } + assertNull(install(cancel, callback)) + assertTrue(File(root, "game.bin").exists()) + assertFalse(File(root, "game.bin.part").exists()) + assertEquals(1, requests.get()) + assertNotNull(install()) + assertEquals(1, requests.get()) + } + + @Test fun cancellationDuringChunkDeliveryRemovesIncompleteOutput() { + val cancel = AtomicBoolean() + val callback = object : EpicDownloadManager.Callback { + override fun onProgress(message: String, pct: Int) { + if (message.startsWith("Downloading chunks")) cancel.set(true) + } + } + assertNull(install(cancel, callback)) + assertFalse(File(root, "game.bin").exists()) + assertNotNull(install()) + assertArrayEquals(bytes.copyOfRange(0, bytes.size / 2), File(root, "game.bin").readBytes()) + } + + @Test fun fileHashFailureIsExplicitAndRetryRepairsIt() { + val expected = manifest.files[0].sha1 + manifest.files[0].sha1 = ByteArray(20) { 1 } + val error = assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertTrue(error.message.orEmpty().contains("SHA-1 mismatch")) + assertFalse(File(root, "game.bin").exists()) + manifest.files[0].sha1 = expected + assertNotNull(install()) + } + + @Test fun corruptCacheIsRefetchedAndScratchCacheIsRemovedOnSuccess() { + val cache = temp.newFolder("scratch") + File(cache, manifest.uniqueChunks[0].guidStr()).writeBytes(ByteArray(bytes.size)) + assertNotNull(install(cache = cache.path)) + assertEquals(1, requests.get()) + assertFalse(cache.exists()) + } + + @Test fun networkFailureLeavesNoCompletedFileAndCanBeRetried() { + reject.set(true) + assertThrows(EpicDownloadManager.InstallException::class.java) { install() } + assertFalse(File(root, "game.bin").exists()) + reject.set(false) + assertNotNull(install()) + } + + @Test fun hogwartsSizedSequentialPlanFitsAvailableSpaceWithoutFullChunkCache() { + val gib = 1024L * 1024 * 1024 + val file = EpicDownloadManager.FileInfo() + val chunks = LinkedHashMap() + repeat(72) { index -> + val chunk = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(index + 1, 0, 0, 0) + windowSize = gib.toInt() + } + chunks[chunk.guidStr()] = chunk + file.parts.add(EpicDownloadManager.ChunkPart().apply { + guid = chunk.guid + size = chunk.windowSize + }) + } + val peak = EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true) + assertEquals(73 * gib, peak) + assertTrue(peak < 96 * gib) + } +} diff --git a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt index 4d085013fa..8c34c125d1 100644 --- a/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt +++ b/app/src/test/java/com/droiddeck/launcher/stores/epic/EpicJsonManifestTest.kt @@ -1,8 +1,10 @@ package com.droiddeck.launcher.stores.epic import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner @@ -12,6 +14,103 @@ import org.robolectric.annotation.Config @RunWith(RobolectricTestRunner::class) @Config(sdk = [33]) class EpicJsonManifestTest { + @Test + fun boundedInstallPeakCountsOnlyOneLiveChunkForSequentialChunks() { + val first = chunk(1) + val second = chunk(2) + val file = file(part(1, 0, 100), part(2, 0, 100)) + val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) + + assertEquals(300L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), true)) + assertEquals(100L, EpicDownloadManager.estimatePeakAdditionalBytes(listOf(file), chunks, emptyMap(), false)) + } + + @Test + fun sharedCachedChunkIsReclaimedOnlyAfterItsLastPart() { + val shared = chunk(1) + val files = listOf(file(part(1, 0, 60)), file(part(1, 60, 40))) + + assertEquals( + 100L, + EpicDownloadManager.estimatePeakAdditionalBytes( + files, mapOf(shared.guidStr() to shared), mapOf(shared.guidStr() to 100L), true, + ), + ) + assertEquals( + 200L, + EpicDownloadManager.estimatePeakAdditionalBytes( + files, mapOf(shared.guidStr() to shared), emptyMap(), true, + ), + ) + } + + @Test + fun distantSharedChunksAreEvictedAndRefetchedWithinTheCacheCeiling() { + val first = chunk(1).apply { windowSize = 40 * 1024 * 1024 } + val second = chunk(2).apply { windowSize = 40 * 1024 * 1024 } + val files = listOf( + file(part(1, 0, 40 * 1024 * 1024)), + file(part(2, 0, 40 * 1024 * 1024)), + file(part(1, 0, 40 * 1024 * 1024)), + file(part(2, 0, 40 * 1024 * 1024)), + ) + val chunks = mapOf(first.guidStr() to first, second.guidStr() to second) + + val gamePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), true) + val cachePeak = EpicDownloadManager.estimatePeakAdditionalBytes(files, chunks, emptyMap(), false) + + assertEquals(200L * 1024 * 1024, gamePeak) + assertTrue(cachePeak <= EpicDownloadManager.MAX_RETAINED_CHUNK_CACHE_BYTES + 40L * 1024 * 1024) + } + + @Test + fun resumeCacheDropsCorruptAndUnneededEntriesButKeepsVerifiedChunks() { + val dir = java.io.File(System.getProperty("user.dir"), "build/epic-cache-test-${System.nanoTime()}") + assertTrue(dir.mkdirs()) + try { + val bytes = ByteArray(100) { (it % 251).toByte() } + val expectedSha = java.security.MessageDigest.getInstance("SHA-1").digest(bytes) + val valid = chunk(1).apply { sha1 = expectedSha } + val corrupt = chunk(2).apply { sha1 = expectedSha } + java.io.File(dir, valid.guidStr()).writeBytes(bytes) + java.io.File(dir, corrupt.guidStr()).writeBytes(ByteArray(100) { 1 }) + java.io.File(dir, "orphan").writeBytes(bytes) + + val retained = EpicDownloadManager.prepareBoundedCache(dir, listOf(valid, corrupt)) + + assertEquals(mapOf(valid.guidStr() to 100L), retained) + assertTrue(java.io.File(dir, valid.guidStr()).exists()) + assertFalse(java.io.File(dir, corrupt.guidStr()).exists()) + assertFalse(java.io.File(dir, "orphan").exists()) + } finally { + EpicDownloadManager.deleteDir(dir) + } + } + + @Test + fun referenceCountsIncludeRepeatedPartsSoCancelledFilesCanBeReplanned() { + val pending = listOf(file(part(1, 0, 20), part(1, 20, 20)), file(part(1, 40, 60))) + + assertEquals(3, EpicDownloadManager.chunkReferences(pending)["00000001000000000000000000000000"]) + } + + private fun chunk(id: Int) = EpicDownloadManager.ChunkInfo().apply { + guid = intArrayOf(id, 0, 0, 0) + windowSize = 100 + fileSize = 100 + } + + private fun part(id: Int, offset: Int, size: Int) = EpicDownloadManager.ChunkPart().apply { + guid = intArrayOf(id, 0, 0, 0) + this.offset = offset + this.size = size + } + + private fun file(vararg parts: EpicDownloadManager.ChunkPart) = EpicDownloadManager.FileInfo().apply { + filename = "test.bin" + this.parts.addAll(parts) + } + @Test fun blobNumbersAreDecimalBytesLeastSignificantFirst() { assertEquals(1L, EpicDownloadManager.blobToNum("001000000000")) diff --git a/docs/development/stores.md b/docs/development/stores.md index 4ce7d1aab3..07f078037c 100644 --- a/docs/development/stores.md +++ b/docs/development/stores.md @@ -24,15 +24,25 @@ scanned (`StoreInstallRoot.roots`): internal, every card, and the Steam library' installs an earlier build put there. `` is `GOG`, `Epic` or `Amazon`; `` is the title with unsafe characters dropped, at most 60 characters (`StoreInstallRoot.folderName`), chosen once; a rerun (repair, update) lands on the folder whose sidecar carries the game's id, wherever it is. -An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`: the -`chunkCacheDir` of both native calls (`EpicNative.run` fetches into it, `EpicNative.assemble` -writes the files from it and drops each chunk after its last use, then the folder), and of the -manager's own loops when the engine is not there; `""` keeps the cache beside the game, as an -internal install has it. The scratch folder is removed on cancel-with-delete and on uninstall. -The cache holds whole ~1 MiB chunk windows, shared with files this device does not install, so it -is often larger than the game (Metalstorm: 9.1 GB of chunks for 4.4 GB of files); the free-space -check, made after the delta pass, counts the missing chunks and the missing files on their own -volumes. A successful run removes the cache. +An Epic install on a card keeps its in-flight chunks in `cacheDir/stores/epic/<id>/`; +`""` keeps the cache beside the game, as an internal install has it. The scratch folder is +removed on cancel-with-delete and on uninstall. + +Epic installs use the same bounded Java writer whether the native engine is available or not. +It fetches one chunk at a time, verifies it, and writes the required parts into the current +game file. Chunks are deleted after their last use; reusable chunks are retained up to 64 MiB, +with at most one additional incoming chunk before eviction. Evicted shared chunks may be +downloaded again. This trades parallel-download throughput and potentially extra network +traffic for bounded temporary disk usage. The native batch APIs remain available but are not +used by this install path. + +The space check follows the manifest delta pass and simulates file writes, chunk fetches, +and evictions, rather than adding the entire cache to the entire game. Completed files and +verified cached chunks are accounted for on resume. On a card, game and cache requirements +are checked separately. File SHA-1 hashes are checked where supplied by the manifest; failed +or cancelled output files are removed, and an interrupted process is repaired by the next +delta pass. A successful run removes the cache. A resume within a large incomplete file +restarts that file, not its individual parts. ## The sidecar