This commit is contained in:
Kurt Himebauch authored and GitHub committed 2026-10-13 21:27:42 -04:00
1 parent 5fd47421f9
commit aecd2ebe1d
852 files changed
+4844 -46144

No files matched your search

-2
View File
@@ -1,5 +1,3 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
-15
View File
@@ -2,9 +2,6 @@
name = "droiddeckstores"
version = "0.1.0"
edition = "2021"
# The engines are a port of Bannerlator's bl-steam-client store modules (GPL-3.0-or-later, derived
# from WinNative's wn-steam-client; the Epic selection/verify rules trace back to GameNative and
# Legendary, both GPL-3.0). DroidDeck is GPL-3.0, so the crate keeps the licence it came with.
license = "GPL-3.0-or-later"
publish = false
@@ -14,24 +11,12 @@ crate-type = ["cdylib"]
[dependencies]
jni = "0.21"
# setpriority/gettid: every engine thread runs at nice +10 (see src/priority.rs). Already in the
# dependency tree through tokio, at the version Cargo.lock pins.
libc = "0.2"
# zlib inflate for GOG gen2 chunks and Epic chunk bodies. The pure-Rust backend keeps the .so free
# of a C zlib and of the NDK sysroot's libz, which the apk does not ship.
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
# The fetch core drives every request through ONE tokio current-thread runtime and feeds a sync
# process pool; rustls (via ring) is the TLS stack so there is no OpenSSL to cross-compile. `gzip`
# transparently inflates a CDN answer that comes back content-encoded. No `blocking` client: the
# three engines are async-fetch only.
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip"] }
tokio = { version = "1", default-features = false, features = ["rt", "net", "time", "sync"] }
futures-util = { version = "0.3", default-features = false, features = ["std"] }
# preserve_order keeps JSON object keys in source order, as in Bannerlator, so the GOG depot plan
# walks a manifest's files in the order the manifest lists them (largest-first sorting happens
# after, explicitly).
serde_json = { version = "1", features = ["preserve_order"] }
# Epic chunks are SHA-1 verified, Amazon files SHA-256; GOG's MD5 is the in-tree md5_small.
sha1 = "0.10"
sha2 = "0.10"
-102
View File
@@ -1,22 +1,3 @@
//! Amazon Games download adapter over the shared fetch core.
//!
//! Replaces ONLY the byte-fetching pool of `AmazonDownloadManager.install()` (Step 4).
//! Java still resolves the download spec, fetches and parses `manifest.proto`, and hands
//! this module the resolved file list (one JSON entry per manifest file with the exact
//! URL the Java loop would have opened). The adapter mirrors the Java rules 1:1 where the
//! core allows it — see Bannerlator's `docs/RUST_AMAZON_PARITY.md` §4 / §9 for the table and the
//! documented deviations:
//!
//! * one `FetchItem` per WHOLE file, streamed (`FetchOptions.stream = true`): pieces are
//! appended to `<dest>.tmp` as they arrive, 8 files wide like Java's pool, memory bounded
//! to the pieces not yet written;
//! * resume-skip = `st_size == manifest size`, no hash check on skip (Java `:250`);
//! * SHA-256 is accumulated while streaming and checked in `on_finish` when the manifest
//! carries a hash; a mismatch deletes the tmp and is a retryable failure (Java `:269`);
//! * `<dest>.tmp` → delete existing dest → rename (Java `:280`); rename failure is fatal
//! without retry (Java `:281`);
//! * a piece credits its wire length to progress as it is written (Java `:347`), and pieces
//! of an attempt that later fails stay counted (Java never rolls `totalDownloaded` back).
pub mod jni;
@@ -30,34 +11,20 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
/// Java `AmazonDownloadManager.MAX_PARALLEL`.
pub const MAX_PARALLEL: usize = 8;
/// Java `AmazonDownloadManager.DOWNLOAD_USER_AGENT`.
pub const DOWNLOAD_USER_AGENT: &str = "nile/0.1 Amazon";
/// Java `conn.setReadTimeout(120000)`; in stream mode the core applies this to the response
/// headers and then as an idle deadline per received piece — the same shape as a read timeout.
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
/// Java `new File(installDir, file.unixPath() + ".tmp")`.
pub const TMP_SUFFIX: &str = ".tmp";
/// Log label used in the core's `fetch-window` lines.
pub const LABEL: &str = "amazon";
/// One resolved manifest file as serialised by Java (`AmazonDownloadManager.buildRustPlan`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PlanEntry {
/// `ManifestFile.unixPath()` — relative to the install dir, forward slashes.
pub rel_path: String,
/// `AmazonApiClient.appendPath(downloadUrl, "files/" + hashHex)` — signed, whole file.
pub url: String,
/// `ManifestFile.size`.
pub size: u64,
/// SHA-256 to verify against; EMPTY = Java would not verify (`hashAlgorithm != 0` or
/// no hash bytes).
pub sha256: Vec<u8>,
}
/// Parse the JSON array `[{relPath, url, size, sha256hex}]`. `relPath` is normalised the same
/// way as `ManifestFile.unixPath()` so a caller that forgot is still byte-identical.
pub fn parse_plan(json: &str) -> Result<Vec<PlanEntry>, String> {
let value: serde_json::Value =
serde_json::from_str(json).map_err(|err| format!("plan json: {err}"))?;
@@ -106,12 +73,10 @@ pub fn parse_plan(json: &str) -> Result<Vec<PlanEntry>, String> {
Ok(entries)
}
/// `ManifestFile.unixPath()`.
pub fn unix_path(path: &str) -> String {
path.replace('\\', "/")
}
/// Lowercase/uppercase hex → bytes; `None` on odd length or a non-hex digit.
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
let bytes = hex.as_bytes();
if bytes.len() % 2 != 0 {
@@ -132,29 +97,21 @@ pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
Some(out)
}
/// `new File(installDir, rel)` — Java resolves the child textually under the parent, so
/// build the path by concatenation (never `Path::join`, which would let a leading `/` escape).
pub fn dest_path(install_dir: &str, rel_path: &str) -> PathBuf {
let base = install_dir.trim_end_matches('/');
PathBuf::from(format!("{base}/{rel_path}"))
}
/// `new File(installDir, rel + ".tmp")`.
pub fn tmp_path(install_dir: &str, rel_path: &str) -> PathBuf {
dest_path(install_dir, &format!("{rel_path}{TMP_SUFFIX}"))
}
/// Java `:250`: `destFile.exists() && destFile.length() == file.size`. `File.length()` is
/// `st_size`, which is what `Metadata::len()` returns — including for a directory, so the
/// (degenerate) behaviours match too. No hash is checked on skip, on purpose.
pub fn is_present_and_complete(install_dir: &str, entry: &PlanEntry) -> bool {
fs::metadata(dest_path(install_dir, &entry.rel_path))
.map(|meta| meta.len() == entry.size)
.unwrap_or(false)
}
/// `scheme://host[:port]` of a URL — the per-host cap key. Amazon serves every file from the
/// one signed CDN base, so this yields a single host.
pub fn host_key(url: &str) -> String {
let (scheme, rest) = match url.find("://") {
Some(idx) => (&url[..idx], &url[idx + 3..]),
@@ -164,33 +121,22 @@ pub fn host_key(url: &str) -> String {
format!("{scheme}://{}", &rest[..end])
}
/// Improvements round 1: `max(6, ceil(max_workers / distinct_hosts))` — a single-host store
/// (Amazon's one signed a2z base) gets the whole ceiling on its one host; the core clamps the
/// window to `hosts × per_host_cap`, so anything smaller would silently cut the ceiling.
pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize {
let hosts = distinct_hosts.max(1);
let per_host = max_workers.max(1).div_ceil(hosts);
per_host.max(crate::fetch_tuning::PER_HOST_CAP)
}
/// The resolved work for one run.
#[derive(Debug, Default)]
pub struct Plan {
/// Items to fetch (index into `entries` via `FetchItem.id`).
pub items: Vec<FetchItem>,
/// Every manifest file, in manifest order (fetched and skipped alike).
pub entries: Vec<PlanEntry>,
/// Σ size of all entries = `ParsedManifest.totalInstallSize`.
pub total_bytes: u64,
/// Bytes credited up front for files that passed the resume check.
pub skipped_bytes: u64,
/// Number of files that passed the resume check.
pub skipped_files: u64,
/// Distinct host keys in fetch order.
pub hosts: Vec<String>,
}
/// Apply the Java resume-skip rule to every entry and turn the rest into whole-file items.
pub fn build_plan(entries: Vec<PlanEntry>, install_dir: &str) -> Plan {
let mut plan = Plan {
total_bytes: entries.iter().map(|e| e.size).sum(),
@@ -220,23 +166,17 @@ pub fn build_plan(entries: Vec<PlanEntry>, install_dir: &str) -> Plan {
plan
}
/// One `<dest>.tmp` being streamed: the open handle, the running hash and the bytes written
/// by the current attempt.
struct OpenTmp {
file: File,
hasher: Sha256,
written: u64,
}
/// Sink: stream pieces into `<dest>.tmp` (SHA-256 accumulated on the fly), verify and rename
/// in `on_finish`. `process` (whole-body mode) does the same in one step.
pub struct AmazonSink {
install_dir: String,
entries: Vec<PlanEntry>,
cancel: Arc<AtomicBool>,
files_done: AtomicU64,
/// Per-item open tmp files. The map lock is held only to look the entry up; the write
/// itself happens under the per-item lock so 8 streams never serialise on one mutex.
open: Mutex<HashMap<u64, Arc<Mutex<OpenTmp>>>>,
}
@@ -261,16 +201,12 @@ impl AmazonSink {
.ok_or_else(|| SinkError::Fatal(format!("plan index {} out of range", item.id)))
}
/// Java `:260` — `destFile.getParentFile().mkdirs()` before each attempt; a failure
/// surfaces on the create/write below, like Java's.
fn ensure_parent(&self, entry: &PlanEntry) {
if let Some(parent) = dest_path(&self.install_dir, &entry.rel_path).parent() {
let _ = fs::create_dir_all(parent);
}
}
/// Java `:280-284`: delete an existing dest, rename tmp → dest; rename failure fails the
/// file with NO retry (→ whole install fails).
fn rename_into_place(&self, entry: &PlanEntry) -> Result<(), SinkError> {
let dest = dest_path(&self.install_dir, &entry.rel_path);
let tmp = tmp_path(&self.install_dir, &entry.rel_path);
@@ -288,8 +224,6 @@ impl AmazonSink {
Ok(())
}
/// Java `:267-277`: SHA-256 check when the manifest carries a hash; a mismatch deletes the
/// tmp and is retried.
fn verify(&self, entry: &PlanEntry, digest: &[u8]) -> Result<(), SinkError> {
if !entry.sha256.is_empty() && digest != entry.sha256.as_slice() {
let _ = fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path));
@@ -301,7 +235,6 @@ impl AmazonSink {
Ok(())
}
/// Whole-body path (`FetchOptions.stream = false`): verify → write tmp → rename.
pub fn commit(&self, entry: &PlanEntry, body: &[u8]) -> Result<u64, SinkError> {
if !entry.sha256.is_empty() {
let digest = Sha256::digest(body);
@@ -322,10 +255,6 @@ impl AmazonSink {
Ok(body.len() as u64)
}
/// After the run: close and delete every tmp that never reached `on_finish` (a stream that
/// failed after max attempts, or was cut by cancel / a fatal error). Java deletes the tmp
/// of every failed or cancelled attempt (`:271`, `:288`, `:293`, `:297`), so no `.tmp`
/// may survive a run on either engine. Returns how many were removed.
pub fn cleanup_partials(&self) -> usize {
let leftovers: Vec<(u64, Arc<Mutex<OpenTmp>>)> = match self.open.lock() {
Ok(mut open) => open.drain().collect(),
@@ -355,13 +284,11 @@ impl FetchSink for AmazonSink {
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
if self.cancel.load(Ordering::Relaxed) {
// Java `:342-344`: cancel inside the read loop → tmp deleted (cleanup_partials).
return Err(SinkError::Fatal("cancelled".to_string()));
}
let entry = self.entry(item)?;
let tmp = tmp_path(&self.install_dir, &entry.rel_path);
let open = if offset == 0 {
// (Re)start of an attempt: Java opens a fresh FileOutputStream (truncating).
self.ensure_parent(entry);
let file = File::create(&tmp).map_err(|err| {
let _ = fs::remove_file(&tmp);
@@ -405,7 +332,6 @@ impl FetchSink for AmazonSink {
)));
}
if let Err(err) = state.file.write_all(data) {
// Java: IOException inside downloadFile → tmp deleted → next attempt.
let _ = fs::remove_file(&tmp);
return Err(SinkError::Retry(format!("write {}: {err}", tmp.display())));
}
@@ -435,7 +361,6 @@ impl FetchSink for AmazonSink {
(written, digest)
}
None => {
// Zero-byte body: no piece ever arrived. Java would have created an empty tmp.
if total_len != 0 {
return Err(SinkError::Retry(format!(
"finish without a stream for: {}",
@@ -462,22 +387,16 @@ impl FetchSink for AmazonSink {
}
}
/// Outcome of one `run_download`.
#[derive(Clone, Debug, Default)]
pub struct RunResult {
pub success: bool,
pub cancelled: bool,
pub error: String,
/// Bytes fetched by this run (skipped files excluded; failed attempts' pieces included,
/// like Java's `totalDownloaded`).
pub bytes_written: u64,
/// Files committed by this run plus files skipped by the resume check.
pub files_done: u64,
pub files_total: u64,
}
/// Throughput bookkeeping for the final summary line (avg over the run, peak over ≥500 ms
/// progress intervals — the same sampling window the Java speed label uses).
struct SpeedMeter {
started: Instant,
last_at: Instant,
@@ -516,10 +435,6 @@ impl SpeedMeter {
}
}
/// Blocking: plan → stream-fetch → verify/rename, calling `progress(bytes_done, bytes_total,
/// files_done, files_total)` after each written piece and each finished file (skipped files
/// are pre-credited) and `log` for every diagnostic line. `bytes_done` includes skipped bytes,
/// matching Java's `totalDownloaded` (which credits skipped files at `:251`).
pub fn run_download(
plan_json: &str,
install_dir: &str,
@@ -582,16 +497,12 @@ pub fn run_download(
let opts = FetchOptions {
max_workers,
// Every Amazon file comes from the one signed CDN base: give that host the whole
// ceiling (`per_host_cap_for`), or the core would clamp the window to hosts × cap.
per_host_cap,
timeout: REQUEST_TIMEOUT,
headers: vec![("User-Agent".to_string(), DOWNLOAD_USER_AGENT.to_string())],
ca_bundle_path: ca_bundle_path.to_string(),
process_workers,
label: LABEL.to_string(),
// Whole files, streamed into the tmp as they arrive: 8 wide like Java's pool, memory
// bounded to the pieces not yet written.
stream: true,
..FetchOptions::default()
};
@@ -753,10 +664,8 @@ mod tests {
fn build_plan_skips_only_size_matched_files() {
let dir = scratch_dir();
let entries = parse_plan(PLAN).unwrap();
// Entry 1 present with the right size → skipped (no hash check even if wrong bytes).
fs::create_dir_all(format!("{dir}/data")).unwrap();
fs::write(format!("{dir}/data/level1.pak"), b"zz").unwrap();
// Entry 0 present with the wrong size → fetched.
fs::create_dir_all(format!("{dir}/Binaries/Win64")).unwrap();
fs::write(format!("{dir}/Binaries/Win64/Game.exe"), b"tes").unwrap();
let plan = build_plan(entries, &dir);
@@ -803,7 +712,6 @@ mod tests {
let cancel = Arc::new(AtomicBool::new(false));
let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::clone(&cancel));
let item = item_for(&entry, 0);
// Mismatch → Retry, nothing on disk.
match sink.process(&item, b"nope".to_vec()) {
Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")),
Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"),
@@ -811,14 +719,11 @@ mod tests {
}
assert!(!dest_path(&dir, "sub/Game.exe").exists());
assert!(!tmp_path(&dir, "sub/Game.exe").exists());
// Match → written, renamed, tmp gone, credited body length.
assert_eq!(sink.process(&item, body.clone()).unwrap(), 4);
assert_eq!(fs::read(dest_path(&dir, "sub/Game.exe")).unwrap(), body);
assert!(!tmp_path(&dir, "sub/Game.exe").exists());
assert_eq!(sink.files_done(), 1);
// Existing dest is replaced (Java `:280`).
assert_eq!(sink.process(&item, body.clone()).unwrap(), 4);
// Cancel → Fatal, no write.
cancel.store(true, Ordering::Relaxed);
assert!(matches!(
sink.process(&item, body),
@@ -863,7 +768,6 @@ mod tests {
};
let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false)));
let item = item_for(&entry, 0);
// Attempt 1 delivers garbage, then the core restarts at offset 0.
sink.on_chunk(&item, 0, b"zzzz").unwrap();
sink.on_chunk(&item, 0, b"abc").unwrap();
sink.on_chunk(&item, 3, b"xyz").unwrap();
@@ -874,21 +778,17 @@ mod tests {
}
assert!(!tmp_path(&dir, "f.bin").exists());
assert!(!dest_path(&dir, "f.bin").exists());
// Out-of-order piece → Retry.
sink.on_chunk(&item, 0, b"abc").unwrap();
assert!(matches!(
sink.on_chunk(&item, 5, b"f"),
Err(SinkError::Retry(_))
));
// Length mismatch at finish → Retry.
sink.on_chunk(&item, 0, b"abc").unwrap();
assert!(matches!(sink.on_finish(&item, 6), Err(SinkError::Retry(_))));
// Piece without a start → Retry.
assert!(matches!(
sink.on_chunk(&item, 3, b"def"),
Err(SinkError::Retry(_))
));
// Good attempt.
sink.on_chunk(&item, 0, b"abc").unwrap();
sink.on_chunk(&item, 3, b"def").unwrap();
assert_eq!(sink.on_finish(&item, 6).unwrap(), 0);
@@ -915,10 +815,8 @@ mod tests {
let sink = AmazonSink::new(&dir, vec![big.clone(), empty.clone()], Arc::clone(&cancel));
sink.on_chunk(&item_for(&big, 0), 0, b"12345").unwrap();
assert!(tmp_path(&dir, "cut.bin").exists());
// Zero-byte body: on_finish with no pieces creates and commits an empty file.
assert_eq!(sink.on_finish(&item_for(&empty, 1), 0).unwrap(), 0);
assert_eq!(fs::read(dest_path(&dir, "empty.bin")).unwrap(), b"");
// Cancel mid-stream → Fatal; cleanup deletes the partial tmp like Java does.
cancel.store(true, Ordering::Relaxed);
assert!(matches!(
sink.on_chunk(&item_for(&big, 0), 5, b"6"),
@@ -1,12 +1,3 @@
//! JNI facade for the Amazon download adapter.
//!
//! Kotlin side: `com.droiddeck.launcher.stores.amazon.AmazonNative` (`nativeStart` /
//! `nativeCancel` / `nativeRelease`) with an `AmazonNative.Listener`
//! (`onProgress(JJJJ)V`, `onLog(String)V`, `onComplete(ZString;J)V`). Mirrors the shape of
//! the GOG and Epic facades: the run happens on a spawned native
//! thread, callbacks attach the JavaVM as daemon threads and hold a `GlobalRef` to the
//! listener; cancel flips an `AtomicBool`. This module keeps its own `JavaVM` handle (taken
//! from the calling `JNIEnv` on first use) so it depends on nothing outside this module.
use crate::amazon::{run_download, MAX_PARALLEL};
use jni::objects::{GlobalRef, JClass, JObject, JString, JValue};
@@ -16,10 +7,6 @@ use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::thread;
/// Logcat tag (`adb logcat -s AmazonNative`). Every engine line reaches logcat exactly ONCE:
/// this module only forwards lines to the listener (`onLog`), and the Java manager is the one
/// place that writes them to `android.util.Log` + its debug file. (Round 1 fix: the native side
/// used to also call `__android_log_write`, so each line appeared twice.)
pub const LOG_TAG: &str = "AmazonNative";
static JVM: OnceLock<JavaVM> = OnceLock::new();
@@ -36,7 +23,6 @@ fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option<String> {
.map(|s| s.to_string_lossy().into_owned())
}
/// One live run; boxed and handed to Kotlin as an opaque `jlong`.
struct AmazonRun {
cancel: Arc<AtomicBool>,
}
@@ -115,7 +101,6 @@ fn call_complete(
clear_pending_exception(env);
}
/// Attach the current (native) thread and run `f` with the listener object.
fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)) {
let Some(vm) = JVM.get() else {
return;
@@ -126,9 +111,6 @@ fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)
f(&mut env, listener.as_obj());
}
/// `AmazonNative.nativeStart(planJson, installDir, caBundlePath, maxWorkers,
/// processWorkers, listener) -> handle`. Returns 0 (and fires `onComplete(false, …)`) when the
/// inputs are unusable. The run itself happens on a spawned thread.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart(
mut env: JNIEnv,
@@ -200,8 +182,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_na
to_handle(run)
}
/// `AmazonNative.nativeCancel(handle)` — flips the run's cancel flag; the core stops
/// issuing requests, in-flight bodies are dropped, `onComplete(false, "cancelled", …)` follows.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel(
_env: JNIEnv,
@@ -213,8 +193,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_na
}
}
/// `AmazonNative.nativeRelease(handle)` — frees the handle. Safe at any time: the worker
/// thread owns its own `Arc` of the cancel flag and the listener `GlobalRef`.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease(
_env: JNIEnv,
+1 -30
View File
@@ -1,29 +1,4 @@
//! Epic Games Store download adapter for the shared fetch core (`crate::fetch_core`).
//!
//! Scope = the chunk-fetch inner loop of `EpicDownloadManager.java` (the fixed 8-thread pool that
//! fills `<installDir>/.chunks/<GUID>`), nothing more. The Java manager still parses the manifest
//! API JSON, downloads the manifest, selects files (install tags), runs the delta/verify pass,
//! assembles files from the chunk cache and does every post-install step. The adapter re-parses
//! the same manifest bytes, rebuilds the same chunk plan for the pending files Java hands it,
//! skips chunks already in the cache and writes verified chunks with the same `.part` + rename
//! protocol, so the on-disk state is identical whichever engine ran (see
//! Bannerlator's `docs/RUST_EPIC_PARITY.md` for the rule-by-rule table).
//!
//! Submodules:
//! - [`manifest`] — ChunksV4 binary manifest + legacy JSON manifest parsers (1:1 with the Java
//! `parseManifest` / `parseJsonManifest`).
//! - [`plan`] — install-tag selection, unique-chunk plan, chunk paths / CDN URLs.
//! - [`chunk`] — chunk header parse, zlib inflate, SHA-1 verify, cache write.
//! - [`driver`] — plan → `FetchItem`s → `fetch_core::run_fetch` with the chunk-cache sink.
//! - [`assemble`] — the files from the cache, one sequential write each, SHA-1 checked as they
//! are written; with a scratch cache (one the caller placed apart from the install) each chunk
//! is deleted as soon as the last file that uses it is done.
//! - [`jni`] — `Java_com_droiddeck_launcher_stores_epic_EpicNative_native*` exports.
//!
//! The selective-install-tag, delta/resume and chunk verification rules mirrored here were ported
//! into the Java manager from GameNative (`service/epic/EpicDownloadManager.kt`,
//! `manifest/ManifestUtils.kt`), itself derived from Legendary; both are GPL-3.0 and this crate
//! carries the same licence (see `Cargo.toml`).
//! Derived from GameNative's Epic manager (itself derived from Legendary); both are GPL-3.0 and this crate carries the same licence (see Cargo.toml).
pub mod assemble;
pub mod chunk;
@@ -32,15 +7,11 @@ pub mod jni;
pub mod manifest;
pub mod plan;
/// User-Agent the Java manager sends on every chunk request (`EpicDownloadManager.UA`).
pub const USER_AGENT: &str =
"UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit";
/// Name of the chunk cache directory under the install dir (`new File(installDir, ".chunks")`).
pub const CHUNK_CACHE_DIR: &str = ".chunks";
/// Fixed pool width of the Java chunk downloader (`Executors.newFixedThreadPool(8)`).
pub const JAVA_POOL_THREADS: usize = 8;
/// logcat tag for every engine line (the Kotlin manager mirrors it into its own debug log).
pub const LOG_TAG: &str = "EpicNative";
@@ -1,26 +1,3 @@
//! Files from the chunk cache: the last stage of an Epic install, as a native run.
//!
//! The Kotlin manager's own loop does this too (`EpicDownloadManager.install`, after the chunk
//! fetch): for every pending file, open it, and for every part read the cached chunk and write
//! `chunk[offset..offset+size]`. This module does the same thing, in the same order, with the
//! same wording on a missing chunk, plus what the Kotlin loop cannot do without re-reading the
//! finished file from a slow card:
//!
//! - the file's SHA-1 is computed from the bytes as they are written (no extra I/O) and compared
//! with the manifest's, when the manifest has one; a mismatch deletes the file and fails the
//! run, so a bad assembly can never pass the next delta pass by size alone;
//! - with a scratch cache — one the caller placed apart from the install directory — every chunk
//! is reference-counted across the pending files and deleted the moment the last file that
//! needs it is written and verified, so the cache never holds more than what is still owed.
//!
//! The cache directory itself goes at the end of a successful run, scratch or not, which is what
//! the Kotlin loop does with `<installDir>/.chunks`. Cancellation and failure leave the cache
//! exactly as it stands, so the next run's delta pass skips the finished files and the fetch
//! skips the chunks still cached — on either engine.
//!
//! Why the cache may live elsewhere at all: an install to an SD card wrote every byte twice to
//! the card (chunk cache, then assembled file). With the cache on internal storage the card sees
//! each byte once, and the fetchers stop waiting on the card's write speed.
use std::collections::HashMap;
use std::fs::{self, File};
@@ -33,33 +10,25 @@ use sha1::{Digest, Sha1};
use super::manifest::{parse_manifest, FileInfo, Manifest};
use super::plan::resolve_cache_dir;
/// Everything the JNI layer hands over for one assembly run.
pub struct AssembleRequest {
pub manifest_bytes: Vec<u8>,
pub install_dir: String,
/// `""` = `<installDir>/.chunks` (no scratch semantics); anything else = scratch cache.
pub chunk_cache_dir: String,
/// Indices into `manifest.files` of the files to assemble, in order.
pub file_indices: Vec<usize>,
}
/// Terminal result of an assembly run.
#[derive(Clone, Debug, Default)]
pub struct AssembleOutcome {
pub success: bool,
pub cancelled: bool,
pub error: String,
/// Bytes written into assembled files this run.
pub bytes_written: u64,
pub files_done: u64,
pub files_total: u64,
pub bytes_total: u64,
/// Chunk files deleted early because no remaining file needed them (scratch cache only).
pub chunks_removed: u64,
}
/// The parsed plan: manifest, the files to write, and how many of them still need each chunk.
/// Built up front so a bad index fails before anything is written.
#[derive(Debug)]
pub struct AssemblePlan {
pub manifest: Manifest,
@@ -68,12 +37,9 @@ pub struct AssemblePlan {
pub scratch_cache: bool,
pub files: Vec<usize>,
pub bytes_total: u64,
/// GUID → number of files in `files` whose parts reference it (a file counts once per GUID,
/// however many of its parts use the chunk).
pub refs: HashMap<String, u32>,
}
/// Parse + plan. `Err` = nothing was written.
pub fn build_assemble_plan(req: &AssembleRequest) -> Result<AssemblePlan, String> {
if req.install_dir.is_empty() {
return Err("assemble: empty install dir".to_string());
@@ -104,7 +70,6 @@ pub fn build_assemble_plan(req: &AssembleRequest) -> Result<AssemblePlan, String
})
}
/// How many of `file_indices` reference each chunk GUID.
pub fn chunk_refs(m: &Manifest, file_indices: &[usize]) -> HashMap<String, u32> {
let mut refs: HashMap<String, u32> = HashMap::new();
for &fi in file_indices {
@@ -118,7 +83,6 @@ pub fn chunk_refs(m: &Manifest, file_indices: &[usize]) -> HashMap<String, u32>
refs
}
/// The GUIDs a file's parts reference, each once, in first-seen order.
fn distinct_guids(f: &FileInfo) -> Vec<String> {
let mut out: Vec<String> = Vec::with_capacity(f.parts.len());
for p in &f.parts {
@@ -130,19 +94,14 @@ fn distinct_guids(f: &FileInfo) -> Vec<String> {
out
}
/// The manifest's relative path with Windows separators normalised, as the Kotlin loop does
/// (`filename.replace("\\", "/")`).
pub fn relative_path(f: &FileInfo) -> String {
f.filename.replace('\\', "/")
}
/// A manifest SHA-1 that is all zero means "none" (the Kotlin delta pass treats it the same way).
fn expected_sha1(f: &FileInfo) -> Option<&[u8; 20]> {
f.sha1.as_ref().filter(|h| h.iter().any(|&b| b != 0))
}
/// Chunk bytes, read whole, with the last chunk kept: consecutive parts of one file usually come
/// from the same chunk, and a file's parts are the only place a chunk is read.
struct ChunkReader {
cache_dir: PathBuf,
last: Option<(String, Vec<u8>)>,
@@ -161,7 +120,6 @@ impl ChunkReader {
Ok(self.last.as_ref().map(|(_, d)| d.as_slice()).unwrap_or(&[]))
}
/// Drop the kept copy of a chunk that is about to be deleted.
fn forget(&mut self, guid: &str) {
if self.last.as_ref().map(|(g, _)| g.as_str()) == Some(guid) {
self.last = None;
@@ -169,8 +127,6 @@ impl ChunkReader {
}
}
/// Write one file from the cache. Returns the bytes written. The output is removed on any error,
/// so a file only ever exists complete (and, when the manifest hashes it, verified) or not at all.
fn write_file(reader: &mut ChunkReader, f: &FileInfo, out_path: &Path) -> Result<u64, String> {
let rel_path = relative_path(f);
if let Some(parent) = out_path.parent() {
@@ -219,8 +175,6 @@ fn write_file(reader: &mut ChunkReader, f: &FileInfo, out_path: &Path) -> Result
result
}
/// Run the plan. `progress(bytes_done, bytes_total, files_done, files_total)` fires after every
/// file; `log` receives the start line, failures and the terminal summary.
pub fn run_assemble(
plan: &AssemblePlan,
cancel: &AtomicBool,
@@ -270,9 +224,6 @@ pub fn run_assemble(
return outcome;
}
}
// Scratch cache: this file was the last user of a chunk → the chunk goes now, not at the
// end, so the cache holds only what is still owed to files not yet written. Released
// before the progress event, so "file done" also means "its chunks are freed".
if plan.scratch_cache {
for guid in distinct_guids(f) {
let Some(count) = refs.get_mut(&guid) else {
@@ -297,8 +248,6 @@ pub fn run_assemble(
}
outcome.success = true;
// Done with the cache: remove it whole (a scratch cache is already down to the chunks no
// pending file referenced; the default one is what the Kotlin loop deleted at this point).
match fs::remove_dir_all(&plan.cache_dir) {
Ok(()) => log(&format!("cache removed {}", plan.cache_dir.display())),
Err(e) => log(&format!(
@@ -337,7 +286,6 @@ mod tests {
out
}
/// file 0 = A[0..100] ++ B[10..60]; file 1 = A[100..300]. A is shared, B is file 0's alone.
fn expected_files() -> (Vec<u8>, Vec<u8>) {
let (a, b) = (chunk_a(), chunk_b());
let mut f0 = a[0..100].to_vec();
@@ -426,7 +374,6 @@ mod tests {
fill_cache(&cache);
let plan = build_assemble_plan(&request(&install, cache.to_str().unwrap())).unwrap();
assert!(plan.scratch_cache);
// After file 0: B (only file 0 used it) must be gone, A (file 1 still needs it) must stay.
let seen = Mutex::new(Vec::new());
let progress = |_: u64, _: u64, files_done: u64, _: u64| {
if files_done == 1 {
+5 -40
View File
@@ -1,16 +1,3 @@
//! One chunk: header parse → (zlib inflate | copy) → SHA-1 → `.part` → rename into the cache.
//!
//! Port of `EpicDownloadManager.downloadChunkStreaming` minus the HTTP part (the fetch core
//! hands us the whole response body). Java's protocol, kept exactly:
//! - 41-byte header: magic(4) headerVersion(4) headerSize(4) compressedSize(4) GUID(16) hash(8)
//! storedAs(1); payload starts at `max(headerSize, 41)` (Java never seeks back);
//! - payload = the next `compressedSize` bytes, or whatever the body still has (a short body
//! is NOT an error — the SHA-1 check decides);
//! - `storedAs & 1` → zlib; corrupt zlib = failure; a stream that ends early = partial output;
//! - the decompressed bytes go to `<final>.part`; the SHA-1 (when the manifest has one) must
//! match or the `.part` is deleted; then an atomic rename publishes `<final>`.
//! A cache file therefore exists under its final name ONLY if it fully downloaded and verified,
//! on either engine.
use std::fs::{self, File};
use std::io::{self, BufWriter, Write};
@@ -18,10 +5,8 @@ use std::path::{Path, PathBuf};
use sha1::{Digest, Sha1};
/// Chunk file magic (`0xB1FE3AA2`).
pub const CHUNK_MAGIC: u32 = 0xB1FE_3AA2;
/// Bytes Java reads before looking at the header (`byte[] hdrBuf = new byte[41]`).
pub const CHUNK_HEADER_PREFIX: usize = 41;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -37,7 +22,6 @@ impl ChunkHeader {
(self.stored_as & 1) != 0
}
/// Where the payload begins in the body (`if (headerSize > 41) skipFully(headerSize - 41)`).
pub fn payload_start(&self) -> usize {
if self.header_size > CHUNK_HEADER_PREFIX as i32 {
self.header_size as usize
@@ -47,8 +31,6 @@ impl ChunkHeader {
}
}
/// `readFully(hdrBuf)` + header decode. `Err` = the Java `continue`/exception paths
/// (short stream, bad magic).
pub fn parse_chunk_header(body: &[u8]) -> Result<ChunkHeader, String> {
if body.len() < CHUNK_HEADER_PREFIX {
return Err(format!(
@@ -70,9 +52,6 @@ pub fn parse_chunk_header(body: &[u8]) -> Result<ChunkHeader, String> {
})
}
/// The payload slice Java's read loop would consume: `compressedSize` bytes after the header,
/// truncated to what the body holds (`if (n <= 0) break;`). `Err` when the header skip runs
/// past the body (`skipFully` throws).
pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8], String> {
let start = hdr.payload_start();
if start > body.len() {
@@ -88,7 +67,6 @@ pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8],
Ok(&body[start..start + take])
}
/// `Writer` that mirrors `fos.write(...)` + `sha.update(...)` on every block.
struct HashingWriter<W: Write> {
inner: W,
sha: Option<Sha1>,
@@ -110,23 +88,18 @@ impl<W: Write> Write for HashingWriter<W> {
}
}
/// `new File(outFile.getPath() + ".part")`.
pub fn part_path(final_path: &Path) -> PathBuf {
let mut s = final_path.as_os_str().to_os_string();
s.push(".part");
PathBuf::from(s)
}
/// Process one downloaded body into `final_path`. Returns the number of DECOMPRESSED bytes
/// written. `Err(reason)` = Java's "try the next CDN" outcome; the `.part` never survives an
/// error and `final_path` is only ever created by the rename after verification.
pub fn write_verified_chunk(
body: &[u8],
expected_sha1: Option<&[u8; 20]>,
final_path: &Path,
) -> Result<u64, String> {
let tmp = part_path(final_path);
// `tmp.delete()` — clear any stale partial from a prior interrupted attempt.
let _ = fs::remove_file(&tmp);
let result = write_part(body, expected_sha1, &tmp, final_path);
@@ -153,9 +126,6 @@ fn write_part(
};
if hdr.is_compressed() {
// Java: inflate until the stream finishes or the input runs out; a corrupt stream throws
// (DataFormatException) → this CDN attempt fails. flate2 behaves the same way: EOF on a
// truncated stream is a clean end (partial output), corruption is an error.
let mut dec = flate2::read::ZlibDecoder::new(payload);
io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?;
} else {
@@ -164,7 +134,7 @@ fn write_part(
out.flush().map_err(|e| format!("flush: {e}"))?;
let written = out.written;
let digest = out.sha.take().map(|s| s.finalize());
drop(out); // close the file before the rename
drop(out);
if let (Some(expected), Some(actual)) = (expected_sha1, digest) {
if actual.as_slice() != &expected[..] {
@@ -172,7 +142,6 @@ fn write_part(
}
}
// Verified → publish atomically (same directory → same filesystem).
fs::rename(tmp, final_path).map_err(|e| format!("Chunk rename failed: {e}"))?;
Ok(written)
}
@@ -181,7 +150,6 @@ fn write_part(
pub(crate) mod test_support {
use super::*;
/// Build a chunk body the way the CDN serves it.
pub fn build_chunk_body(
decompressed: &[u8],
compress: bool,
@@ -199,15 +167,15 @@ pub(crate) mod test_support {
let header_size = (CHUNK_HEADER_PREFIX + extra_header) as i32;
let mut out = Vec::new();
out.extend_from_slice(&CHUNK_MAGIC.to_le_bytes());
out.extend_from_slice(&3i32.to_le_bytes()); // headerVersion
out.extend_from_slice(&3i32.to_le_bytes());
out.extend_from_slice(&header_size.to_le_bytes());
out.extend_from_slice(
&declared_compressed_size
.unwrap_or(payload.len() as i32)
.to_le_bytes(),
);
out.extend_from_slice(&[0xAB; 16]); // GUID (unused by the reader)
out.extend_from_slice(&[0xCD; 8]); // hash (unused by the reader)
out.extend_from_slice(&[0xAB; 16]);
out.extend_from_slice(&[0xCD; 8]);
out.push(if compress { 1 } else { 0 });
out.extend(std::iter::repeat(0x5A).take(extra_header));
assert_eq!(out.len(), header_size as usize);
@@ -280,7 +248,6 @@ mod tests {
let neg = build_chunk_body(b"abcdef", false, 0, Some(-1));
let h = parse_chunk_header(&neg).unwrap();
assert_eq!(chunk_payload(&neg, &h).unwrap(), b"");
// headerSize past the body → skipFully throws.
let mut far = build_chunk_body(b"abcdef", false, 0, None);
far[8..12].copy_from_slice(&5000i32.to_le_bytes());
let h = parse_chunk_header(&far).unwrap();
@@ -345,12 +312,10 @@ mod tests {
let data = sample_data();
let mut body = build_chunk_body(&data, true, 0, None);
let final_path = dir.join("chunk");
// Truncate the body: inflate ends early → partial output → SHA-1 mismatch.
let cut = body.len() - 1000;
let err = write_verified_chunk(&body[..cut], Some(&sha1_of(&data)), &final_path).unwrap_err();
assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}");
assert!(!final_path.exists());
// Corrupt the deflate stream itself.
for b in body.iter_mut().skip(60).take(64) {
*b ^= 0x55;
}
@@ -366,7 +331,7 @@ mod tests {
let dir = temp_dir("stale");
let final_path = dir.join("chunk");
fs::write(part_path(&final_path), b"stale").unwrap();
let body = [0u8; 10]; // too short → header failure
let body = [0u8; 10];
assert!(write_verified_chunk(&body, None, &final_path).is_err());
assert!(!part_path(&final_path).exists());
assert!(!final_path.exists());
@@ -1,12 +1,3 @@
//! Plan → `FetchItem`s → `crate::fetch_core::run_fetch` with a sink that fills the chunk cache.
//!
//! This is the replacement for the body of the Java pool block in `EpicDownloadManager.install`
//! ("Download unique chunks — 8 parallel threads"). Inputs are exactly what that block sees:
//! the parsed manifest (re-parsed here from the same bytes), the pending file set (indices Java
//! computed after its delta/verify pass), the CDN prefixes (`baseUrl + cloudDir`, cloudflare
//! already skipped) and the chunk cache dir. Output = `<cache>/<GUID>` for every needed chunk,
//! where `<cache>` is `<installDir>/.chunks` unless the caller named another directory; the
//! files are assembled from it afterwards (by the Kotlin manager, or by [`super::assemble`]).
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
@@ -21,65 +12,44 @@ use super::plan::{
total_credit_bytes, unique_chunks_for_files,
};
/// Java `conn.setReadTimeout(60000)` — the longer of the two Java timeouts (connect was 30 s).
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
/// Everything the JNI layer hands over for one run.
pub struct EpicRequest {
pub manifest_bytes: Vec<u8>,
pub install_dir: String,
/// Chunk cache directory; `""` = `<installDir>/.chunks`. See [`resolve_cache_dir`].
pub chunk_cache_dir: String,
/// `cdn.baseUrl + cdn.cloudDir` per manifest-API CDN entry, in Java's order.
pub cdn_prefixes: Vec<String>,
/// Indices into `manifest.files` of Java's `pendingFiles` (post delta/verify), in order.
pub pending_file_indices: Vec<usize>,
/// Java's `neededChunks.size()` / `totalBytes` for the plan cross-check (`None` = skip).
pub expected_chunks: Option<u64>,
pub expected_bytes: Option<u64>,
pub ca_bundle_path: String,
/// Window ceiling: the Steam speed tier's network window (Fast = 32) since improvements
/// round 1; the Java fallback pool keeps its fixed 8 (`super::JAVA_POOL_THREADS`).
pub max_workers: usize,
pub process_workers: usize,
pub label: String,
}
/// The chunk plan for one run.
#[derive(Debug)]
pub struct EpicPlan {
pub manifest: Manifest,
pub cache_dir: PathBuf,
/// The cache is a directory the caller named, apart from the install: scratch space that
/// the assembler empties as it goes and removes when it is done.
pub scratch_cache: bool,
/// Chunk indices (into `manifest.unique_chunks`) in Java's submission order.
pub needed: Vec<usize>,
/// `Σ max(fileSize, 1)` over `needed`.
pub total_bytes: u64,
/// Distinct CDN prefixes = fetch-core host keys.
pub hosts: Vec<String>,
}
/// Terminal result of a run, in the shape Java's pool block needs to reproduce its own exit
/// paths (`CANCELLED during chunk download (n/total chunks)`, `N chunks failed`, `chunksOK=`).
#[derive(Clone, Debug, Default)]
pub struct EpicOutcome {
pub success: bool,
pub cancelled: bool,
pub error: String,
/// Credited (manifest `max(fileSize,1)`) bytes: skipped-cached + fetched.
pub bytes_credited: u64,
/// Chunks accounted for (skipped-cached + fetched), Java's `completedCount`.
pub chunks_done: u64,
pub chunks_total: u64,
pub bytes_total: u64,
/// Decompressed bytes actually written to the cache this run.
pub decompressed_written: u64,
}
/// Parse + plan (no I/O beyond reading the cache dir). `Err` = "engine could not start"; Java
/// then runs its own pool, since nothing has been fetched yet.
pub fn build_plan(req: &EpicRequest) -> Result<EpicPlan, String> {
let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("plan: {e}"))?;
for &i in &req.pending_file_indices {
@@ -124,10 +94,8 @@ pub fn build_plan(req: &EpicRequest) -> Result<EpicPlan, String> {
})
}
/// Sink: one downloaded body → verified cache file. `id` indexes `fetch_chunks`.
struct ChunkCacheSink<'a> {
plan: &'a EpicPlan,
/// Chunk indices (into `manifest.unique_chunks`) of the items actually being fetched.
fetch_chunks: &'a [usize],
decompressed: AtomicU64,
}
@@ -144,15 +112,11 @@ impl<'a> FetchSink for ChunkCacheSink<'a> {
self.decompressed.fetch_add(written, Ordering::Relaxed);
Ok(chunk.credit_bytes())
}
// Every per-attempt failure in Java is "try the next CDN"; the core's Retry rotates
// hosts and backs off the same way (bounded at its attempt cap).
Err(reason) => Err(SinkError::Retry(format!("{} {reason}", chunk.guid_str()))),
}
}
}
/// Rolling speed sampler for the end-of-run summary: samples at ≥500 ms like the Java pool's
/// `lastSpeedMs` logic, keeps the peak.
struct SpeedMeter {
start: Instant,
last_at: Instant,
@@ -197,9 +161,6 @@ impl SpeedMeter {
}
}
/// Run the plan. `progress(bytes_done, chunks_done)` fires once per accounted chunk (cached-skip
/// or fetched), exactly the cadence of the Java pool's per-task `progress(...)` call. `log`
/// receives engine lines (`fetch-window`, `summary`, failures).
pub fn run_plan(
plan: &EpicPlan,
req: &EpicRequest,
@@ -228,8 +189,6 @@ pub fn run_plan(
req.process_workers
));
// Java: each pool task first checks the cancel flag, then `cachedFile.exists()` → credit
// without fetching. Account for the cached chunks up front (same credit, same callback).
let mut fetch_chunks: Vec<usize> = Vec::with_capacity(plan.needed.len());
let mut pre_bytes: u64 = 0;
let mut pre_chunks: u64 = 0;
@@ -293,16 +252,12 @@ pub fn run_plan(
let opts = FetchOptions {
max_workers: req.max_workers.max(1),
// Improvements round 1: the tier ceiling split across the distinct CDNs (floor 6), so
// the whole window is reachable on 1 host or on all 3.
per_host_cap: host_cap,
timeout: REQUEST_TIMEOUT,
headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())],
ca_bundle_path: req.ca_bundle_path.clone(),
process_workers: req.process_workers.max(1),
label: req.label.clone(),
// Whole-body mode (Epic chunks are ≤ ~1 MiB compressed; the core's byte budget scales
// with the window); `stream` and any future field keep the core's defaults.
..FetchOptions::default()
};
-34
View File
@@ -1,25 +1,3 @@
//! JNI exports for `com.droiddeck.launcher.stores.epic.EpicNative`.
//!
//! ```text
//! nativeStart(manifest: byte[], installDir: String, chunkCacheDir: String, cdnPrefixes: String[],
//! pendingFileIdx: int[], expectedChunks: int, expectedBytes: long,
//! caBundlePath: String, maxWorkers: int, processWorkers: int,
//! listener: EpicNative.Listener) -> long handle (0 = not started)
//! nativeAssemble(manifest: byte[], installDir: String, chunkCacheDir: String, fileIdx: int[],
//! listener: EpicNative.Listener) -> long handle (0 = not started)
//! nativeCancel(handle) — flips the run's AtomicBool; fetch or assembly stops promptly
//! nativeRelease(handle) — frees the handle (after onComplete)
//! ```
//! `chunkCacheDir` = "" keeps the cache at `<installDir>/.chunks`; any other directory is a
//! scratch cache (see `assemble.rs`). Listener (all methods run on native threads):
//! `onPlan(int chunksTotal, long bytesTotal, String chunkDir)` once before any fetch,
//! `onProgress(long bytesDone, long bytesTotal, int done, int total)` per chunk fetched (or, in
//! an assembly run, per file written), `onLog(String line)`,
//! `onComplete(boolean success, String error, long bytes)`.
//!
//! Planning (manifest parse + cross-check) runs synchronously on the calling thread so a plan
//! failure returns 0 BEFORE any fetch and Java can fall back to its own pool; the fetch runs on
//! a new thread. JavaVM attach / GlobalRef handling is the same shape as the GOG facade.
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
@@ -59,8 +37,6 @@ fn android_log(message: &str) {
}
}
/// Per-run native state behind the `jlong` handle — the same type for a fetch run and an
/// assembly run, so one `nativeCancel` / `nativeRelease` pair serves both.
pub struct EpicDownloadHandle {
cancel: Arc<AtomicBool>,
}
@@ -200,7 +176,6 @@ fn call_on_complete(env: &mut JNIEnv, listener: &JObject, success: bool, error:
clear_pending_exception(env);
}
/// Log to logcat AND to the listener (the Kotlin manager mirrors it into its own debug log).
fn log_both(env: &mut JNIEnv, listener: &JObject, line: &str) {
android_log(line);
call_on_log(env, listener, line);
@@ -334,8 +309,6 @@ fn run_on_thread(
let bytes_total = plan.total_bytes;
let chunks_total = plan.needed.len() as u64;
// Progress fires from the core's process-pool threads; attach each as a daemon like the
// Steam engine's `dispatch_download_progress` does.
let progress = |bytes_done: u64, chunks_done: u64| {
let Ok(mut env) = vm.attach_current_thread_as_daemon() else {
return;
@@ -371,11 +344,6 @@ fn run_on_thread(
);
}
/// `EpicNative.nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, listener)`: write
/// the files `fileIdx` names from the chunk cache (see `assemble.rs`). Parse + plan run on the
/// calling thread, so bad inputs return 0 (after `onComplete(false, reason, 0)`) before anything
/// is written; the assembly runs on a new thread and reports per file through `onProgress(bytes,
/// bytesTotal, filesDone, filesTotal)`, then `onComplete(success, error, bytesWritten)`.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble(
mut env: JNIEnv,
@@ -481,7 +449,5 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
if handle == 0 {
return;
}
// The worker thread owns its own Arc clone of the cancel flag and its own GlobalRef, so
// freeing the handle is safe even if a (cancelled) run is still winding down.
drop(unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) });
}
+13 -95
View File
@@ -1,43 +1,24 @@
//! Epic build manifest parsers — a 1:1 port of `EpicDownloadManager.parseManifest` (binary,
//! magic `0x44BEC00C`) and `parseJsonManifest` (legacy JSON manifests).
//!
//! Every quirk of the Java reader is kept on purpose (the Java manager stays the reference and
//! the two engines must derive the same plan from the same bytes): section skips by declared
//! size, `readFString` UTF-16LE/ASCII decoding, duplicate-GUID handling (last value, first
//! position), the JSON blob decoding (`blobToNum`) and its fallbacks, and "any exception →
//! parse failed".
use std::collections::HashMap;
use std::io::Read;
/// Binary manifest magic (`EpicDownloadManager.parseManifest`).
pub const MANIFEST_MAGIC: u32 = 0x44BE_C00C;
/// One entry of the ChunkDataList section (`EpicDownloadManager.ChunkInfo`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ChunkInfo {
/// Four uint32 in binary read order.
pub guid: [u32; 4],
/// uint64 "hash" that forms the first half of the chunk file name.
pub hash: u64,
/// SHA-1 of the DECOMPRESSED chunk; `None` for JSON manifests (Java: `sha1 == null`).
pub sha1: Option<[u8; 20]>,
/// Subfolder number (DECIMAL `%02d`). `i32` so the JSON parser's `Integer.parseInt` range is
/// representable exactly (binary manifests always yield 0..=255).
pub group_num: i32,
/// Uncompressed size (0 for JSON manifests).
pub window_size: i32,
/// Compressed (wire) size. Signed like the Java `long`; JSON manifests may yield anything.
pub file_size: i64,
}
impl ChunkInfo {
/// `String.format("%08X%08X%08X%08X", guid[0..4])`.
pub fn guid_str(&self) -> String {
guid_to_string(&self.guid)
}
/// `ChunkInfo.getPath`: `"<chunkDir>/<%02d groupNum>/<%016X hash>_<GUID>.chunk"`.
pub fn path(&self, chunk_dir: &str) -> String {
format!(
"{}/{:02}/{:016X}_{}.chunk",
@@ -48,8 +29,6 @@ impl ChunkInfo {
)
}
/// Java `chunk.sha1 != null && length == 20 && !allZero` — the condition under which the
/// streaming downloader verifies the decompressed chunk.
pub fn verifiable_sha1(&self) -> Option<&[u8; 20]> {
match &self.sha1 {
Some(h) if h.iter().any(|b| *b != 0) => Some(h),
@@ -57,7 +36,6 @@ impl ChunkInfo {
}
}
/// `Math.max(chunk.fileSize, 1)` — the byte credit the Java pool uses for totals/progress.
pub fn credit_bytes(&self) -> u64 {
if self.file_size < 1 {
1
@@ -67,13 +45,10 @@ impl ChunkInfo {
}
}
/// One file part referencing a chunk (`EpicDownloadManager.ChunkPart`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ChunkPart {
pub guid: [u32; 4],
/// Offset into the DECOMPRESSED chunk (Java `int`).
pub offset: i32,
/// Part length (Java `int`; summed as `size & 0xFFFFFFFFL`).
pub size: i32,
}
@@ -83,26 +58,20 @@ impl ChunkPart {
}
}
/// One entry of the FileManifestList section (`EpicDownloadManager.FileInfo`).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct FileInfo {
pub filename: String,
pub parts: Vec<ChunkPart>,
/// Install-tag names; empty = required/base file.
pub install_tags: Vec<String>,
/// Full-file SHA-1; `None` for JSON manifests.
pub sha1: Option<[u8; 20]>,
}
impl FileInfo {
/// `FileInfo.fileSize()` = Σ `(part.size & 0xFFFFFFFFL)`.
pub fn file_size(&self) -> u64 {
self.parts.iter().map(|p| p.size as u32 as u64).sum()
}
}
/// Parsed manifest (`EpicDownloadManager.EpicManifest.ParsedManifest`, minus the CDN list which
/// the Java side still owns).
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Manifest {
pub chunk_dir: String,
@@ -111,8 +80,6 @@ pub struct Manifest {
}
impl Manifest {
/// `byGuid` map as `uniqueChunksForFiles` builds it: `LinkedHashMap.put` per chunk, so a
/// duplicate GUID resolves to the LAST `ChunkInfo`.
pub fn chunk_index_by_guid(&self) -> HashMap<String, usize> {
let mut map = HashMap::with_capacity(self.unique_chunks.len());
for (i, c) in self.unique_chunks.iter().enumerate() {
@@ -126,7 +93,6 @@ pub fn guid_to_string(guid: &[u32; 4]) -> String {
format!("{:08X}{:08X}{:08X}{:08X}", guid[0], guid[1], guid[2], guid[3])
}
/// `chunkDir` from the binary header version (`parseManifest`).
pub fn chunk_dir_for_binary_version(version: i32) -> &'static str {
if version >= 15 {
"ChunksV4"
@@ -139,7 +105,6 @@ pub fn chunk_dir_for_binary_version(version: i32) -> &'static str {
}
}
/// `chunkDir` from `ManifestFileVersion` (`parseJsonManifest`; note the different `< 3` default).
pub fn chunk_dir_for_json_version(version: i32) -> &'static str {
if version >= 15 {
"ChunksV4"
@@ -152,10 +117,7 @@ pub fn chunk_dir_for_json_version(version: i32) -> &'static str {
}
}
// ── little-endian cursor with ByteBuffer-like failure semantics ─────────────────────────────
/// Mirrors the `ByteBuffer` calls the Java parser makes: every under-run or out-of-range
/// `position()` is an exception there, which the Java code turns into "parse failed" (null).
struct Cursor<'a> {
buf: &'a [u8],
pos: usize,
@@ -170,7 +132,6 @@ impl<'a> Cursor<'a> {
self.pos
}
/// `ByteBuffer.position(p)` — `p` must be within `0..=limit`.
fn set_position(&mut self, p: i64) -> Result<(), String> {
if p < 0 || p as u64 > self.buf.len() as u64 {
return Err(format!("position {p} out of range (limit {})", self.buf.len()));
@@ -223,7 +184,6 @@ impl<'a> Cursor<'a> {
Ok(out)
}
/// `ByteBuffer.position(position + n)` with the same range check.
fn skip(&mut self, n: i64) -> Result<(), String> {
let target = self.pos as i64 + n;
self.set_position(target)
@@ -233,23 +193,19 @@ impl<'a> Cursor<'a> {
&self.buf[self.pos..]
}
/// `EpicDownloadManager.readFString`: int32 length; `0` → ""; negative → `(-len)-1` UTF-16LE
/// code units + 2-byte terminator; positive → `len-1` US-ASCII bytes + 1-byte terminator.
fn fstring(&mut self) -> Result<String, String> {
let length = self.i32()?;
if length == 0 {
return Ok(String::new());
}
if length < 0 {
// Java: `int chars = (-length) - 1;` — `-Integer.MIN_VALUE` wraps to itself, then -1
// gives Integer.MAX_VALUE; `new byte[chars * 2]` overflows negative → exception.
let chars = length.wrapping_neg().wrapping_sub(1);
let byte_len = chars.checked_mul(2).ok_or("fstring length overflow")?;
if byte_len < 0 {
return Err("negative fstring length".to_string());
}
let bytes = self.take(byte_len as usize)?;
self.take(2)?; // null terminator
self.take(2)?;
let units: Vec<u16> = bytes
.chunks_exact(2)
.map(|p| u16::from_le_bytes([p[0], p[1]]))
@@ -257,8 +213,8 @@ impl<'a> Cursor<'a> {
Ok(String::from_utf16_lossy(&units))
} else {
let bytes = self.take((length - 1) as usize)?;
self.take(1)?; // null terminator
// `new String(bytes, US_ASCII)`: every byte > 0x7F decodes to U+FFFD.
self.take(1)?;
Ok(bytes
.iter()
.map(|&b| if b < 0x80 { b as char } else { '\u{FFFD}' })
@@ -267,10 +223,7 @@ impl<'a> Cursor<'a> {
}
}
// ── binary manifest ──────────────────────────────────────────────────────────────────────
/// `EpicDownloadManager.parseManifest(bytes)`: binary when the magic matches, otherwise the JSON
/// parser. `Err` wherever Java returns `null`.
pub fn parse_manifest(bytes: &[u8]) -> Result<Manifest, String> {
let mut cur = Cursor::new(bytes);
let magic = cur.u32()?;
@@ -284,7 +237,7 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
let header_size = cur.i32()?;
let size_uncompressed = cur.i32()?;
let _size_compressed = cur.i32()?;
cur.skip(20)?; // SHA-1 of the body, not checked by the Java parser
cur.skip(20)?;
let stored_as = cur.u8()?;
let version = cur.i32()?;
let chunk_dir = chunk_dir_for_binary_version(version);
@@ -297,8 +250,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
if size_uncompressed < 0 {
return Err("negative sizeUncompressed".to_string());
}
// Java: `inflater.inflate(new byte[sizeUncompressed])` — a single call that fills at most
// `sizeUncompressed` bytes; anything else (short, corrupt) is a size mismatch → null.
let mut out = Vec::with_capacity(size_uncompressed as usize);
let mut dec = flate2::read::ZlibDecoder::new(body_slice).take(size_uncompressed as u64);
dec.read_to_end(&mut out)
@@ -317,18 +268,16 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
let mut b = Cursor::new(body);
// ManifestMeta: skipped by its declared size (the size field counts itself).
let meta_size = b.i32()?;
let meta_end = b.position() as i64 - 4 + meta_size as i64;
b.set_position(meta_end)?;
// ChunkDataList
let cdl_start = b.position();
let cdl_size = b.i32()?;
let _cdl_version = b.u8()?;
let chunk_count = b.i32()?;
if chunk_count < 0 {
return Err("negative chunkCount".to_string()); // `new ArrayList<>(negative)` throws
return Err("negative chunkCount".to_string());
}
let chunk_count = chunk_count as usize;
let mut chunks: Vec<ChunkInfo> = (0..chunk_count)
@@ -361,7 +310,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
}
b.set_position(cdl_start as i64 + cdl_size as i64)?;
// FileManifestList
let fml_start = b.position();
let fml_size = b.i32()?;
let _fml_version = b.u8()?;
@@ -375,12 +323,12 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
f.filename = b.fstring()?;
}
for _ in 0..file_count {
b.fstring()?; // symlink targets
b.fstring()?;
}
for f in files.iter_mut() {
f.sha1 = Some(b.sha1()?);
}
b.skip(file_count as i64)?; // per-file flags byte
b.skip(file_count as i64)?;
for f in files.iter_mut() {
let tag_count = b.i32()?;
let mut j = 0;
@@ -415,8 +363,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
})
}
/// `LinkedHashMap<String, ChunkInfo>` keyed by GUID string: a repeated key keeps its FIRST
/// position but takes the LAST value.
fn dedup_last_wins_first_position(chunks: Vec<ChunkInfo>) -> Vec<ChunkInfo> {
let mut order: Vec<ChunkInfo> = Vec::with_capacity(chunks.len());
let mut index: HashMap<String, usize> = HashMap::with_capacity(chunks.len());
@@ -433,10 +379,7 @@ fn dedup_last_wins_first_position(chunks: Vec<ChunkInfo>) -> Vec<ChunkInfo> {
order
}
// ── JSON manifest (older games) ──────────────────────────────────────────────────────────
/// `org.json` value → string coercion as `optString` / `getString` do on Android (numbers and
/// booleans print as-is, JSON null prints "null", containers print their JSON).
fn json_to_string(v: &serde_json::Value) -> String {
match v {
serde_json::Value::String(s) => s.clone(),
@@ -445,9 +388,6 @@ fn json_to_string(v: &serde_json::Value) -> String {
}
}
/// `EpicDownloadManager.blobToNum`: a JSON manifest writes every number as a "blob", each byte
/// as three decimal digits, least significant first ("013000000000" = 13). Bits past 64 are
/// dropped; `None` when a group is not a number (Java: `NumberFormatException`).
pub fn blob_to_num(s: &str) -> Option<u64> {
let bytes = s.as_bytes();
let mut num: u64 = 0;
@@ -464,7 +404,6 @@ pub fn blob_to_num(s: &str) -> Option<u64> {
Some(num)
}
/// `obj.optString(key, fallback)`.
fn opt_string(obj: &serde_json::Map<String, serde_json::Value>, key: &str, fallback: &str) -> String {
match obj.get(key) {
Some(v) => json_to_string(v),
@@ -472,16 +411,12 @@ fn opt_string(obj: &serde_json::Map<String, serde_json::Value>, key: &str, fallb
}
}
/// `(int) Long.parseLong(hex8, 16)` — the Java GUID quarter parse; invalid hex throws out of the
/// whole parse.
fn parse_guid_quarter(s: &str) -> Result<u32, String> {
let v = i64::from_str_radix(s, 16).map_err(|e| format!("bad guid hex '{s}': {e}"))?;
Ok(v as i32 as u32)
}
fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> {
// Java substrings by UTF-16 unit and `parseLong` throws on anything non-hex; a non-ASCII
// prefix can never parse, so refuse it before slicing (keeps the slice on char boundaries).
if s.len() < 32 || !s.as_bytes()[..32].is_ascii() {
return Err(format!("bad guid hex '{s}'"));
}
@@ -493,7 +428,6 @@ fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> {
])
}
/// `EpicDownloadManager.parseJsonManifest`.
pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
let text = String::from_utf8_lossy(bytes);
let root: serde_json::Value =
@@ -513,7 +447,6 @@ pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
let data_group_list = root.get("DataGroupList").and_then(|v| v.as_object());
let chunk_filesize_list = root.get("ChunkFilesizeList").and_then(|v| v.as_object());
// `LinkedHashMap<String, ChunkInfo>` keyed by the RAW key string (not the normalized GUID).
let mut chunk_order: Vec<ChunkInfo> = Vec::new();
let mut chunk_index: HashMap<String, usize> = HashMap::new();
for (guid_hex, hash_value) in chunk_hash_list.iter() {
@@ -598,12 +531,9 @@ pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
})
}
// ── tests ────────────────────────────────────────────────────────────────────────────────
#[cfg(test)]
pub(crate) mod test_support {
//! Synthetic binary-manifest builder used by this module's tests and by `plan`/`driver`
//! tests. Emits exactly the layout `parseManifest` walks (meta / CDL / FML with sizes).
use super::*;
use std::io::Write;
@@ -621,7 +551,6 @@ pub(crate) mod test_support {
pub name: String,
pub sha1: [u8; 20],
pub tags: Vec<String>,
/// (guid, offset, size)
pub parts: Vec<([u32; 4], i32, i32)>,
}
@@ -648,17 +577,14 @@ pub(crate) mod test_support {
v
}
/// Body (meta + CDL + FML) without the outer header.
pub fn build_body(chunks: &[TestChunk], files: &[TestFile], meta_padding: usize) -> Vec<u8> {
let mut body = Vec::new();
// ManifestMeta: size field counts itself.
let meta_size = 4 + meta_padding;
body.extend_from_slice(&(meta_size as i32).to_le_bytes());
body.extend(std::iter::repeat(0xEEu8).take(meta_padding));
// ChunkDataList
let mut cdl = Vec::new();
cdl.push(0u8); // version
cdl.push(0u8);
cdl.extend_from_slice(&(chunks.len() as i32).to_le_bytes());
for c in chunks {
for g in c.guid {
@@ -680,14 +606,12 @@ pub(crate) mod test_support {
for c in chunks {
cdl.extend_from_slice(&c.file_size.to_le_bytes());
}
// Trailing padding the parser must skip via cdlSize.
cdl.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
body.extend_from_slice(&((cdl.len() + 4) as i32).to_le_bytes());
body.extend_from_slice(&cdl);
// FileManifestList
let mut fml = Vec::new();
fml.push(0u8); // version
fml.push(0u8);
fml.extend_from_slice(&(files.len() as i32).to_le_bytes());
for f in files {
if f.name.is_ascii() {
@@ -697,13 +621,13 @@ pub(crate) mod test_support {
}
}
for _ in files {
fml.extend_from_slice(&fstring_ascii("")); // symlink target
fml.extend_from_slice(&fstring_ascii(""));
}
for f in files {
fml.extend_from_slice(&f.sha1);
}
for _ in files {
fml.push(0); // flags
fml.push(0);
}
for f in files {
fml.extend_from_slice(&(f.tags.len() as i32).to_le_bytes());
@@ -714,7 +638,6 @@ pub(crate) mod test_support {
for f in files {
fml.extend_from_slice(&(f.parts.len() as i32).to_le_bytes());
for (guid, offset, size) in &f.parts {
// struct size 4 + 16 + 4 + 4 = 28, plus 2 bytes of padding the parser skips.
fml.extend_from_slice(&30i32.to_le_bytes());
for g in guid {
fml.extend_from_slice(&g.to_le_bytes());
@@ -729,7 +652,6 @@ pub(crate) mod test_support {
body
}
/// Full manifest bytes: header + (optionally zlib-compressed) body.
pub fn build_manifest(
chunks: &[TestChunk],
files: &[TestFile],
@@ -745,7 +667,7 @@ pub(crate) mod test_support {
} else {
body.clone()
};
let header_size: i32 = 41 + 3; // 3 bytes of extra header the parser must jump over
let header_size: i32 = 41 + 3;
let mut out = Vec::new();
out.extend_from_slice(&MANIFEST_MAGIC.to_le_bytes());
out.extend_from_slice(&header_size.to_le_bytes());
@@ -916,13 +838,9 @@ mod tests {
fn bad_uncompressed_size_is_a_parse_failure() {
let good = build_manifest(&sample_chunks(), &sample_files(), 21, true);
let real = i32::from_le_bytes([good[8], good[9], good[10], good[11]]);
// Claim MORE than the body inflates to: Java's single `inflate()` returns fewer bytes
// than `sizeUncompressed` → "Decomp size mismatch" → null.
let mut bigger = good.clone();
bigger[8..12].copy_from_slice(&(real + 1).to_le_bytes());
assert!(parse_manifest(&bigger).is_err());
// Claim LESS: Java fills exactly `sizeUncompressed` bytes (silently truncating), then the
// FileManifestList end position lands past the limit → exception → null.
let mut smaller = good;
smaller[8..12].copy_from_slice(&(real - 1).to_le_bytes());
assert!(parse_manifest(&smaller).is_err());
@@ -933,7 +851,7 @@ mod tests {
let mut v = fstring_ascii("abc");
v.extend_from_slice(&fstring_utf16("dé"));
v.extend_from_slice(&fstring_ascii(""));
v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]); // non-ASCII byte → U+FFFD
v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]);
let mut c = Cursor::new(&v);
assert_eq!(c.fstring().unwrap(), "abc");
assert_eq!(c.fstring().unwrap(), "dé");
-49
View File
@@ -1,19 +1,9 @@
//! Chunk plan — the part of `EpicDownloadManager.install` between "manifest parsed" and "submit
//! to the pool", ported 1:1: install-tag file selection (`resolveInstallFiles` /
//! `getRequiredInstallFiles`), the unique-chunk list for a file set (`uniqueChunksForFiles`),
//! the byte total (`Σ max(fileSize, 1)`), chunk cache paths and CDN URLs.
//!
//! At runtime the Java manager passes the INDICES of its `pendingFiles` (post delta/verify), so
//! the file selection functions here are exercised by the unit tests (and available to a future
//! caller) while the driver only runs `unique_chunks_for_files` on Java's pending set — the same
//! function Java runs, on the same manifest, in the same order.
use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use super::manifest::{ChunkInfo, Manifest};
/// `getRequiredInstallFiles`: files with no install tags; if NONE carries a tag, every file.
pub fn required_install_files(m: &Manifest) -> Vec<usize> {
let required: Vec<usize> = m
.files
@@ -29,11 +19,6 @@ pub fn required_install_files(m: &Manifest) -> Vec<usize> {
}
}
/// `resolveInstallFiles(manifest, selectedTags)`:
/// - `None` → every file (legacy / DLC / overlay path);
/// - `Some([])` → required(base) files only;
/// - otherwise → base files + any file carrying at least one selected tag, falling back to
/// required-only when nothing matched.
pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) -> Vec<usize> {
let Some(tags) = selected_tags else {
return (0..m.files.len()).collect();
@@ -59,8 +44,6 @@ pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) ->
}
}
/// `uniqueChunksForFiles(manifest, files)`: chunk indices (into `unique_chunks`) in first-seen
/// order over `files[*].parts`, each GUID once, GUIDs absent from the chunk list dropped.
pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec<usize> {
let by_guid: HashMap<String, usize> = m.chunk_index_by_guid();
let mut seen: HashSet<String> = HashSet::new();
@@ -81,7 +64,6 @@ pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec<usiz
chunks
}
/// `totalBytes += Math.max(chunk.fileSize, 1)` over the needed chunks.
pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 {
chunk_indices
.iter()
@@ -90,15 +72,10 @@ pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 {
.sum()
}
/// `<installDir>/.chunks`.
pub fn chunk_cache_dir(install_dir: &str) -> PathBuf {
Path::new(install_dir).join(super::CHUNK_CACHE_DIR)
}
/// Where this run's chunks go: `cache_dir` when the caller named one, else the default
/// `<installDir>/.chunks`. A separate directory exists so the cache can sit on internal storage
/// while the game installs to an SD card: a chunk is written once into the cache and once into
/// the assembled file, and with both on a 26 MB/s card the second write starved the fetchers.
pub fn resolve_cache_dir(install_dir: &str, cache_dir: &str) -> PathBuf {
if cache_dir.is_empty() {
chunk_cache_dir(install_dir)
@@ -107,37 +84,23 @@ pub fn resolve_cache_dir(install_dir: &str, cache_dir: &str) -> PathBuf {
}
}
/// Final cache file for a chunk: `new File(chunkCacheDir, chunk.guidStr())`.
pub fn cached_chunk_path(cache_dir: &Path, chunk: &ChunkInfo) -> PathBuf {
cache_dir.join(chunk.guid_str())
}
/// The Java pool's skip rule: a chunk whose cache file EXISTS (any size) is not fetched.
pub fn is_chunk_cached(cache_dir: &Path, chunk: &ChunkInfo) -> bool {
cached_chunk_path(cache_dir, chunk).exists()
}
/// One CDN entry as the Java manager passes it: `cdn.baseUrl + cdn.cloudDir` (auth params are
/// never used on chunk URLs). Chunk URL = `prefix + "/" + chunk.getPath(chunkDir)`.
pub fn chunk_url(cdn_prefix: &str, chunk_dir: &str, chunk: &ChunkInfo) -> String {
format!("{}/{}", cdn_prefix, chunk.path(chunk_dir))
}
/// Undo the JSON string escaping the Java CDN scanner passes through. `EpicApiClient` re-serializes
/// the manifest API response with Android's `org.json`, which escapes every `/` as `\/`, and
/// `parseCdnUrls` takes the raw substrings — so a base URL arrives as `https:\/\/host\` and a cloud
/// dir as `/Builds\/Org\/…\/default\`. Java's `HttpURLConnection` (OkHttp) treats `\` as `/` and
/// only ever reached the first CDN, so it never noticed; reqwest's URL parser does the same
/// mapping, which leaves empty `//` path segments that Fastly/Akamai tolerate but CloudFront
/// rejects outright (immediate 4xx). Produce the URL Java intended: `\/` → `/`, any other `\`
/// dropped, and no trailing slash (the chunk path is joined with one).
pub fn normalize_prefix(raw: &str) -> String {
let unescaped = raw.replace("\\/", "/").replace('\\', "");
unescaped.trim_end_matches('/').to_string()
}
/// Distinct CDN prefixes in first-seen order, normalized (the fetch core wants one host key per
/// distinct endpoint; Java would simply try an identical URL twice, which changes nothing).
pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec<String> {
let mut seen = HashSet::new();
let mut out = Vec::new();
@@ -153,9 +116,6 @@ pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec<String> {
out
}
/// Improvements round 1: split the window across the distinct CDNs so the whole ceiling is
/// reachable whatever the host count — `max(6, ceil(max_workers / hosts))` (3 Epic CDNs at a
/// ceiling of 32 → 11 each; a single host gets all 32).
pub fn per_host_cap(max_workers: usize, host_count: usize) -> usize {
let hosts = host_count.max(1);
let workers = max_workers.max(1);
@@ -176,14 +136,10 @@ mod tests {
#[test]
fn install_tag_rules_match_java() {
let m = manifest();
// null → all
assert_eq!(resolve_install_files(&m, None), vec![0, 1, 2]);
// empty → required only (file 0 has no tags)
assert_eq!(resolve_install_files(&m, Some(&[])), vec![0]);
// German → base + the German file
let de = vec!["German".to_string(), "de-DE".to_string()];
assert_eq!(resolve_install_files(&m, Some(&de)), vec![0, 1]);
// unknown tag → nothing extra matched, but base files matched, so base only
let xx = vec!["Klingon".to_string()];
assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0]);
}
@@ -204,7 +160,6 @@ mod tests {
let mut files = sample_files();
files[0].tags = vec!["English".to_string()];
let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap();
// No untagged file, unknown tag selected → matched nothing → required → (none) → all.
let xx = vec!["Klingon".to_string()];
assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0, 1, 2]);
}
@@ -212,12 +167,10 @@ mod tests {
#[test]
fn unique_chunks_follow_first_seen_part_order() {
let m = manifest();
// file 2 first: chunk c[2] then c[0]; then file 0: c[0] (dup), c[1].
assert_eq!(unique_chunks_for_files(&m, &[2, 0]), vec![2, 0, 1]);
assert_eq!(unique_chunks_for_files(&m, &[1]), vec![1]);
assert!(unique_chunks_for_files(&m, &[]).is_empty());
assert!(unique_chunks_for_files(&m, &[99]).is_empty(), "out-of-range index ignored");
// total = 700000 + max(0,1) + 9
assert_eq!(total_credit_bytes(&m, &[0, 1, 2]), 700_000 + 1 + 9);
}
@@ -256,13 +209,11 @@ mod tests {
#[test]
fn prefixes_are_unescaped_from_the_java_json_scan() {
// Exactly what Bannerlator's Epic debug log shows its Java manager extracting for a real title.
let raw = "https:\\/\\/egdownload.fastly-edge.com\\/Builds\\/Org\\/o-83e8\\/04b7\\/default\\";
assert_eq!(
normalize_prefix(raw),
"https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default"
);
// Already-clean input is untouched (minus a trailing slash).
assert_eq!(
normalize_prefix("https://download.epicgames.com/Builds/x/default/"),
"https://download.epicgames.com/Builds/x/default"
+8 -256
View File
@@ -1,28 +1,3 @@
//! Shared adaptive HTTP fetch core for the store download engines (Epic / GOG / Amazon).
//!
//! This is the generic half of the fetch layer Bannerlator's Steam depot writer grew: an
//! adaptive in-flight window (slow-start → hold at plateau → shrink on REAL errors only), a
//! speed-ranked per-host-capped scheduler, an in-flight RAW byte budget sized from the window
//! ceiling, per-item retry with exponential back-off + host rotation, one current-thread tokio
//! runtime driving every request through a [`FuturesUnordered`], and an mpsc hand-off to a sync
//! process pool that calls the adapter's [`FetchSink::process`] (inflate + hash + write).
//!
//! The tunables come from [`crate::fetch_tuning`], copied number for number from Bannerlator's
//! depot writer, so a `fetch-window` line from either app reads against the same tuning notes.
//!
//! What the core does NOT know: manifests, resume/skip rules, output layout, hashing. The adapter
//! builds the item list (already excluding present+verified items, exactly as its Java manager
//! does), maps host index → URL, and does everything with the body inside its sink.
//!
//! Log grammar (all lines go through the `log` callback; `label` is adapter-chosen, e.g.
//! `epic app=Fortnite`):
//! - `fetch-start label=… items=… bytes_reserved=… hosts=… ceiling=… budget=…MiB tier_max=… per_host_cap=… distinct_hosts=… window=… mode=body|stream`
//! - `fetch-window label=… window=N (min=… max=…) in_flight=… last=…MB/s ewma=…MB/s best=…MB/s reason=… cooldown=…ms err_rate=…% phase=… rtt=…ms budget_stalls=… host_stalls=…`
//! (same fields as the Steam engine's `fetch-window depot=…` line, so the two are A/B-comparable)
//! - `throughput label=… overall=…MB/s total=…MB elapsed=…s used=x/y servers: [host …MB/s …MB] …` every 5 s
//! - `fetch-end label=… items_ok=… bytes=… credited=… elapsed_ms=… avg_mbps=… peak_mbps=… result=ok|cancelled|error [error=…]`
//!
//! `MB/s` here means MiB/s (1024²), matching the Steam engine's lines.
use crate::fetch_tuning::{
budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError,
@@ -46,95 +21,42 @@ use std::thread;
use std::time::{Duration, Instant};
use tokio::sync::{Notify, Semaphore};
/// Maximum attempts per item (fetch failures AND `SinkError::Retry` both count). Same as Steam.
pub const MAX_ITEM_ATTEMPTS: u32 = MAX_CHUNK_ATTEMPTS;
/// How often the throughput reporter samples (and logs) — the samples feed `peak_mbps`.
pub const THROUGHPUT_SAMPLE_INTERVAL_MS: u64 = 5_000;
/// Upper bound on how long the driver sits in one `await` before re-checking cancel/error/feedback.
const DRIVER_POLL_MS: u64 = 250;
// ── Delay awareness (bufferbloat) ──
// On a 5G hotspot the window kept growing (31 → 84) while the per-request delay went from 3 s to
// 11 s and throughput stayed at 10-20 MB/s: every extra request only queued behind the others in
// the link's buffer. The signal used here is time-to-first-byte (request sent → response headers),
// which on a link with a deep buffer grows with the queue, while on a fast link it stays at a few
// round trips however wide the window gets. The baseline is the lowest per-probe mean seen.
/// A probe needs at least this many TTFB samples before its mean is trusted.
pub const LATENCY_MIN_SAMPLES: u32 = 3;
/// Mean TTFB at or above `baseline ×` this counts as queueing delay…
pub const LATENCY_HOLD_FACTOR: f64 = 2.5;
/// …and only when it is also this far above the baseline in absolute terms: a fast link whose
/// TTFB goes from 20 ms to 60 ms (new connections' TLS handshakes) is not bufferbloat.
pub const LATENCY_MIN_EXCESS_MS: f64 = 250.0;
/// Throughput above `best × (1 + this)` is a gain that justifies the extra delay: hold, but never
/// shrink, while it lasts.
pub const LATENCY_GAIN_EPS: f64 = 0.25;
/// Gentle proportional shrink while the delay keeps rising (gentler than the error shrink).
pub const LATENCY_SHRINK_FACTOR: f64 = 0.85;
// ── Stalled hosts ──
// A CDN host can stop delivering while its connections stay open: on the device the Fastly host
// froze at 49 MB for the rest of the run while the other host carried everything, and its requests
// sat until their idle timeout. A host whose in-flight requests receive no byte for this long,
// while another host is receiving, has its requests cancelled and re-queued and is put to rest.
/// No byte from any of a host's in-flight requests for this long (while another host delivers).
pub const HOST_STALL_MS: u64 = if cfg!(test) { 1_500 } else { 10_000 };
/// First rest for a stalled host; doubles per further stall of the same host, reset on success.
pub const HOST_STALL_COOLDOWN_MS: u64 = 30_000;
pub const HOST_STALL_COOLDOWN_MAX_MS: u64 = 300_000;
/// How often the driver looks for stalled hosts.
const STALL_CHECK_MS: u64 = 250;
/// Stream mode: the longest a request may sit with no byte arriving before it is failed (the
/// store's own timeout when it is shorter). A dead connection on a single-host run, where the
/// stall detector has no healthy host to compare with, never holds an item longer than this.
pub const STREAM_IDLE_CAP: Duration = Duration::from_secs(30);
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Public contract
// ─────────────────────────────────────────────────────────────────────────────────────────────
/// One unit of work: a body to fetch from one of the hosts and hand to the sink.
#[derive(Clone, Debug)]
pub struct FetchItem {
/// Adapter-defined (index into the adapter's own table). Only echoed in errors / logs.
pub id: u64,
/// `len == hosts.len()` → `urls[host_idx]`; `len == 1` → the same URL on every host. Any other
/// non-zero length falls back to `urls.get(host_idx)` then `urls[0]`. Empty = fatal.
pub urls: Vec<String>,
/// Expected RAW (wire) bytes, for the in-flight byte budget; `0` → nominal 1 MiB. Ignored in
/// stream mode (streams are budgeted piece by piece as they arrive).
pub reserve: u64,
/// Inclusive HTTP `Range` (`bytes=a-b`); `None` = whole body.
pub range: Option<(u64, u64)>,
}
#[derive(Clone, Debug)]
pub struct FetchOptions {
/// Window ceiling (tier); the core clamps it to `hosts × per_host_cap` and the hard cap.
pub max_workers: usize,
/// Max concurrent requests per distinct host (6 unless the store needs otherwise). Honoured
/// as given: 1 host × `per_host_cap` 8 = a ceiling of 8.
pub per_host_cap: usize,
/// Whole-body mode: the deadline for the entire request (connect + headers + body). Stream
/// mode: the deadline for the response headers, then an IDLE deadline per received piece.
pub timeout: Duration,
/// Sent on EVERY request (applied on the request builder, so they override the client-level
/// defaults — e.g. a store's own `User-Agent`, or `Authorization: Bearer …`). Validated once
/// when the client is built.
pub headers: Vec<(String, String)>,
/// PEM bundle path; `""` = the platform roots reqwest/rustls ships with.
pub ca_bundle_path: String,
/// Sync process-pool threads (inflate + hash + write).
pub process_workers: usize,
/// Log prefix, e.g. `"epic app=Fortnite"`.
pub label: String,
/// `false` (default): each item's whole body is buffered and handed to
/// [`FetchSink::process`]. `true`: the body is streamed — pieces go to
/// [`FetchSink::on_chunk`] in order as they arrive, then [`FetchSink::on_finish`]; for
/// multi-GB files (Amazon) that keeps memory at "pieces not yet written", not whole files.
/// Applies to every item of the run.
pub stream: bool,
}
@@ -153,36 +75,20 @@ impl Default for FetchOptions {
}
}
/// What the sink says about a body it was handed.
#[derive(Clone, Debug)]
pub enum SinkError {
/// Counts as a failed attempt for that item: host cooled + demoted, back-off, re-dispatch on a
/// different host, up to [`MAX_ITEM_ATTEMPTS`]. In stream mode the whole item is re-streamed
/// (`on_chunk` sees `offset == 0` again).
Retry(String),
/// Abort the whole run; this becomes the run's first error.
Fatal(String),
}
/// Runs on a process-pool thread. Whole-body mode uses [`FetchSink::process`]; stream mode uses
/// [`FetchSink::on_chunk`] + [`FetchSink::on_finish`] (default impls refuse, so a whole-body sink
/// is unaffected by the stream option existing).
pub trait FetchSink: Send + Sync {
/// Whole-body mode: the raw wire body. Do inflate + hash verify + write here. Returns the
/// number of bytes to credit to progress (adapter's choice, e.g. decompressed bytes).
fn process(&self, item: &FetchItem, body: Vec<u8>) -> Result<u64, SinkError>;
/// Stream mode: one piece of the body, delivered in order (`offset` = bytes of this attempt
/// delivered before this piece). `offset == 0` marks the (re)start of an attempt — truncate any
/// partial output from a previous attempt. Each processed piece credits `data.len()` bytes.
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
let _ = (item, offset, data);
Err(SinkError::Fatal("streaming not supported by this sink".to_string()))
}
/// Stream mode: the body ended after `total_len` bytes. Finalize (rename `.tmp`, verify the
/// hash, …). Returns EXTRA bytes to credit (pieces were already credited as they were
/// written; return 0 for none).
fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result<u64, SinkError> {
let _ = (item, total_len);
Err(SinkError::Fatal("streaming not supported by this sink".to_string()))
@@ -191,15 +97,9 @@ pub trait FetchSink: Send + Sync {
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct FetchOutcome {
/// Sum of what the sink returned for every successfully processed item (stream mode: every
/// piece written + every `on_finish` extra; pieces of an attempt that later failed stay
/// counted).
pub bytes_credited: u64,
/// First error (fetch after max attempts, sink `Fatal`, client build failure).
pub error: Option<String>,
/// `cancel` was observed set.
pub cancelled: bool,
/// Items the sink accepted.
pub items_ok: u64,
}
@@ -209,10 +109,6 @@ impl FetchOutcome {
}
}
/// Blocking: spawns its own current-thread tokio runtime + process pool and returns when every
/// item is processed, `cancel` is set, or the first fatal error is recorded. `progress` fires from
/// a process-pool thread after each successful `process` / each written piece + `on_finish` (the
/// caller throttles).
pub fn run_fetch(
items: Vec<FetchItem>,
hosts: &[String],
@@ -254,8 +150,6 @@ mode={}",
let error_slot: Mutex<Option<String>> = Mutex::new(None);
let reporter_done = AtomicBool::new(false);
let meter = BandwidthMeter::new(&hosts);
// Stream mode: per-item "attempt N was rejected by the sink mid-stream" flags, so the driver's
// in-flight stream stops reading instead of feeding pieces nobody wants.
let abandon: Vec<AtomicU32> = if stream {
items.iter().map(|_| AtomicU32::new(NO_ATTEMPT)).collect()
} else {
@@ -268,11 +162,6 @@ mode={}",
let abandon = &abandon;
thread::scope(|scope| {
// Async(fetch) → sync(process): tokio unbounded mpsc drained with `blocking_recv`. Memory is
// bounded by the RAW byte budget enforced at DISPATCH (whole-body) / per piece (stream), not
// by channel capacity. Whole-body mode: ONE channel shared by every worker (any worker takes
// the next body). Stream mode: one channel PER worker and an item's pieces always go to the
// same worker (`idx % workers`), so pieces are written in order.
let channel_count = if stream { proc_count } else { 1 };
let mut txs: Vec<tokio::sync::mpsc::UnboundedSender<ProcessMsg>> =
Vec::with_capacity(channel_count);
@@ -283,7 +172,6 @@ mode={}",
txs.push(tx);
rxs.push(Arc::new(Mutex::new(rx)));
}
// Sync(process) → async(driver): sink verdicts, polled non-blocking by the driver.
let (fb_tx, fb_rx) = std_mpsc::channel::<Feedback>();
let bytes_credited = &bytes_credited;
let items_ok = &items_ok;
@@ -291,8 +179,6 @@ mode={}",
let error_slot = &error_slot;
let reporter_done = &reporter_done;
// Periodic throughput reporter — atomics only, never on the fetch hot path. Its samples are
// also the `peak_mbps` source.
let reporter = scope.spawn(move || {
crate::priority::background();
loop {
@@ -312,15 +198,12 @@ mode={}",
}
});
// ── Process pool ──
let mut proc_handles = Vec::with_capacity(proc_count);
for worker in 0..proc_count {
let rx = Arc::clone(&rxs[if stream { worker } else { 0 }]);
let fb = fb_tx.clone();
proc_handles.push(scope.spawn(move || {
crate::priority::background();
// Stream mode: (item → attempt, message) whose pieces the sink rejected; the rest
// of that attempt's pieces are dropped and its Finish becomes a Retry verdict.
let mut poisoned: HashMap<usize, (u32, String)> = HashMap::new();
loop {
if cancel.load(Ordering::Relaxed) {
@@ -344,8 +227,6 @@ mode={}",
body,
} => {
let raw_len = body.len() as u64;
// A body that was already queued when cancel / the first error landed
// must not be written any more.
if cancel.load(Ordering::Relaxed)
|| error_slot.lock().expect("err slot poisoned").is_some()
{
@@ -354,7 +235,6 @@ mode={}",
}
let item = &items[idx];
let res = sink.process(item, body);
// Free the budget on PROCESS-COMPLETE (actual raw bytes).
in_flight.fetch_sub(raw_len, Ordering::Relaxed);
let verdict = match res {
Ok(credited) => {
@@ -390,7 +270,7 @@ mode={}",
let len = data.len() as u64;
in_flight.fetch_sub(len, Ordering::Relaxed);
if poisoned.get(&idx).is_some_and(|(a, _)| *a == attempts) {
continue; // rest of a rejected attempt
continue;
}
if cancel.load(Ordering::Relaxed)
|| error_slot.lock().expect("err slot poisoned").is_some()
@@ -471,11 +351,6 @@ mode={}",
}
drop(fb_tx);
// ── Async fetch driver on one current-thread runtime. `txs` are moved in and dropped when
// the driver returns, which closes the channels so the pool's `blocking_recv` yields
// `None`. ──
// The runtime drives on this (already lowered) thread; `on_thread_start` covers the
// blocking pool it starts for DNS lookups.
match tokio::runtime::Builder::new_current_thread()
.enable_all()
.on_thread_start(crate::priority::background)
@@ -515,7 +390,6 @@ mode={}",
let _ = reporter.join();
});
// Final partial-interval sample so a short run still has a peak.
meter.sample();
let elapsed = started.elapsed();
let elapsed_secs = elapsed.as_secs_f64().max(0.001);
@@ -552,11 +426,7 @@ avg_mbps={avg_mbps:.2} peak_mbps={peak_mbps:.2} result={result}",
outcome
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Pure helpers (unit-tested)
// ─────────────────────────────────────────────────────────────────────────────────────────────
/// Bytes reserved against the in-flight budget at DISPATCH (nominal when the adapter has none).
pub fn reserve_bytes(item: &FetchItem) -> u64 {
if item.reserve == 0 {
NOMINAL_CHUNK_RESERVE_BYTES
@@ -565,8 +435,6 @@ pub fn reserve_bytes(item: &FetchItem) -> u64 {
}
}
/// Window bounds: `max` = tier ceiling clamped to distinct-hosts × per-host-cap and the hard cap;
/// `min` = the floor; `bootstrap` = the start. Same arithmetic as the Steam depot writer.
pub fn window_bounds(
max_workers: usize,
distinct_hosts: usize,
@@ -579,7 +447,6 @@ pub fn window_bounds(
(bootstrap, min, max)
}
/// Distinct host count (string equality) — the multiplier for the window's host ceiling.
pub fn distinct_host_count(hosts: &[String]) -> usize {
let mut keys: Vec<&str> = Vec::new();
for h in hosts {
@@ -590,8 +457,6 @@ pub fn distinct_host_count(hosts: &[String]) -> usize {
keys.len().max(1)
}
/// The URL to use for `item` on host `host_idx` (see [`FetchItem::urls`]). `None` when the item
/// carries no URL at all.
pub fn url_for_host(item: &FetchItem, host_idx: usize) -> Option<&str> {
if item.urls.is_empty() {
return None;
@@ -607,7 +472,6 @@ pub fn url_for_host(item: &FetchItem, host_idx: usize) -> Option<&str> {
)
}
/// Byte count an inclusive range must deliver; `None` for an inverted range.
pub fn range_len(range: (u64, u64)) -> Option<u64> {
let (a, b) = range;
if b < a {
@@ -617,8 +481,6 @@ pub fn range_len(range: (u64, u64)) -> Option<u64> {
}
}
/// Validates a received body's length: for a range request it must be exactly `b-a+1`; otherwise
/// it must match `Content-Length` when the server sent one (as the Steam client does).
pub fn validate_body_len(
body_len: u64,
content_length: Option<u64>,
@@ -653,13 +515,10 @@ pub fn validate_body_len(
Ok(())
}
/// Whether an HTTP status is acceptable for this request shape (200 always; 206 only when a range
/// was asked for).
pub fn status_accepted(status: u16, ranged: bool) -> bool {
status == 200 || (ranged && status == 206)
}
/// Classifies a rejected HTTP status the same way the Steam async client does.
pub fn classify_status(status: u16) -> FetchFailKind {
if status == 429 {
FetchFailKind::RateLimited
@@ -670,10 +529,6 @@ pub fn classify_status(status: u16) -> FetchFailKind {
}
}
/// The HTTP status carried by a `send_checked` rejection message, if that is what the error was.
/// A 4xx other than 429 is a CLIENT rejection (bad/expired signed URL, wrong host for this path,
/// 404): it says nothing about congestion, so the driver demotes the host but does not shrink the
/// shared window for it — one CDN that refuses a path must not throttle the healthy ones.
pub fn rejected_status(message: &str) -> Option<u16> {
let rest = message.strip_prefix("unexpected HTTP status (")?;
let digits = rest.strip_suffix(')')?;
@@ -684,8 +539,6 @@ pub fn is_client_rejection(message: &str) -> bool {
matches!(rejected_status(message), Some(s) if (400..500).contains(&s) && s != 429)
}
/// How many individual fetch failures a run logs verbatim before going quiet (the window and
/// throughput lines keep reporting the aggregate).
pub const FETCH_ERROR_LOG_LIMIT: u32 = 12;
fn record_first_error(slot: &Mutex<Option<String>>, err: String) {
@@ -695,11 +548,7 @@ fn record_first_error(slot: &Mutex<Option<String>>, err: String) {
}
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// HTTP client (one pooled reqwest client + per-request headers + optional Range + streaming)
// ─────────────────────────────────────────────────────────────────────────────────────────────
/// The pooled client plus the pre-validated per-request headers.
struct Http {
client: reqwest::Client,
headers: Vec<(reqwest::header::HeaderName, reqwest::header::HeaderValue)>,
@@ -732,8 +581,6 @@ fn build_http(opts: &FetchOptions, pool_max_idle_per_host: usize) -> Result<Http
}
impl Http {
/// `GET url` with the run's headers (set per request, so they override the client defaults —
/// a store's own `User-Agent` wins over the Steam one) and the optional `Range`.
fn request(&self, url: &str, range: Option<(u64, u64)>) -> reqwest::RequestBuilder {
let mut request = self.client.get(url);
for (name, value) in &self.headers {
@@ -756,7 +603,6 @@ fn classify_reqwest_error(err: &reqwest::Error) -> FetchFailKind {
}
}
/// Sends the request and checks the status; shared by both modes.
async fn send_checked(
http: &Http,
url: &str,
@@ -786,9 +632,6 @@ async fn send_checked(
Ok(response)
}
/// Whole-body mode: one attempt for one item (the driver owns retry/rotation). `timeout` bounds the
/// whole request (headers + body). The body is read piece by piece so `host_rx` sees every byte as
/// it arrives (the stall detector's signal); `ttfb` is set when the response headers arrive.
async fn fetch_once(
http: &Http,
url: &str,
@@ -835,10 +678,6 @@ async fn fetch_once(
}
}
/// Stream mode: one attempt for one item. Pieces are pushed to the item's pool worker as they
/// arrive (each one added to the in-flight budget until the pool consumes it); the future waits
/// while the budget is full, and stops early on cancel or when the sink rejected this attempt.
/// Returns the total body length after the last piece was queued.
#[allow(clippy::too_many_arguments)]
async fn fetch_stream(
http: &Http,
@@ -871,7 +710,6 @@ async fn fetch_stream(
let content_length = response.content_length();
let mut offset = 0u64;
loop {
// Budget gate: don't read ahead of the pool by more than the budget.
while in_flight.load(Ordering::Relaxed) >= budget {
if cancel.load(Ordering::Relaxed) {
return Err(AsyncFetchError {
@@ -931,29 +769,22 @@ async fn fetch_stream(
Ok(offset)
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Driver plumbing
// ─────────────────────────────────────────────────────────────────────────────────────────────
/// `abandon` sentinel: no attempt of this item has been rejected by the sink.
const NO_ATTEMPT: u32 = u32::MAX;
enum ProcessMsg {
/// Whole-body mode: the complete raw body.
Body {
idx: usize,
attempts: u32,
server_idx: usize,
body: Vec<u8>,
},
/// Stream mode: one in-order piece of attempt `attempts` of item `idx`.
Piece {
idx: usize,
attempts: u32,
offset: u64,
data: Vec<u8>,
},
/// Stream mode: the body ended; the pool finalizes and reports a verdict.
Finish {
idx: usize,
attempts: u32,
@@ -975,7 +806,6 @@ struct Feedback {
verdict: SinkVerdict,
}
/// An item awaiting (re)dispatch: `attempts` already spent; not eligible before `not_before`.
#[derive(Clone, Copy, Debug)]
struct PendingItem {
idx: usize,
@@ -988,18 +818,14 @@ enum Fetched {
Streamed(u64),
}
/// The result the driver awaits for each dispatched fetch.
struct FetchDone {
idx: usize,
attempts: u32,
reserve: u64,
server_idx: usize,
elapsed: Duration,
/// Request sent → response headers, when the headers arrived.
ttfb: Option<Duration>,
/// Key in the driver's registry of running requests.
req_id: u64,
/// The request was cancelled because its host stalled (not a failure of the item).
stalled: bool,
res: Result<Fetched, AsyncFetchError>,
}
@@ -1061,11 +887,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
let mut retry: VecDeque<PendingItem> = VecDeque::new();
let mut next_item = 0usize;
let mut errors_logged: u32 = 0;
// Bodies / finishes handed to the pool whose verdict has not come back yet.
let mut outstanding = 0usize;
// Stall detection: bytes received per server (bumped by the request futures as pieces
// arrive), when that count last moved, since when the server has had requests to answer,
// how many requests it is running, and each running request's abort switch.
let host_rx: Vec<AtomicU64> = hosts.iter().map(|_| AtomicU64::new(0)).collect();
let mut last_rx: Vec<u64> = vec![0; hosts.len()];
let mut last_rx_at: Vec<Option<Instant>> = vec![None; hosts.len()];
@@ -1075,7 +897,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
let mut aborts: HashMap<u64, (usize, Arc<Notify>)> = HashMap::new();
let mut next_req_id: u64 = 0;
let mut last_stall_check = Instant::now();
// Declared AFTER `txs`/`http` so it is dropped BEFORE them (its futures borrow both).
let mut inflight: FuturesUnordered<_> = FuturesUnordered::new();
'outer: loop {
@@ -1085,8 +906,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
break;
}
// ── Sink verdicts: Retry = counts as a failed attempt (cool + demote host, back-off,
// rotate); Fatal was already recorded in the error slot by the pool. ──
while let Ok(fb) = fb_rx.try_recv() {
outstanding = outstanding.saturating_sub(1);
match fb.verdict {
@@ -1123,7 +942,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
log(&window.summary_line(label, inflight.len(), probe_now));
}
// ── Stalled hosts: requests that receive nothing while another host delivers. ──
if probe_now.duration_since(last_stall_check) >= Duration::from_millis(STALL_CHECK_MS) {
last_stall_check = probe_now;
for s in 0..hosts.len() {
@@ -1163,12 +981,10 @@ async fn run_driver(ctx: DriverCtx<'_>) {
hosts[s],
probe_now.duration_since(silent_since).as_millis()
));
// Do not fire again for the same silence while the cancelled requests wind down.
watch_from[s] = probe_now;
}
}
// ── Dispatch up to the current window ──
while inflight.len() < window.current {
if cancel.load(Ordering::Relaxed)
|| error_slot.lock().expect("err slot poisoned").is_some()
@@ -1177,7 +993,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
}
let now = Instant::now();
// A due retry first, else the next fresh item.
let use_retry = retry.front().is_some_and(|p| p.not_before <= now);
let (idx, attempts) = if use_retry {
let p = *retry.front().expect("retry nonempty");
@@ -1185,23 +1000,19 @@ async fn run_driver(ctx: DriverCtx<'_>) {
} else if next_item < items.len() {
(next_item, 0u32)
} else {
break; // nothing dispatchable now (retry not due, or all dispatched)
break;
};
let item = &items[idx];
// Byte-budget gate (hard memory bound): reserve at DISPATCH. Always admits when nothing
// is in flight, so an item bigger than the whole budget can't deadlock. Streams reserve
// nothing here — their pieces are budgeted as they arrive.
let reserve = if stream { 0 } else { reserve_bytes(item) };
if !budget_admits(in_flight.load(Ordering::Relaxed), reserve, budget) {
window.note_budget_stall();
break; // wait for a completion to free budget
break;
}
// Speed-ranked, per-host-capped host pick.
let Some(server_idx) = sched.pick(now) else {
window.note_host_stall();
break; // every host at cap or cooling → wait for a completion
break;
};
let permit = match sched.host_sem(server_idx).clone().try_acquire_owned() {
Ok(permit) => permit,
@@ -1218,7 +1029,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
let url = url.to_string();
let range = item.range;
// Commit: pop the work item, reserve budget, launch the fetch future.
if use_retry {
retry.pop_front();
} else {
@@ -1231,7 +1041,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
let rx_counter = &host_rx[server_idx];
let started = Instant::now();
if running[server_idx] == 0 {
// The host had nothing to answer until now: its silence is measured from here.
watch_from[server_idx] = started;
}
running[server_idx] += 1;
@@ -1268,7 +1077,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
.map(Fetched::Body)
}
};
// The stall detector's switch: dropping the fetch future closes its connection.
match select(pin!(fetch), pin!(abort.notified())).await {
Either::Left((res, _)) => (res, false),
Either::Right(((), _)) => (
@@ -1280,7 +1088,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
),
}
};
drop(permit); // RAII per-host permit release (also on future drop / cancel-abort)
drop(permit);
FetchDone {
idx,
attempts: attempts + 1,
@@ -1295,17 +1103,14 @@ async fn run_driver(ctx: DriverCtx<'_>) {
});
}
// ── Nothing in flight? finished, or waiting on retry-not-due / host cooldown / the pool. ──
if inflight.is_empty() {
if next_item >= items.len() && retry.is_empty() && outstanding == 0 {
break; // every item fetched and processed
break;
}
tokio::time::sleep(Duration::from_millis(5)).await;
continue;
}
// ── Await one completion (drives ALL in-flight requests cooperatively), bounded so
// cancel / a pool Fatal / a Retry verdict are noticed promptly even mid-fetch. ──
let done = match tokio::time::timeout(Duration::from_millis(DRIVER_POLL_MS), inflight.next())
.await
{
@@ -1317,9 +1122,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
aborts.remove(&done.req_id);
running[done.server_idx] = running[done.server_idx].saturating_sub(1);
if done.stalled {
// Not the item's fault: straight back into the queue for a healthy host, no window
// or host penalty beyond the rest the detector already gave the host. It still counts
// as an attempt, which keeps each stream attempt's pieces distinguishable.
in_flight.fetch_sub(done.reserve, Ordering::Relaxed);
retry.push_front(PendingItem {
idx: done.idx,
@@ -1340,8 +1142,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
meter.record(done.server_idx, raw_len);
sched.on_success(done.server_idx, raw_len, done.elapsed);
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
// Reconcile the dispatch reservation to the actual raw size; the pool subtracts the
// actual size on process-complete, netting this item's budget to zero.
if raw_len >= done.reserve {
in_flight.fetch_add(raw_len - done.reserve, Ordering::Relaxed);
} else {
@@ -1357,7 +1157,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
})
.is_err()
{
break; // process side gone
break;
}
}
Ok(Fetched::Streamed(total_len)) => {
@@ -1374,7 +1174,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
})
.is_err()
{
break; // process side gone
break;
}
}
Err(err) => {
@@ -1423,14 +1223,10 @@ window_shrink={} msg={}",
}
}
// Dropping `inflight` aborts any still-running requests (the cancel / error / final-failure
// teardown path); dropping the senders then closes the channels so the pool drains and exits.
drop(inflight);
drop(txs);
}
/// Since when a server has been silent: its last byte, or when it was last given work to do if
/// that came later.
fn silent_since(last_rx_at: Option<Instant>, watch_from: Instant) -> Instant {
match last_rx_at {
Some(at) if at > watch_from => at,
@@ -1438,16 +1234,12 @@ fn silent_since(last_rx_at: Option<Instant>, watch_from: Instant) -> Instant {
}
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Bandwidth meter (per-host bytes + 5-s samples for peak)
// ─────────────────────────────────────────────────────────────────────────────────────────────
pub struct BandwidthMeter {
start: Instant,
total_bytes: AtomicU64,
per_server_bytes: Vec<AtomicU64>,
hosts: Vec<String>,
/// (bytes, instant) at the last sample; peak = max sample rate in bytes/s.
last_sample: Mutex<(u64, Instant)>,
peak_bps: Mutex<f64>,
}
@@ -1478,8 +1270,6 @@ impl BandwidthMeter {
self.total_bytes.load(Ordering::Relaxed)
}
/// Take a throughput sample since the previous one (≥ 1 s apart to be counted) and fold it
/// into the peak.
pub fn sample(&self) {
let now = Instant::now();
let total = self.total_bytes();
@@ -1530,9 +1320,6 @@ impl BandwidthMeter {
}
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Scheduler + adaptive window — a copy of the Steam depot writer's (keyed on host strings).
// ─────────────────────────────────────────────────────────────────────────────────────────────
#[derive(Clone, Debug, Default)]
struct ServerHealth {
@@ -1542,10 +1329,6 @@ struct ServerHealth {
cooldown_until: Option<Instant>,
}
/// Speed-ranked, per-host-capped host picker. Cold hosts are probed first, then selection is
/// epsilon-greedy over the measured EWMA so a demoted-but-recovered host returns. Per-host caps key
/// on the host STRING (one `tokio::sync::Semaphore` each) so two entries that share a host still
/// share one cap.
struct FetchScheduler {
health: Vec<ServerHealth>,
server_host: Vec<usize>,
@@ -1635,8 +1418,6 @@ impl FetchScheduler {
h.cooldown_until = None;
}
/// A stalled host: out of the rotation for `rest`, and its speed estimate cut so it comes
/// back behind the hosts that kept delivering.
fn rest(&mut self, server_idx: usize, now: Instant, rest: Duration) {
let h = &mut self.health[server_idx];
h.cooldown_until = Some(now + rest);
@@ -1710,8 +1491,6 @@ impl WindowReason {
}
}
/// Adaptive in-flight window: slow-start doubles every healthy probe until the first real back-off,
/// then grows additively; holds on plateau; shrinks proportionally only on real errors.
#[derive(Clone, Debug)]
struct AdaptiveWindow {
current: usize,
@@ -1733,14 +1512,10 @@ struct AdaptiveWindow {
last_budget_stalls: u32,
last_host_stalls: u32,
latency_ewma_ms: f64,
/// TTFB samples of the current probe interval.
ttfb_sum_ms: f64,
ttfb_samples: u32,
/// Lowest per-probe mean TTFB seen this run (0 = none yet): the no-queue baseline.
ttfb_base_ms: f64,
/// Mean TTFB of the last probe that had enough samples.
ttfb_last_ms: f64,
/// Consecutive probes with queueing delay.
latency_streak: u32,
stalled_hosts: u32,
cooldown_until: Option<Instant>,
@@ -1803,7 +1578,6 @@ impl AdaptiveWindow {
self.stalled_hosts = self.stalled_hosts.saturating_add(1);
}
/// The latency brake's shrink: proportional and gentle, never below the floor, at least −1.
fn shrink_latency(&mut self, now: Instant) {
let scaled = (self.current as f64 * LATENCY_SHRINK_FACTOR).floor() as usize;
self.current = scaled.min(self.current.saturating_sub(1)).max(self.min);
@@ -1884,7 +1658,6 @@ impl AdaptiveWindow {
let before = self.current;
let cooling = self.cooldown_until.is_some_and(|t| now < t);
// Queueing delay: this probe's mean TTFB against the run's baseline.
let probe_ttfb = (self.ttfb_samples >= LATENCY_MIN_SAMPLES)
.then(|| self.ttfb_sum_ms / self.ttfb_samples as f64);
let mut queueing = false;
@@ -1909,9 +1682,6 @@ impl AdaptiveWindow {
} else if cooling {
self.last_reason = WindowReason::HoldCooldown;
} else if queueing {
// More requests only made each one wait longer. Hold while a real gain still comes
// with the delay; once the delay keeps rising without one, give some of the window
// back, gently, and let the cooldown show whether that helped.
let gain = self.bps_ewma > self.best_bps * (1.0 + LATENCY_GAIN_EPS);
if gain {
self.best_bps = self.bps_ewma;
@@ -2012,9 +1782,6 @@ stalled_hosts={}",
}
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// Tests
// ─────────────────────────────────────────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
@@ -2154,8 +1921,6 @@ mod tests {
const MIB: u64 = 1024 * 1024;
/// Drive the window through probes 2.1 s apart, each with `samples` TTFB values and the
/// bytes that arrived in it; returns (reason, window) after every probe.
fn run_probes(w: &mut AdaptiveWindow, t0: Instant, probes: &[(f64, u64)]) -> Vec<(WindowReason, usize)> {
let mut total = 0u64;
let mut out = Vec::new();
@@ -2174,7 +1939,6 @@ mod tests {
fn rising_delay_without_more_throughput_holds_then_shrinks_gently() {
let t0 = Instant::now();
let mut w = AdaptiveWindow::new(8, 2, 256, t0);
// Flat ~15 MB/s while the time to first byte climbs from 0.4 s to 12 s (the 5G log).
let flat = 32 * MIB;
let probes: Vec<(f64, u64)> = [400.0, 400.0, 450.0, 900.0, 1500.0, 2500.0, 4000.0, 6000.0, 9000.0, 12000.0]
.iter()
@@ -2186,7 +1950,6 @@ mod tests {
assert!(first_hold < first_shrink, "{seen:?}");
let before = seen[first_shrink - 1].1;
assert_eq!(seen[first_shrink].1, (before as f64 * LATENCY_SHRINK_FACTOR).floor() as usize, "{seen:?}");
// From the first sign of queueing on, the window never grows again.
for pair in seen[first_hold..].windows(2) {
assert!(pair[1].1 <= pair[0].1, "grew under queueing delay: {seen:?}");
}
@@ -2199,7 +1962,6 @@ mod tests {
fn a_fast_link_keeps_growing_while_its_delay_stays_low() {
let t0 = Instant::now();
let mut w = AdaptiveWindow::new(8, 2, 256, t0);
// Throughput doubles with the window; TTFB wobbles between 25 and 70 ms (new connections).
let probes: Vec<(f64, u64)> = (0..6)
.map(|k| ([25.0, 70.0, 40.0, 60.0, 30.0, 65.0][k], (8 * MIB) << k))
.collect();
@@ -2211,8 +1973,6 @@ mod tests {
assert!(w.current >= 128, "{seen:?}");
}
/// Two-behaviour stub for the stall test: `freeze` answers the headers and 100 bytes, then
/// never sends another byte; otherwise every path gets 10 000 bytes in ten paced slices.
fn start_paced_stub(freeze: bool) -> String {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
let port = listener.local_addr().expect("addr").port();
@@ -2296,12 +2056,10 @@ mod tests {
.unwrap_or_else(|| panic!("no host-stall line: {lines:?}"));
assert!(stall.contains(&format!(" host={frozen} ")), "{stall}");
assert!(!stall.contains(" requeued=0 "), "{stall}");
// Well inside the 10 s request timeout that would otherwise have freed the items.
assert!(elapsed < Duration::from_secs(8), "took {elapsed:?}: {lines:?}");
assert!(!lines.iter().any(|l| l.contains(&format!("host={healthy}")) && l.starts_with("host-stall")));
}
// ── Minimal HTTP/1.1 stub: one thread per connection, `Connection: close`, Range-aware. ──
struct Stub {
base: String,
hits: Arc<Mutex<HashMap<String, usize>>>,
@@ -2444,11 +2202,9 @@ mod tests {
}
}
/// Stream-mode sink: appends pieces per item (offset 0 restarts), finalizes by length check.
struct StreamSink {
got: Mutex<HashMap<u64, Vec<u8>>>,
finished: Mutex<HashMap<u64, u64>>,
/// Reject the first piece of this item once (exercises the mid-stream Retry path).
retry_once_id: Option<u64>,
retried: AtomicUsize,
}
@@ -2534,8 +2290,6 @@ mod tests {
assert_eq!(got[&5], b"nile/0.1 Amazon".to_vec());
let finished = sink.finished.lock().unwrap();
assert_eq!(finished.len(), 5);
// Pieces credited as written + 7 extra per on_finish; the rejected attempt of /big
// contributed nothing (its first piece was refused before any write).
let written: u64 = got.values().map(|b| b.len() as u64).sum();
assert_eq!(outcome.bytes_credited, written + 5 * 7);
let hits = stub.hits.lock().unwrap();
@@ -2575,7 +2329,6 @@ mod tests {
&sink,
&cancel,
&|bytes: u64, ok: u64| {
// Pool threads race on the callback order; keep the high-water mark.
let mut cur = last_progress.lock().unwrap();
if bytes >= cur.0 {
*cur = (bytes, ok);
@@ -2595,7 +2348,6 @@ mod tests {
let expected_credit: u64 = got.values().map(|b| b.len() as u64 * 2).sum();
assert_eq!(outcome.bytes_credited, expected_credit);
assert_eq!(*last_progress.lock().unwrap(), (expected_credit, 4));
// /c was served 503 once, then 200; /a was re-fetched once after the sink's Retry.
let hits = stub.hits.lock().unwrap();
assert_eq!(hits["/c"], 2);
assert_eq!(hits["/a"], 2);
@@ -1,32 +1,12 @@
//! The numbers and the two small types [`fetch_core`](crate::fetch_core) shares with the Steam
//! depot writer it was carved out of.
//!
//! In Bannerlator these live in `depot_writer.rs` and `cdn_client.rs`, next to the Steam CM and
//! depot code, so that the Steam engine and the store engines run on ONE set of tunables and their
//! `fetch-window` log lines stay A/B-comparable. DroidDeck ships no Steam depot engine (the Steam
//! client runs natively in the Linux runtime), so only this slice came across: the values are
//! copied verbatim and keep their names, which is what lets a GOG/Epic/Amazon log from either app
//! be read against the same tuning notes.
//!
//! The important behaviours are structural (slow-start ramp -> hold at plateau -> back off on REAL
//! errors, per-host cap, hard byte budget), not the exact numbers. The r3/r4 notes below record
//! why each number is what it is, because every one of them was arrived at by a device run that
//! went wrong the other way.
use std::time::Duration;
/// Default `User-Agent` of the pooled client. Every store engine overrides it per request with
/// the agent its own launcher sends (`FetchOptions::headers`), so this value is only what a
/// request without an explicit header would carry.
pub const USER_AGENT: &str = "Valve/Steam HTTP Client 1.0";
/// TCP connect deadline for every request the pooled client makes.
pub fn connect_timeout() -> Duration {
Duration::from_secs(15)
}
/// Classified fetch failure so the fetch driver can back off proportionally (429/5xx/timeout/reset
/// shrink the window + cool the host harder than a one-off protocol error).
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum FetchFailKind {
Timeout,
@@ -42,53 +22,17 @@ pub struct AsyncFetchError {
pub kind: FetchFailKind,
}
/// Attempts per item before the run gives up on it (fetch failures and sink retries both count).
pub const MAX_CHUNK_ATTEMPTS: u32 = 5;
/// Floor for the in-flight byte budget — the pre-r4 fixed value, so no run ever gets *less*
/// buffered memory than it had before the budget started scaling. See [`inflight_budget_bytes`].
pub const FETCH_INFLIGHT_BUDGET_FLOOR_BYTES: u64 = 24 * 1024 * 1024;
/// Hard ceiling for the in-flight byte budget. This is THE OOM guard: whatever the tier, the host
/// count or the chunk size, the RAW bytes in memory for one run can never exceed this. 96 MiB = 4×
/// the old fixed budget and still a small fraction of the heap an Android game installer runs in,
/// and it is only ever reached by a fast-tier run whose CDN pool is large enough to support a 64+
/// window. Raising this is the one change in the fetch layer that can reintroduce OOM, so it stays
/// fixed and small.
pub const FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES: u64 = 96 * 1024 * 1024;
/// Assumed RAW (on-the-wire, still-compressed) size of one chunk when sizing the budget. Store
/// chunks are ~1 MiB; this is the same nominal [`NOMINAL_CHUNK_RESERVE_BYTES`] uses at dispatch.
pub const TYPICAL_CHUNK_BYTES: u64 = NOMINAL_CHUNK_RESERVE_BYTES;
/// Headroom the budget carries over "one chunk per window slot" (3/2 = 1.5×), as a rational so the
/// arithmetic stays integer. The window is a count of concurrent *requests*; a slot's bytes are held
/// from DISPATCH until its WRITE completes, so at any instant some slots hold a fetched-but-not-yet-
/// written chunk while others hold a reservation for a fetch still on the wire. 1.5× covers that
/// overlap (and the occasional above-nominal chunk) without doubling the memory bound.
pub const BUDGET_HEADROOM_NUM: u64 = 3;
pub const BUDGET_HEADROOM_DEN: u64 = 2;
/// Byte budget for the *compressed* bytes buffered in the fetch→process channel, scaled to the
/// window ceiling this run can actually reach.
///
/// The decouple channel is bounded by BYTES, not chunk count: a count cap makes in-flight memory
/// swing wildly with chunk size (a 4-chunk cap is ~4 MB for 1 MB chunks but ~120 MB for 30 MB
/// chunks). Budgeting bytes keeps the heap bounded regardless of game size while giving the fetch
/// pool bandwidth-delay-product headroom so the socket stays busy while process workers
/// decompress+write.
///
/// r4: a device run proved a fixed 24 MiB budget becomes the binding constraint the moment the
/// adaptive window works — on a 1 Gbps line the window grew to 42 but the budget admitted only ~24
/// chunks, so `in_flight` stalled at 23–28 with `budget_stalls` in the hundreds and
/// `host_stalls=0`. A budget that refuses slots the window has already decided are safe is not a
/// memory guard, it is a throttle. So the budget scales WITH the effective ceiling (tier ∧
/// hosts×per-host-cap ∧ hard cap) and is clamped between the old fixed value and a hard cap. Low
/// tiers are unaffected; a fast link gets a budget that stops binding before the window does.
///
/// Budget only the RAW bytes actually queued (reserved at DISPATCH, released on WRITE-COMPLETE); the
/// decoded expansion is transient inside a process worker and never accumulates. See
/// [`budget_admits`] for the single-chunk deadlock guard.
pub fn inflight_budget_bytes(effective_ceiling: usize) -> u64 {
(effective_ceiling as u64)
.saturating_mul(TYPICAL_CHUNK_BYTES)
@@ -100,71 +44,30 @@ pub fn inflight_budget_bytes(effective_ceiling: usize) -> u64 {
)
}
// ─── Adaptive-window tuning ────────────────────────────────────────────────────────────────────
// r3 retune: the first device run pinned the window at the floor on a 1 Gbps link. The controller
// shrank on throughput dips and latency jitter (normal CDN noise) as well as on errors, shrank a flat
// −6 on every single error, and only grew when throughput rose >10% in a 700 ms sample. The shrinks
// dominated, so the window ratcheted to the floor and stayed there. r3 makes growth eager and makes
// ERRORS the only shrink signal.
/// Max concurrent requests to any single CDN host. CDNs throttle/reset per host, so a big window
/// is reached by spreading across MORE hosts, never by piling onto one.
pub const PER_HOST_CAP: usize = 6;
/// Initial in-flight window. Slow-start doubles from here, so this only sets how fast the first
/// couple of probes get useful — the ceiling still comes from the tier × hosts clamp.
pub const BOOTSTRAP_WINDOW: usize = 8;
/// Hard safety ceiling on the window regardless of tier (also clamped to distinct-hosts × per-host).
pub const WINDOW_HARD_CAP: usize = 256;
/// The window never shrinks below this (keeps a little pipelining even on a rough link).
pub const WINDOW_MIN_FLOOR: usize = 2;
/// Additive growth step once slow-start has ended (after the first real back-off).
pub const WINDOW_STEP_UP: usize = 4;
/// Slow-start multiplier: until the first back-off the window DOUBLES each healthy probe, so a fast
/// link reaches a useful window in seconds instead of never.
pub const WINDOW_SLOW_START_FACTOR: usize = 2;
/// Proportional shrink on a real error signal (never a flat step: a flat −6 slammed a small window
/// straight to the floor). Always at least −1 while above the floor.
pub const WINDOW_SHRINK_FACTOR: f64 = 0.75;
/// How often the adaptive window re-evaluates. Long enough that one jittery sample cannot drive a
/// decision; decisions use the EWMA, not the raw sample.
pub const WINDOW_PROBE_INTERVAL_MS: u64 = 2_000;
/// After a back-off, hold the window (no growth) for this long — hysteresis against oscillation.
pub const WINDOW_COOLDOWN_MS: u64 = 3_000;
/// EWMA smoothing for the measured throughput samples (higher = more responsive).
pub const WINDOW_BPS_EWMA_ALPHA: f64 = 0.4;
/// Throughput above `best × (1 + this)` counts as a genuine improvement (keeps slow-start climbing).
pub const WINDOW_IMPROVE_EPS: f64 = 0.05;
/// Throughput below `best × (1 − this)` counts as falling — growth STOPS (hold). It never shrinks:
/// only real errors shrink.
pub const WINDOW_DECLINE_EPS: f64 = 0.15;
/// Error-rate thresholds over one probe window: above HIGH shrinks; growth needs at/below LOW
/// (in between = hold).
pub const WINDOW_ERR_RATE_HIGH: f64 = 0.10;
pub const WINDOW_ERR_RATE_LOW: f64 = 0.05;
/// Errors inside ONE probe interval that trigger an immediate (don't-wait-for-the-probe) shrink. A
/// single stray timeout no longer moves the window; a burst does. A 429 always shrinks immediately.
pub const WINDOW_ERR_BURST_IMMEDIATE: u32 = 3;
/// Consecutive non-improving probes still allowed to grow before declaring a plateau — throughput
/// lags a window change by a probe or two, so one flat sample must not stop the ramp.
pub const WINDOW_PLATEAU_PATIENCE: u32 = 2;
/// Once plateaued, re-arm the ramp this long later (doubling, capped) to re-probe for headroom if the
/// link improved. Bounded re-probing, not a creep: a weak link answers with errors and shrinks back.
pub const WINDOW_PLATEAU_REARM_MS: u64 = 30_000;
pub const WINDOW_PLATEAU_REARM_MAX_MS: u64 = 300_000;
/// Even when the window does not change, emit one `fetch-window` line every N probes so a stuck
/// window explains itself in the log (`reason=` says why it is not moving).
pub const WINDOW_LOG_EVERY_PROBES: u32 = 5;
/// Every Nth exploit dispatch is instead an exploration pick (epsilon-greedy ≈ 1/N) so a
/// demoted-but-recovered host is retried rather than starved forever.
pub const SERVER_EXPLORE_EVERY: u64 = 8;
/// Reservation for an item whose compressed size the manifest didn't carry (keeps the byte budget
/// honest at dispatch time).
pub const NOMINAL_CHUNK_RESERVE_BYTES: u64 = 1024 * 1024;
/// Extra host cooldown floor when a host answers 429 (rate-limited) — back off harder than a
/// one-off timeout.
pub const RATE_LIMIT_COOLDOWN_MS: u64 = 5_000;
/// Exponential back-off between attempts of one item: 0, 300, 600, 1200, 2400 ms … capped at 4 s.
pub fn retry_backoff_millis(attempt: u32) -> u64 {
if attempt == 0 {
return 0;
@@ -172,9 +75,6 @@ pub fn retry_backoff_millis(attempt: u32) -> u64 {
(300u64 << (attempt - 1)).min(4000)
}
/// Byte-budget admission test for the fetch→process channel. Always admits when nothing is in
/// flight (`in_flight == 0`) so a single item LARGER than the whole budget can never deadlock — it
/// simply waits until the channel drains, then goes through alone.
#[inline]
pub fn budget_admits(in_flight: u64, raw_len: u64, budget: u64) -> bool {
in_flight == 0 || in_flight.saturating_add(raw_len) <= budget
-31
View File
@@ -1,21 +1,3 @@
//! GOG gen2 chunk download engine — the Rust replacement for the byte-fetching inner loop of
//! `GogDownloadManager.java` (`assembleDepotFile` / `fetchChunkVerified` driven from the
//! per-file thread pool). Everything around that loop — token/builds/manifest head, base-product
//! + language depot selection, secure-link resolution, disk guard, `_gog_manifest.json` marker,
//! prefs, exe resolution, DLC markers, redist orchestration — stays in Java and is untouched.
//!
//! Parity contract: Bannerlator's `docs/RUST_GOG_PARITY.md`. The adapter is fed the SAME inflated depot-manifest
//! JSON strings Java already parsed, the SAME resolved CDN base (secure link) and the SAME
//! install dir, and produces the SAME on-disk layout (`<file>.bhtmp` staging, atomic rename,
//! size+MD5-verified finals) so a download can be resumed by either engine.
//!
//! Layout:
//! - [`plan`] — depot-manifest JSON → file/chunk plan (mirrors `parseDepotManifest`), CDN path
//! and chunk-URL builders, largest-first ordering.
//! - [`engine`] — resume/skip pass (mirrors `fileVerified`), `FetchItem` list, the `FetchSink`
//! (compressed size/MD5 → inflate → decompressed size/MD5 → positioned write →
//! whole-file size/MD5 → rename), summary logging.
//! - [`jni`] — `GogNative` JNI exports (`nativeStart` / `nativeCancel` / `nativeRelease`).
pub mod engine;
pub mod jni;
@@ -25,7 +7,6 @@ use std::fs::File;
use std::io::Read;
use std::path::Path;
/// Lowercase hex of `bytes` (matches `GogDownloadManager.toHex`).
pub fn to_hex(bytes: &[u8]) -> String {
const DIGITS: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(bytes.len() * 2);
@@ -36,15 +17,12 @@ pub fn to_hex(bytes: &[u8]) -> String {
out
}
/// Lowercase hex MD5 of an in-memory buffer (`GogDownloadManager.md5Hex`).
pub fn md5_hex(data: &[u8]) -> String {
let mut hasher = crate::md5_small::Md5::new();
hasher.update(data);
to_hex(&hasher.finalize())
}
/// Streaming lowercase hex MD5 of a file, never buffering the whole file
/// (`GogDownloadManager.md5HexFile`). `None` when the file cannot be read.
pub fn md5_hex_file(path: &Path) -> Option<String> {
let mut file = File::open(path).ok()?;
let mut hasher = crate::md5_small::Md5::new();
@@ -59,11 +37,6 @@ pub fn md5_hex_file(path: &Path) -> Option<String> {
Some(to_hex(&hasher.finalize()))
}
/// zlib-inflate a chunk body (`GogDownloadManager.inflateZlib`): `None` when the body is not a
/// zlib stream (first byte != 0x78) or fails to inflate — the caller then treats the body as a
/// stored (non-compressed) chunk, exactly like Java's `if (inflated == null) inflated = raw`.
/// A truncated stream yields the partial output (Java's loop breaks on `n == 0` the same way);
/// the decompressed-size check that follows rejects it.
pub fn inflate_zlib(data: &[u8]) -> Option<Vec<u8>> {
if data.len() < 2 || data[0] != 0x78 {
return None;
@@ -96,16 +69,12 @@ mod tests {
let compressed = enc.finish().unwrap();
assert_eq!(compressed[0], 0x78);
assert_eq!(inflate_zlib(&compressed).unwrap(), payload);
// Not zlib → None (stored chunk path).
assert!(inflate_zlib(b"\x1f\x8bnot zlib").is_none());
assert!(inflate_zlib(b"\x78").is_none());
// Truncated zlib → partial output, like Java's `while (!inf.finished()) { if (n == 0) break; }`
// loop; the decompressed-size check downstream rejects it either way.
let mut truncated = compressed.clone();
let mid = truncated.len() / 2;
truncated.truncate(mid);
assert!(inflate_zlib(&truncated).map(|out| out != payload).unwrap_or(true));
// Corrupt zlib body → None (Java: DataFormatException → null → raw fallback).
let mut corrupt = compressed.clone();
for b in corrupt.iter_mut().skip(2) {
*b = !*b;
+6 -147
View File
@@ -1,17 +1,3 @@
//! The GOG gen2 fetch loop on top of the shared fetch core. One call = one Java pool run
//! (`runGen2` base loop, `doInstallDlc` loop, or one `assembleDependencyInstaller`).
//!
//! What this mirrors, and where (see Bannerlator's `docs/RUST_GOG_PARITY.md`):
//! - resume/skip: [`file_verified`] = `GogDownloadManager.fileVerified` (exists, non-empty, size
//! when known, MD5 when known) — a verified file is reported as "Verified…" progress with NO
//! bytes credited, exactly like the Java task;
//! - per chunk: compressed size → compressed MD5 → inflate (stored fallback) → decompressed size →
//! decompressed MD5 (`fetchChunkVerified`); a mismatch is a retryable failure (Java: hard fail
//! ≤3 with backoff; the core: ≤5 attempts with backoff);
//! - per file: write into `<file>.bhtmp` (positioned, so chunks may land out of order), then
//! whole-file size + MD5, delete any existing final, rename (`assembleDepotFile`);
//! - failure of any file aborts the run (`anyFailed`), cancel aborts the run; in both cases every
//! unfinished `.bhtmp` is deleted (Java deletes its tmp on the failing/cancelled thread).
use std::collections::HashSet;
use std::fs::{self, File, OpenOptions};
@@ -30,35 +16,25 @@ use super::plan::{
};
use super::{inflate_zlib, md5_hex, md5_hex_file};
/// Per-request timeout on chunk fetches (Java: `TIMEOUT = 30_000` connect + read).
pub const CHUNK_TIMEOUT: Duration = Duration::from_secs(30);
/// Java sends this on every content-system / CDN request (`fetchBytesEx`).
pub const USER_AGENT: &str = "GOG Galaxy";
/// Java staging suffix (`assembleDepotFile`).
pub const TMP_SUFFIX: &str = ".bhtmp";
/// Floor for the per-host cap (the core's default); a single-host store gets the whole ceiling.
pub const PER_HOST_CAP_FLOOR: usize = 6;
/// `max(6, ceil(max_workers / distinct_hosts))` — one CDN host gets the whole ceiling, several
/// share it.
pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize {
let hosts = distinct_hosts.max(1);
let workers = max_workers.max(1);
PER_HOST_CAP_FLOOR.max(workers.div_ceil(hosts))
}
/// Which Java loop this run replaces.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum PlanKind {
/// gen2: depot manifests → chunks → `assembleDepotFile` (base install, DLC, dependencies).
#[default]
Gen2Chunks,
/// gen1: build manifest → per-file HTTP Range GET streamed to disk (`runGen1`).
Gen1Ranges,
}
impl PlanKind {
/// JNI mapping (`GogNative.KIND_*`): 0 = gen2 chunks, 1 = gen1 ranges, anything else → gen2.
pub fn from_i32(v: i32) -> Self {
if v == 1 {
PlanKind::Gen1Ranges
@@ -68,39 +44,21 @@ impl PlanKind {
}
}
/// Inputs — exactly what the Java manager has in hand after manifest + secure-link resolution.
#[derive(Clone, Debug, Default)]
pub struct GogRequest {
pub kind: PlanKind,
/// gen2: inflated depot-manifest JSON strings, in the order Java fetched them (already filtered
/// by base-product / DLC-product and language in Java). gen1: the inflated build manifest.
pub depot_manifests: Vec<String>,
/// gen2: the resolved CDN bases from the secure-link answer (every `urls[]` entry, query
/// strings kept, in the order GOG lists them), or the one unauthenticated dependency store
/// base. Several bases on distinct hosts let the fetch core spread the window across CDNs
/// with its per-host cap; a base on a host already listed is dropped (the first one wins).
/// gen1: unused (file URLs live in the manifest).
pub cdn_bases: Vec<String>,
pub install_dir: String,
/// Files already completed by an earlier run of this same download (secure-link refresh
/// re-run): counted as done WITHOUT re-hashing and WITHOUT a progress event.
pub skip_paths: Vec<String>,
pub ca_bundle_path: String,
/// Java pool size: `resolveDownloadThreads` (base), 8 (DLC), 1 (dependency).
pub max_workers: usize,
/// Inflate/hash/write threads; Java did that on the same pool threads, so pass the same count.
pub process_workers: usize,
/// Base install: largest-first (LPT) ordering; DLC / dependency: manifest order.
pub sort_largest_first: bool,
/// Log prefix, e.g. `gog base=1207658930`.
pub label: String,
}
/// Callbacks out of the engine (JNI listener or a test recorder). Must be callable from the
/// process-pool threads.
pub trait GogEvents: Sync {
/// One file reached its final state. `verified` = resume-skip (no bytes credited); otherwise a
/// freshly assembled + verified file (`file_bytes` credited). `files_done` counts both.
#[allow(clippy::too_many_arguments)]
fn on_file_done(
&self,
@@ -113,9 +71,6 @@ pub trait GogEvents: Sync {
bytes_total: u64,
);
fn on_log(&self, line: &str);
/// Periodic byte progress while files are in flight (~4/s, only when the value moved):
/// assembled bytes plus the bytes of files not yet finished. Same `bytes_total` as
/// `on_file_done`. Default: ignored (test recorders).
fn on_bytes(&self, _bytes_done: u64, _bytes_total: u64) {}
}
@@ -123,20 +78,14 @@ pub trait GogEvents: Sync {
pub struct GogRunResult {
pub success: bool,
pub cancelled: bool,
/// The run died on an HTTP code Java treats as an expired secure link (401/403/404/500) —
/// the Java caller refreshes the link (cap 5) and re-runs with `skip_paths` filled in.
pub link_expiry: bool,
pub error: String,
/// Decompressed bytes of files assembled by THIS run (Java `totalBytes`).
pub bytes_written: u64,
/// Files done = verified-skipped + assembled (Java `doneCount`), excluding `skip_paths`.
pub files_done: u32,
pub files_verified: u32,
pub files_total: u32,
}
/// `GogDownloadManager.fileVerified`: true only if the file exists, is non-empty, matches the
/// expected size when known and the expected MD5 when known.
pub fn file_verified(path: &Path, expected_size: u64, expected_md5: &str) -> bool {
let Ok(meta) = fs::metadata(path) else {
return false;
@@ -156,8 +105,6 @@ pub fn file_verified(path: &Path, expected_size: u64, expected_md5: &str) -> boo
true
}
/// Finds an HTTP status in a fetch-core error string (`non-200 HTTP status (403)`, `status 403`,
/// `HTTP 403`, `http=403`, `code=403`). The link-expiry set is Java's `fetchChunkVerified` list.
pub fn http_status_in(message: &str) -> Option<u16> {
let lower = message.to_ascii_lowercase();
for marker in ["status (", "status ", "http ", "http=", "code=", "code "] {
@@ -178,7 +125,6 @@ pub fn http_status_in(message: &str) -> Option<u16> {
None
}
/// Java: `code == 401 || code == 403 || code == 404 || code == 500` → refresh the secure link.
pub fn is_link_expiry_status(code: u16) -> bool {
matches!(code, 401 | 403 | 404 | 500)
}
@@ -186,8 +132,6 @@ pub fn is_link_expiry_status(code: u16) -> bool {
struct FileState {
handle: Option<Arc<File>>,
chunks_done: usize,
/// Decompressed bytes of this file's finished chunks, credited to the sink's `partial`
/// counter until the file itself is finalized and moves into `bytes_done`.
partial: u64,
finalized: bool,
failed: bool,
@@ -240,11 +184,6 @@ impl SpeedSampler {
}
}
/// Positioned writer for ONE chunk's decompressed byte range inside the file's `.bhtmp`: hashes
/// everything it is given (the decompressed MD5) and counts it, but only writes bytes inside the
/// chunk's expected range when the size is known — an inflate overshoot then fails the size
/// check instead of clobbering the neighbouring chunk. An I/O error is remembered so the sink can
/// tell "disk failed" (fatal) from "stream failed" (retry).
struct RangeWriter {
file: Arc<File>,
base: u64,
@@ -294,13 +233,10 @@ enum ChunkMode {
Stored(RangeWriter),
}
/// Per-item streaming state for one attempt (reset whenever a piece arrives at `offset == 0`).
struct ChunkStream {
comp_md5: crate::md5_small::Md5,
comp_len: u64,
/// Decided on the first non-empty piece (`0x78` → zlib, else stored — Java's `inflateZlib`).
mode: Option<ChunkMode>,
/// The writer until the mode is decided (an empty body never decides).
writer: Option<RangeWriter>,
}
@@ -315,17 +251,10 @@ impl ChunkStream {
}
}
/// The `FetchSink`. Stream mode (what `run_gen2` uses): pieces of a chunk are inflated as they
/// arrive straight into the chunk's byte range of the file, the compressed MD5 and the
/// decompressed size/MD5 accumulate on the fly, and `on_finish` runs the checks in Java's order
/// (compressed size → compressed MD5 → inflate → size → MD5). Body mode (`process`) is the same
/// pipeline on a whole buffer and stays available for tests / a body-mode fallback.
struct GogSink<'a> {
files: &'a [PlannedFile],
/// item id → (file index, chunk index)
table: Vec<(usize, usize)>,
states: Vec<Mutex<FileState>>,
/// item id → in-flight stream attempt (stream mode only)
streams: Vec<Mutex<Option<ChunkStream>>>,
files_total: u32,
bytes_total: u64,
@@ -333,8 +262,6 @@ struct GogSink<'a> {
bytes_done: &'a AtomicU64,
events: &'a dyn GogEvents,
speed: Mutex<SpeedSampler>,
/// Bytes of files still in flight (finished chunks; for gen1, the current attempt's pieces).
/// `bytes_done + partial` is what the byte ticker reports between file completions.
partial: AtomicU64,
}
@@ -343,8 +270,6 @@ impl<'a> GogSink<'a> {
self.events.on_log(line);
}
/// `assembleDepotFile` head: `parent.mkdirs(); tmpFile.delete(); new FileOutputStream(tmp)` —
/// once per file, on its first chunk. Returns a shared handle for positioned writes.
fn file_handle(&self, file_idx: usize) -> Result<Arc<File>, SinkError> {
let file = &self.files[file_idx];
let mut st = self
@@ -385,9 +310,6 @@ impl<'a> GogSink<'a> {
}
}
/// One verified chunk landed: count it; on the file's last chunk run the whole-file verify +
/// rename and fire the per-file progress event (Java: doneCount++, totalBytes += df.totalSize,
/// "Downloading: <name> <speed>").
fn chunk_done(&self, file_idx: usize, chunk_bytes: u64) -> Result<(), SinkError> {
let file = &self.files[file_idx];
let mut st = self
@@ -431,7 +353,6 @@ impl<'a> GogSink<'a> {
Ok(())
}
/// `assembleDepotFile` tail: whole-file size + MD5 on the tmp, delete existing final, rename.
fn finalize_file(&self, file_idx: usize, st: &mut FileState) -> Result<(), String> {
let file = &self.files[file_idx];
if let Some(handle) = st.handle.take() {
@@ -469,7 +390,6 @@ impl<'a> GogSink<'a> {
Ok(())
}
/// Delete every unfinished `.bhtmp` (Java: `tmpFile.delete()` on the failing/cancelled task).
fn cleanup_partials(&self) {
for st in &self.states {
let Ok(mut st) = st.lock() else {
@@ -496,8 +416,6 @@ impl<'a> GogSink<'a> {
}
impl<'a> FetchSink for GogSink<'a> {
/// Body mode: `fetchChunkVerified` on a whole buffer, then a positioned write of the inflated
/// bytes (used by tests and available as a non-stream fallback).
fn process(&self, item: &FetchItem, body: Vec<u8>) -> Result<u64, SinkError> {
let Some(&(file_idx, chunk_idx)) = self.table.get(item.id as usize) else {
return Err(SinkError::Fatal(format!("unknown item id {}", item.id)));
@@ -508,7 +426,6 @@ impl<'a> FetchSink for GogSink<'a> {
speed.record(body.len() as u64);
}
// fetchChunkVerified, in order: compressed size → compressed MD5 → inflate → size → MD5.
if chunk.compressed_size > 0 && body.len() as u64 != chunk.compressed_size {
let msg = format!(
"chunk compressed-size mismatch chunk={} exp={} got={}",
@@ -528,7 +445,7 @@ impl<'a> FetchSink for GogSink<'a> {
}
let inflated = match inflate_zlib(&body) {
Some(out) => out,
None => body, // stored (non-zlib) chunk
None => body,
};
if chunk.size > 0 && inflated.len() as u64 != chunk.size {
let msg = format!(
@@ -563,8 +480,6 @@ impl<'a> FetchSink for GogSink<'a> {
Ok(credited)
}
/// Stream mode: hash the compressed piece, inflate it straight into the chunk's byte range
/// (or pass it through for a stored chunk), hashing + counting the decompressed output.
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
let id = item.id as usize;
let Some(&(file_idx, chunk_idx)) = self.table.get(id) else {
@@ -581,7 +496,6 @@ impl<'a> FetchSink for GogSink<'a> {
.and_then(|m| m.lock().ok())
.ok_or_else(|| SinkError::Fatal("chunk stream poisoned".to_string()))?;
if offset == 0 || slot.is_none() {
// (Re)start of an attempt: fresh hashers + inflater; positioned writes re-cover the range.
let handle = self.file_handle(file_idx)?;
*slot = Some(ChunkStream::new(handle, chunk.offset, chunk.size));
}
@@ -601,7 +515,7 @@ impl<'a> FetchSink for GogSink<'a> {
stream.mode = Some(if data[0] == 0x78 {
ChunkMode::Zlib(flate2::write::ZlibDecoder::new(writer))
} else {
ChunkMode::Stored(writer) // stored (non-zlib) chunk — Java's `inflated = raw`
ChunkMode::Stored(writer)
});
}
let result = match stream.mode.as_mut() {
@@ -616,7 +530,6 @@ impl<'a> FetchSink for GogSink<'a> {
match result {
Ok(()) => Ok(()),
Err((_, Some(io))) => {
// Disk, not stream: Java's `fos.write` exception → file fails → run aborts.
if let Some(mut st) = self.states.get(file_idx).and_then(|m| m.lock().ok()) {
st.failed = true;
let _ = st.handle.take();
@@ -627,7 +540,6 @@ impl<'a> FetchSink for GogSink<'a> {
Err(SinkError::Fatal(msg))
}
Err((err, None)) => {
// Corrupt zlib body. Java: inflate → null → raw → decompressed-size mismatch → retry.
let msg = format!("chunk inflate failed chunk={} err={err}", chunk.hash);
self.log(&msg);
*slot = None;
@@ -636,7 +548,6 @@ impl<'a> FetchSink for GogSink<'a> {
}
}
/// Stream mode: the body ended — run Java's checks in order and count the chunk.
fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result<u64, SinkError> {
let id = item.id as usize;
let Some(&(file_idx, chunk_idx)) = self.table.get(id) else {
@@ -652,7 +563,6 @@ impl<'a> FetchSink for GogSink<'a> {
.take();
let stream = match taken {
Some(s) => s,
// Empty body with no pieces: same checks on zero bytes.
None => ChunkStream::new(self.file_handle(file_idx)?, chunk.offset, chunk.size),
};
@@ -717,11 +627,8 @@ impl<'a> FetchSink for GogSink<'a> {
}
}
/// How often the byte ticker reports while files are in flight.
pub const BYTE_TICK: Duration = Duration::from_millis(250);
/// The CDN bases to fetch from: empty entries dropped, then one per distinct host (`host_key`),
/// the first base seen for a host kept, order preserved.
pub fn distinct_bases(bases: &[String]) -> Vec<String> {
let mut seen: HashSet<String> = HashSet::new();
bases
@@ -732,10 +639,6 @@ pub fn distinct_bases(bases: &[String]) -> Vec<String> {
.collect()
}
/// Runs `fetch` with a reporter thread beside it that calls `on_bytes(bytes_done + partial,
/// bytes_total)` every [`BYTE_TICK`] when the value moved. `on_file_done` only fires when a whole
/// file is finished, so a game made of a few large files gave the UI a progress bar that sat
/// still and then jumped; this fills the gaps. One last report is made when `fetch` returns.
fn with_byte_ticker<T>(
events: &dyn GogEvents,
bytes_done: &AtomicU64,
@@ -765,9 +668,6 @@ fn with_byte_ticker<T>(
})
}
/// Runs one download loop. Blocking; returns when every pending file is finalized, on the first
/// fatal file failure, or on cancel. Safe to call again with `skip_paths` extended by the files
/// this run reported done (that is how the Java side re-runs after a secure-link refresh).
pub fn run(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult {
match req.kind {
PlanKind::Gen2Chunks => run_gen2(req, cancel, events),
@@ -808,8 +708,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
let files_verified = AtomicU32::new(0);
if files.is_empty() {
// Java: the caller already returned "no depot files collected" before the loop; reaching
// here means the two parsers disagree — surface it loudly rather than pretend success.
return GogRunResult {
success: false,
error: "no files in plan (manifest parse mismatch)".to_string(),
@@ -818,8 +716,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
};
}
// Resume / repair pass — Java does this per task on the pool threads; here it runs up front
// on `process_workers` threads so the fetch item list can exclude verified files.
let pending: Vec<AtomicBool> = files.iter().map(|_| AtomicBool::new(false)).collect();
let next = AtomicUsize::new(0);
let verify_threads = process_workers.min(files.len().max(1));
@@ -870,7 +766,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
};
}
// Fetch items: one per chunk of every pending file, in plan order (chunks in manifest order).
let mut table: Vec<(usize, usize)> = Vec::new();
let mut items: Vec<FetchItem> = Vec::new();
let mut states: Vec<Mutex<FileState>> = Vec::new();
@@ -884,7 +779,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
handle: None,
chunks_done: 0,
partial: 0,
// Files not pending (verified or skip-listed) are already final for this run.
finalized: !is_pending,
failed: false,
tmp_path,
@@ -899,7 +793,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
let path = build_cdn_path(&chunk.hash);
items.push(FetchItem {
id: table.len() as u64,
// One URL per host, in `hosts` order: the core picks `urls[host_idx]`.
urls: bases.iter().map(|b| build_chunk_url(b, &path)).collect(),
reserve: chunk.compressed_size,
range: None,
@@ -943,15 +836,12 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
};
let opts = FetchOptions {
max_workers,
// The ceiling split across the distinct CDN hosts (floor 6); one host gets all of it.
per_host_cap,
timeout: CHUNK_TIMEOUT,
headers: vec![("User-Agent".to_string(), USER_AGENT.to_string())],
ca_bundle_path: req.ca_bundle_path.clone(),
process_workers,
label: req.label.clone(),
// Stream mode: pieces inflate straight to disk; no whole-body byte budget (the 24 MiB
// body budget + largest-first big chunks left 1-3 requests in flight at window 16).
stream: true,
};
let log = |line: &str| events.on_log(line);
@@ -1010,23 +900,14 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
}
}
// ─────────────────────────────────────────────────────────────────────────────────────────────
// gen1: per-file HTTP Range GET streamed to disk (`runGen1` / `downloadRange`)
// ─────────────────────────────────────────────────────────────────────────────────────────────
struct Gen1State {
handle: Option<File>,
/// Bytes this file's current attempt has written (credited to the sink's `partial`); a new
/// attempt starts over, so a retry never counts its bytes twice.
attempt_bytes: u64,
out_path: PathBuf,
finalized: bool,
}
/// Stream-mode sink for gen1. Java's `downloadRange`: `outFile.delete()` before EVERY attempt,
/// `new FileOutputStream(out)`, copy the Range body, no size/hash check; success → `doneG1++`,
/// `totalBytesG1 += size`, `"Downloading: <name> <speed>"`. Pieces arrive in order with their
/// offset; `offset == 0` marks the (re)start of an attempt, exactly where Java deletes the file.
struct Gen1Sink<'a> {
files: &'a [Gen1File],
states: Vec<Mutex<Gen1State>>,
@@ -1036,8 +917,6 @@ struct Gen1Sink<'a> {
bytes_done: &'a AtomicU64,
events: &'a dyn GogEvents,
speed: Mutex<SpeedSampler>,
/// Bytes of files still in flight (finished chunks; for gen1, the current attempt's pieces).
/// `bytes_done + partial` is what the byte ticker reports between file completions.
partial: AtomicU64,
}
@@ -1061,7 +940,6 @@ impl<'a> FetchSink for Gen1Sink<'a> {
.lock()
.map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?;
if offset == 0 {
// New attempt: Java `outFile.delete()` + `new FileOutputStream(out)`.
self.partial
.fetch_sub(std::mem::take(&mut st.attempt_bytes), Ordering::Relaxed);
st.handle = None;
@@ -1109,7 +987,7 @@ impl<'a> FetchSink for Gen1Sink<'a> {
let mut st = state
.lock()
.map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?;
st.handle = None; // close
st.handle = None;
st.finalized = true;
let attempt = std::mem::take(&mut st.attempt_bytes);
self.bytes_done.fetch_add(file.size, Ordering::Relaxed);
@@ -1130,10 +1008,6 @@ impl<'a> FetchSink for Gen1Sink<'a> {
}
}
/// gen1 loop. Resume = `exists && length == size` ("Resuming…", no bytes). Items = one Range GET
/// per pending file; retries/back-off by the core (Java: 3 attempts, 1 s / 2 s). Failed or
/// cancelled attempts leave whatever was written, like Java (a wrong-size partial fails the
/// size-match resume and is re-pulled next time; a base-install cancel deletes the whole dir).
fn run_gen1(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult {
let mut files: Vec<Gen1File> = Vec::new();
for manifest in &req.depot_manifests {
@@ -1298,7 +1172,7 @@ fn run_gen1(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
GogRunResult {
success,
cancelled,
link_expiry: false, // gen1 has no secure-link refresh in Java
link_expiry: false,
error: if cancelled && error.is_empty() { "cancelled".to_string() } else { error },
bytes_written: bytes_done.load(Ordering::Relaxed),
files_done: files_done.load(Ordering::Relaxed),
@@ -1412,8 +1286,6 @@ mod tests {
assert!(peak >= 8.0, "2 MB across at most two buckets → ≥8 Mbps peak, got {peak}");
}
/// Whole-file finalize through the sink with a synthetic body (no network): verifies the
/// chunk → positioned write → whole-file MD5 → rename path and the progress event shape.
#[test]
fn sink_assembles_verifies_and_renames() {
use flate2::write::ZlibEncoder;
@@ -1468,10 +1340,8 @@ mod tests {
};
let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None };
// Second chunk first: positioned write must still produce the right bytes.
assert_eq!(sink.process(&item(1), cb.clone()).unwrap(), part_b.len() as u64);
assert!(tmp_path.exists() && !out_path.exists(), "still staging after 1 of 2 chunks");
// A corrupt body is a retryable failure, not fatal.
match sink.process(&item(0), b"\x78garbage".to_vec()) {
Err(SinkError::Retry(msg)) => assert!(msg.contains("mismatch"), "{msg}"),
Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"),
@@ -1580,16 +1450,14 @@ mod tests {
assert_eq!(per_host_cap_for(0, 0), 6);
}
/// Stream mode: zlib chunk split into pieces (with a poisoned first attempt restarting at
/// offset 0), a stored chunk, corrupt data → Retry, then the whole-file finalize.
#[test]
fn stream_sink_inflates_pieces_verifies_and_finalizes() {
use flate2::write::ZlibEncoder;
use flate2::Compression;
let dir = temp_dir("stream");
let part_a = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".repeat(50); // 2000 bytes, compresses well
let part_b = b"\x01\x02stored bytes".to_vec(); // first byte != 0x78 → stored
let part_a = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".repeat(50);
let part_b = b"\x01\x02stored bytes".to_vec();
let whole = [part_a.clone(), part_b.clone()].concat();
let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
enc.write_all(&part_a).unwrap();
@@ -1632,7 +1500,6 @@ mod tests {
};
let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None };
// Attempt 1 of the zlib chunk: corrupt tail → Retry from on_chunk.
let mut corrupt = ca.clone();
for b in corrupt.iter_mut().skip(2) {
*b = !*b;
@@ -1642,7 +1509,6 @@ mod tests {
Err(SinkError::Retry(m)) => assert!(m.contains("inflate failed"), "{m}"),
Err(SinkError::Fatal(m)) => panic!("expected Retry, got Fatal({m})"),
Ok(()) => {
// Some corruptions only surface at finish: that must also be a Retry.
match sink.on_finish(&item(0), corrupt.len() as u64) {
Err(SinkError::Retry(_)) => {}
other => panic!("expected Retry at finish, got {}", match other {
@@ -1653,7 +1519,6 @@ mod tests {
}
}
}
// Attempt 2: restart at offset 0 in three pieces.
let cut1 = ca.len() / 3;
let cut2 = 2 * ca.len() / 3;
sink.on_chunk(&item(0), 0, &ca[..cut1]).unwrap();
@@ -1661,7 +1526,6 @@ mod tests {
sink.on_chunk(&item(0), cut2 as u64, &ca[cut2..]).unwrap();
assert_eq!(sink.on_finish(&item(0), ca.len() as u64).unwrap(), 0);
assert!(tmp_path.exists() && !out_path.exists());
// Stored chunk, one piece; wrong length first (→ Retry), then the real one.
sink.on_chunk(&item(1), 0, &part_b[..3]).unwrap();
match sink.on_finish(&item(1), 3) {
Err(SinkError::Retry(m)) => assert!(m.contains("compressed-size mismatch"), "{m}"),
@@ -1705,7 +1569,6 @@ mod tests {
partial: AtomicU64::new(0),
};
let item = FetchItem { id: 0, urls: vec!["x".into()], reserve: 8, range: Some((100, 107)) };
// Attempt 1 writes a partial, attempt 2 restarts from offset 0 and truncates it.
sink.on_chunk(&item, 0, b"ZZZZZZZZZZZZ").unwrap();
sink.on_chunk(&item, 0, b"abcd").unwrap();
sink.on_chunk(&item, 4, b"efgh").unwrap();
@@ -1737,11 +1600,7 @@ mod tests {
..Default::default()
};
let rec = Recorder { events: StdMutex::new(Vec::new()) };
// Cancel before the fetch so no network is touched: the resume pass still runs.
let cancel = AtomicBool::new(false);
// Use a pre-set cancel via a second request: run the resume pass, then expect the fetch
// to be attempted for short.bin only. We can't reach the network here, so assert on the
// plan log line instead of the outcome.
cancel.store(true, Ordering::Relaxed);
let res = run(&req, &cancel, &rec);
assert!(res.cancelled, "{res:?}");
-25
View File
@@ -1,15 +1,3 @@
//! JNI facade for the GOG Rust download engine — the native half of
//! `com.droiddeck.launcher.stores.gog.GogNative` (Kotlin). Symbol names are bound to that
//! package/class; the `@JvmStatic external` declarations there are the other side of this file.
//!
//! Shape mirrors the Steam download JNI (`jni.rs` `nativeDownloadApp`): `nativeStart` validates
//! its inputs, takes a `GlobalRef` on the listener, spawns the worker thread and returns a handle
//! immediately; callbacks are delivered from the worker / process-pool threads through a
//! `JavaVM` attach; `nativeCancel` flips an `AtomicBool`; `nativeRelease` drops the Java side's
//! reference to the handle (the worker keeps its own until it exits).
//!
//! Log tag `GogNative`: every engine log line goes to logcat here AND to the listener's `onLog`
//! (which the Kotlin manager folds into its own debug log).
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
@@ -29,7 +17,6 @@ unsafe extern "C" {
fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32;
}
/// `android.util.Log.i(LOG_TAG, message)`; a no-op on non-Android hosts (cargo test).
pub fn android_log(message: &str) {
#[cfg(target_os = "android")]
{
@@ -51,8 +38,6 @@ pub fn android_log(message: &str) {
}
}
/// One in-flight run. The Java side owns a `Box<Arc<..>>` (the `jlong`); the worker thread owns a
/// second `Arc` so `nativeRelease` is safe at any time after `nativeStart` returned.
pub struct GogRunHandle {
pub cancel: AtomicBool,
}
@@ -137,8 +122,6 @@ impl GogEvents for JniEvents {
}
fn on_bytes(&self, bytes_done: u64, bytes_total: u64) {
// A listener without `onBytes` (an app built against the older contract) makes this a
// NoSuchMethodError, which clear_exception drops: the run is unaffected.
self.with_env(|env| {
let _ = env.call_method(
self.listener.as_obj(),
@@ -209,7 +192,6 @@ unsafe fn handle_ref(handle: jlong) -> Option<&'static Arc<GogRunHandle>> {
}
}
/// Symbol-binding probe (see `GogNative.isAvailable`). Returns 1.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeProbe(
_env: JNIEnv,
@@ -218,9 +200,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativePr
1
}
/// Starts one download loop on a worker thread and returns a handle (0 = could not start; the
/// listener then receives NO callbacks). Inputs are exactly what `GogDownloadManager` holds after
/// manifest + secure-link resolution — see `GogRequest`.
#[no_mangle]
#[allow(clippy::too_many_arguments)]
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart(
@@ -290,8 +269,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeSt
Box::into_raw(Box::new(handle)) as jlong
}
/// Requests cancellation; the run stops at its next scheduling point and `onComplete` fires with
/// `cancelled = true`. Idempotent.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCancel(
_env: JNIEnv,
@@ -303,8 +280,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCa
}
}
/// Drops the Java side's reference. Call once, after `onComplete` (or after cancelling — the worker
/// keeps the run alive until it exits either way).
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeRelease(
_env: JNIEnv,
-46
View File
@@ -1,50 +1,24 @@
//! Depot-manifest JSON → download plan. Mirrors `GogDownloadManager.parseDepotManifest` rule for
//! rule (see Bannerlator's `docs/RUST_GOG_PARITY.md` §3):
//! - root `depot.items[]`; `path` with `\` → `/` and one leading `/` stripped;
//! - a file is kept only when `path` is non-empty AND `chunks[]` is non-empty;
//! - per chunk: CDN hash = `compressedMd5` when non-empty, else `md5`; a chunk with neither is
//! dropped; `compressedSize` / `size` default 0;
//! - file `md5` kept when non-empty; `total_size` = Σ chunk `size` (decompressed);
//! - a file whose chunks all dropped is not added.
//! `org.json` coercion (`optString` turns numbers/bools into their text, `optLong` parses numeric
//! strings) is reproduced by [`opt_string`] / [`opt_u64`].
//!
//! Additionally each chunk carries `offset` = prefix-sum of the preceding chunks' decompressed
//! sizes — Java appends sequentially so it never needs one; the Rust sink writes chunks at that
//! offset as they arrive so the final bytes are identical.
use serde_json::Value;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ChunkRef {
/// CDN-path hash (`compressedMd5` preferred, else `md5`).
pub hash: String,
/// Verify compressed bytes before inflate (may be empty).
pub compressed_md5: String,
/// Verify decompressed bytes after inflate (may be empty).
pub md5: String,
/// Expected compressed size, 0 if absent.
pub compressed_size: u64,
/// Expected decompressed size, 0 if absent.
pub size: u64,
/// Decompressed offset inside the file (prefix-sum of preceding `size`s).
pub offset: u64,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PlannedFile {
/// Install-dir-relative path, forward slashes, no leading slash.
pub relative_path: String,
/// Whole-file MD5 from the manifest ("" when absent).
pub md5: String,
/// Σ chunk decompressed sizes (0 if unknown).
pub total_size: u64,
pub chunks: Vec<ChunkRef>,
}
/// `JSONObject.optString(key, "")` semantics: strings as-is, numbers/bools rendered, null/absent
/// → "". (`org.json` renders `JSONObject.NULL` as "null"; GOG manifests never carry JSON nulls in
/// these fields, and treating one as absent is the only safe reading.)
pub fn opt_string(obj: &Value, key: &str) -> String {
match obj.get(key) {
Some(Value::String(s)) => s.clone(),
@@ -54,8 +28,6 @@ pub fn opt_string(obj: &Value, key: &str) -> String {
}
}
/// `JSONObject.optLong(key, 0)` semantics for a size field (numbers, or numeric strings; a
/// negative/float value that Java would truncate is clamped to 0 — sizes are never negative).
pub fn opt_u64(obj: &Value, key: &str) -> u64 {
match obj.get(key) {
Some(Value::Number(n)) => n
@@ -68,8 +40,6 @@ pub fn opt_u64(obj: &Value, key: &str) -> u64 {
}
}
/// Parses one inflated gen2 depot manifest and appends its files to `out`
/// (`GogDownloadManager.parseDepotManifest`). Malformed JSON adds nothing, like Java's catch-all.
pub fn parse_depot_manifest(json: &str, out: &mut Vec<PlannedFile>) {
let Ok(root) = serde_json::from_str::<Value>(json) else {
return;
@@ -137,20 +107,14 @@ pub fn parse_depot_manifest(json: &str, out: &mut Vec<PlannedFile>) {
}
}
/// One gen1 file: a byte range of a (pre-signed) depot blob (`GogDownloadManager.Gen1File`).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Gen1File {
/// Install-dir-relative path, used verbatim (Java's gen1 path does not normalise slashes).
pub path: String,
pub url: String,
pub offset: u64,
pub size: u64,
}
/// Parses a gen1 build manifest (`runGen1`): every `depot[]` entry that is not `support: true`,
/// its `files[]` → (`path`, `url`, `offset` default 0, `size` default 0); an entry with
/// `size == 0` is dropped (Java's `path == null || url == null` checks never fire — `optString`
/// returns "" — so `size == 0` is the only effective filter, mirrored here).
pub fn parse_gen1_manifest(json: &str, out: &mut Vec<Gen1File>) {
let Ok(root) = serde_json::from_str::<Value>(json) else {
return;
@@ -184,10 +148,6 @@ pub fn parse_gen1_manifest(json: &str, out: &mut Vec<Gen1File>) {
}
}
/// Parses every manifest in order (the order Java fetched the depots) and optionally applies the
/// base-install largest-first (LPT) ordering — a STABLE sort by descending `total_size`, exactly
/// `Collections.sort(files, (a, b) -> Long.compare(b.totalSize, a.totalSize))`. The DLC and
/// dependency paths in Java do not sort, so they pass `sort_largest_first = false`.
pub fn build_plan(manifests: &[String], sort_largest_first: bool) -> Vec<PlannedFile> {
let mut files = Vec::new();
for manifest in manifests {
@@ -199,8 +159,6 @@ pub fn build_plan(manifests: &[String], sort_largest_first: bool) -> Vec<Planned
files
}
/// `GogDownloadManager.buildCdnPath`: `"ab/cd/abcdef..."`. A hash shorter than 4 chars would
/// throw in Java (and fail that file); here it is returned unchanged and fails at fetch time.
pub fn build_cdn_path(hash: &str) -> String {
if !hash.is_char_boundary(2) || !hash.is_char_boundary(4) || hash.len() < 4 {
return hash.to_string();
@@ -208,8 +166,6 @@ pub fn build_cdn_path(hash: &str) -> String {
format!("{}/{}/{}", &hash[0..2], &hash[2..4], hash)
}
/// `GogDownloadManager.buildChunkUrl`: appends the chunk path BEFORE any query string so the
/// secure-link token survives.
pub fn build_chunk_url(base: &str, chunk_path: &str) -> String {
match base.find('?') {
Some(q) => format!("{}/{}{}", &base[..q], chunk_path, &base[q..]),
@@ -217,8 +173,6 @@ pub fn build_chunk_url(base: &str, chunk_path: &str) -> String {
}
}
/// Per-host key for the fetch core's host semaphores: `scheme://host[:port]` of `url` (lowercased
/// host). Java has one resolved CDN base per run, so the host list has exactly one entry.
pub fn host_key(url: &str) -> String {
let (scheme, rest) = match url.find("://") {
Some(i) => (&url[..i], &url[i + 3..]),
+1 -18
View File
@@ -1,21 +1,4 @@
//! DroidDeck's native store download engines (`libdroiddeckstores.so`).
//!
//! The GOG, Epic Games Store and Amazon Games download loops, ported from Bannerlator's
//! `bl-steam-client` crate (GPL-3.0-or-later, itself derived from WinNative's `wn-steam-client`).
//! Only the store engines and the fetch core they run on came across: DroidDeck's Steam client is
//! the real one, running in the Linux runtime, so none of the Steam CM / depot code is here.
//!
//! Each store module owns its manifest → plan → `FetchItem` mapping, its `FetchSink` (inflate +
//! hash + write in the exact output layout of the Kotlin manager that drives it) and its JNI facade.
//! The JNI contract — classes, method names, signatures, threading — is written down in `JNI.md`
//! next to `Cargo.toml`; the Kotlin side mirrors it exactly.
//!
//! What a run looks like: Kotlin hands over a manifest it already parsed (or, for Amazon, the
//! plan it built from one), the resolved CDN base(s) and the install directory, calls `nativeStart`
//! and gets a handle back at once. Everything after that happens on native threads: the fetch core
//! keeps an adaptive window of HTTP requests in flight on one tokio runtime, hands bodies to a sync
//! process pool that inflates, hashes and writes them, and reports progress through the listener
//! object Kotlin passed in. `nativeCancel` flips a flag; `onComplete` fires exactly once.
//! Ported from Bannerlator's bl-steam-client crate (GPL-3.0-or-later), itself derived from WinNative's wn-steam-client.
#![allow(clippy::missing_safety_doc, clippy::result_large_err)]
-10
View File
@@ -1,17 +1,10 @@
//! Minimal MD5 (RFC 1321) for the store download engines (GOG chunk / file verification).
//!
//! In-crate on purpose: the crate is built `--locked` in CI with no way to add a dependency from
//! this environment, and the algorithm is 100 lines. Not for anything security-sensitive — it only
//! mirrors the checksum the Java managers already compare against.
/// One-shot digest.
pub fn md5(data: &[u8]) -> [u8; 16] {
let mut h = Md5::new();
h.update(data);
h.finalize()
}
/// Lower-case hex of a 16-byte digest.
pub fn md5_hex(digest: &[u8; 16]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = String::with_capacity(32);
@@ -22,7 +15,6 @@ pub fn md5_hex(digest: &[u8; 16]) -> String {
out
}
/// Streaming MD5: `update` any number of times, then `finalize`.
#[derive(Clone, Debug)]
pub struct Md5 {
state: [u32; 4],
@@ -93,7 +85,6 @@ impl Md5 {
let bit_len = self.total_len.wrapping_mul(8);
let mut pad = [0u8; 72];
pad[0] = 0x80;
// Pad to 56 mod 64, then append the 64-bit little-endian bit length.
let pad_len = if self.buffer_len < 56 {
56 - self.buffer_len
} else {
@@ -189,7 +180,6 @@ mod tests {
#[test]
fn padding_edge_lengths() {
// 55 / 56 / 64 bytes exercise the "pad fits in one block" vs "needs a second block" paths.
assert_eq!(hex(&[b'a'; 55]), "ef1772b6dff9a122358552954ad0df65");
assert_eq!(hex(&[b'a'; 56]), "3b0c8ac703f828b04c6c197006d17218");
assert_eq!(hex(&[b'a'; 64]), "014842d480b571495a4a0363793f7367");
-18
View File
@@ -1,22 +1,5 @@
//! Background priority for every thread the store engines create.
//!
//! A download keeps several threads busy at once (the run's worker, the tokio runtime and its
//! blocking pool for DNS, the inflate/hash/write pool, the throughput reporter, GOG's verify pass,
//! the Epic assembler). At the default nice 0 they compete as equals with the app's UI and input
//! threads, and on the device the menus' animations and controller navigation slowed visibly for
//! as long as a store download ran. At nice +10 the scheduler hands them only what the UI leaves,
//! which for a download bound by network and storage costs next to nothing.
//!
//! Linux applies `setpriority(PRIO_PROCESS, tid, …)` to the one thread `tid`, not the process, and
//! a new thread inherits its creator's nice value; every thread is still lowered explicitly at its
//! start so the guarantee does not depend on who spawned it. Only threads this library creates are
//! touched: the JNI calling thread belongs to the app and keeps its priority.
/// The nice value store-engine threads run at.
pub const BACKGROUND_NICE: i32 = 10;
/// Lower the calling thread to [`BACKGROUND_NICE`]. Best effort: a refusal leaves the thread at
/// its current priority and the download runs as before.
pub fn background() {
#[cfg(any(target_os = "android", target_os = "linux"))]
unsafe {
@@ -25,7 +8,6 @@ pub fn background() {
}
}
/// The calling thread's nice value as a log field, e.g. `priority=nice10`.
pub fn log_field() -> String {
#[cfg(any(target_os = "android", target_os = "linux"))]
{
@@ -1,19 +1,10 @@
//! The one export that is not a store engine: `com.droiddeck.launcher.stores.StoresNative`.
//!
//! `nativeVersion()` exists so the app can prove, at Setup time, that `libdroiddeckstores.so`
//! loaded and its JNI exports bind — before a user is mid-download and an
//! `UnsatisfiedLinkError` would surface as a failed install. It returns the crate version from
//! `Cargo.toml`, which the app can show next to the other component versions.
use jni::objects::JClass;
use jni::sys::jstring;
use jni::JNIEnv;
/// The crate version, as `Cargo.toml` declares it.
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
/// `StoresNative.nativeVersion(): String` — the crate version (`"0.1.0"`). A null return means the
/// JVM could not allocate the string, which the caller treats like a failed load.
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion(
env: JNIEnv,