mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-20 13:26:02 +00:00
cleanup (#764)
This commit is contained in:
1 parent
5fd47421f9
commit
aecd2ebe1d
852 files changed
+4844
-46144
No files matched your search
Generated
-2
@@ -1,5 +1,3 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -2,9 +2,6 @@
|
||||
name = "droiddeckstores"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
# The engines are a port of Bannerlator's bl-steam-client store modules (GPL-3.0-or-later, derived
|
||||
# from WinNative's wn-steam-client; the Epic selection/verify rules trace back to GameNative and
|
||||
# Legendary, both GPL-3.0). DroidDeck is GPL-3.0, so the crate keeps the licence it came with.
|
||||
license = "GPL-3.0-or-later"
|
||||
publish = false
|
||||
|
||||
@@ -14,24 +11,12 @@ crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
jni = "0.21"
|
||||
# setpriority/gettid: every engine thread runs at nice +10 (see src/priority.rs). Already in the
|
||||
# dependency tree through tokio, at the version Cargo.lock pins.
|
||||
libc = "0.2"
|
||||
# zlib inflate for GOG gen2 chunks and Epic chunk bodies. The pure-Rust backend keeps the .so free
|
||||
# of a C zlib and of the NDK sysroot's libz, which the apk does not ship.
|
||||
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
|
||||
# The fetch core drives every request through ONE tokio current-thread runtime and feeds a sync
|
||||
# process pool; rustls (via ring) is the TLS stack so there is no OpenSSL to cross-compile. `gzip`
|
||||
# transparently inflates a CDN answer that comes back content-encoded. No `blocking` client: the
|
||||
# three engines are async-fetch only.
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip"] }
|
||||
tokio = { version = "1", default-features = false, features = ["rt", "net", "time", "sync"] }
|
||||
futures-util = { version = "0.3", default-features = false, features = ["std"] }
|
||||
# preserve_order keeps JSON object keys in source order, as in Bannerlator, so the GOG depot plan
|
||||
# walks a manifest's files in the order the manifest lists them (largest-first sorting happens
|
||||
# after, explicitly).
|
||||
serde_json = { version = "1", features = ["preserve_order"] }
|
||||
# Epic chunks are SHA-1 verified, Amazon files SHA-256; GOG's MD5 is the in-tree md5_small.
|
||||
sha1 = "0.10"
|
||||
sha2 = "0.10"
|
||||
|
||||
|
||||
@@ -1,22 +1,3 @@
|
||||
//! Amazon Games download adapter over the shared fetch core.
|
||||
//!
|
||||
//! Replaces ONLY the byte-fetching pool of `AmazonDownloadManager.install()` (Step 4).
|
||||
//! Java still resolves the download spec, fetches and parses `manifest.proto`, and hands
|
||||
//! this module the resolved file list (one JSON entry per manifest file with the exact
|
||||
//! URL the Java loop would have opened). The adapter mirrors the Java rules 1:1 where the
|
||||
//! core allows it — see Bannerlator's `docs/RUST_AMAZON_PARITY.md` §4 / §9 for the table and the
|
||||
//! documented deviations:
|
||||
//!
|
||||
//! * one `FetchItem` per WHOLE file, streamed (`FetchOptions.stream = true`): pieces are
|
||||
//! appended to `<dest>.tmp` as they arrive, 8 files wide like Java's pool, memory bounded
|
||||
//! to the pieces not yet written;
|
||||
//! * resume-skip = `st_size == manifest size`, no hash check on skip (Java `:250`);
|
||||
//! * SHA-256 is accumulated while streaming and checked in `on_finish` when the manifest
|
||||
//! carries a hash; a mismatch deletes the tmp and is a retryable failure (Java `:269`);
|
||||
//! * `<dest>.tmp` → delete existing dest → rename (Java `:280`); rename failure is fatal
|
||||
//! without retry (Java `:281`);
|
||||
//! * a piece credits its wire length to progress as it is written (Java `:347`), and pieces
|
||||
//! of an attempt that later fails stay counted (Java never rolls `totalDownloaded` back).
|
||||
|
||||
pub mod jni;
|
||||
|
||||
@@ -30,34 +11,20 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Java `AmazonDownloadManager.MAX_PARALLEL`.
|
||||
pub const MAX_PARALLEL: usize = 8;
|
||||
/// Java `AmazonDownloadManager.DOWNLOAD_USER_AGENT`.
|
||||
pub const DOWNLOAD_USER_AGENT: &str = "nile/0.1 Amazon";
|
||||
/// Java `conn.setReadTimeout(120000)`; in stream mode the core applies this to the response
|
||||
/// headers and then as an idle deadline per received piece — the same shape as a read timeout.
|
||||
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
/// Java `new File(installDir, file.unixPath() + ".tmp")`.
|
||||
pub const TMP_SUFFIX: &str = ".tmp";
|
||||
/// Log label used in the core's `fetch-window` lines.
|
||||
pub const LABEL: &str = "amazon";
|
||||
|
||||
/// One resolved manifest file as serialised by Java (`AmazonDownloadManager.buildRustPlan`).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PlanEntry {
|
||||
/// `ManifestFile.unixPath()` — relative to the install dir, forward slashes.
|
||||
pub rel_path: String,
|
||||
/// `AmazonApiClient.appendPath(downloadUrl, "files/" + hashHex)` — signed, whole file.
|
||||
pub url: String,
|
||||
/// `ManifestFile.size`.
|
||||
pub size: u64,
|
||||
/// SHA-256 to verify against; EMPTY = Java would not verify (`hashAlgorithm != 0` or
|
||||
/// no hash bytes).
|
||||
pub sha256: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Parse the JSON array `[{relPath, url, size, sha256hex}]`. `relPath` is normalised the same
|
||||
/// way as `ManifestFile.unixPath()` so a caller that forgot is still byte-identical.
|
||||
pub fn parse_plan(json: &str) -> Result<Vec<PlanEntry>, String> {
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(json).map_err(|err| format!("plan json: {err}"))?;
|
||||
@@ -106,12 +73,10 @@ pub fn parse_plan(json: &str) -> Result<Vec<PlanEntry>, String> {
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// `ManifestFile.unixPath()`.
|
||||
pub fn unix_path(path: &str) -> String {
|
||||
path.replace('\\', "/")
|
||||
}
|
||||
|
||||
/// Lowercase/uppercase hex → bytes; `None` on odd length or a non-hex digit.
|
||||
pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
||||
let bytes = hex.as_bytes();
|
||||
if bytes.len() % 2 != 0 {
|
||||
@@ -132,29 +97,21 @@ pub fn hex_decode(hex: &str) -> Option<Vec<u8>> {
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// `new File(installDir, rel)` — Java resolves the child textually under the parent, so
|
||||
/// build the path by concatenation (never `Path::join`, which would let a leading `/` escape).
|
||||
pub fn dest_path(install_dir: &str, rel_path: &str) -> PathBuf {
|
||||
let base = install_dir.trim_end_matches('/');
|
||||
PathBuf::from(format!("{base}/{rel_path}"))
|
||||
}
|
||||
|
||||
/// `new File(installDir, rel + ".tmp")`.
|
||||
pub fn tmp_path(install_dir: &str, rel_path: &str) -> PathBuf {
|
||||
dest_path(install_dir, &format!("{rel_path}{TMP_SUFFIX}"))
|
||||
}
|
||||
|
||||
/// Java `:250`: `destFile.exists() && destFile.length() == file.size`. `File.length()` is
|
||||
/// `st_size`, which is what `Metadata::len()` returns — including for a directory, so the
|
||||
/// (degenerate) behaviours match too. No hash is checked on skip, on purpose.
|
||||
pub fn is_present_and_complete(install_dir: &str, entry: &PlanEntry) -> bool {
|
||||
fs::metadata(dest_path(install_dir, &entry.rel_path))
|
||||
.map(|meta| meta.len() == entry.size)
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// `scheme://host[:port]` of a URL — the per-host cap key. Amazon serves every file from the
|
||||
/// one signed CDN base, so this yields a single host.
|
||||
pub fn host_key(url: &str) -> String {
|
||||
let (scheme, rest) = match url.find("://") {
|
||||
Some(idx) => (&url[..idx], &url[idx + 3..]),
|
||||
@@ -164,33 +121,22 @@ pub fn host_key(url: &str) -> String {
|
||||
format!("{scheme}://{}", &rest[..end])
|
||||
}
|
||||
|
||||
/// Improvements round 1: `max(6, ceil(max_workers / distinct_hosts))` — a single-host store
|
||||
/// (Amazon's one signed a2z base) gets the whole ceiling on its one host; the core clamps the
|
||||
/// window to `hosts × per_host_cap`, so anything smaller would silently cut the ceiling.
|
||||
pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize {
|
||||
let hosts = distinct_hosts.max(1);
|
||||
let per_host = max_workers.max(1).div_ceil(hosts);
|
||||
per_host.max(crate::fetch_tuning::PER_HOST_CAP)
|
||||
}
|
||||
|
||||
/// The resolved work for one run.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Plan {
|
||||
/// Items to fetch (index into `entries` via `FetchItem.id`).
|
||||
pub items: Vec<FetchItem>,
|
||||
/// Every manifest file, in manifest order (fetched and skipped alike).
|
||||
pub entries: Vec<PlanEntry>,
|
||||
/// Σ size of all entries = `ParsedManifest.totalInstallSize`.
|
||||
pub total_bytes: u64,
|
||||
/// Bytes credited up front for files that passed the resume check.
|
||||
pub skipped_bytes: u64,
|
||||
/// Number of files that passed the resume check.
|
||||
pub skipped_files: u64,
|
||||
/// Distinct host keys in fetch order.
|
||||
pub hosts: Vec<String>,
|
||||
}
|
||||
|
||||
/// Apply the Java resume-skip rule to every entry and turn the rest into whole-file items.
|
||||
pub fn build_plan(entries: Vec<PlanEntry>, install_dir: &str) -> Plan {
|
||||
let mut plan = Plan {
|
||||
total_bytes: entries.iter().map(|e| e.size).sum(),
|
||||
@@ -220,23 +166,17 @@ pub fn build_plan(entries: Vec<PlanEntry>, install_dir: &str) -> Plan {
|
||||
plan
|
||||
}
|
||||
|
||||
/// One `<dest>.tmp` being streamed: the open handle, the running hash and the bytes written
|
||||
/// by the current attempt.
|
||||
struct OpenTmp {
|
||||
file: File,
|
||||
hasher: Sha256,
|
||||
written: u64,
|
||||
}
|
||||
|
||||
/// Sink: stream pieces into `<dest>.tmp` (SHA-256 accumulated on the fly), verify and rename
|
||||
/// in `on_finish`. `process` (whole-body mode) does the same in one step.
|
||||
pub struct AmazonSink {
|
||||
install_dir: String,
|
||||
entries: Vec<PlanEntry>,
|
||||
cancel: Arc<AtomicBool>,
|
||||
files_done: AtomicU64,
|
||||
/// Per-item open tmp files. The map lock is held only to look the entry up; the write
|
||||
/// itself happens under the per-item lock so 8 streams never serialise on one mutex.
|
||||
open: Mutex<HashMap<u64, Arc<Mutex<OpenTmp>>>>,
|
||||
}
|
||||
|
||||
@@ -261,16 +201,12 @@ impl AmazonSink {
|
||||
.ok_or_else(|| SinkError::Fatal(format!("plan index {} out of range", item.id)))
|
||||
}
|
||||
|
||||
/// Java `:260` — `destFile.getParentFile().mkdirs()` before each attempt; a failure
|
||||
/// surfaces on the create/write below, like Java's.
|
||||
fn ensure_parent(&self, entry: &PlanEntry) {
|
||||
if let Some(parent) = dest_path(&self.install_dir, &entry.rel_path).parent() {
|
||||
let _ = fs::create_dir_all(parent);
|
||||
}
|
||||
}
|
||||
|
||||
/// Java `:280-284`: delete an existing dest, rename tmp → dest; rename failure fails the
|
||||
/// file with NO retry (→ whole install fails).
|
||||
fn rename_into_place(&self, entry: &PlanEntry) -> Result<(), SinkError> {
|
||||
let dest = dest_path(&self.install_dir, &entry.rel_path);
|
||||
let tmp = tmp_path(&self.install_dir, &entry.rel_path);
|
||||
@@ -288,8 +224,6 @@ impl AmazonSink {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Java `:267-277`: SHA-256 check when the manifest carries a hash; a mismatch deletes the
|
||||
/// tmp and is retried.
|
||||
fn verify(&self, entry: &PlanEntry, digest: &[u8]) -> Result<(), SinkError> {
|
||||
if !entry.sha256.is_empty() && digest != entry.sha256.as_slice() {
|
||||
let _ = fs::remove_file(tmp_path(&self.install_dir, &entry.rel_path));
|
||||
@@ -301,7 +235,6 @@ impl AmazonSink {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whole-body path (`FetchOptions.stream = false`): verify → write tmp → rename.
|
||||
pub fn commit(&self, entry: &PlanEntry, body: &[u8]) -> Result<u64, SinkError> {
|
||||
if !entry.sha256.is_empty() {
|
||||
let digest = Sha256::digest(body);
|
||||
@@ -322,10 +255,6 @@ impl AmazonSink {
|
||||
Ok(body.len() as u64)
|
||||
}
|
||||
|
||||
/// After the run: close and delete every tmp that never reached `on_finish` (a stream that
|
||||
/// failed after max attempts, or was cut by cancel / a fatal error). Java deletes the tmp
|
||||
/// of every failed or cancelled attempt (`:271`, `:288`, `:293`, `:297`), so no `.tmp`
|
||||
/// may survive a run on either engine. Returns how many were removed.
|
||||
pub fn cleanup_partials(&self) -> usize {
|
||||
let leftovers: Vec<(u64, Arc<Mutex<OpenTmp>>)> = match self.open.lock() {
|
||||
Ok(mut open) => open.drain().collect(),
|
||||
@@ -355,13 +284,11 @@ impl FetchSink for AmazonSink {
|
||||
|
||||
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
if self.cancel.load(Ordering::Relaxed) {
|
||||
// Java `:342-344`: cancel inside the read loop → tmp deleted (cleanup_partials).
|
||||
return Err(SinkError::Fatal("cancelled".to_string()));
|
||||
}
|
||||
let entry = self.entry(item)?;
|
||||
let tmp = tmp_path(&self.install_dir, &entry.rel_path);
|
||||
let open = if offset == 0 {
|
||||
// (Re)start of an attempt: Java opens a fresh FileOutputStream (truncating).
|
||||
self.ensure_parent(entry);
|
||||
let file = File::create(&tmp).map_err(|err| {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
@@ -405,7 +332,6 @@ impl FetchSink for AmazonSink {
|
||||
)));
|
||||
}
|
||||
if let Err(err) = state.file.write_all(data) {
|
||||
// Java: IOException inside downloadFile → tmp deleted → next attempt.
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(SinkError::Retry(format!("write {}: {err}", tmp.display())));
|
||||
}
|
||||
@@ -435,7 +361,6 @@ impl FetchSink for AmazonSink {
|
||||
(written, digest)
|
||||
}
|
||||
None => {
|
||||
// Zero-byte body: no piece ever arrived. Java would have created an empty tmp.
|
||||
if total_len != 0 {
|
||||
return Err(SinkError::Retry(format!(
|
||||
"finish without a stream for: {}",
|
||||
@@ -462,22 +387,16 @@ impl FetchSink for AmazonSink {
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of one `run_download`.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct RunResult {
|
||||
pub success: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: String,
|
||||
/// Bytes fetched by this run (skipped files excluded; failed attempts' pieces included,
|
||||
/// like Java's `totalDownloaded`).
|
||||
pub bytes_written: u64,
|
||||
/// Files committed by this run plus files skipped by the resume check.
|
||||
pub files_done: u64,
|
||||
pub files_total: u64,
|
||||
}
|
||||
|
||||
/// Throughput bookkeeping for the final summary line (avg over the run, peak over ≥500 ms
|
||||
/// progress intervals — the same sampling window the Java speed label uses).
|
||||
struct SpeedMeter {
|
||||
started: Instant,
|
||||
last_at: Instant,
|
||||
@@ -516,10 +435,6 @@ impl SpeedMeter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Blocking: plan → stream-fetch → verify/rename, calling `progress(bytes_done, bytes_total,
|
||||
/// files_done, files_total)` after each written piece and each finished file (skipped files
|
||||
/// are pre-credited) and `log` for every diagnostic line. `bytes_done` includes skipped bytes,
|
||||
/// matching Java's `totalDownloaded` (which credits skipped files at `:251`).
|
||||
pub fn run_download(
|
||||
plan_json: &str,
|
||||
install_dir: &str,
|
||||
@@ -582,16 +497,12 @@ pub fn run_download(
|
||||
|
||||
let opts = FetchOptions {
|
||||
max_workers,
|
||||
// Every Amazon file comes from the one signed CDN base: give that host the whole
|
||||
// ceiling (`per_host_cap_for`), or the core would clamp the window to hosts × cap.
|
||||
per_host_cap,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
headers: vec![("User-Agent".to_string(), DOWNLOAD_USER_AGENT.to_string())],
|
||||
ca_bundle_path: ca_bundle_path.to_string(),
|
||||
process_workers,
|
||||
label: LABEL.to_string(),
|
||||
// Whole files, streamed into the tmp as they arrive: 8 wide like Java's pool, memory
|
||||
// bounded to the pieces not yet written.
|
||||
stream: true,
|
||||
..FetchOptions::default()
|
||||
};
|
||||
@@ -753,10 +664,8 @@ mod tests {
|
||||
fn build_plan_skips_only_size_matched_files() {
|
||||
let dir = scratch_dir();
|
||||
let entries = parse_plan(PLAN).unwrap();
|
||||
// Entry 1 present with the right size → skipped (no hash check even if wrong bytes).
|
||||
fs::create_dir_all(format!("{dir}/data")).unwrap();
|
||||
fs::write(format!("{dir}/data/level1.pak"), b"zz").unwrap();
|
||||
// Entry 0 present with the wrong size → fetched.
|
||||
fs::create_dir_all(format!("{dir}/Binaries/Win64")).unwrap();
|
||||
fs::write(format!("{dir}/Binaries/Win64/Game.exe"), b"tes").unwrap();
|
||||
let plan = build_plan(entries, &dir);
|
||||
@@ -803,7 +712,6 @@ mod tests {
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::clone(&cancel));
|
||||
let item = item_for(&entry, 0);
|
||||
// Mismatch → Retry, nothing on disk.
|
||||
match sink.process(&item, b"nope".to_vec()) {
|
||||
Err(SinkError::Retry(msg)) => assert!(msg.contains("SHA-256 mismatch")),
|
||||
Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"),
|
||||
@@ -811,14 +719,11 @@ mod tests {
|
||||
}
|
||||
assert!(!dest_path(&dir, "sub/Game.exe").exists());
|
||||
assert!(!tmp_path(&dir, "sub/Game.exe").exists());
|
||||
// Match → written, renamed, tmp gone, credited body length.
|
||||
assert_eq!(sink.process(&item, body.clone()).unwrap(), 4);
|
||||
assert_eq!(fs::read(dest_path(&dir, "sub/Game.exe")).unwrap(), body);
|
||||
assert!(!tmp_path(&dir, "sub/Game.exe").exists());
|
||||
assert_eq!(sink.files_done(), 1);
|
||||
// Existing dest is replaced (Java `:280`).
|
||||
assert_eq!(sink.process(&item, body.clone()).unwrap(), 4);
|
||||
// Cancel → Fatal, no write.
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
assert!(matches!(
|
||||
sink.process(&item, body),
|
||||
@@ -863,7 +768,6 @@ mod tests {
|
||||
};
|
||||
let sink = AmazonSink::new(&dir, vec![entry.clone()], Arc::new(AtomicBool::new(false)));
|
||||
let item = item_for(&entry, 0);
|
||||
// Attempt 1 delivers garbage, then the core restarts at offset 0.
|
||||
sink.on_chunk(&item, 0, b"zzzz").unwrap();
|
||||
sink.on_chunk(&item, 0, b"abc").unwrap();
|
||||
sink.on_chunk(&item, 3, b"xyz").unwrap();
|
||||
@@ -874,21 +778,17 @@ mod tests {
|
||||
}
|
||||
assert!(!tmp_path(&dir, "f.bin").exists());
|
||||
assert!(!dest_path(&dir, "f.bin").exists());
|
||||
// Out-of-order piece → Retry.
|
||||
sink.on_chunk(&item, 0, b"abc").unwrap();
|
||||
assert!(matches!(
|
||||
sink.on_chunk(&item, 5, b"f"),
|
||||
Err(SinkError::Retry(_))
|
||||
));
|
||||
// Length mismatch at finish → Retry.
|
||||
sink.on_chunk(&item, 0, b"abc").unwrap();
|
||||
assert!(matches!(sink.on_finish(&item, 6), Err(SinkError::Retry(_))));
|
||||
// Piece without a start → Retry.
|
||||
assert!(matches!(
|
||||
sink.on_chunk(&item, 3, b"def"),
|
||||
Err(SinkError::Retry(_))
|
||||
));
|
||||
// Good attempt.
|
||||
sink.on_chunk(&item, 0, b"abc").unwrap();
|
||||
sink.on_chunk(&item, 3, b"def").unwrap();
|
||||
assert_eq!(sink.on_finish(&item, 6).unwrap(), 0);
|
||||
@@ -915,10 +815,8 @@ mod tests {
|
||||
let sink = AmazonSink::new(&dir, vec![big.clone(), empty.clone()], Arc::clone(&cancel));
|
||||
sink.on_chunk(&item_for(&big, 0), 0, b"12345").unwrap();
|
||||
assert!(tmp_path(&dir, "cut.bin").exists());
|
||||
// Zero-byte body: on_finish with no pieces creates and commits an empty file.
|
||||
assert_eq!(sink.on_finish(&item_for(&empty, 1), 0).unwrap(), 0);
|
||||
assert_eq!(fs::read(dest_path(&dir, "empty.bin")).unwrap(), b"");
|
||||
// Cancel mid-stream → Fatal; cleanup deletes the partial tmp like Java does.
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
assert!(matches!(
|
||||
sink.on_chunk(&item_for(&big, 0), 5, b"6"),
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
//! JNI facade for the Amazon download adapter.
|
||||
//!
|
||||
//! Kotlin side: `com.droiddeck.launcher.stores.amazon.AmazonNative` (`nativeStart` /
|
||||
//! `nativeCancel` / `nativeRelease`) with an `AmazonNative.Listener`
|
||||
//! (`onProgress(JJJJ)V`, `onLog(String)V`, `onComplete(ZString;J)V`). Mirrors the shape of
|
||||
//! the GOG and Epic facades: the run happens on a spawned native
|
||||
//! thread, callbacks attach the JavaVM as daemon threads and hold a `GlobalRef` to the
|
||||
//! listener; cancel flips an `AtomicBool`. This module keeps its own `JavaVM` handle (taken
|
||||
//! from the calling `JNIEnv` on first use) so it depends on nothing outside this module.
|
||||
|
||||
use crate::amazon::{run_download, MAX_PARALLEL};
|
||||
use jni::objects::{GlobalRef, JClass, JObject, JString, JValue};
|
||||
@@ -16,10 +7,6 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::thread;
|
||||
|
||||
/// Logcat tag (`adb logcat -s AmazonNative`). Every engine line reaches logcat exactly ONCE:
|
||||
/// this module only forwards lines to the listener (`onLog`), and the Java manager is the one
|
||||
/// place that writes them to `android.util.Log` + its debug file. (Round 1 fix: the native side
|
||||
/// used to also call `__android_log_write`, so each line appeared twice.)
|
||||
pub const LOG_TAG: &str = "AmazonNative";
|
||||
|
||||
static JVM: OnceLock<JavaVM> = OnceLock::new();
|
||||
@@ -36,7 +23,6 @@ fn jstring_to_string(env: &mut JNIEnv, value: &JString) -> Option<String> {
|
||||
.map(|s| s.to_string_lossy().into_owned())
|
||||
}
|
||||
|
||||
/// One live run; boxed and handed to Kotlin as an opaque `jlong`.
|
||||
struct AmazonRun {
|
||||
cancel: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -115,7 +101,6 @@ fn call_complete(
|
||||
clear_pending_exception(env);
|
||||
}
|
||||
|
||||
/// Attach the current (native) thread and run `f` with the listener object.
|
||||
fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)) {
|
||||
let Some(vm) = JVM.get() else {
|
||||
return;
|
||||
@@ -126,9 +111,6 @@ fn with_attached_env(listener: &GlobalRef, f: impl FnOnce(&mut JNIEnv, &JObject)
|
||||
f(&mut env, listener.as_obj());
|
||||
}
|
||||
|
||||
/// `AmazonNative.nativeStart(planJson, installDir, caBundlePath, maxWorkers,
|
||||
/// processWorkers, listener) -> handle`. Returns 0 (and fires `onComplete(false, …)`) when the
|
||||
/// inputs are unusable. The run itself happens on a spawned thread.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeStart(
|
||||
mut env: JNIEnv,
|
||||
@@ -200,8 +182,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_na
|
||||
to_handle(run)
|
||||
}
|
||||
|
||||
/// `AmazonNative.nativeCancel(handle)` — flips the run's cancel flag; the core stops
|
||||
/// issuing requests, in-flight bodies are dropped, `onComplete(false, "cancelled", …)` follows.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeCancel(
|
||||
_env: JNIEnv,
|
||||
@@ -213,8 +193,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_na
|
||||
}
|
||||
}
|
||||
|
||||
/// `AmazonNative.nativeRelease(handle)` — frees the handle. Safe at any time: the worker
|
||||
/// thread owns its own `Arc` of the cancel flag and the listener `GlobalRef`.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_amazon_AmazonNative_nativeRelease(
|
||||
_env: JNIEnv,
|
||||
|
||||
@@ -1,29 +1,4 @@
|
||||
//! Epic Games Store download adapter for the shared fetch core (`crate::fetch_core`).
|
||||
//!
|
||||
//! Scope = the chunk-fetch inner loop of `EpicDownloadManager.java` (the fixed 8-thread pool that
|
||||
//! fills `<installDir>/.chunks/<GUID>`), nothing more. The Java manager still parses the manifest
|
||||
//! API JSON, downloads the manifest, selects files (install tags), runs the delta/verify pass,
|
||||
//! assembles files from the chunk cache and does every post-install step. The adapter re-parses
|
||||
//! the same manifest bytes, rebuilds the same chunk plan for the pending files Java hands it,
|
||||
//! skips chunks already in the cache and writes verified chunks with the same `.part` + rename
|
||||
//! protocol, so the on-disk state is identical whichever engine ran (see
|
||||
//! Bannerlator's `docs/RUST_EPIC_PARITY.md` for the rule-by-rule table).
|
||||
//!
|
||||
//! Submodules:
|
||||
//! - [`manifest`] — ChunksV4 binary manifest + legacy JSON manifest parsers (1:1 with the Java
|
||||
//! `parseManifest` / `parseJsonManifest`).
|
||||
//! - [`plan`] — install-tag selection, unique-chunk plan, chunk paths / CDN URLs.
|
||||
//! - [`chunk`] — chunk header parse, zlib inflate, SHA-1 verify, cache write.
|
||||
//! - [`driver`] — plan → `FetchItem`s → `fetch_core::run_fetch` with the chunk-cache sink.
|
||||
//! - [`assemble`] — the files from the cache, one sequential write each, SHA-1 checked as they
|
||||
//! are written; with a scratch cache (one the caller placed apart from the install) each chunk
|
||||
//! is deleted as soon as the last file that uses it is done.
|
||||
//! - [`jni`] — `Java_com_droiddeck_launcher_stores_epic_EpicNative_native*` exports.
|
||||
//!
|
||||
//! The selective-install-tag, delta/resume and chunk verification rules mirrored here were ported
|
||||
//! into the Java manager from GameNative (`service/epic/EpicDownloadManager.kt`,
|
||||
//! `manifest/ManifestUtils.kt`), itself derived from Legendary; both are GPL-3.0 and this crate
|
||||
//! carries the same licence (see `Cargo.toml`).
|
||||
//! Derived from GameNative's Epic manager (itself derived from Legendary); both are GPL-3.0 and this crate carries the same licence (see Cargo.toml).
|
||||
|
||||
pub mod assemble;
|
||||
pub mod chunk;
|
||||
@@ -32,15 +7,11 @@ pub mod jni;
|
||||
pub mod manifest;
|
||||
pub mod plan;
|
||||
|
||||
/// User-Agent the Java manager sends on every chunk request (`EpicDownloadManager.UA`).
|
||||
pub const USER_AGENT: &str =
|
||||
"UELauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit";
|
||||
|
||||
/// Name of the chunk cache directory under the install dir (`new File(installDir, ".chunks")`).
|
||||
pub const CHUNK_CACHE_DIR: &str = ".chunks";
|
||||
|
||||
/// Fixed pool width of the Java chunk downloader (`Executors.newFixedThreadPool(8)`).
|
||||
pub const JAVA_POOL_THREADS: usize = 8;
|
||||
|
||||
/// logcat tag for every engine line (the Kotlin manager mirrors it into its own debug log).
|
||||
pub const LOG_TAG: &str = "EpicNative";
|
||||
@@ -1,26 +1,3 @@
|
||||
//! Files from the chunk cache: the last stage of an Epic install, as a native run.
|
||||
//!
|
||||
//! The Kotlin manager's own loop does this too (`EpicDownloadManager.install`, after the chunk
|
||||
//! fetch): for every pending file, open it, and for every part read the cached chunk and write
|
||||
//! `chunk[offset..offset+size]`. This module does the same thing, in the same order, with the
|
||||
//! same wording on a missing chunk, plus what the Kotlin loop cannot do without re-reading the
|
||||
//! finished file from a slow card:
|
||||
//!
|
||||
//! - the file's SHA-1 is computed from the bytes as they are written (no extra I/O) and compared
|
||||
//! with the manifest's, when the manifest has one; a mismatch deletes the file and fails the
|
||||
//! run, so a bad assembly can never pass the next delta pass by size alone;
|
||||
//! - with a scratch cache — one the caller placed apart from the install directory — every chunk
|
||||
//! is reference-counted across the pending files and deleted the moment the last file that
|
||||
//! needs it is written and verified, so the cache never holds more than what is still owed.
|
||||
//!
|
||||
//! The cache directory itself goes at the end of a successful run, scratch or not, which is what
|
||||
//! the Kotlin loop does with `<installDir>/.chunks`. Cancellation and failure leave the cache
|
||||
//! exactly as it stands, so the next run's delta pass skips the finished files and the fetch
|
||||
//! skips the chunks still cached — on either engine.
|
||||
//!
|
||||
//! Why the cache may live elsewhere at all: an install to an SD card wrote every byte twice to
|
||||
//! the card (chunk cache, then assembled file). With the cache on internal storage the card sees
|
||||
//! each byte once, and the fetchers stop waiting on the card's write speed.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File};
|
||||
@@ -33,33 +10,25 @@ use sha1::{Digest, Sha1};
|
||||
use super::manifest::{parse_manifest, FileInfo, Manifest};
|
||||
use super::plan::resolve_cache_dir;
|
||||
|
||||
/// Everything the JNI layer hands over for one assembly run.
|
||||
pub struct AssembleRequest {
|
||||
pub manifest_bytes: Vec<u8>,
|
||||
pub install_dir: String,
|
||||
/// `""` = `<installDir>/.chunks` (no scratch semantics); anything else = scratch cache.
|
||||
pub chunk_cache_dir: String,
|
||||
/// Indices into `manifest.files` of the files to assemble, in order.
|
||||
pub file_indices: Vec<usize>,
|
||||
}
|
||||
|
||||
/// Terminal result of an assembly run.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct AssembleOutcome {
|
||||
pub success: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: String,
|
||||
/// Bytes written into assembled files this run.
|
||||
pub bytes_written: u64,
|
||||
pub files_done: u64,
|
||||
pub files_total: u64,
|
||||
pub bytes_total: u64,
|
||||
/// Chunk files deleted early because no remaining file needed them (scratch cache only).
|
||||
pub chunks_removed: u64,
|
||||
}
|
||||
|
||||
/// The parsed plan: manifest, the files to write, and how many of them still need each chunk.
|
||||
/// Built up front so a bad index fails before anything is written.
|
||||
#[derive(Debug)]
|
||||
pub struct AssemblePlan {
|
||||
pub manifest: Manifest,
|
||||
@@ -68,12 +37,9 @@ pub struct AssemblePlan {
|
||||
pub scratch_cache: bool,
|
||||
pub files: Vec<usize>,
|
||||
pub bytes_total: u64,
|
||||
/// GUID → number of files in `files` whose parts reference it (a file counts once per GUID,
|
||||
/// however many of its parts use the chunk).
|
||||
pub refs: HashMap<String, u32>,
|
||||
}
|
||||
|
||||
/// Parse + plan. `Err` = nothing was written.
|
||||
pub fn build_assemble_plan(req: &AssembleRequest) -> Result<AssemblePlan, String> {
|
||||
if req.install_dir.is_empty() {
|
||||
return Err("assemble: empty install dir".to_string());
|
||||
@@ -104,7 +70,6 @@ pub fn build_assemble_plan(req: &AssembleRequest) -> Result<AssemblePlan, String
|
||||
})
|
||||
}
|
||||
|
||||
/// How many of `file_indices` reference each chunk GUID.
|
||||
pub fn chunk_refs(m: &Manifest, file_indices: &[usize]) -> HashMap<String, u32> {
|
||||
let mut refs: HashMap<String, u32> = HashMap::new();
|
||||
for &fi in file_indices {
|
||||
@@ -118,7 +83,6 @@ pub fn chunk_refs(m: &Manifest, file_indices: &[usize]) -> HashMap<String, u32>
|
||||
refs
|
||||
}
|
||||
|
||||
/// The GUIDs a file's parts reference, each once, in first-seen order.
|
||||
fn distinct_guids(f: &FileInfo) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::with_capacity(f.parts.len());
|
||||
for p in &f.parts {
|
||||
@@ -130,19 +94,14 @@ fn distinct_guids(f: &FileInfo) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// The manifest's relative path with Windows separators normalised, as the Kotlin loop does
|
||||
/// (`filename.replace("\\", "/")`).
|
||||
pub fn relative_path(f: &FileInfo) -> String {
|
||||
f.filename.replace('\\', "/")
|
||||
}
|
||||
|
||||
/// A manifest SHA-1 that is all zero means "none" (the Kotlin delta pass treats it the same way).
|
||||
fn expected_sha1(f: &FileInfo) -> Option<&[u8; 20]> {
|
||||
f.sha1.as_ref().filter(|h| h.iter().any(|&b| b != 0))
|
||||
}
|
||||
|
||||
/// Chunk bytes, read whole, with the last chunk kept: consecutive parts of one file usually come
|
||||
/// from the same chunk, and a file's parts are the only place a chunk is read.
|
||||
struct ChunkReader {
|
||||
cache_dir: PathBuf,
|
||||
last: Option<(String, Vec<u8>)>,
|
||||
@@ -161,7 +120,6 @@ impl ChunkReader {
|
||||
Ok(self.last.as_ref().map(|(_, d)| d.as_slice()).unwrap_or(&[]))
|
||||
}
|
||||
|
||||
/// Drop the kept copy of a chunk that is about to be deleted.
|
||||
fn forget(&mut self, guid: &str) {
|
||||
if self.last.as_ref().map(|(g, _)| g.as_str()) == Some(guid) {
|
||||
self.last = None;
|
||||
@@ -169,8 +127,6 @@ impl ChunkReader {
|
||||
}
|
||||
}
|
||||
|
||||
/// Write one file from the cache. Returns the bytes written. The output is removed on any error,
|
||||
/// so a file only ever exists complete (and, when the manifest hashes it, verified) or not at all.
|
||||
fn write_file(reader: &mut ChunkReader, f: &FileInfo, out_path: &Path) -> Result<u64, String> {
|
||||
let rel_path = relative_path(f);
|
||||
if let Some(parent) = out_path.parent() {
|
||||
@@ -219,8 +175,6 @@ fn write_file(reader: &mut ChunkReader, f: &FileInfo, out_path: &Path) -> Result
|
||||
result
|
||||
}
|
||||
|
||||
/// Run the plan. `progress(bytes_done, bytes_total, files_done, files_total)` fires after every
|
||||
/// file; `log` receives the start line, failures and the terminal summary.
|
||||
pub fn run_assemble(
|
||||
plan: &AssemblePlan,
|
||||
cancel: &AtomicBool,
|
||||
@@ -270,9 +224,6 @@ pub fn run_assemble(
|
||||
return outcome;
|
||||
}
|
||||
}
|
||||
// Scratch cache: this file was the last user of a chunk → the chunk goes now, not at the
|
||||
// end, so the cache holds only what is still owed to files not yet written. Released
|
||||
// before the progress event, so "file done" also means "its chunks are freed".
|
||||
if plan.scratch_cache {
|
||||
for guid in distinct_guids(f) {
|
||||
let Some(count) = refs.get_mut(&guid) else {
|
||||
@@ -297,8 +248,6 @@ pub fn run_assemble(
|
||||
}
|
||||
|
||||
outcome.success = true;
|
||||
// Done with the cache: remove it whole (a scratch cache is already down to the chunks no
|
||||
// pending file referenced; the default one is what the Kotlin loop deleted at this point).
|
||||
match fs::remove_dir_all(&plan.cache_dir) {
|
||||
Ok(()) => log(&format!("cache removed {}", plan.cache_dir.display())),
|
||||
Err(e) => log(&format!(
|
||||
@@ -337,7 +286,6 @@ mod tests {
|
||||
out
|
||||
}
|
||||
|
||||
/// file 0 = A[0..100] ++ B[10..60]; file 1 = A[100..300]. A is shared, B is file 0's alone.
|
||||
fn expected_files() -> (Vec<u8>, Vec<u8>) {
|
||||
let (a, b) = (chunk_a(), chunk_b());
|
||||
let mut f0 = a[0..100].to_vec();
|
||||
@@ -426,7 +374,6 @@ mod tests {
|
||||
fill_cache(&cache);
|
||||
let plan = build_assemble_plan(&request(&install, cache.to_str().unwrap())).unwrap();
|
||||
assert!(plan.scratch_cache);
|
||||
// After file 0: B (only file 0 used it) must be gone, A (file 1 still needs it) must stay.
|
||||
let seen = Mutex::new(Vec::new());
|
||||
let progress = |_: u64, _: u64, files_done: u64, _: u64| {
|
||||
if files_done == 1 {
|
||||
|
||||
@@ -1,16 +1,3 @@
|
||||
//! One chunk: header parse → (zlib inflate | copy) → SHA-1 → `.part` → rename into the cache.
|
||||
//!
|
||||
//! Port of `EpicDownloadManager.downloadChunkStreaming` minus the HTTP part (the fetch core
|
||||
//! hands us the whole response body). Java's protocol, kept exactly:
|
||||
//! - 41-byte header: magic(4) headerVersion(4) headerSize(4) compressedSize(4) GUID(16) hash(8)
|
||||
//! storedAs(1); payload starts at `max(headerSize, 41)` (Java never seeks back);
|
||||
//! - payload = the next `compressedSize` bytes, or whatever the body still has (a short body
|
||||
//! is NOT an error — the SHA-1 check decides);
|
||||
//! - `storedAs & 1` → zlib; corrupt zlib = failure; a stream that ends early = partial output;
|
||||
//! - the decompressed bytes go to `<final>.part`; the SHA-1 (when the manifest has one) must
|
||||
//! match or the `.part` is deleted; then an atomic rename publishes `<final>`.
|
||||
//! A cache file therefore exists under its final name ONLY if it fully downloaded and verified,
|
||||
//! on either engine.
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, BufWriter, Write};
|
||||
@@ -18,10 +5,8 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
/// Chunk file magic (`0xB1FE3AA2`).
|
||||
pub const CHUNK_MAGIC: u32 = 0xB1FE_3AA2;
|
||||
|
||||
/// Bytes Java reads before looking at the header (`byte[] hdrBuf = new byte[41]`).
|
||||
pub const CHUNK_HEADER_PREFIX: usize = 41;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -37,7 +22,6 @@ impl ChunkHeader {
|
||||
(self.stored_as & 1) != 0
|
||||
}
|
||||
|
||||
/// Where the payload begins in the body (`if (headerSize > 41) skipFully(headerSize - 41)`).
|
||||
pub fn payload_start(&self) -> usize {
|
||||
if self.header_size > CHUNK_HEADER_PREFIX as i32 {
|
||||
self.header_size as usize
|
||||
@@ -47,8 +31,6 @@ impl ChunkHeader {
|
||||
}
|
||||
}
|
||||
|
||||
/// `readFully(hdrBuf)` + header decode. `Err` = the Java `continue`/exception paths
|
||||
/// (short stream, bad magic).
|
||||
pub fn parse_chunk_header(body: &[u8]) -> Result<ChunkHeader, String> {
|
||||
if body.len() < CHUNK_HEADER_PREFIX {
|
||||
return Err(format!(
|
||||
@@ -70,9 +52,6 @@ pub fn parse_chunk_header(body: &[u8]) -> Result<ChunkHeader, String> {
|
||||
})
|
||||
}
|
||||
|
||||
/// The payload slice Java's read loop would consume: `compressedSize` bytes after the header,
|
||||
/// truncated to what the body holds (`if (n <= 0) break;`). `Err` when the header skip runs
|
||||
/// past the body (`skipFully` throws).
|
||||
pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8], String> {
|
||||
let start = hdr.payload_start();
|
||||
if start > body.len() {
|
||||
@@ -88,7 +67,6 @@ pub fn chunk_payload<'a>(body: &'a [u8], hdr: &ChunkHeader) -> Result<&'a [u8],
|
||||
Ok(&body[start..start + take])
|
||||
}
|
||||
|
||||
/// `Writer` that mirrors `fos.write(...)` + `sha.update(...)` on every block.
|
||||
struct HashingWriter<W: Write> {
|
||||
inner: W,
|
||||
sha: Option<Sha1>,
|
||||
@@ -110,23 +88,18 @@ impl<W: Write> Write for HashingWriter<W> {
|
||||
}
|
||||
}
|
||||
|
||||
/// `new File(outFile.getPath() + ".part")`.
|
||||
pub fn part_path(final_path: &Path) -> PathBuf {
|
||||
let mut s = final_path.as_os_str().to_os_string();
|
||||
s.push(".part");
|
||||
PathBuf::from(s)
|
||||
}
|
||||
|
||||
/// Process one downloaded body into `final_path`. Returns the number of DECOMPRESSED bytes
|
||||
/// written. `Err(reason)` = Java's "try the next CDN" outcome; the `.part` never survives an
|
||||
/// error and `final_path` is only ever created by the rename after verification.
|
||||
pub fn write_verified_chunk(
|
||||
body: &[u8],
|
||||
expected_sha1: Option<&[u8; 20]>,
|
||||
final_path: &Path,
|
||||
) -> Result<u64, String> {
|
||||
let tmp = part_path(final_path);
|
||||
// `tmp.delete()` — clear any stale partial from a prior interrupted attempt.
|
||||
let _ = fs::remove_file(&tmp);
|
||||
|
||||
let result = write_part(body, expected_sha1, &tmp, final_path);
|
||||
@@ -153,9 +126,6 @@ fn write_part(
|
||||
};
|
||||
|
||||
if hdr.is_compressed() {
|
||||
// Java: inflate until the stream finishes or the input runs out; a corrupt stream throws
|
||||
// (DataFormatException) → this CDN attempt fails. flate2 behaves the same way: EOF on a
|
||||
// truncated stream is a clean end (partial output), corruption is an error.
|
||||
let mut dec = flate2::read::ZlibDecoder::new(payload);
|
||||
io::copy(&mut dec, &mut out).map_err(|e| format!("inflate: {e}"))?;
|
||||
} else {
|
||||
@@ -164,7 +134,7 @@ fn write_part(
|
||||
out.flush().map_err(|e| format!("flush: {e}"))?;
|
||||
let written = out.written;
|
||||
let digest = out.sha.take().map(|s| s.finalize());
|
||||
drop(out); // close the file before the rename
|
||||
drop(out);
|
||||
|
||||
if let (Some(expected), Some(actual)) = (expected_sha1, digest) {
|
||||
if actual.as_slice() != &expected[..] {
|
||||
@@ -172,7 +142,6 @@ fn write_part(
|
||||
}
|
||||
}
|
||||
|
||||
// Verified → publish atomically (same directory → same filesystem).
|
||||
fs::rename(tmp, final_path).map_err(|e| format!("Chunk rename failed: {e}"))?;
|
||||
Ok(written)
|
||||
}
|
||||
@@ -181,7 +150,6 @@ fn write_part(
|
||||
pub(crate) mod test_support {
|
||||
use super::*;
|
||||
|
||||
/// Build a chunk body the way the CDN serves it.
|
||||
pub fn build_chunk_body(
|
||||
decompressed: &[u8],
|
||||
compress: bool,
|
||||
@@ -199,15 +167,15 @@ pub(crate) mod test_support {
|
||||
let header_size = (CHUNK_HEADER_PREFIX + extra_header) as i32;
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&CHUNK_MAGIC.to_le_bytes());
|
||||
out.extend_from_slice(&3i32.to_le_bytes()); // headerVersion
|
||||
out.extend_from_slice(&3i32.to_le_bytes());
|
||||
out.extend_from_slice(&header_size.to_le_bytes());
|
||||
out.extend_from_slice(
|
||||
&declared_compressed_size
|
||||
.unwrap_or(payload.len() as i32)
|
||||
.to_le_bytes(),
|
||||
);
|
||||
out.extend_from_slice(&[0xAB; 16]); // GUID (unused by the reader)
|
||||
out.extend_from_slice(&[0xCD; 8]); // hash (unused by the reader)
|
||||
out.extend_from_slice(&[0xAB; 16]);
|
||||
out.extend_from_slice(&[0xCD; 8]);
|
||||
out.push(if compress { 1 } else { 0 });
|
||||
out.extend(std::iter::repeat(0x5A).take(extra_header));
|
||||
assert_eq!(out.len(), header_size as usize);
|
||||
@@ -280,7 +248,6 @@ mod tests {
|
||||
let neg = build_chunk_body(b"abcdef", false, 0, Some(-1));
|
||||
let h = parse_chunk_header(&neg).unwrap();
|
||||
assert_eq!(chunk_payload(&neg, &h).unwrap(), b"");
|
||||
// headerSize past the body → skipFully throws.
|
||||
let mut far = build_chunk_body(b"abcdef", false, 0, None);
|
||||
far[8..12].copy_from_slice(&5000i32.to_le_bytes());
|
||||
let h = parse_chunk_header(&far).unwrap();
|
||||
@@ -345,12 +312,10 @@ mod tests {
|
||||
let data = sample_data();
|
||||
let mut body = build_chunk_body(&data, true, 0, None);
|
||||
let final_path = dir.join("chunk");
|
||||
// Truncate the body: inflate ends early → partial output → SHA-1 mismatch.
|
||||
let cut = body.len() - 1000;
|
||||
let err = write_verified_chunk(&body[..cut], Some(&sha1_of(&data)), &final_path).unwrap_err();
|
||||
assert!(err.contains("SHA-1 mismatch") || err.contains("inflate"), "{err}");
|
||||
assert!(!final_path.exists());
|
||||
// Corrupt the deflate stream itself.
|
||||
for b in body.iter_mut().skip(60).take(64) {
|
||||
*b ^= 0x55;
|
||||
}
|
||||
@@ -366,7 +331,7 @@ mod tests {
|
||||
let dir = temp_dir("stale");
|
||||
let final_path = dir.join("chunk");
|
||||
fs::write(part_path(&final_path), b"stale").unwrap();
|
||||
let body = [0u8; 10]; // too short → header failure
|
||||
let body = [0u8; 10];
|
||||
assert!(write_verified_chunk(&body, None, &final_path).is_err());
|
||||
assert!(!part_path(&final_path).exists());
|
||||
assert!(!final_path.exists());
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
//! Plan → `FetchItem`s → `crate::fetch_core::run_fetch` with a sink that fills the chunk cache.
|
||||
//!
|
||||
//! This is the replacement for the body of the Java pool block in `EpicDownloadManager.install`
|
||||
//! ("Download unique chunks — 8 parallel threads"). Inputs are exactly what that block sees:
|
||||
//! the parsed manifest (re-parsed here from the same bytes), the pending file set (indices Java
|
||||
//! computed after its delta/verify pass), the CDN prefixes (`baseUrl + cloudDir`, cloudflare
|
||||
//! already skipped) and the chunk cache dir. Output = `<cache>/<GUID>` for every needed chunk,
|
||||
//! where `<cache>` is `<installDir>/.chunks` unless the caller named another directory; the
|
||||
//! files are assembled from it afterwards (by the Kotlin manager, or by [`super::assemble`]).
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
@@ -21,65 +12,44 @@ use super::plan::{
|
||||
total_credit_bytes, unique_chunks_for_files,
|
||||
};
|
||||
|
||||
/// Java `conn.setReadTimeout(60000)` — the longer of the two Java timeouts (connect was 30 s).
|
||||
pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Everything the JNI layer hands over for one run.
|
||||
pub struct EpicRequest {
|
||||
pub manifest_bytes: Vec<u8>,
|
||||
pub install_dir: String,
|
||||
/// Chunk cache directory; `""` = `<installDir>/.chunks`. See [`resolve_cache_dir`].
|
||||
pub chunk_cache_dir: String,
|
||||
/// `cdn.baseUrl + cdn.cloudDir` per manifest-API CDN entry, in Java's order.
|
||||
pub cdn_prefixes: Vec<String>,
|
||||
/// Indices into `manifest.files` of Java's `pendingFiles` (post delta/verify), in order.
|
||||
pub pending_file_indices: Vec<usize>,
|
||||
/// Java's `neededChunks.size()` / `totalBytes` for the plan cross-check (`None` = skip).
|
||||
pub expected_chunks: Option<u64>,
|
||||
pub expected_bytes: Option<u64>,
|
||||
pub ca_bundle_path: String,
|
||||
/// Window ceiling: the Steam speed tier's network window (Fast = 32) since improvements
|
||||
/// round 1; the Java fallback pool keeps its fixed 8 (`super::JAVA_POOL_THREADS`).
|
||||
pub max_workers: usize,
|
||||
pub process_workers: usize,
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
/// The chunk plan for one run.
|
||||
#[derive(Debug)]
|
||||
pub struct EpicPlan {
|
||||
pub manifest: Manifest,
|
||||
pub cache_dir: PathBuf,
|
||||
/// The cache is a directory the caller named, apart from the install: scratch space that
|
||||
/// the assembler empties as it goes and removes when it is done.
|
||||
pub scratch_cache: bool,
|
||||
/// Chunk indices (into `manifest.unique_chunks`) in Java's submission order.
|
||||
pub needed: Vec<usize>,
|
||||
/// `Σ max(fileSize, 1)` over `needed`.
|
||||
pub total_bytes: u64,
|
||||
/// Distinct CDN prefixes = fetch-core host keys.
|
||||
pub hosts: Vec<String>,
|
||||
}
|
||||
|
||||
/// Terminal result of a run, in the shape Java's pool block needs to reproduce its own exit
|
||||
/// paths (`CANCELLED during chunk download (n/total chunks)`, `N chunks failed`, `chunksOK=`).
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct EpicOutcome {
|
||||
pub success: bool,
|
||||
pub cancelled: bool,
|
||||
pub error: String,
|
||||
/// Credited (manifest `max(fileSize,1)`) bytes: skipped-cached + fetched.
|
||||
pub bytes_credited: u64,
|
||||
/// Chunks accounted for (skipped-cached + fetched), Java's `completedCount`.
|
||||
pub chunks_done: u64,
|
||||
pub chunks_total: u64,
|
||||
pub bytes_total: u64,
|
||||
/// Decompressed bytes actually written to the cache this run.
|
||||
pub decompressed_written: u64,
|
||||
}
|
||||
|
||||
/// Parse + plan (no I/O beyond reading the cache dir). `Err` = "engine could not start"; Java
|
||||
/// then runs its own pool, since nothing has been fetched yet.
|
||||
pub fn build_plan(req: &EpicRequest) -> Result<EpicPlan, String> {
|
||||
let manifest = parse_manifest(&req.manifest_bytes).map_err(|e| format!("plan: {e}"))?;
|
||||
for &i in &req.pending_file_indices {
|
||||
@@ -124,10 +94,8 @@ pub fn build_plan(req: &EpicRequest) -> Result<EpicPlan, String> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Sink: one downloaded body → verified cache file. `id` indexes `fetch_chunks`.
|
||||
struct ChunkCacheSink<'a> {
|
||||
plan: &'a EpicPlan,
|
||||
/// Chunk indices (into `manifest.unique_chunks`) of the items actually being fetched.
|
||||
fetch_chunks: &'a [usize],
|
||||
decompressed: AtomicU64,
|
||||
}
|
||||
@@ -144,15 +112,11 @@ impl<'a> FetchSink for ChunkCacheSink<'a> {
|
||||
self.decompressed.fetch_add(written, Ordering::Relaxed);
|
||||
Ok(chunk.credit_bytes())
|
||||
}
|
||||
// Every per-attempt failure in Java is "try the next CDN"; the core's Retry rotates
|
||||
// hosts and backs off the same way (bounded at its attempt cap).
|
||||
Err(reason) => Err(SinkError::Retry(format!("{} {reason}", chunk.guid_str()))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Rolling speed sampler for the end-of-run summary: samples at ≥500 ms like the Java pool's
|
||||
/// `lastSpeedMs` logic, keeps the peak.
|
||||
struct SpeedMeter {
|
||||
start: Instant,
|
||||
last_at: Instant,
|
||||
@@ -197,9 +161,6 @@ impl SpeedMeter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the plan. `progress(bytes_done, chunks_done)` fires once per accounted chunk (cached-skip
|
||||
/// or fetched), exactly the cadence of the Java pool's per-task `progress(...)` call. `log`
|
||||
/// receives engine lines (`fetch-window`, `summary`, failures).
|
||||
pub fn run_plan(
|
||||
plan: &EpicPlan,
|
||||
req: &EpicRequest,
|
||||
@@ -228,8 +189,6 @@ pub fn run_plan(
|
||||
req.process_workers
|
||||
));
|
||||
|
||||
// Java: each pool task first checks the cancel flag, then `cachedFile.exists()` → credit
|
||||
// without fetching. Account for the cached chunks up front (same credit, same callback).
|
||||
let mut fetch_chunks: Vec<usize> = Vec::with_capacity(plan.needed.len());
|
||||
let mut pre_bytes: u64 = 0;
|
||||
let mut pre_chunks: u64 = 0;
|
||||
@@ -293,16 +252,12 @@ pub fn run_plan(
|
||||
|
||||
let opts = FetchOptions {
|
||||
max_workers: req.max_workers.max(1),
|
||||
// Improvements round 1: the tier ceiling split across the distinct CDNs (floor 6), so
|
||||
// the whole window is reachable on 1 host or on all 3.
|
||||
per_host_cap: host_cap,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
headers: vec![("User-Agent".to_string(), super::USER_AGENT.to_string())],
|
||||
ca_bundle_path: req.ca_bundle_path.clone(),
|
||||
process_workers: req.process_workers.max(1),
|
||||
label: req.label.clone(),
|
||||
// Whole-body mode (Epic chunks are ≤ ~1 MiB compressed; the core's byte budget scales
|
||||
// with the window); `stream` and any future field keep the core's defaults.
|
||||
..FetchOptions::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -1,25 +1,3 @@
|
||||
//! JNI exports for `com.droiddeck.launcher.stores.epic.EpicNative`.
|
||||
//!
|
||||
//! ```text
|
||||
//! nativeStart(manifest: byte[], installDir: String, chunkCacheDir: String, cdnPrefixes: String[],
|
||||
//! pendingFileIdx: int[], expectedChunks: int, expectedBytes: long,
|
||||
//! caBundlePath: String, maxWorkers: int, processWorkers: int,
|
||||
//! listener: EpicNative.Listener) -> long handle (0 = not started)
|
||||
//! nativeAssemble(manifest: byte[], installDir: String, chunkCacheDir: String, fileIdx: int[],
|
||||
//! listener: EpicNative.Listener) -> long handle (0 = not started)
|
||||
//! nativeCancel(handle) — flips the run's AtomicBool; fetch or assembly stops promptly
|
||||
//! nativeRelease(handle) — frees the handle (after onComplete)
|
||||
//! ```
|
||||
//! `chunkCacheDir` = "" keeps the cache at `<installDir>/.chunks`; any other directory is a
|
||||
//! scratch cache (see `assemble.rs`). Listener (all methods run on native threads):
|
||||
//! `onPlan(int chunksTotal, long bytesTotal, String chunkDir)` once before any fetch,
|
||||
//! `onProgress(long bytesDone, long bytesTotal, int done, int total)` per chunk fetched (or, in
|
||||
//! an assembly run, per file written), `onLog(String line)`,
|
||||
//! `onComplete(boolean success, String error, long bytes)`.
|
||||
//!
|
||||
//! Planning (manifest parse + cross-check) runs synchronously on the calling thread so a plan
|
||||
//! failure returns 0 BEFORE any fetch and Java can fall back to its own pool; the fetch runs on
|
||||
//! a new thread. JavaVM attach / GlobalRef handling is the same shape as the GOG facade.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
@@ -59,8 +37,6 @@ fn android_log(message: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-run native state behind the `jlong` handle — the same type for a fetch run and an
|
||||
/// assembly run, so one `nativeCancel` / `nativeRelease` pair serves both.
|
||||
pub struct EpicDownloadHandle {
|
||||
cancel: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -200,7 +176,6 @@ fn call_on_complete(env: &mut JNIEnv, listener: &JObject, success: bool, error:
|
||||
clear_pending_exception(env);
|
||||
}
|
||||
|
||||
/// Log to logcat AND to the listener (the Kotlin manager mirrors it into its own debug log).
|
||||
fn log_both(env: &mut JNIEnv, listener: &JObject, line: &str) {
|
||||
android_log(line);
|
||||
call_on_log(env, listener, line);
|
||||
@@ -334,8 +309,6 @@ fn run_on_thread(
|
||||
let bytes_total = plan.total_bytes;
|
||||
let chunks_total = plan.needed.len() as u64;
|
||||
|
||||
// Progress fires from the core's process-pool threads; attach each as a daemon like the
|
||||
// Steam engine's `dispatch_download_progress` does.
|
||||
let progress = |bytes_done: u64, chunks_done: u64| {
|
||||
let Ok(mut env) = vm.attach_current_thread_as_daemon() else {
|
||||
return;
|
||||
@@ -371,11 +344,6 @@ fn run_on_thread(
|
||||
);
|
||||
}
|
||||
|
||||
/// `EpicNative.nativeAssemble(manifest, installDir, chunkCacheDir, fileIdx, listener)`: write
|
||||
/// the files `fileIdx` names from the chunk cache (see `assemble.rs`). Parse + plan run on the
|
||||
/// calling thread, so bad inputs return 0 (after `onComplete(false, reason, 0)`) before anything
|
||||
/// is written; the assembly runs on a new thread and reports per file through `onProgress(bytes,
|
||||
/// bytesTotal, filesDone, filesTotal)`, then `onComplete(success, error, bytesWritten)`.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_nativeAssemble(
|
||||
mut env: JNIEnv,
|
||||
@@ -481,7 +449,5 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_epic_EpicNative_native
|
||||
if handle == 0 {
|
||||
return;
|
||||
}
|
||||
// The worker thread owns its own Arc clone of the cancel flag and its own GlobalRef, so
|
||||
// freeing the handle is safe even if a (cancelled) run is still winding down.
|
||||
drop(unsafe { Box::from_raw(handle as *mut EpicDownloadHandle) });
|
||||
}
|
||||
@@ -1,43 +1,24 @@
|
||||
//! Epic build manifest parsers — a 1:1 port of `EpicDownloadManager.parseManifest` (binary,
|
||||
//! magic `0x44BEC00C`) and `parseJsonManifest` (legacy JSON manifests).
|
||||
//!
|
||||
//! Every quirk of the Java reader is kept on purpose (the Java manager stays the reference and
|
||||
//! the two engines must derive the same plan from the same bytes): section skips by declared
|
||||
//! size, `readFString` UTF-16LE/ASCII decoding, duplicate-GUID handling (last value, first
|
||||
//! position), the JSON blob decoding (`blobToNum`) and its fallbacks, and "any exception →
|
||||
//! parse failed".
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
|
||||
/// Binary manifest magic (`EpicDownloadManager.parseManifest`).
|
||||
pub const MANIFEST_MAGIC: u32 = 0x44BE_C00C;
|
||||
|
||||
/// One entry of the ChunkDataList section (`EpicDownloadManager.ChunkInfo`).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ChunkInfo {
|
||||
/// Four uint32 in binary read order.
|
||||
pub guid: [u32; 4],
|
||||
/// uint64 "hash" that forms the first half of the chunk file name.
|
||||
pub hash: u64,
|
||||
/// SHA-1 of the DECOMPRESSED chunk; `None` for JSON manifests (Java: `sha1 == null`).
|
||||
pub sha1: Option<[u8; 20]>,
|
||||
/// Subfolder number (DECIMAL `%02d`). `i32` so the JSON parser's `Integer.parseInt` range is
|
||||
/// representable exactly (binary manifests always yield 0..=255).
|
||||
pub group_num: i32,
|
||||
/// Uncompressed size (0 for JSON manifests).
|
||||
pub window_size: i32,
|
||||
/// Compressed (wire) size. Signed like the Java `long`; JSON manifests may yield anything.
|
||||
pub file_size: i64,
|
||||
}
|
||||
|
||||
impl ChunkInfo {
|
||||
/// `String.format("%08X%08X%08X%08X", guid[0..4])`.
|
||||
pub fn guid_str(&self) -> String {
|
||||
guid_to_string(&self.guid)
|
||||
}
|
||||
|
||||
/// `ChunkInfo.getPath`: `"<chunkDir>/<%02d groupNum>/<%016X hash>_<GUID>.chunk"`.
|
||||
pub fn path(&self, chunk_dir: &str) -> String {
|
||||
format!(
|
||||
"{}/{:02}/{:016X}_{}.chunk",
|
||||
@@ -48,8 +29,6 @@ impl ChunkInfo {
|
||||
)
|
||||
}
|
||||
|
||||
/// Java `chunk.sha1 != null && length == 20 && !allZero` — the condition under which the
|
||||
/// streaming downloader verifies the decompressed chunk.
|
||||
pub fn verifiable_sha1(&self) -> Option<&[u8; 20]> {
|
||||
match &self.sha1 {
|
||||
Some(h) if h.iter().any(|b| *b != 0) => Some(h),
|
||||
@@ -57,7 +36,6 @@ impl ChunkInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// `Math.max(chunk.fileSize, 1)` — the byte credit the Java pool uses for totals/progress.
|
||||
pub fn credit_bytes(&self) -> u64 {
|
||||
if self.file_size < 1 {
|
||||
1
|
||||
@@ -67,13 +45,10 @@ impl ChunkInfo {
|
||||
}
|
||||
}
|
||||
|
||||
/// One file part referencing a chunk (`EpicDownloadManager.ChunkPart`).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ChunkPart {
|
||||
pub guid: [u32; 4],
|
||||
/// Offset into the DECOMPRESSED chunk (Java `int`).
|
||||
pub offset: i32,
|
||||
/// Part length (Java `int`; summed as `size & 0xFFFFFFFFL`).
|
||||
pub size: i32,
|
||||
}
|
||||
|
||||
@@ -83,26 +58,20 @@ impl ChunkPart {
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of the FileManifestList section (`EpicDownloadManager.FileInfo`).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct FileInfo {
|
||||
pub filename: String,
|
||||
pub parts: Vec<ChunkPart>,
|
||||
/// Install-tag names; empty = required/base file.
|
||||
pub install_tags: Vec<String>,
|
||||
/// Full-file SHA-1; `None` for JSON manifests.
|
||||
pub sha1: Option<[u8; 20]>,
|
||||
}
|
||||
|
||||
impl FileInfo {
|
||||
/// `FileInfo.fileSize()` = Σ `(part.size & 0xFFFFFFFFL)`.
|
||||
pub fn file_size(&self) -> u64 {
|
||||
self.parts.iter().map(|p| p.size as u32 as u64).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed manifest (`EpicDownloadManager.EpicManifest.ParsedManifest`, minus the CDN list which
|
||||
/// the Java side still owns).
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Manifest {
|
||||
pub chunk_dir: String,
|
||||
@@ -111,8 +80,6 @@ pub struct Manifest {
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
/// `byGuid` map as `uniqueChunksForFiles` builds it: `LinkedHashMap.put` per chunk, so a
|
||||
/// duplicate GUID resolves to the LAST `ChunkInfo`.
|
||||
pub fn chunk_index_by_guid(&self) -> HashMap<String, usize> {
|
||||
let mut map = HashMap::with_capacity(self.unique_chunks.len());
|
||||
for (i, c) in self.unique_chunks.iter().enumerate() {
|
||||
@@ -126,7 +93,6 @@ pub fn guid_to_string(guid: &[u32; 4]) -> String {
|
||||
format!("{:08X}{:08X}{:08X}{:08X}", guid[0], guid[1], guid[2], guid[3])
|
||||
}
|
||||
|
||||
/// `chunkDir` from the binary header version (`parseManifest`).
|
||||
pub fn chunk_dir_for_binary_version(version: i32) -> &'static str {
|
||||
if version >= 15 {
|
||||
"ChunksV4"
|
||||
@@ -139,7 +105,6 @@ pub fn chunk_dir_for_binary_version(version: i32) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// `chunkDir` from `ManifestFileVersion` (`parseJsonManifest`; note the different `< 3` default).
|
||||
pub fn chunk_dir_for_json_version(version: i32) -> &'static str {
|
||||
if version >= 15 {
|
||||
"ChunksV4"
|
||||
@@ -152,10 +117,7 @@ pub fn chunk_dir_for_json_version(version: i32) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
// ── little-endian cursor with ByteBuffer-like failure semantics ─────────────────────────────
|
||||
|
||||
/// Mirrors the `ByteBuffer` calls the Java parser makes: every under-run or out-of-range
|
||||
/// `position()` is an exception there, which the Java code turns into "parse failed" (null).
|
||||
struct Cursor<'a> {
|
||||
buf: &'a [u8],
|
||||
pos: usize,
|
||||
@@ -170,7 +132,6 @@ impl<'a> Cursor<'a> {
|
||||
self.pos
|
||||
}
|
||||
|
||||
/// `ByteBuffer.position(p)` — `p` must be within `0..=limit`.
|
||||
fn set_position(&mut self, p: i64) -> Result<(), String> {
|
||||
if p < 0 || p as u64 > self.buf.len() as u64 {
|
||||
return Err(format!("position {p} out of range (limit {})", self.buf.len()));
|
||||
@@ -223,7 +184,6 @@ impl<'a> Cursor<'a> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// `ByteBuffer.position(position + n)` with the same range check.
|
||||
fn skip(&mut self, n: i64) -> Result<(), String> {
|
||||
let target = self.pos as i64 + n;
|
||||
self.set_position(target)
|
||||
@@ -233,23 +193,19 @@ impl<'a> Cursor<'a> {
|
||||
&self.buf[self.pos..]
|
||||
}
|
||||
|
||||
/// `EpicDownloadManager.readFString`: int32 length; `0` → ""; negative → `(-len)-1` UTF-16LE
|
||||
/// code units + 2-byte terminator; positive → `len-1` US-ASCII bytes + 1-byte terminator.
|
||||
fn fstring(&mut self) -> Result<String, String> {
|
||||
let length = self.i32()?;
|
||||
if length == 0 {
|
||||
return Ok(String::new());
|
||||
}
|
||||
if length < 0 {
|
||||
// Java: `int chars = (-length) - 1;` — `-Integer.MIN_VALUE` wraps to itself, then -1
|
||||
// gives Integer.MAX_VALUE; `new byte[chars * 2]` overflows negative → exception.
|
||||
let chars = length.wrapping_neg().wrapping_sub(1);
|
||||
let byte_len = chars.checked_mul(2).ok_or("fstring length overflow")?;
|
||||
if byte_len < 0 {
|
||||
return Err("negative fstring length".to_string());
|
||||
}
|
||||
let bytes = self.take(byte_len as usize)?;
|
||||
self.take(2)?; // null terminator
|
||||
self.take(2)?;
|
||||
let units: Vec<u16> = bytes
|
||||
.chunks_exact(2)
|
||||
.map(|p| u16::from_le_bytes([p[0], p[1]]))
|
||||
@@ -257,8 +213,8 @@ impl<'a> Cursor<'a> {
|
||||
Ok(String::from_utf16_lossy(&units))
|
||||
} else {
|
||||
let bytes = self.take((length - 1) as usize)?;
|
||||
self.take(1)?; // null terminator
|
||||
// `new String(bytes, US_ASCII)`: every byte > 0x7F decodes to U+FFFD.
|
||||
self.take(1)?;
|
||||
|
||||
Ok(bytes
|
||||
.iter()
|
||||
.map(|&b| if b < 0x80 { b as char } else { '\u{FFFD}' })
|
||||
@@ -267,10 +223,7 @@ impl<'a> Cursor<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
// ── binary manifest ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// `EpicDownloadManager.parseManifest(bytes)`: binary when the magic matches, otherwise the JSON
|
||||
/// parser. `Err` wherever Java returns `null`.
|
||||
pub fn parse_manifest(bytes: &[u8]) -> Result<Manifest, String> {
|
||||
let mut cur = Cursor::new(bytes);
|
||||
let magic = cur.u32()?;
|
||||
@@ -284,7 +237,7 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
let header_size = cur.i32()?;
|
||||
let size_uncompressed = cur.i32()?;
|
||||
let _size_compressed = cur.i32()?;
|
||||
cur.skip(20)?; // SHA-1 of the body, not checked by the Java parser
|
||||
cur.skip(20)?;
|
||||
let stored_as = cur.u8()?;
|
||||
let version = cur.i32()?;
|
||||
let chunk_dir = chunk_dir_for_binary_version(version);
|
||||
@@ -297,8 +250,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
if size_uncompressed < 0 {
|
||||
return Err("negative sizeUncompressed".to_string());
|
||||
}
|
||||
// Java: `inflater.inflate(new byte[sizeUncompressed])` — a single call that fills at most
|
||||
// `sizeUncompressed` bytes; anything else (short, corrupt) is a size mismatch → null.
|
||||
let mut out = Vec::with_capacity(size_uncompressed as usize);
|
||||
let mut dec = flate2::read::ZlibDecoder::new(body_slice).take(size_uncompressed as u64);
|
||||
dec.read_to_end(&mut out)
|
||||
@@ -317,18 +268,16 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
|
||||
let mut b = Cursor::new(body);
|
||||
|
||||
// ManifestMeta: skipped by its declared size (the size field counts itself).
|
||||
let meta_size = b.i32()?;
|
||||
let meta_end = b.position() as i64 - 4 + meta_size as i64;
|
||||
b.set_position(meta_end)?;
|
||||
|
||||
// ChunkDataList
|
||||
let cdl_start = b.position();
|
||||
let cdl_size = b.i32()?;
|
||||
let _cdl_version = b.u8()?;
|
||||
let chunk_count = b.i32()?;
|
||||
if chunk_count < 0 {
|
||||
return Err("negative chunkCount".to_string()); // `new ArrayList<>(negative)` throws
|
||||
return Err("negative chunkCount".to_string());
|
||||
}
|
||||
let chunk_count = chunk_count as usize;
|
||||
let mut chunks: Vec<ChunkInfo> = (0..chunk_count)
|
||||
@@ -361,7 +310,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
}
|
||||
b.set_position(cdl_start as i64 + cdl_size as i64)?;
|
||||
|
||||
// FileManifestList
|
||||
let fml_start = b.position();
|
||||
let fml_size = b.i32()?;
|
||||
let _fml_version = b.u8()?;
|
||||
@@ -375,12 +323,12 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
f.filename = b.fstring()?;
|
||||
}
|
||||
for _ in 0..file_count {
|
||||
b.fstring()?; // symlink targets
|
||||
b.fstring()?;
|
||||
}
|
||||
for f in files.iter_mut() {
|
||||
f.sha1 = Some(b.sha1()?);
|
||||
}
|
||||
b.skip(file_count as i64)?; // per-file flags byte
|
||||
b.skip(file_count as i64)?;
|
||||
for f in files.iter_mut() {
|
||||
let tag_count = b.i32()?;
|
||||
let mut j = 0;
|
||||
@@ -415,8 +363,6 @@ fn parse_binary_after_magic(cur: &mut Cursor<'_>) -> Result<Manifest, String> {
|
||||
})
|
||||
}
|
||||
|
||||
/// `LinkedHashMap<String, ChunkInfo>` keyed by GUID string: a repeated key keeps its FIRST
|
||||
/// position but takes the LAST value.
|
||||
fn dedup_last_wins_first_position(chunks: Vec<ChunkInfo>) -> Vec<ChunkInfo> {
|
||||
let mut order: Vec<ChunkInfo> = Vec::with_capacity(chunks.len());
|
||||
let mut index: HashMap<String, usize> = HashMap::with_capacity(chunks.len());
|
||||
@@ -433,10 +379,7 @@ fn dedup_last_wins_first_position(chunks: Vec<ChunkInfo>) -> Vec<ChunkInfo> {
|
||||
order
|
||||
}
|
||||
|
||||
// ── JSON manifest (older games) ──────────────────────────────────────────────────────────
|
||||
|
||||
/// `org.json` value → string coercion as `optString` / `getString` do on Android (numbers and
|
||||
/// booleans print as-is, JSON null prints "null", containers print their JSON).
|
||||
fn json_to_string(v: &serde_json::Value) -> String {
|
||||
match v {
|
||||
serde_json::Value::String(s) => s.clone(),
|
||||
@@ -445,9 +388,6 @@ fn json_to_string(v: &serde_json::Value) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// `EpicDownloadManager.blobToNum`: a JSON manifest writes every number as a "blob", each byte
|
||||
/// as three decimal digits, least significant first ("013000000000" = 13). Bits past 64 are
|
||||
/// dropped; `None` when a group is not a number (Java: `NumberFormatException`).
|
||||
pub fn blob_to_num(s: &str) -> Option<u64> {
|
||||
let bytes = s.as_bytes();
|
||||
let mut num: u64 = 0;
|
||||
@@ -464,7 +404,6 @@ pub fn blob_to_num(s: &str) -> Option<u64> {
|
||||
Some(num)
|
||||
}
|
||||
|
||||
/// `obj.optString(key, fallback)`.
|
||||
fn opt_string(obj: &serde_json::Map<String, serde_json::Value>, key: &str, fallback: &str) -> String {
|
||||
match obj.get(key) {
|
||||
Some(v) => json_to_string(v),
|
||||
@@ -472,16 +411,12 @@ fn opt_string(obj: &serde_json::Map<String, serde_json::Value>, key: &str, fallb
|
||||
}
|
||||
}
|
||||
|
||||
/// `(int) Long.parseLong(hex8, 16)` — the Java GUID quarter parse; invalid hex throws out of the
|
||||
/// whole parse.
|
||||
fn parse_guid_quarter(s: &str) -> Result<u32, String> {
|
||||
let v = i64::from_str_radix(s, 16).map_err(|e| format!("bad guid hex '{s}': {e}"))?;
|
||||
Ok(v as i32 as u32)
|
||||
}
|
||||
|
||||
fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> {
|
||||
// Java substrings by UTF-16 unit and `parseLong` throws on anything non-hex; a non-ASCII
|
||||
// prefix can never parse, so refuse it before slicing (keeps the slice on char boundaries).
|
||||
if s.len() < 32 || !s.as_bytes()[..32].is_ascii() {
|
||||
return Err(format!("bad guid hex '{s}'"));
|
||||
}
|
||||
@@ -493,7 +428,6 @@ fn parse_guid_from_hex(s: &str) -> Result<[u32; 4], String> {
|
||||
])
|
||||
}
|
||||
|
||||
/// `EpicDownloadManager.parseJsonManifest`.
|
||||
pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
|
||||
let text = String::from_utf8_lossy(bytes);
|
||||
let root: serde_json::Value =
|
||||
@@ -513,7 +447,6 @@ pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
|
||||
let data_group_list = root.get("DataGroupList").and_then(|v| v.as_object());
|
||||
let chunk_filesize_list = root.get("ChunkFilesizeList").and_then(|v| v.as_object());
|
||||
|
||||
// `LinkedHashMap<String, ChunkInfo>` keyed by the RAW key string (not the normalized GUID).
|
||||
let mut chunk_order: Vec<ChunkInfo> = Vec::new();
|
||||
let mut chunk_index: HashMap<String, usize> = HashMap::new();
|
||||
for (guid_hex, hash_value) in chunk_hash_list.iter() {
|
||||
@@ -598,12 +531,9 @@ pub fn parse_json_manifest(bytes: &[u8]) -> Result<Manifest, String> {
|
||||
})
|
||||
}
|
||||
|
||||
// ── tests ────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_support {
|
||||
//! Synthetic binary-manifest builder used by this module's tests and by `plan`/`driver`
|
||||
//! tests. Emits exactly the layout `parseManifest` walks (meta / CDL / FML with sizes).
|
||||
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
@@ -621,7 +551,6 @@ pub(crate) mod test_support {
|
||||
pub name: String,
|
||||
pub sha1: [u8; 20],
|
||||
pub tags: Vec<String>,
|
||||
/// (guid, offset, size)
|
||||
pub parts: Vec<([u32; 4], i32, i32)>,
|
||||
}
|
||||
|
||||
@@ -648,17 +577,14 @@ pub(crate) mod test_support {
|
||||
v
|
||||
}
|
||||
|
||||
/// Body (meta + CDL + FML) without the outer header.
|
||||
pub fn build_body(chunks: &[TestChunk], files: &[TestFile], meta_padding: usize) -> Vec<u8> {
|
||||
let mut body = Vec::new();
|
||||
// ManifestMeta: size field counts itself.
|
||||
let meta_size = 4 + meta_padding;
|
||||
body.extend_from_slice(&(meta_size as i32).to_le_bytes());
|
||||
body.extend(std::iter::repeat(0xEEu8).take(meta_padding));
|
||||
|
||||
// ChunkDataList
|
||||
let mut cdl = Vec::new();
|
||||
cdl.push(0u8); // version
|
||||
cdl.push(0u8);
|
||||
cdl.extend_from_slice(&(chunks.len() as i32).to_le_bytes());
|
||||
for c in chunks {
|
||||
for g in c.guid {
|
||||
@@ -680,14 +606,12 @@ pub(crate) mod test_support {
|
||||
for c in chunks {
|
||||
cdl.extend_from_slice(&c.file_size.to_le_bytes());
|
||||
}
|
||||
// Trailing padding the parser must skip via cdlSize.
|
||||
cdl.extend_from_slice(&[0xAA, 0xBB, 0xCC]);
|
||||
body.extend_from_slice(&((cdl.len() + 4) as i32).to_le_bytes());
|
||||
body.extend_from_slice(&cdl);
|
||||
|
||||
// FileManifestList
|
||||
let mut fml = Vec::new();
|
||||
fml.push(0u8); // version
|
||||
fml.push(0u8);
|
||||
fml.extend_from_slice(&(files.len() as i32).to_le_bytes());
|
||||
for f in files {
|
||||
if f.name.is_ascii() {
|
||||
@@ -697,13 +621,13 @@ pub(crate) mod test_support {
|
||||
}
|
||||
}
|
||||
for _ in files {
|
||||
fml.extend_from_slice(&fstring_ascii("")); // symlink target
|
||||
fml.extend_from_slice(&fstring_ascii(""));
|
||||
}
|
||||
for f in files {
|
||||
fml.extend_from_slice(&f.sha1);
|
||||
}
|
||||
for _ in files {
|
||||
fml.push(0); // flags
|
||||
fml.push(0);
|
||||
}
|
||||
for f in files {
|
||||
fml.extend_from_slice(&(f.tags.len() as i32).to_le_bytes());
|
||||
@@ -714,7 +638,6 @@ pub(crate) mod test_support {
|
||||
for f in files {
|
||||
fml.extend_from_slice(&(f.parts.len() as i32).to_le_bytes());
|
||||
for (guid, offset, size) in &f.parts {
|
||||
// struct size 4 + 16 + 4 + 4 = 28, plus 2 bytes of padding the parser skips.
|
||||
fml.extend_from_slice(&30i32.to_le_bytes());
|
||||
for g in guid {
|
||||
fml.extend_from_slice(&g.to_le_bytes());
|
||||
@@ -729,7 +652,6 @@ pub(crate) mod test_support {
|
||||
body
|
||||
}
|
||||
|
||||
/// Full manifest bytes: header + (optionally zlib-compressed) body.
|
||||
pub fn build_manifest(
|
||||
chunks: &[TestChunk],
|
||||
files: &[TestFile],
|
||||
@@ -745,7 +667,7 @@ pub(crate) mod test_support {
|
||||
} else {
|
||||
body.clone()
|
||||
};
|
||||
let header_size: i32 = 41 + 3; // 3 bytes of extra header the parser must jump over
|
||||
let header_size: i32 = 41 + 3;
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(&MANIFEST_MAGIC.to_le_bytes());
|
||||
out.extend_from_slice(&header_size.to_le_bytes());
|
||||
@@ -916,13 +838,9 @@ mod tests {
|
||||
fn bad_uncompressed_size_is_a_parse_failure() {
|
||||
let good = build_manifest(&sample_chunks(), &sample_files(), 21, true);
|
||||
let real = i32::from_le_bytes([good[8], good[9], good[10], good[11]]);
|
||||
// Claim MORE than the body inflates to: Java's single `inflate()` returns fewer bytes
|
||||
// than `sizeUncompressed` → "Decomp size mismatch" → null.
|
||||
let mut bigger = good.clone();
|
||||
bigger[8..12].copy_from_slice(&(real + 1).to_le_bytes());
|
||||
assert!(parse_manifest(&bigger).is_err());
|
||||
// Claim LESS: Java fills exactly `sizeUncompressed` bytes (silently truncating), then the
|
||||
// FileManifestList end position lands past the limit → exception → null.
|
||||
let mut smaller = good;
|
||||
smaller[8..12].copy_from_slice(&(real - 1).to_le_bytes());
|
||||
assert!(parse_manifest(&smaller).is_err());
|
||||
@@ -933,7 +851,7 @@ mod tests {
|
||||
let mut v = fstring_ascii("abc");
|
||||
v.extend_from_slice(&fstring_utf16("dé"));
|
||||
v.extend_from_slice(&fstring_ascii(""));
|
||||
v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]); // non-ASCII byte → U+FFFD
|
||||
v.extend_from_slice(&[4, 0, 0, 0, b'a', 0xFF, b'b', 0]);
|
||||
let mut c = Cursor::new(&v);
|
||||
assert_eq!(c.fstring().unwrap(), "abc");
|
||||
assert_eq!(c.fstring().unwrap(), "dé");
|
||||
|
||||
@@ -1,19 +1,9 @@
|
||||
//! Chunk plan — the part of `EpicDownloadManager.install` between "manifest parsed" and "submit
|
||||
//! to the pool", ported 1:1: install-tag file selection (`resolveInstallFiles` /
|
||||
//! `getRequiredInstallFiles`), the unique-chunk list for a file set (`uniqueChunksForFiles`),
|
||||
//! the byte total (`Σ max(fileSize, 1)`), chunk cache paths and CDN URLs.
|
||||
//!
|
||||
//! At runtime the Java manager passes the INDICES of its `pendingFiles` (post delta/verify), so
|
||||
//! the file selection functions here are exercised by the unit tests (and available to a future
|
||||
//! caller) while the driver only runs `unique_chunks_for_files` on Java's pending set — the same
|
||||
//! function Java runs, on the same manifest, in the same order.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::manifest::{ChunkInfo, Manifest};
|
||||
|
||||
/// `getRequiredInstallFiles`: files with no install tags; if NONE carries a tag, every file.
|
||||
pub fn required_install_files(m: &Manifest) -> Vec<usize> {
|
||||
let required: Vec<usize> = m
|
||||
.files
|
||||
@@ -29,11 +19,6 @@ pub fn required_install_files(m: &Manifest) -> Vec<usize> {
|
||||
}
|
||||
}
|
||||
|
||||
/// `resolveInstallFiles(manifest, selectedTags)`:
|
||||
/// - `None` → every file (legacy / DLC / overlay path);
|
||||
/// - `Some([])` → required(base) files only;
|
||||
/// - otherwise → base files + any file carrying at least one selected tag, falling back to
|
||||
/// required-only when nothing matched.
|
||||
pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) -> Vec<usize> {
|
||||
let Some(tags) = selected_tags else {
|
||||
return (0..m.files.len()).collect();
|
||||
@@ -59,8 +44,6 @@ pub fn resolve_install_files(m: &Manifest, selected_tags: Option<&[String]>) ->
|
||||
}
|
||||
}
|
||||
|
||||
/// `uniqueChunksForFiles(manifest, files)`: chunk indices (into `unique_chunks`) in first-seen
|
||||
/// order over `files[*].parts`, each GUID once, GUIDs absent from the chunk list dropped.
|
||||
pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec<usize> {
|
||||
let by_guid: HashMap<String, usize> = m.chunk_index_by_guid();
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
@@ -81,7 +64,6 @@ pub fn unique_chunks_for_files(m: &Manifest, file_indices: &[usize]) -> Vec<usiz
|
||||
chunks
|
||||
}
|
||||
|
||||
/// `totalBytes += Math.max(chunk.fileSize, 1)` over the needed chunks.
|
||||
pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 {
|
||||
chunk_indices
|
||||
.iter()
|
||||
@@ -90,15 +72,10 @@ pub fn total_credit_bytes(m: &Manifest, chunk_indices: &[usize]) -> u64 {
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// `<installDir>/.chunks`.
|
||||
pub fn chunk_cache_dir(install_dir: &str) -> PathBuf {
|
||||
Path::new(install_dir).join(super::CHUNK_CACHE_DIR)
|
||||
}
|
||||
|
||||
/// Where this run's chunks go: `cache_dir` when the caller named one, else the default
|
||||
/// `<installDir>/.chunks`. A separate directory exists so the cache can sit on internal storage
|
||||
/// while the game installs to an SD card: a chunk is written once into the cache and once into
|
||||
/// the assembled file, and with both on a 26 MB/s card the second write starved the fetchers.
|
||||
pub fn resolve_cache_dir(install_dir: &str, cache_dir: &str) -> PathBuf {
|
||||
if cache_dir.is_empty() {
|
||||
chunk_cache_dir(install_dir)
|
||||
@@ -107,37 +84,23 @@ pub fn resolve_cache_dir(install_dir: &str, cache_dir: &str) -> PathBuf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Final cache file for a chunk: `new File(chunkCacheDir, chunk.guidStr())`.
|
||||
pub fn cached_chunk_path(cache_dir: &Path, chunk: &ChunkInfo) -> PathBuf {
|
||||
cache_dir.join(chunk.guid_str())
|
||||
}
|
||||
|
||||
/// The Java pool's skip rule: a chunk whose cache file EXISTS (any size) is not fetched.
|
||||
pub fn is_chunk_cached(cache_dir: &Path, chunk: &ChunkInfo) -> bool {
|
||||
cached_chunk_path(cache_dir, chunk).exists()
|
||||
}
|
||||
|
||||
/// One CDN entry as the Java manager passes it: `cdn.baseUrl + cdn.cloudDir` (auth params are
|
||||
/// never used on chunk URLs). Chunk URL = `prefix + "/" + chunk.getPath(chunkDir)`.
|
||||
pub fn chunk_url(cdn_prefix: &str, chunk_dir: &str, chunk: &ChunkInfo) -> String {
|
||||
format!("{}/{}", cdn_prefix, chunk.path(chunk_dir))
|
||||
}
|
||||
|
||||
/// Undo the JSON string escaping the Java CDN scanner passes through. `EpicApiClient` re-serializes
|
||||
/// the manifest API response with Android's `org.json`, which escapes every `/` as `\/`, and
|
||||
/// `parseCdnUrls` takes the raw substrings — so a base URL arrives as `https:\/\/host\` and a cloud
|
||||
/// dir as `/Builds\/Org\/…\/default\`. Java's `HttpURLConnection` (OkHttp) treats `\` as `/` and
|
||||
/// only ever reached the first CDN, so it never noticed; reqwest's URL parser does the same
|
||||
/// mapping, which leaves empty `//` path segments that Fastly/Akamai tolerate but CloudFront
|
||||
/// rejects outright (immediate 4xx). Produce the URL Java intended: `\/` → `/`, any other `\`
|
||||
/// dropped, and no trailing slash (the chunk path is joined with one).
|
||||
pub fn normalize_prefix(raw: &str) -> String {
|
||||
let unescaped = raw.replace("\\/", "/").replace('\\', "");
|
||||
unescaped.trim_end_matches('/').to_string()
|
||||
}
|
||||
|
||||
/// Distinct CDN prefixes in first-seen order, normalized (the fetch core wants one host key per
|
||||
/// distinct endpoint; Java would simply try an identical URL twice, which changes nothing).
|
||||
pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec<String> {
|
||||
let mut seen = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
@@ -153,9 +116,6 @@ pub fn distinct_prefixes(cdn_prefixes: &[String]) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Improvements round 1: split the window across the distinct CDNs so the whole ceiling is
|
||||
/// reachable whatever the host count — `max(6, ceil(max_workers / hosts))` (3 Epic CDNs at a
|
||||
/// ceiling of 32 → 11 each; a single host gets all 32).
|
||||
pub fn per_host_cap(max_workers: usize, host_count: usize) -> usize {
|
||||
let hosts = host_count.max(1);
|
||||
let workers = max_workers.max(1);
|
||||
@@ -176,14 +136,10 @@ mod tests {
|
||||
#[test]
|
||||
fn install_tag_rules_match_java() {
|
||||
let m = manifest();
|
||||
// null → all
|
||||
assert_eq!(resolve_install_files(&m, None), vec![0, 1, 2]);
|
||||
// empty → required only (file 0 has no tags)
|
||||
assert_eq!(resolve_install_files(&m, Some(&[])), vec![0]);
|
||||
// German → base + the German file
|
||||
let de = vec!["German".to_string(), "de-DE".to_string()];
|
||||
assert_eq!(resolve_install_files(&m, Some(&de)), vec![0, 1]);
|
||||
// unknown tag → nothing extra matched, but base files matched, so base only
|
||||
let xx = vec!["Klingon".to_string()];
|
||||
assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0]);
|
||||
}
|
||||
@@ -204,7 +160,6 @@ mod tests {
|
||||
let mut files = sample_files();
|
||||
files[0].tags = vec!["English".to_string()];
|
||||
let m = parse_manifest(&build_manifest(&sample_chunks(), &files, 21, false)).unwrap();
|
||||
// No untagged file, unknown tag selected → matched nothing → required → (none) → all.
|
||||
let xx = vec!["Klingon".to_string()];
|
||||
assert_eq!(resolve_install_files(&m, Some(&xx)), vec![0, 1, 2]);
|
||||
}
|
||||
@@ -212,12 +167,10 @@ mod tests {
|
||||
#[test]
|
||||
fn unique_chunks_follow_first_seen_part_order() {
|
||||
let m = manifest();
|
||||
// file 2 first: chunk c[2] then c[0]; then file 0: c[0] (dup), c[1].
|
||||
assert_eq!(unique_chunks_for_files(&m, &[2, 0]), vec![2, 0, 1]);
|
||||
assert_eq!(unique_chunks_for_files(&m, &[1]), vec![1]);
|
||||
assert!(unique_chunks_for_files(&m, &[]).is_empty());
|
||||
assert!(unique_chunks_for_files(&m, &[99]).is_empty(), "out-of-range index ignored");
|
||||
// total = 700000 + max(0,1) + 9
|
||||
assert_eq!(total_credit_bytes(&m, &[0, 1, 2]), 700_000 + 1 + 9);
|
||||
}
|
||||
|
||||
@@ -256,13 +209,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn prefixes_are_unescaped_from_the_java_json_scan() {
|
||||
// Exactly what Bannerlator's Epic debug log shows its Java manager extracting for a real title.
|
||||
let raw = "https:\\/\\/egdownload.fastly-edge.com\\/Builds\\/Org\\/o-83e8\\/04b7\\/default\\";
|
||||
assert_eq!(
|
||||
normalize_prefix(raw),
|
||||
"https://egdownload.fastly-edge.com/Builds/Org/o-83e8/04b7/default"
|
||||
);
|
||||
// Already-clean input is untouched (minus a trailing slash).
|
||||
assert_eq!(
|
||||
normalize_prefix("https://download.epicgames.com/Builds/x/default/"),
|
||||
"https://download.epicgames.com/Builds/x/default"
|
||||
|
||||
@@ -1,28 +1,3 @@
|
||||
//! Shared adaptive HTTP fetch core for the store download engines (Epic / GOG / Amazon).
|
||||
//!
|
||||
//! This is the generic half of the fetch layer Bannerlator's Steam depot writer grew: an
|
||||
//! adaptive in-flight window (slow-start → hold at plateau → shrink on REAL errors only), a
|
||||
//! speed-ranked per-host-capped scheduler, an in-flight RAW byte budget sized from the window
|
||||
//! ceiling, per-item retry with exponential back-off + host rotation, one current-thread tokio
|
||||
//! runtime driving every request through a [`FuturesUnordered`], and an mpsc hand-off to a sync
|
||||
//! process pool that calls the adapter's [`FetchSink::process`] (inflate + hash + write).
|
||||
//!
|
||||
//! The tunables come from [`crate::fetch_tuning`], copied number for number from Bannerlator's
|
||||
//! depot writer, so a `fetch-window` line from either app reads against the same tuning notes.
|
||||
//!
|
||||
//! What the core does NOT know: manifests, resume/skip rules, output layout, hashing. The adapter
|
||||
//! builds the item list (already excluding present+verified items, exactly as its Java manager
|
||||
//! does), maps host index → URL, and does everything with the body inside its sink.
|
||||
//!
|
||||
//! Log grammar (all lines go through the `log` callback; `label` is adapter-chosen, e.g.
|
||||
//! `epic app=Fortnite`):
|
||||
//! - `fetch-start label=… items=… bytes_reserved=… hosts=… ceiling=… budget=…MiB tier_max=… per_host_cap=… distinct_hosts=… window=… mode=body|stream`
|
||||
//! - `fetch-window label=… window=N (min=… max=…) in_flight=… last=…MB/s ewma=…MB/s best=…MB/s reason=… cooldown=…ms err_rate=…% phase=… rtt=…ms budget_stalls=… host_stalls=…`
|
||||
//! (same fields as the Steam engine's `fetch-window depot=…` line, so the two are A/B-comparable)
|
||||
//! - `throughput label=… overall=…MB/s total=…MB elapsed=…s used=x/y servers: [host …MB/s …MB] …` every 5 s
|
||||
//! - `fetch-end label=… items_ok=… bytes=… credited=… elapsed_ms=… avg_mbps=… peak_mbps=… result=ok|cancelled|error [error=…]`
|
||||
//!
|
||||
//! `MB/s` here means MiB/s (1024²), matching the Steam engine's lines.
|
||||
|
||||
use crate::fetch_tuning::{
|
||||
budget_admits, connect_timeout, inflight_budget_bytes, retry_backoff_millis, AsyncFetchError,
|
||||
@@ -46,95 +21,42 @@ use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
|
||||
/// Maximum attempts per item (fetch failures AND `SinkError::Retry` both count). Same as Steam.
|
||||
pub const MAX_ITEM_ATTEMPTS: u32 = MAX_CHUNK_ATTEMPTS;
|
||||
/// How often the throughput reporter samples (and logs) — the samples feed `peak_mbps`.
|
||||
pub const THROUGHPUT_SAMPLE_INTERVAL_MS: u64 = 5_000;
|
||||
/// Upper bound on how long the driver sits in one `await` before re-checking cancel/error/feedback.
|
||||
const DRIVER_POLL_MS: u64 = 250;
|
||||
|
||||
// ── Delay awareness (bufferbloat) ──
|
||||
// On a 5G hotspot the window kept growing (31 → 84) while the per-request delay went from 3 s to
|
||||
// 11 s and throughput stayed at 10-20 MB/s: every extra request only queued behind the others in
|
||||
// the link's buffer. The signal used here is time-to-first-byte (request sent → response headers),
|
||||
// which on a link with a deep buffer grows with the queue, while on a fast link it stays at a few
|
||||
// round trips however wide the window gets. The baseline is the lowest per-probe mean seen.
|
||||
|
||||
/// A probe needs at least this many TTFB samples before its mean is trusted.
|
||||
pub const LATENCY_MIN_SAMPLES: u32 = 3;
|
||||
/// Mean TTFB at or above `baseline ×` this counts as queueing delay…
|
||||
pub const LATENCY_HOLD_FACTOR: f64 = 2.5;
|
||||
/// …and only when it is also this far above the baseline in absolute terms: a fast link whose
|
||||
/// TTFB goes from 20 ms to 60 ms (new connections' TLS handshakes) is not bufferbloat.
|
||||
pub const LATENCY_MIN_EXCESS_MS: f64 = 250.0;
|
||||
/// Throughput above `best × (1 + this)` is a gain that justifies the extra delay: hold, but never
|
||||
/// shrink, while it lasts.
|
||||
pub const LATENCY_GAIN_EPS: f64 = 0.25;
|
||||
/// Gentle proportional shrink while the delay keeps rising (gentler than the error shrink).
|
||||
pub const LATENCY_SHRINK_FACTOR: f64 = 0.85;
|
||||
|
||||
// ── Stalled hosts ──
|
||||
// A CDN host can stop delivering while its connections stay open: on the device the Fastly host
|
||||
// froze at 49 MB for the rest of the run while the other host carried everything, and its requests
|
||||
// sat until their idle timeout. A host whose in-flight requests receive no byte for this long,
|
||||
// while another host is receiving, has its requests cancelled and re-queued and is put to rest.
|
||||
|
||||
/// No byte from any of a host's in-flight requests for this long (while another host delivers).
|
||||
pub const HOST_STALL_MS: u64 = if cfg!(test) { 1_500 } else { 10_000 };
|
||||
/// First rest for a stalled host; doubles per further stall of the same host, reset on success.
|
||||
pub const HOST_STALL_COOLDOWN_MS: u64 = 30_000;
|
||||
pub const HOST_STALL_COOLDOWN_MAX_MS: u64 = 300_000;
|
||||
/// How often the driver looks for stalled hosts.
|
||||
const STALL_CHECK_MS: u64 = 250;
|
||||
/// Stream mode: the longest a request may sit with no byte arriving before it is failed (the
|
||||
/// store's own timeout when it is shorter). A dead connection on a single-host run, where the
|
||||
/// stall detector has no healthy host to compare with, never holds an item longer than this.
|
||||
pub const STREAM_IDLE_CAP: Duration = Duration::from_secs(30);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Public contract
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// One unit of work: a body to fetch from one of the hosts and hand to the sink.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FetchItem {
|
||||
/// Adapter-defined (index into the adapter's own table). Only echoed in errors / logs.
|
||||
pub id: u64,
|
||||
/// `len == hosts.len()` → `urls[host_idx]`; `len == 1` → the same URL on every host. Any other
|
||||
/// non-zero length falls back to `urls.get(host_idx)` then `urls[0]`. Empty = fatal.
|
||||
pub urls: Vec<String>,
|
||||
/// Expected RAW (wire) bytes, for the in-flight byte budget; `0` → nominal 1 MiB. Ignored in
|
||||
/// stream mode (streams are budgeted piece by piece as they arrive).
|
||||
pub reserve: u64,
|
||||
/// Inclusive HTTP `Range` (`bytes=a-b`); `None` = whole body.
|
||||
pub range: Option<(u64, u64)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FetchOptions {
|
||||
/// Window ceiling (tier); the core clamps it to `hosts × per_host_cap` and the hard cap.
|
||||
pub max_workers: usize,
|
||||
/// Max concurrent requests per distinct host (6 unless the store needs otherwise). Honoured
|
||||
/// as given: 1 host × `per_host_cap` 8 = a ceiling of 8.
|
||||
pub per_host_cap: usize,
|
||||
/// Whole-body mode: the deadline for the entire request (connect + headers + body). Stream
|
||||
/// mode: the deadline for the response headers, then an IDLE deadline per received piece.
|
||||
pub timeout: Duration,
|
||||
/// Sent on EVERY request (applied on the request builder, so they override the client-level
|
||||
/// defaults — e.g. a store's own `User-Agent`, or `Authorization: Bearer …`). Validated once
|
||||
/// when the client is built.
|
||||
pub headers: Vec<(String, String)>,
|
||||
/// PEM bundle path; `""` = the platform roots reqwest/rustls ships with.
|
||||
pub ca_bundle_path: String,
|
||||
/// Sync process-pool threads (inflate + hash + write).
|
||||
pub process_workers: usize,
|
||||
/// Log prefix, e.g. `"epic app=Fortnite"`.
|
||||
pub label: String,
|
||||
/// `false` (default): each item's whole body is buffered and handed to
|
||||
/// [`FetchSink::process`]. `true`: the body is streamed — pieces go to
|
||||
/// [`FetchSink::on_chunk`] in order as they arrive, then [`FetchSink::on_finish`]; for
|
||||
/// multi-GB files (Amazon) that keeps memory at "pieces not yet written", not whole files.
|
||||
/// Applies to every item of the run.
|
||||
pub stream: bool,
|
||||
}
|
||||
|
||||
@@ -153,36 +75,20 @@ impl Default for FetchOptions {
|
||||
}
|
||||
}
|
||||
|
||||
/// What the sink says about a body it was handed.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum SinkError {
|
||||
/// Counts as a failed attempt for that item: host cooled + demoted, back-off, re-dispatch on a
|
||||
/// different host, up to [`MAX_ITEM_ATTEMPTS`]. In stream mode the whole item is re-streamed
|
||||
/// (`on_chunk` sees `offset == 0` again).
|
||||
Retry(String),
|
||||
/// Abort the whole run; this becomes the run's first error.
|
||||
Fatal(String),
|
||||
}
|
||||
|
||||
/// Runs on a process-pool thread. Whole-body mode uses [`FetchSink::process`]; stream mode uses
|
||||
/// [`FetchSink::on_chunk`] + [`FetchSink::on_finish`] (default impls refuse, so a whole-body sink
|
||||
/// is unaffected by the stream option existing).
|
||||
pub trait FetchSink: Send + Sync {
|
||||
/// Whole-body mode: the raw wire body. Do inflate + hash verify + write here. Returns the
|
||||
/// number of bytes to credit to progress (adapter's choice, e.g. decompressed bytes).
|
||||
fn process(&self, item: &FetchItem, body: Vec<u8>) -> Result<u64, SinkError>;
|
||||
|
||||
/// Stream mode: one piece of the body, delivered in order (`offset` = bytes of this attempt
|
||||
/// delivered before this piece). `offset == 0` marks the (re)start of an attempt — truncate any
|
||||
/// partial output from a previous attempt. Each processed piece credits `data.len()` bytes.
|
||||
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
let _ = (item, offset, data);
|
||||
Err(SinkError::Fatal("streaming not supported by this sink".to_string()))
|
||||
}
|
||||
|
||||
/// Stream mode: the body ended after `total_len` bytes. Finalize (rename `.tmp`, verify the
|
||||
/// hash, …). Returns EXTRA bytes to credit (pieces were already credited as they were
|
||||
/// written; return 0 for none).
|
||||
fn on_finish(&self, item: &FetchItem, total_len: u64) -> Result<u64, SinkError> {
|
||||
let _ = (item, total_len);
|
||||
Err(SinkError::Fatal("streaming not supported by this sink".to_string()))
|
||||
@@ -191,15 +97,9 @@ pub trait FetchSink: Send + Sync {
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct FetchOutcome {
|
||||
/// Sum of what the sink returned for every successfully processed item (stream mode: every
|
||||
/// piece written + every `on_finish` extra; pieces of an attempt that later failed stay
|
||||
/// counted).
|
||||
pub bytes_credited: u64,
|
||||
/// First error (fetch after max attempts, sink `Fatal`, client build failure).
|
||||
pub error: Option<String>,
|
||||
/// `cancel` was observed set.
|
||||
pub cancelled: bool,
|
||||
/// Items the sink accepted.
|
||||
pub items_ok: u64,
|
||||
}
|
||||
|
||||
@@ -209,10 +109,6 @@ impl FetchOutcome {
|
||||
}
|
||||
}
|
||||
|
||||
/// Blocking: spawns its own current-thread tokio runtime + process pool and returns when every
|
||||
/// item is processed, `cancel` is set, or the first fatal error is recorded. `progress` fires from
|
||||
/// a process-pool thread after each successful `process` / each written piece + `on_finish` (the
|
||||
/// caller throttles).
|
||||
pub fn run_fetch(
|
||||
items: Vec<FetchItem>,
|
||||
hosts: &[String],
|
||||
@@ -254,8 +150,6 @@ mode={}",
|
||||
let error_slot: Mutex<Option<String>> = Mutex::new(None);
|
||||
let reporter_done = AtomicBool::new(false);
|
||||
let meter = BandwidthMeter::new(&hosts);
|
||||
// Stream mode: per-item "attempt N was rejected by the sink mid-stream" flags, so the driver's
|
||||
// in-flight stream stops reading instead of feeding pieces nobody wants.
|
||||
let abandon: Vec<AtomicU32> = if stream {
|
||||
items.iter().map(|_| AtomicU32::new(NO_ATTEMPT)).collect()
|
||||
} else {
|
||||
@@ -268,11 +162,6 @@ mode={}",
|
||||
let abandon = &abandon;
|
||||
|
||||
thread::scope(|scope| {
|
||||
// Async(fetch) → sync(process): tokio unbounded mpsc drained with `blocking_recv`. Memory is
|
||||
// bounded by the RAW byte budget enforced at DISPATCH (whole-body) / per piece (stream), not
|
||||
// by channel capacity. Whole-body mode: ONE channel shared by every worker (any worker takes
|
||||
// the next body). Stream mode: one channel PER worker and an item's pieces always go to the
|
||||
// same worker (`idx % workers`), so pieces are written in order.
|
||||
let channel_count = if stream { proc_count } else { 1 };
|
||||
let mut txs: Vec<tokio::sync::mpsc::UnboundedSender<ProcessMsg>> =
|
||||
Vec::with_capacity(channel_count);
|
||||
@@ -283,7 +172,6 @@ mode={}",
|
||||
txs.push(tx);
|
||||
rxs.push(Arc::new(Mutex::new(rx)));
|
||||
}
|
||||
// Sync(process) → async(driver): sink verdicts, polled non-blocking by the driver.
|
||||
let (fb_tx, fb_rx) = std_mpsc::channel::<Feedback>();
|
||||
let bytes_credited = &bytes_credited;
|
||||
let items_ok = &items_ok;
|
||||
@@ -291,8 +179,6 @@ mode={}",
|
||||
let error_slot = &error_slot;
|
||||
let reporter_done = &reporter_done;
|
||||
|
||||
// Periodic throughput reporter — atomics only, never on the fetch hot path. Its samples are
|
||||
// also the `peak_mbps` source.
|
||||
let reporter = scope.spawn(move || {
|
||||
crate::priority::background();
|
||||
loop {
|
||||
@@ -312,15 +198,12 @@ mode={}",
|
||||
}
|
||||
});
|
||||
|
||||
// ── Process pool ──
|
||||
let mut proc_handles = Vec::with_capacity(proc_count);
|
||||
for worker in 0..proc_count {
|
||||
let rx = Arc::clone(&rxs[if stream { worker } else { 0 }]);
|
||||
let fb = fb_tx.clone();
|
||||
proc_handles.push(scope.spawn(move || {
|
||||
crate::priority::background();
|
||||
// Stream mode: (item → attempt, message) whose pieces the sink rejected; the rest
|
||||
// of that attempt's pieces are dropped and its Finish becomes a Retry verdict.
|
||||
let mut poisoned: HashMap<usize, (u32, String)> = HashMap::new();
|
||||
loop {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
@@ -344,8 +227,6 @@ mode={}",
|
||||
body,
|
||||
} => {
|
||||
let raw_len = body.len() as u64;
|
||||
// A body that was already queued when cancel / the first error landed
|
||||
// must not be written any more.
|
||||
if cancel.load(Ordering::Relaxed)
|
||||
|| error_slot.lock().expect("err slot poisoned").is_some()
|
||||
{
|
||||
@@ -354,7 +235,6 @@ mode={}",
|
||||
}
|
||||
let item = &items[idx];
|
||||
let res = sink.process(item, body);
|
||||
// Free the budget on PROCESS-COMPLETE (actual raw bytes).
|
||||
in_flight.fetch_sub(raw_len, Ordering::Relaxed);
|
||||
let verdict = match res {
|
||||
Ok(credited) => {
|
||||
@@ -390,7 +270,7 @@ mode={}",
|
||||
let len = data.len() as u64;
|
||||
in_flight.fetch_sub(len, Ordering::Relaxed);
|
||||
if poisoned.get(&idx).is_some_and(|(a, _)| *a == attempts) {
|
||||
continue; // rest of a rejected attempt
|
||||
continue;
|
||||
}
|
||||
if cancel.load(Ordering::Relaxed)
|
||||
|| error_slot.lock().expect("err slot poisoned").is_some()
|
||||
@@ -471,11 +351,6 @@ mode={}",
|
||||
}
|
||||
drop(fb_tx);
|
||||
|
||||
// ── Async fetch driver on one current-thread runtime. `txs` are moved in and dropped when
|
||||
// the driver returns, which closes the channels so the pool's `blocking_recv` yields
|
||||
// `None`. ──
|
||||
// The runtime drives on this (already lowered) thread; `on_thread_start` covers the
|
||||
// blocking pool it starts for DNS lookups.
|
||||
match tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.on_thread_start(crate::priority::background)
|
||||
@@ -515,7 +390,6 @@ mode={}",
|
||||
let _ = reporter.join();
|
||||
});
|
||||
|
||||
// Final partial-interval sample so a short run still has a peak.
|
||||
meter.sample();
|
||||
let elapsed = started.elapsed();
|
||||
let elapsed_secs = elapsed.as_secs_f64().max(0.001);
|
||||
@@ -552,11 +426,7 @@ avg_mbps={avg_mbps:.2} peak_mbps={peak_mbps:.2} result={result}",
|
||||
outcome
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Pure helpers (unit-tested)
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Bytes reserved against the in-flight budget at DISPATCH (nominal when the adapter has none).
|
||||
pub fn reserve_bytes(item: &FetchItem) -> u64 {
|
||||
if item.reserve == 0 {
|
||||
NOMINAL_CHUNK_RESERVE_BYTES
|
||||
@@ -565,8 +435,6 @@ pub fn reserve_bytes(item: &FetchItem) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Window bounds: `max` = tier ceiling clamped to distinct-hosts × per-host-cap and the hard cap;
|
||||
/// `min` = the floor; `bootstrap` = the start. Same arithmetic as the Steam depot writer.
|
||||
pub fn window_bounds(
|
||||
max_workers: usize,
|
||||
distinct_hosts: usize,
|
||||
@@ -579,7 +447,6 @@ pub fn window_bounds(
|
||||
(bootstrap, min, max)
|
||||
}
|
||||
|
||||
/// Distinct host count (string equality) — the multiplier for the window's host ceiling.
|
||||
pub fn distinct_host_count(hosts: &[String]) -> usize {
|
||||
let mut keys: Vec<&str> = Vec::new();
|
||||
for h in hosts {
|
||||
@@ -590,8 +457,6 @@ pub fn distinct_host_count(hosts: &[String]) -> usize {
|
||||
keys.len().max(1)
|
||||
}
|
||||
|
||||
/// The URL to use for `item` on host `host_idx` (see [`FetchItem::urls`]). `None` when the item
|
||||
/// carries no URL at all.
|
||||
pub fn url_for_host(item: &FetchItem, host_idx: usize) -> Option<&str> {
|
||||
if item.urls.is_empty() {
|
||||
return None;
|
||||
@@ -607,7 +472,6 @@ pub fn url_for_host(item: &FetchItem, host_idx: usize) -> Option<&str> {
|
||||
)
|
||||
}
|
||||
|
||||
/// Byte count an inclusive range must deliver; `None` for an inverted range.
|
||||
pub fn range_len(range: (u64, u64)) -> Option<u64> {
|
||||
let (a, b) = range;
|
||||
if b < a {
|
||||
@@ -617,8 +481,6 @@ pub fn range_len(range: (u64, u64)) -> Option<u64> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates a received body's length: for a range request it must be exactly `b-a+1`; otherwise
|
||||
/// it must match `Content-Length` when the server sent one (as the Steam client does).
|
||||
pub fn validate_body_len(
|
||||
body_len: u64,
|
||||
content_length: Option<u64>,
|
||||
@@ -653,13 +515,10 @@ pub fn validate_body_len(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an HTTP status is acceptable for this request shape (200 always; 206 only when a range
|
||||
/// was asked for).
|
||||
pub fn status_accepted(status: u16, ranged: bool) -> bool {
|
||||
status == 200 || (ranged && status == 206)
|
||||
}
|
||||
|
||||
/// Classifies a rejected HTTP status the same way the Steam async client does.
|
||||
pub fn classify_status(status: u16) -> FetchFailKind {
|
||||
if status == 429 {
|
||||
FetchFailKind::RateLimited
|
||||
@@ -670,10 +529,6 @@ pub fn classify_status(status: u16) -> FetchFailKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// The HTTP status carried by a `send_checked` rejection message, if that is what the error was.
|
||||
/// A 4xx other than 429 is a CLIENT rejection (bad/expired signed URL, wrong host for this path,
|
||||
/// 404): it says nothing about congestion, so the driver demotes the host but does not shrink the
|
||||
/// shared window for it — one CDN that refuses a path must not throttle the healthy ones.
|
||||
pub fn rejected_status(message: &str) -> Option<u16> {
|
||||
let rest = message.strip_prefix("unexpected HTTP status (")?;
|
||||
let digits = rest.strip_suffix(')')?;
|
||||
@@ -684,8 +539,6 @@ pub fn is_client_rejection(message: &str) -> bool {
|
||||
matches!(rejected_status(message), Some(s) if (400..500).contains(&s) && s != 429)
|
||||
}
|
||||
|
||||
/// How many individual fetch failures a run logs verbatim before going quiet (the window and
|
||||
/// throughput lines keep reporting the aggregate).
|
||||
pub const FETCH_ERROR_LOG_LIMIT: u32 = 12;
|
||||
|
||||
fn record_first_error(slot: &Mutex<Option<String>>, err: String) {
|
||||
@@ -695,11 +548,7 @@ fn record_first_error(slot: &Mutex<Option<String>>, err: String) {
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// HTTP client (one pooled reqwest client + per-request headers + optional Range + streaming)
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// The pooled client plus the pre-validated per-request headers.
|
||||
struct Http {
|
||||
client: reqwest::Client,
|
||||
headers: Vec<(reqwest::header::HeaderName, reqwest::header::HeaderValue)>,
|
||||
@@ -732,8 +581,6 @@ fn build_http(opts: &FetchOptions, pool_max_idle_per_host: usize) -> Result<Http
|
||||
}
|
||||
|
||||
impl Http {
|
||||
/// `GET url` with the run's headers (set per request, so they override the client defaults —
|
||||
/// a store's own `User-Agent` wins over the Steam one) and the optional `Range`.
|
||||
fn request(&self, url: &str, range: Option<(u64, u64)>) -> reqwest::RequestBuilder {
|
||||
let mut request = self.client.get(url);
|
||||
for (name, value) in &self.headers {
|
||||
@@ -756,7 +603,6 @@ fn classify_reqwest_error(err: &reqwest::Error) -> FetchFailKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends the request and checks the status; shared by both modes.
|
||||
async fn send_checked(
|
||||
http: &Http,
|
||||
url: &str,
|
||||
@@ -786,9 +632,6 @@ async fn send_checked(
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Whole-body mode: one attempt for one item (the driver owns retry/rotation). `timeout` bounds the
|
||||
/// whole request (headers + body). The body is read piece by piece so `host_rx` sees every byte as
|
||||
/// it arrives (the stall detector's signal); `ttfb` is set when the response headers arrive.
|
||||
async fn fetch_once(
|
||||
http: &Http,
|
||||
url: &str,
|
||||
@@ -835,10 +678,6 @@ async fn fetch_once(
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream mode: one attempt for one item. Pieces are pushed to the item's pool worker as they
|
||||
/// arrive (each one added to the in-flight budget until the pool consumes it); the future waits
|
||||
/// while the budget is full, and stops early on cancel or when the sink rejected this attempt.
|
||||
/// Returns the total body length after the last piece was queued.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn fetch_stream(
|
||||
http: &Http,
|
||||
@@ -871,7 +710,6 @@ async fn fetch_stream(
|
||||
let content_length = response.content_length();
|
||||
let mut offset = 0u64;
|
||||
loop {
|
||||
// Budget gate: don't read ahead of the pool by more than the budget.
|
||||
while in_flight.load(Ordering::Relaxed) >= budget {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(AsyncFetchError {
|
||||
@@ -931,29 +769,22 @@ async fn fetch_stream(
|
||||
Ok(offset)
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Driver plumbing
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// `abandon` sentinel: no attempt of this item has been rejected by the sink.
|
||||
const NO_ATTEMPT: u32 = u32::MAX;
|
||||
|
||||
enum ProcessMsg {
|
||||
/// Whole-body mode: the complete raw body.
|
||||
Body {
|
||||
idx: usize,
|
||||
attempts: u32,
|
||||
server_idx: usize,
|
||||
body: Vec<u8>,
|
||||
},
|
||||
/// Stream mode: one in-order piece of attempt `attempts` of item `idx`.
|
||||
Piece {
|
||||
idx: usize,
|
||||
attempts: u32,
|
||||
offset: u64,
|
||||
data: Vec<u8>,
|
||||
},
|
||||
/// Stream mode: the body ended; the pool finalizes and reports a verdict.
|
||||
Finish {
|
||||
idx: usize,
|
||||
attempts: u32,
|
||||
@@ -975,7 +806,6 @@ struct Feedback {
|
||||
verdict: SinkVerdict,
|
||||
}
|
||||
|
||||
/// An item awaiting (re)dispatch: `attempts` already spent; not eligible before `not_before`.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct PendingItem {
|
||||
idx: usize,
|
||||
@@ -988,18 +818,14 @@ enum Fetched {
|
||||
Streamed(u64),
|
||||
}
|
||||
|
||||
/// The result the driver awaits for each dispatched fetch.
|
||||
struct FetchDone {
|
||||
idx: usize,
|
||||
attempts: u32,
|
||||
reserve: u64,
|
||||
server_idx: usize,
|
||||
elapsed: Duration,
|
||||
/// Request sent → response headers, when the headers arrived.
|
||||
ttfb: Option<Duration>,
|
||||
/// Key in the driver's registry of running requests.
|
||||
req_id: u64,
|
||||
/// The request was cancelled because its host stalled (not a failure of the item).
|
||||
stalled: bool,
|
||||
res: Result<Fetched, AsyncFetchError>,
|
||||
}
|
||||
@@ -1061,11 +887,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
let mut retry: VecDeque<PendingItem> = VecDeque::new();
|
||||
let mut next_item = 0usize;
|
||||
let mut errors_logged: u32 = 0;
|
||||
// Bodies / finishes handed to the pool whose verdict has not come back yet.
|
||||
let mut outstanding = 0usize;
|
||||
// Stall detection: bytes received per server (bumped by the request futures as pieces
|
||||
// arrive), when that count last moved, since when the server has had requests to answer,
|
||||
// how many requests it is running, and each running request's abort switch.
|
||||
let host_rx: Vec<AtomicU64> = hosts.iter().map(|_| AtomicU64::new(0)).collect();
|
||||
let mut last_rx: Vec<u64> = vec![0; hosts.len()];
|
||||
let mut last_rx_at: Vec<Option<Instant>> = vec![None; hosts.len()];
|
||||
@@ -1075,7 +897,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
let mut aborts: HashMap<u64, (usize, Arc<Notify>)> = HashMap::new();
|
||||
let mut next_req_id: u64 = 0;
|
||||
let mut last_stall_check = Instant::now();
|
||||
// Declared AFTER `txs`/`http` so it is dropped BEFORE them (its futures borrow both).
|
||||
let mut inflight: FuturesUnordered<_> = FuturesUnordered::new();
|
||||
|
||||
'outer: loop {
|
||||
@@ -1085,8 +906,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
break;
|
||||
}
|
||||
|
||||
// ── Sink verdicts: Retry = counts as a failed attempt (cool + demote host, back-off,
|
||||
// rotate); Fatal was already recorded in the error slot by the pool. ──
|
||||
while let Ok(fb) = fb_rx.try_recv() {
|
||||
outstanding = outstanding.saturating_sub(1);
|
||||
match fb.verdict {
|
||||
@@ -1123,7 +942,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
log(&window.summary_line(label, inflight.len(), probe_now));
|
||||
}
|
||||
|
||||
// ── Stalled hosts: requests that receive nothing while another host delivers. ──
|
||||
if probe_now.duration_since(last_stall_check) >= Duration::from_millis(STALL_CHECK_MS) {
|
||||
last_stall_check = probe_now;
|
||||
for s in 0..hosts.len() {
|
||||
@@ -1163,12 +981,10 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
hosts[s],
|
||||
probe_now.duration_since(silent_since).as_millis()
|
||||
));
|
||||
// Do not fire again for the same silence while the cancelled requests wind down.
|
||||
watch_from[s] = probe_now;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Dispatch up to the current window ──
|
||||
while inflight.len() < window.current {
|
||||
if cancel.load(Ordering::Relaxed)
|
||||
|| error_slot.lock().expect("err slot poisoned").is_some()
|
||||
@@ -1177,7 +993,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
}
|
||||
let now = Instant::now();
|
||||
|
||||
// A due retry first, else the next fresh item.
|
||||
let use_retry = retry.front().is_some_and(|p| p.not_before <= now);
|
||||
let (idx, attempts) = if use_retry {
|
||||
let p = *retry.front().expect("retry nonempty");
|
||||
@@ -1185,23 +1000,19 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
} else if next_item < items.len() {
|
||||
(next_item, 0u32)
|
||||
} else {
|
||||
break; // nothing dispatchable now (retry not due, or all dispatched)
|
||||
break;
|
||||
};
|
||||
let item = &items[idx];
|
||||
|
||||
// Byte-budget gate (hard memory bound): reserve at DISPATCH. Always admits when nothing
|
||||
// is in flight, so an item bigger than the whole budget can't deadlock. Streams reserve
|
||||
// nothing here — their pieces are budgeted as they arrive.
|
||||
let reserve = if stream { 0 } else { reserve_bytes(item) };
|
||||
if !budget_admits(in_flight.load(Ordering::Relaxed), reserve, budget) {
|
||||
window.note_budget_stall();
|
||||
break; // wait for a completion to free budget
|
||||
break;
|
||||
}
|
||||
|
||||
// Speed-ranked, per-host-capped host pick.
|
||||
let Some(server_idx) = sched.pick(now) else {
|
||||
window.note_host_stall();
|
||||
break; // every host at cap or cooling → wait for a completion
|
||||
break;
|
||||
};
|
||||
let permit = match sched.host_sem(server_idx).clone().try_acquire_owned() {
|
||||
Ok(permit) => permit,
|
||||
@@ -1218,7 +1029,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
let url = url.to_string();
|
||||
let range = item.range;
|
||||
|
||||
// Commit: pop the work item, reserve budget, launch the fetch future.
|
||||
if use_retry {
|
||||
retry.pop_front();
|
||||
} else {
|
||||
@@ -1231,7 +1041,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
let rx_counter = &host_rx[server_idx];
|
||||
let started = Instant::now();
|
||||
if running[server_idx] == 0 {
|
||||
// The host had nothing to answer until now: its silence is measured from here.
|
||||
watch_from[server_idx] = started;
|
||||
}
|
||||
running[server_idx] += 1;
|
||||
@@ -1268,7 +1077,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
.map(Fetched::Body)
|
||||
}
|
||||
};
|
||||
// The stall detector's switch: dropping the fetch future closes its connection.
|
||||
match select(pin!(fetch), pin!(abort.notified())).await {
|
||||
Either::Left((res, _)) => (res, false),
|
||||
Either::Right(((), _)) => (
|
||||
@@ -1280,7 +1088,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
),
|
||||
}
|
||||
};
|
||||
drop(permit); // RAII per-host permit release (also on future drop / cancel-abort)
|
||||
drop(permit);
|
||||
FetchDone {
|
||||
idx,
|
||||
attempts: attempts + 1,
|
||||
@@ -1295,17 +1103,14 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
});
|
||||
}
|
||||
|
||||
// ── Nothing in flight? finished, or waiting on retry-not-due / host cooldown / the pool. ──
|
||||
if inflight.is_empty() {
|
||||
if next_item >= items.len() && retry.is_empty() && outstanding == 0 {
|
||||
break; // every item fetched and processed
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Await one completion (drives ALL in-flight requests cooperatively), bounded so
|
||||
// cancel / a pool Fatal / a Retry verdict are noticed promptly even mid-fetch. ──
|
||||
let done = match tokio::time::timeout(Duration::from_millis(DRIVER_POLL_MS), inflight.next())
|
||||
.await
|
||||
{
|
||||
@@ -1317,9 +1122,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
aborts.remove(&done.req_id);
|
||||
running[done.server_idx] = running[done.server_idx].saturating_sub(1);
|
||||
if done.stalled {
|
||||
// Not the item's fault: straight back into the queue for a healthy host, no window
|
||||
// or host penalty beyond the rest the detector already gave the host. It still counts
|
||||
// as an attempt, which keeps each stream attempt's pieces distinguishable.
|
||||
in_flight.fetch_sub(done.reserve, Ordering::Relaxed);
|
||||
retry.push_front(PendingItem {
|
||||
idx: done.idx,
|
||||
@@ -1340,8 +1142,6 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
meter.record(done.server_idx, raw_len);
|
||||
sched.on_success(done.server_idx, raw_len, done.elapsed);
|
||||
window.record_ok(done.elapsed.as_secs_f64() * 1000.0);
|
||||
// Reconcile the dispatch reservation to the actual raw size; the pool subtracts the
|
||||
// actual size on process-complete, netting this item's budget to zero.
|
||||
if raw_len >= done.reserve {
|
||||
in_flight.fetch_add(raw_len - done.reserve, Ordering::Relaxed);
|
||||
} else {
|
||||
@@ -1357,7 +1157,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
break; // process side gone
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(Fetched::Streamed(total_len)) => {
|
||||
@@ -1374,7 +1174,7 @@ async fn run_driver(ctx: DriverCtx<'_>) {
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
break; // process side gone
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -1423,14 +1223,10 @@ window_shrink={} msg={}",
|
||||
}
|
||||
}
|
||||
|
||||
// Dropping `inflight` aborts any still-running requests (the cancel / error / final-failure
|
||||
// teardown path); dropping the senders then closes the channels so the pool drains and exits.
|
||||
drop(inflight);
|
||||
drop(txs);
|
||||
}
|
||||
|
||||
/// Since when a server has been silent: its last byte, or when it was last given work to do if
|
||||
/// that came later.
|
||||
fn silent_since(last_rx_at: Option<Instant>, watch_from: Instant) -> Instant {
|
||||
match last_rx_at {
|
||||
Some(at) if at > watch_from => at,
|
||||
@@ -1438,16 +1234,12 @@ fn silent_since(last_rx_at: Option<Instant>, watch_from: Instant) -> Instant {
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Bandwidth meter (per-host bytes + 5-s samples for peak)
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
pub struct BandwidthMeter {
|
||||
start: Instant,
|
||||
total_bytes: AtomicU64,
|
||||
per_server_bytes: Vec<AtomicU64>,
|
||||
hosts: Vec<String>,
|
||||
/// (bytes, instant) at the last sample; peak = max sample rate in bytes/s.
|
||||
last_sample: Mutex<(u64, Instant)>,
|
||||
peak_bps: Mutex<f64>,
|
||||
}
|
||||
@@ -1478,8 +1270,6 @@ impl BandwidthMeter {
|
||||
self.total_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Take a throughput sample since the previous one (≥ 1 s apart to be counted) and fold it
|
||||
/// into the peak.
|
||||
pub fn sample(&self) {
|
||||
let now = Instant::now();
|
||||
let total = self.total_bytes();
|
||||
@@ -1530,9 +1320,6 @@ impl BandwidthMeter {
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Scheduler + adaptive window — a copy of the Steam depot writer's (keyed on host strings).
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct ServerHealth {
|
||||
@@ -1542,10 +1329,6 @@ struct ServerHealth {
|
||||
cooldown_until: Option<Instant>,
|
||||
}
|
||||
|
||||
/// Speed-ranked, per-host-capped host picker. Cold hosts are probed first, then selection is
|
||||
/// epsilon-greedy over the measured EWMA so a demoted-but-recovered host returns. Per-host caps key
|
||||
/// on the host STRING (one `tokio::sync::Semaphore` each) so two entries that share a host still
|
||||
/// share one cap.
|
||||
struct FetchScheduler {
|
||||
health: Vec<ServerHealth>,
|
||||
server_host: Vec<usize>,
|
||||
@@ -1635,8 +1418,6 @@ impl FetchScheduler {
|
||||
h.cooldown_until = None;
|
||||
}
|
||||
|
||||
/// A stalled host: out of the rotation for `rest`, and its speed estimate cut so it comes
|
||||
/// back behind the hosts that kept delivering.
|
||||
fn rest(&mut self, server_idx: usize, now: Instant, rest: Duration) {
|
||||
let h = &mut self.health[server_idx];
|
||||
h.cooldown_until = Some(now + rest);
|
||||
@@ -1710,8 +1491,6 @@ impl WindowReason {
|
||||
}
|
||||
}
|
||||
|
||||
/// Adaptive in-flight window: slow-start doubles every healthy probe until the first real back-off,
|
||||
/// then grows additively; holds on plateau; shrinks proportionally only on real errors.
|
||||
#[derive(Clone, Debug)]
|
||||
struct AdaptiveWindow {
|
||||
current: usize,
|
||||
@@ -1733,14 +1512,10 @@ struct AdaptiveWindow {
|
||||
last_budget_stalls: u32,
|
||||
last_host_stalls: u32,
|
||||
latency_ewma_ms: f64,
|
||||
/// TTFB samples of the current probe interval.
|
||||
ttfb_sum_ms: f64,
|
||||
ttfb_samples: u32,
|
||||
/// Lowest per-probe mean TTFB seen this run (0 = none yet): the no-queue baseline.
|
||||
ttfb_base_ms: f64,
|
||||
/// Mean TTFB of the last probe that had enough samples.
|
||||
ttfb_last_ms: f64,
|
||||
/// Consecutive probes with queueing delay.
|
||||
latency_streak: u32,
|
||||
stalled_hosts: u32,
|
||||
cooldown_until: Option<Instant>,
|
||||
@@ -1803,7 +1578,6 @@ impl AdaptiveWindow {
|
||||
self.stalled_hosts = self.stalled_hosts.saturating_add(1);
|
||||
}
|
||||
|
||||
/// The latency brake's shrink: proportional and gentle, never below the floor, at least −1.
|
||||
fn shrink_latency(&mut self, now: Instant) {
|
||||
let scaled = (self.current as f64 * LATENCY_SHRINK_FACTOR).floor() as usize;
|
||||
self.current = scaled.min(self.current.saturating_sub(1)).max(self.min);
|
||||
@@ -1884,7 +1658,6 @@ impl AdaptiveWindow {
|
||||
let before = self.current;
|
||||
let cooling = self.cooldown_until.is_some_and(|t| now < t);
|
||||
|
||||
// Queueing delay: this probe's mean TTFB against the run's baseline.
|
||||
let probe_ttfb = (self.ttfb_samples >= LATENCY_MIN_SAMPLES)
|
||||
.then(|| self.ttfb_sum_ms / self.ttfb_samples as f64);
|
||||
let mut queueing = false;
|
||||
@@ -1909,9 +1682,6 @@ impl AdaptiveWindow {
|
||||
} else if cooling {
|
||||
self.last_reason = WindowReason::HoldCooldown;
|
||||
} else if queueing {
|
||||
// More requests only made each one wait longer. Hold while a real gain still comes
|
||||
// with the delay; once the delay keeps rising without one, give some of the window
|
||||
// back, gently, and let the cooldown show whether that helped.
|
||||
let gain = self.bps_ewma > self.best_bps * (1.0 + LATENCY_GAIN_EPS);
|
||||
if gain {
|
||||
self.best_bps = self.bps_ewma;
|
||||
@@ -2012,9 +1782,6 @@ stalled_hosts={}",
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
@@ -2154,8 +1921,6 @@ mod tests {
|
||||
|
||||
const MIB: u64 = 1024 * 1024;
|
||||
|
||||
/// Drive the window through probes 2.1 s apart, each with `samples` TTFB values and the
|
||||
/// bytes that arrived in it; returns (reason, window) after every probe.
|
||||
fn run_probes(w: &mut AdaptiveWindow, t0: Instant, probes: &[(f64, u64)]) -> Vec<(WindowReason, usize)> {
|
||||
let mut total = 0u64;
|
||||
let mut out = Vec::new();
|
||||
@@ -2174,7 +1939,6 @@ mod tests {
|
||||
fn rising_delay_without_more_throughput_holds_then_shrinks_gently() {
|
||||
let t0 = Instant::now();
|
||||
let mut w = AdaptiveWindow::new(8, 2, 256, t0);
|
||||
// Flat ~15 MB/s while the time to first byte climbs from 0.4 s to 12 s (the 5G log).
|
||||
let flat = 32 * MIB;
|
||||
let probes: Vec<(f64, u64)> = [400.0, 400.0, 450.0, 900.0, 1500.0, 2500.0, 4000.0, 6000.0, 9000.0, 12000.0]
|
||||
.iter()
|
||||
@@ -2186,7 +1950,6 @@ mod tests {
|
||||
assert!(first_hold < first_shrink, "{seen:?}");
|
||||
let before = seen[first_shrink - 1].1;
|
||||
assert_eq!(seen[first_shrink].1, (before as f64 * LATENCY_SHRINK_FACTOR).floor() as usize, "{seen:?}");
|
||||
// From the first sign of queueing on, the window never grows again.
|
||||
for pair in seen[first_hold..].windows(2) {
|
||||
assert!(pair[1].1 <= pair[0].1, "grew under queueing delay: {seen:?}");
|
||||
}
|
||||
@@ -2199,7 +1962,6 @@ mod tests {
|
||||
fn a_fast_link_keeps_growing_while_its_delay_stays_low() {
|
||||
let t0 = Instant::now();
|
||||
let mut w = AdaptiveWindow::new(8, 2, 256, t0);
|
||||
// Throughput doubles with the window; TTFB wobbles between 25 and 70 ms (new connections).
|
||||
let probes: Vec<(f64, u64)> = (0..6)
|
||||
.map(|k| ([25.0, 70.0, 40.0, 60.0, 30.0, 65.0][k], (8 * MIB) << k))
|
||||
.collect();
|
||||
@@ -2211,8 +1973,6 @@ mod tests {
|
||||
assert!(w.current >= 128, "{seen:?}");
|
||||
}
|
||||
|
||||
/// Two-behaviour stub for the stall test: `freeze` answers the headers and 100 bytes, then
|
||||
/// never sends another byte; otherwise every path gets 10 000 bytes in ten paced slices.
|
||||
fn start_paced_stub(freeze: bool) -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind");
|
||||
let port = listener.local_addr().expect("addr").port();
|
||||
@@ -2296,12 +2056,10 @@ mod tests {
|
||||
.unwrap_or_else(|| panic!("no host-stall line: {lines:?}"));
|
||||
assert!(stall.contains(&format!(" host={frozen} ")), "{stall}");
|
||||
assert!(!stall.contains(" requeued=0 "), "{stall}");
|
||||
// Well inside the 10 s request timeout that would otherwise have freed the items.
|
||||
assert!(elapsed < Duration::from_secs(8), "took {elapsed:?}: {lines:?}");
|
||||
assert!(!lines.iter().any(|l| l.contains(&format!("host={healthy}")) && l.starts_with("host-stall")));
|
||||
}
|
||||
|
||||
// ── Minimal HTTP/1.1 stub: one thread per connection, `Connection: close`, Range-aware. ──
|
||||
struct Stub {
|
||||
base: String,
|
||||
hits: Arc<Mutex<HashMap<String, usize>>>,
|
||||
@@ -2444,11 +2202,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream-mode sink: appends pieces per item (offset 0 restarts), finalizes by length check.
|
||||
struct StreamSink {
|
||||
got: Mutex<HashMap<u64, Vec<u8>>>,
|
||||
finished: Mutex<HashMap<u64, u64>>,
|
||||
/// Reject the first piece of this item once (exercises the mid-stream Retry path).
|
||||
retry_once_id: Option<u64>,
|
||||
retried: AtomicUsize,
|
||||
}
|
||||
@@ -2534,8 +2290,6 @@ mod tests {
|
||||
assert_eq!(got[&5], b"nile/0.1 Amazon".to_vec());
|
||||
let finished = sink.finished.lock().unwrap();
|
||||
assert_eq!(finished.len(), 5);
|
||||
// Pieces credited as written + 7 extra per on_finish; the rejected attempt of /big
|
||||
// contributed nothing (its first piece was refused before any write).
|
||||
let written: u64 = got.values().map(|b| b.len() as u64).sum();
|
||||
assert_eq!(outcome.bytes_credited, written + 5 * 7);
|
||||
let hits = stub.hits.lock().unwrap();
|
||||
@@ -2575,7 +2329,6 @@ mod tests {
|
||||
&sink,
|
||||
&cancel,
|
||||
&|bytes: u64, ok: u64| {
|
||||
// Pool threads race on the callback order; keep the high-water mark.
|
||||
let mut cur = last_progress.lock().unwrap();
|
||||
if bytes >= cur.0 {
|
||||
*cur = (bytes, ok);
|
||||
@@ -2595,7 +2348,6 @@ mod tests {
|
||||
let expected_credit: u64 = got.values().map(|b| b.len() as u64 * 2).sum();
|
||||
assert_eq!(outcome.bytes_credited, expected_credit);
|
||||
assert_eq!(*last_progress.lock().unwrap(), (expected_credit, 4));
|
||||
// /c was served 503 once, then 200; /a was re-fetched once after the sink's Retry.
|
||||
let hits = stub.hits.lock().unwrap();
|
||||
assert_eq!(hits["/c"], 2);
|
||||
assert_eq!(hits["/a"], 2);
|
||||
|
||||
@@ -1,32 +1,12 @@
|
||||
//! The numbers and the two small types [`fetch_core`](crate::fetch_core) shares with the Steam
|
||||
//! depot writer it was carved out of.
|
||||
//!
|
||||
//! In Bannerlator these live in `depot_writer.rs` and `cdn_client.rs`, next to the Steam CM and
|
||||
//! depot code, so that the Steam engine and the store engines run on ONE set of tunables and their
|
||||
//! `fetch-window` log lines stay A/B-comparable. DroidDeck ships no Steam depot engine (the Steam
|
||||
//! client runs natively in the Linux runtime), so only this slice came across: the values are
|
||||
//! copied verbatim and keep their names, which is what lets a GOG/Epic/Amazon log from either app
|
||||
//! be read against the same tuning notes.
|
||||
//!
|
||||
//! The important behaviours are structural (slow-start ramp -> hold at plateau -> back off on REAL
|
||||
//! errors, per-host cap, hard byte budget), not the exact numbers. The r3/r4 notes below record
|
||||
//! why each number is what it is, because every one of them was arrived at by a device run that
|
||||
//! went wrong the other way.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default `User-Agent` of the pooled client. Every store engine overrides it per request with
|
||||
/// the agent its own launcher sends (`FetchOptions::headers`), so this value is only what a
|
||||
/// request without an explicit header would carry.
|
||||
pub const USER_AGENT: &str = "Valve/Steam HTTP Client 1.0";
|
||||
|
||||
/// TCP connect deadline for every request the pooled client makes.
|
||||
pub fn connect_timeout() -> Duration {
|
||||
Duration::from_secs(15)
|
||||
}
|
||||
|
||||
/// Classified fetch failure so the fetch driver can back off proportionally (429/5xx/timeout/reset
|
||||
/// shrink the window + cool the host harder than a one-off protocol error).
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum FetchFailKind {
|
||||
Timeout,
|
||||
@@ -42,53 +22,17 @@ pub struct AsyncFetchError {
|
||||
pub kind: FetchFailKind,
|
||||
}
|
||||
|
||||
/// Attempts per item before the run gives up on it (fetch failures and sink retries both count).
|
||||
pub const MAX_CHUNK_ATTEMPTS: u32 = 5;
|
||||
|
||||
/// Floor for the in-flight byte budget — the pre-r4 fixed value, so no run ever gets *less*
|
||||
/// buffered memory than it had before the budget started scaling. See [`inflight_budget_bytes`].
|
||||
pub const FETCH_INFLIGHT_BUDGET_FLOOR_BYTES: u64 = 24 * 1024 * 1024;
|
||||
|
||||
/// Hard ceiling for the in-flight byte budget. This is THE OOM guard: whatever the tier, the host
|
||||
/// count or the chunk size, the RAW bytes in memory for one run can never exceed this. 96 MiB = 4×
|
||||
/// the old fixed budget and still a small fraction of the heap an Android game installer runs in,
|
||||
/// and it is only ever reached by a fast-tier run whose CDN pool is large enough to support a 64+
|
||||
/// window. Raising this is the one change in the fetch layer that can reintroduce OOM, so it stays
|
||||
/// fixed and small.
|
||||
pub const FETCH_INFLIGHT_BUDGET_HARD_CAP_BYTES: u64 = 96 * 1024 * 1024;
|
||||
|
||||
/// Assumed RAW (on-the-wire, still-compressed) size of one chunk when sizing the budget. Store
|
||||
/// chunks are ~1 MiB; this is the same nominal [`NOMINAL_CHUNK_RESERVE_BYTES`] uses at dispatch.
|
||||
pub const TYPICAL_CHUNK_BYTES: u64 = NOMINAL_CHUNK_RESERVE_BYTES;
|
||||
|
||||
/// Headroom the budget carries over "one chunk per window slot" (3/2 = 1.5×), as a rational so the
|
||||
/// arithmetic stays integer. The window is a count of concurrent *requests*; a slot's bytes are held
|
||||
/// from DISPATCH until its WRITE completes, so at any instant some slots hold a fetched-but-not-yet-
|
||||
/// written chunk while others hold a reservation for a fetch still on the wire. 1.5× covers that
|
||||
/// overlap (and the occasional above-nominal chunk) without doubling the memory bound.
|
||||
pub const BUDGET_HEADROOM_NUM: u64 = 3;
|
||||
pub const BUDGET_HEADROOM_DEN: u64 = 2;
|
||||
|
||||
/// Byte budget for the *compressed* bytes buffered in the fetch→process channel, scaled to the
|
||||
/// window ceiling this run can actually reach.
|
||||
///
|
||||
/// The decouple channel is bounded by BYTES, not chunk count: a count cap makes in-flight memory
|
||||
/// swing wildly with chunk size (a 4-chunk cap is ~4 MB for 1 MB chunks but ~120 MB for 30 MB
|
||||
/// chunks). Budgeting bytes keeps the heap bounded regardless of game size while giving the fetch
|
||||
/// pool bandwidth-delay-product headroom so the socket stays busy while process workers
|
||||
/// decompress+write.
|
||||
///
|
||||
/// r4: a device run proved a fixed 24 MiB budget becomes the binding constraint the moment the
|
||||
/// adaptive window works — on a 1 Gbps line the window grew to 42 but the budget admitted only ~24
|
||||
/// chunks, so `in_flight` stalled at 23–28 with `budget_stalls` in the hundreds and
|
||||
/// `host_stalls=0`. A budget that refuses slots the window has already decided are safe is not a
|
||||
/// memory guard, it is a throttle. So the budget scales WITH the effective ceiling (tier ∧
|
||||
/// hosts×per-host-cap ∧ hard cap) and is clamped between the old fixed value and a hard cap. Low
|
||||
/// tiers are unaffected; a fast link gets a budget that stops binding before the window does.
|
||||
///
|
||||
/// Budget only the RAW bytes actually queued (reserved at DISPATCH, released on WRITE-COMPLETE); the
|
||||
/// decoded expansion is transient inside a process worker and never accumulates. See
|
||||
/// [`budget_admits`] for the single-chunk deadlock guard.
|
||||
pub fn inflight_budget_bytes(effective_ceiling: usize) -> u64 {
|
||||
(effective_ceiling as u64)
|
||||
.saturating_mul(TYPICAL_CHUNK_BYTES)
|
||||
@@ -100,71 +44,30 @@ pub fn inflight_budget_bytes(effective_ceiling: usize) -> u64 {
|
||||
)
|
||||
}
|
||||
|
||||
// ─── Adaptive-window tuning ────────────────────────────────────────────────────────────────────
|
||||
// r3 retune: the first device run pinned the window at the floor on a 1 Gbps link. The controller
|
||||
// shrank on throughput dips and latency jitter (normal CDN noise) as well as on errors, shrank a flat
|
||||
// −6 on every single error, and only grew when throughput rose >10% in a 700 ms sample. The shrinks
|
||||
// dominated, so the window ratcheted to the floor and stayed there. r3 makes growth eager and makes
|
||||
// ERRORS the only shrink signal.
|
||||
|
||||
/// Max concurrent requests to any single CDN host. CDNs throttle/reset per host, so a big window
|
||||
/// is reached by spreading across MORE hosts, never by piling onto one.
|
||||
pub const PER_HOST_CAP: usize = 6;
|
||||
/// Initial in-flight window. Slow-start doubles from here, so this only sets how fast the first
|
||||
/// couple of probes get useful — the ceiling still comes from the tier × hosts clamp.
|
||||
pub const BOOTSTRAP_WINDOW: usize = 8;
|
||||
/// Hard safety ceiling on the window regardless of tier (also clamped to distinct-hosts × per-host).
|
||||
pub const WINDOW_HARD_CAP: usize = 256;
|
||||
/// The window never shrinks below this (keeps a little pipelining even on a rough link).
|
||||
pub const WINDOW_MIN_FLOOR: usize = 2;
|
||||
/// Additive growth step once slow-start has ended (after the first real back-off).
|
||||
pub const WINDOW_STEP_UP: usize = 4;
|
||||
/// Slow-start multiplier: until the first back-off the window DOUBLES each healthy probe, so a fast
|
||||
/// link reaches a useful window in seconds instead of never.
|
||||
pub const WINDOW_SLOW_START_FACTOR: usize = 2;
|
||||
/// Proportional shrink on a real error signal (never a flat step: a flat −6 slammed a small window
|
||||
/// straight to the floor). Always at least −1 while above the floor.
|
||||
pub const WINDOW_SHRINK_FACTOR: f64 = 0.75;
|
||||
/// How often the adaptive window re-evaluates. Long enough that one jittery sample cannot drive a
|
||||
/// decision; decisions use the EWMA, not the raw sample.
|
||||
pub const WINDOW_PROBE_INTERVAL_MS: u64 = 2_000;
|
||||
/// After a back-off, hold the window (no growth) for this long — hysteresis against oscillation.
|
||||
pub const WINDOW_COOLDOWN_MS: u64 = 3_000;
|
||||
/// EWMA smoothing for the measured throughput samples (higher = more responsive).
|
||||
pub const WINDOW_BPS_EWMA_ALPHA: f64 = 0.4;
|
||||
/// Throughput above `best × (1 + this)` counts as a genuine improvement (keeps slow-start climbing).
|
||||
pub const WINDOW_IMPROVE_EPS: f64 = 0.05;
|
||||
/// Throughput below `best × (1 − this)` counts as falling — growth STOPS (hold). It never shrinks:
|
||||
/// only real errors shrink.
|
||||
pub const WINDOW_DECLINE_EPS: f64 = 0.15;
|
||||
/// Error-rate thresholds over one probe window: above HIGH shrinks; growth needs at/below LOW
|
||||
/// (in between = hold).
|
||||
pub const WINDOW_ERR_RATE_HIGH: f64 = 0.10;
|
||||
pub const WINDOW_ERR_RATE_LOW: f64 = 0.05;
|
||||
/// Errors inside ONE probe interval that trigger an immediate (don't-wait-for-the-probe) shrink. A
|
||||
/// single stray timeout no longer moves the window; a burst does. A 429 always shrinks immediately.
|
||||
pub const WINDOW_ERR_BURST_IMMEDIATE: u32 = 3;
|
||||
/// Consecutive non-improving probes still allowed to grow before declaring a plateau — throughput
|
||||
/// lags a window change by a probe or two, so one flat sample must not stop the ramp.
|
||||
pub const WINDOW_PLATEAU_PATIENCE: u32 = 2;
|
||||
/// Once plateaued, re-arm the ramp this long later (doubling, capped) to re-probe for headroom if the
|
||||
/// link improved. Bounded re-probing, not a creep: a weak link answers with errors and shrinks back.
|
||||
pub const WINDOW_PLATEAU_REARM_MS: u64 = 30_000;
|
||||
pub const WINDOW_PLATEAU_REARM_MAX_MS: u64 = 300_000;
|
||||
/// Even when the window does not change, emit one `fetch-window` line every N probes so a stuck
|
||||
/// window explains itself in the log (`reason=` says why it is not moving).
|
||||
pub const WINDOW_LOG_EVERY_PROBES: u32 = 5;
|
||||
/// Every Nth exploit dispatch is instead an exploration pick (epsilon-greedy ≈ 1/N) so a
|
||||
/// demoted-but-recovered host is retried rather than starved forever.
|
||||
pub const SERVER_EXPLORE_EVERY: u64 = 8;
|
||||
/// Reservation for an item whose compressed size the manifest didn't carry (keeps the byte budget
|
||||
/// honest at dispatch time).
|
||||
pub const NOMINAL_CHUNK_RESERVE_BYTES: u64 = 1024 * 1024;
|
||||
/// Extra host cooldown floor when a host answers 429 (rate-limited) — back off harder than a
|
||||
/// one-off timeout.
|
||||
pub const RATE_LIMIT_COOLDOWN_MS: u64 = 5_000;
|
||||
|
||||
/// Exponential back-off between attempts of one item: 0, 300, 600, 1200, 2400 ms … capped at 4 s.
|
||||
pub fn retry_backoff_millis(attempt: u32) -> u64 {
|
||||
if attempt == 0 {
|
||||
return 0;
|
||||
@@ -172,9 +75,6 @@ pub fn retry_backoff_millis(attempt: u32) -> u64 {
|
||||
(300u64 << (attempt - 1)).min(4000)
|
||||
}
|
||||
|
||||
/// Byte-budget admission test for the fetch→process channel. Always admits when nothing is in
|
||||
/// flight (`in_flight == 0`) so a single item LARGER than the whole budget can never deadlock — it
|
||||
/// simply waits until the channel drains, then goes through alone.
|
||||
#[inline]
|
||||
pub fn budget_admits(in_flight: u64, raw_len: u64, budget: u64) -> bool {
|
||||
in_flight == 0 || in_flight.saturating_add(raw_len) <= budget
|
||||
|
||||
@@ -1,21 +1,3 @@
|
||||
//! GOG gen2 chunk download engine — the Rust replacement for the byte-fetching inner loop of
|
||||
//! `GogDownloadManager.java` (`assembleDepotFile` / `fetchChunkVerified` driven from the
|
||||
//! per-file thread pool). Everything around that loop — token/builds/manifest head, base-product
|
||||
//! + language depot selection, secure-link resolution, disk guard, `_gog_manifest.json` marker,
|
||||
//! prefs, exe resolution, DLC markers, redist orchestration — stays in Java and is untouched.
|
||||
//!
|
||||
//! Parity contract: Bannerlator's `docs/RUST_GOG_PARITY.md`. The adapter is fed the SAME inflated depot-manifest
|
||||
//! JSON strings Java already parsed, the SAME resolved CDN base (secure link) and the SAME
|
||||
//! install dir, and produces the SAME on-disk layout (`<file>.bhtmp` staging, atomic rename,
|
||||
//! size+MD5-verified finals) so a download can be resumed by either engine.
|
||||
//!
|
||||
//! Layout:
|
||||
//! - [`plan`] — depot-manifest JSON → file/chunk plan (mirrors `parseDepotManifest`), CDN path
|
||||
//! and chunk-URL builders, largest-first ordering.
|
||||
//! - [`engine`] — resume/skip pass (mirrors `fileVerified`), `FetchItem` list, the `FetchSink`
|
||||
//! (compressed size/MD5 → inflate → decompressed size/MD5 → positioned write →
|
||||
//! whole-file size/MD5 → rename), summary logging.
|
||||
//! - [`jni`] — `GogNative` JNI exports (`nativeStart` / `nativeCancel` / `nativeRelease`).
|
||||
|
||||
pub mod engine;
|
||||
pub mod jni;
|
||||
@@ -25,7 +7,6 @@ use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
/// Lowercase hex of `bytes` (matches `GogDownloadManager.toHex`).
|
||||
pub fn to_hex(bytes: &[u8]) -> String {
|
||||
const DIGITS: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut out = String::with_capacity(bytes.len() * 2);
|
||||
@@ -36,15 +17,12 @@ pub fn to_hex(bytes: &[u8]) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Lowercase hex MD5 of an in-memory buffer (`GogDownloadManager.md5Hex`).
|
||||
pub fn md5_hex(data: &[u8]) -> String {
|
||||
let mut hasher = crate::md5_small::Md5::new();
|
||||
hasher.update(data);
|
||||
to_hex(&hasher.finalize())
|
||||
}
|
||||
|
||||
/// Streaming lowercase hex MD5 of a file, never buffering the whole file
|
||||
/// (`GogDownloadManager.md5HexFile`). `None` when the file cannot be read.
|
||||
pub fn md5_hex_file(path: &Path) -> Option<String> {
|
||||
let mut file = File::open(path).ok()?;
|
||||
let mut hasher = crate::md5_small::Md5::new();
|
||||
@@ -59,11 +37,6 @@ pub fn md5_hex_file(path: &Path) -> Option<String> {
|
||||
Some(to_hex(&hasher.finalize()))
|
||||
}
|
||||
|
||||
/// zlib-inflate a chunk body (`GogDownloadManager.inflateZlib`): `None` when the body is not a
|
||||
/// zlib stream (first byte != 0x78) or fails to inflate — the caller then treats the body as a
|
||||
/// stored (non-compressed) chunk, exactly like Java's `if (inflated == null) inflated = raw`.
|
||||
/// A truncated stream yields the partial output (Java's loop breaks on `n == 0` the same way);
|
||||
/// the decompressed-size check that follows rejects it.
|
||||
pub fn inflate_zlib(data: &[u8]) -> Option<Vec<u8>> {
|
||||
if data.len() < 2 || data[0] != 0x78 {
|
||||
return None;
|
||||
@@ -96,16 +69,12 @@ mod tests {
|
||||
let compressed = enc.finish().unwrap();
|
||||
assert_eq!(compressed[0], 0x78);
|
||||
assert_eq!(inflate_zlib(&compressed).unwrap(), payload);
|
||||
// Not zlib → None (stored chunk path).
|
||||
assert!(inflate_zlib(b"\x1f\x8bnot zlib").is_none());
|
||||
assert!(inflate_zlib(b"\x78").is_none());
|
||||
// Truncated zlib → partial output, like Java's `while (!inf.finished()) { if (n == 0) break; }`
|
||||
// loop; the decompressed-size check downstream rejects it either way.
|
||||
let mut truncated = compressed.clone();
|
||||
let mid = truncated.len() / 2;
|
||||
truncated.truncate(mid);
|
||||
assert!(inflate_zlib(&truncated).map(|out| out != payload).unwrap_or(true));
|
||||
// Corrupt zlib body → None (Java: DataFormatException → null → raw fallback).
|
||||
let mut corrupt = compressed.clone();
|
||||
for b in corrupt.iter_mut().skip(2) {
|
||||
*b = !*b;
|
||||
|
||||
@@ -1,17 +1,3 @@
|
||||
//! The GOG gen2 fetch loop on top of the shared fetch core. One call = one Java pool run
|
||||
//! (`runGen2` base loop, `doInstallDlc` loop, or one `assembleDependencyInstaller`).
|
||||
//!
|
||||
//! What this mirrors, and where (see Bannerlator's `docs/RUST_GOG_PARITY.md`):
|
||||
//! - resume/skip: [`file_verified`] = `GogDownloadManager.fileVerified` (exists, non-empty, size
|
||||
//! when known, MD5 when known) — a verified file is reported as "Verified…" progress with NO
|
||||
//! bytes credited, exactly like the Java task;
|
||||
//! - per chunk: compressed size → compressed MD5 → inflate (stored fallback) → decompressed size →
|
||||
//! decompressed MD5 (`fetchChunkVerified`); a mismatch is a retryable failure (Java: hard fail
|
||||
//! ≤3 with backoff; the core: ≤5 attempts with backoff);
|
||||
//! - per file: write into `<file>.bhtmp` (positioned, so chunks may land out of order), then
|
||||
//! whole-file size + MD5, delete any existing final, rename (`assembleDepotFile`);
|
||||
//! - failure of any file aborts the run (`anyFailed`), cancel aborts the run; in both cases every
|
||||
//! unfinished `.bhtmp` is deleted (Java deletes its tmp on the failing/cancelled thread).
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
@@ -30,35 +16,25 @@ use super::plan::{
|
||||
};
|
||||
use super::{inflate_zlib, md5_hex, md5_hex_file};
|
||||
|
||||
/// Per-request timeout on chunk fetches (Java: `TIMEOUT = 30_000` connect + read).
|
||||
pub const CHUNK_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
/// Java sends this on every content-system / CDN request (`fetchBytesEx`).
|
||||
pub const USER_AGENT: &str = "GOG Galaxy";
|
||||
/// Java staging suffix (`assembleDepotFile`).
|
||||
pub const TMP_SUFFIX: &str = ".bhtmp";
|
||||
/// Floor for the per-host cap (the core's default); a single-host store gets the whole ceiling.
|
||||
pub const PER_HOST_CAP_FLOOR: usize = 6;
|
||||
|
||||
/// `max(6, ceil(max_workers / distinct_hosts))` — one CDN host gets the whole ceiling, several
|
||||
/// share it.
|
||||
pub fn per_host_cap_for(max_workers: usize, distinct_hosts: usize) -> usize {
|
||||
let hosts = distinct_hosts.max(1);
|
||||
let workers = max_workers.max(1);
|
||||
PER_HOST_CAP_FLOOR.max(workers.div_ceil(hosts))
|
||||
}
|
||||
|
||||
/// Which Java loop this run replaces.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum PlanKind {
|
||||
/// gen2: depot manifests → chunks → `assembleDepotFile` (base install, DLC, dependencies).
|
||||
#[default]
|
||||
Gen2Chunks,
|
||||
/// gen1: build manifest → per-file HTTP Range GET streamed to disk (`runGen1`).
|
||||
Gen1Ranges,
|
||||
}
|
||||
|
||||
impl PlanKind {
|
||||
/// JNI mapping (`GogNative.KIND_*`): 0 = gen2 chunks, 1 = gen1 ranges, anything else → gen2.
|
||||
pub fn from_i32(v: i32) -> Self {
|
||||
if v == 1 {
|
||||
PlanKind::Gen1Ranges
|
||||
@@ -68,39 +44,21 @@ impl PlanKind {
|
||||
}
|
||||
}
|
||||
|
||||
/// Inputs — exactly what the Java manager has in hand after manifest + secure-link resolution.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct GogRequest {
|
||||
pub kind: PlanKind,
|
||||
/// gen2: inflated depot-manifest JSON strings, in the order Java fetched them (already filtered
|
||||
/// by base-product / DLC-product and language in Java). gen1: the inflated build manifest.
|
||||
pub depot_manifests: Vec<String>,
|
||||
/// gen2: the resolved CDN bases from the secure-link answer (every `urls[]` entry, query
|
||||
/// strings kept, in the order GOG lists them), or the one unauthenticated dependency store
|
||||
/// base. Several bases on distinct hosts let the fetch core spread the window across CDNs
|
||||
/// with its per-host cap; a base on a host already listed is dropped (the first one wins).
|
||||
/// gen1: unused (file URLs live in the manifest).
|
||||
pub cdn_bases: Vec<String>,
|
||||
pub install_dir: String,
|
||||
/// Files already completed by an earlier run of this same download (secure-link refresh
|
||||
/// re-run): counted as done WITHOUT re-hashing and WITHOUT a progress event.
|
||||
pub skip_paths: Vec<String>,
|
||||
pub ca_bundle_path: String,
|
||||
/// Java pool size: `resolveDownloadThreads` (base), 8 (DLC), 1 (dependency).
|
||||
pub max_workers: usize,
|
||||
/// Inflate/hash/write threads; Java did that on the same pool threads, so pass the same count.
|
||||
pub process_workers: usize,
|
||||
/// Base install: largest-first (LPT) ordering; DLC / dependency: manifest order.
|
||||
pub sort_largest_first: bool,
|
||||
/// Log prefix, e.g. `gog base=1207658930`.
|
||||
pub label: String,
|
||||
}
|
||||
|
||||
/// Callbacks out of the engine (JNI listener or a test recorder). Must be callable from the
|
||||
/// process-pool threads.
|
||||
pub trait GogEvents: Sync {
|
||||
/// One file reached its final state. `verified` = resume-skip (no bytes credited); otherwise a
|
||||
/// freshly assembled + verified file (`file_bytes` credited). `files_done` counts both.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn on_file_done(
|
||||
&self,
|
||||
@@ -113,9 +71,6 @@ pub trait GogEvents: Sync {
|
||||
bytes_total: u64,
|
||||
);
|
||||
fn on_log(&self, line: &str);
|
||||
/// Periodic byte progress while files are in flight (~4/s, only when the value moved):
|
||||
/// assembled bytes plus the bytes of files not yet finished. Same `bytes_total` as
|
||||
/// `on_file_done`. Default: ignored (test recorders).
|
||||
fn on_bytes(&self, _bytes_done: u64, _bytes_total: u64) {}
|
||||
}
|
||||
|
||||
@@ -123,20 +78,14 @@ pub trait GogEvents: Sync {
|
||||
pub struct GogRunResult {
|
||||
pub success: bool,
|
||||
pub cancelled: bool,
|
||||
/// The run died on an HTTP code Java treats as an expired secure link (401/403/404/500) —
|
||||
/// the Java caller refreshes the link (cap 5) and re-runs with `skip_paths` filled in.
|
||||
pub link_expiry: bool,
|
||||
pub error: String,
|
||||
/// Decompressed bytes of files assembled by THIS run (Java `totalBytes`).
|
||||
pub bytes_written: u64,
|
||||
/// Files done = verified-skipped + assembled (Java `doneCount`), excluding `skip_paths`.
|
||||
pub files_done: u32,
|
||||
pub files_verified: u32,
|
||||
pub files_total: u32,
|
||||
}
|
||||
|
||||
/// `GogDownloadManager.fileVerified`: true only if the file exists, is non-empty, matches the
|
||||
/// expected size when known and the expected MD5 when known.
|
||||
pub fn file_verified(path: &Path, expected_size: u64, expected_md5: &str) -> bool {
|
||||
let Ok(meta) = fs::metadata(path) else {
|
||||
return false;
|
||||
@@ -156,8 +105,6 @@ pub fn file_verified(path: &Path, expected_size: u64, expected_md5: &str) -> boo
|
||||
true
|
||||
}
|
||||
|
||||
/// Finds an HTTP status in a fetch-core error string (`non-200 HTTP status (403)`, `status 403`,
|
||||
/// `HTTP 403`, `http=403`, `code=403`). The link-expiry set is Java's `fetchChunkVerified` list.
|
||||
pub fn http_status_in(message: &str) -> Option<u16> {
|
||||
let lower = message.to_ascii_lowercase();
|
||||
for marker in ["status (", "status ", "http ", "http=", "code=", "code "] {
|
||||
@@ -178,7 +125,6 @@ pub fn http_status_in(message: &str) -> Option<u16> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Java: `code == 401 || code == 403 || code == 404 || code == 500` → refresh the secure link.
|
||||
pub fn is_link_expiry_status(code: u16) -> bool {
|
||||
matches!(code, 401 | 403 | 404 | 500)
|
||||
}
|
||||
@@ -186,8 +132,6 @@ pub fn is_link_expiry_status(code: u16) -> bool {
|
||||
struct FileState {
|
||||
handle: Option<Arc<File>>,
|
||||
chunks_done: usize,
|
||||
/// Decompressed bytes of this file's finished chunks, credited to the sink's `partial`
|
||||
/// counter until the file itself is finalized and moves into `bytes_done`.
|
||||
partial: u64,
|
||||
finalized: bool,
|
||||
failed: bool,
|
||||
@@ -240,11 +184,6 @@ impl SpeedSampler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Positioned writer for ONE chunk's decompressed byte range inside the file's `.bhtmp`: hashes
|
||||
/// everything it is given (the decompressed MD5) and counts it, but only writes bytes inside the
|
||||
/// chunk's expected range when the size is known — an inflate overshoot then fails the size
|
||||
/// check instead of clobbering the neighbouring chunk. An I/O error is remembered so the sink can
|
||||
/// tell "disk failed" (fatal) from "stream failed" (retry).
|
||||
struct RangeWriter {
|
||||
file: Arc<File>,
|
||||
base: u64,
|
||||
@@ -294,13 +233,10 @@ enum ChunkMode {
|
||||
Stored(RangeWriter),
|
||||
}
|
||||
|
||||
/// Per-item streaming state for one attempt (reset whenever a piece arrives at `offset == 0`).
|
||||
struct ChunkStream {
|
||||
comp_md5: crate::md5_small::Md5,
|
||||
comp_len: u64,
|
||||
/// Decided on the first non-empty piece (`0x78` → zlib, else stored — Java's `inflateZlib`).
|
||||
mode: Option<ChunkMode>,
|
||||
/// The writer until the mode is decided (an empty body never decides).
|
||||
writer: Option<RangeWriter>,
|
||||
}
|
||||
|
||||
@@ -315,17 +251,10 @@ impl ChunkStream {
|
||||
}
|
||||
}
|
||||
|
||||
/// The `FetchSink`. Stream mode (what `run_gen2` uses): pieces of a chunk are inflated as they
|
||||
/// arrive straight into the chunk's byte range of the file, the compressed MD5 and the
|
||||
/// decompressed size/MD5 accumulate on the fly, and `on_finish` runs the checks in Java's order
|
||||
/// (compressed size → compressed MD5 → inflate → size → MD5). Body mode (`process`) is the same
|
||||
/// pipeline on a whole buffer and stays available for tests / a body-mode fallback.
|
||||
struct GogSink<'a> {
|
||||
files: &'a [PlannedFile],
|
||||
/// item id → (file index, chunk index)
|
||||
table: Vec<(usize, usize)>,
|
||||
states: Vec<Mutex<FileState>>,
|
||||
/// item id → in-flight stream attempt (stream mode only)
|
||||
streams: Vec<Mutex<Option<ChunkStream>>>,
|
||||
files_total: u32,
|
||||
bytes_total: u64,
|
||||
@@ -333,8 +262,6 @@ struct GogSink<'a> {
|
||||
bytes_done: &'a AtomicU64,
|
||||
events: &'a dyn GogEvents,
|
||||
speed: Mutex<SpeedSampler>,
|
||||
/// Bytes of files still in flight (finished chunks; for gen1, the current attempt's pieces).
|
||||
/// `bytes_done + partial` is what the byte ticker reports between file completions.
|
||||
partial: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -343,8 +270,6 @@ impl<'a> GogSink<'a> {
|
||||
self.events.on_log(line);
|
||||
}
|
||||
|
||||
/// `assembleDepotFile` head: `parent.mkdirs(); tmpFile.delete(); new FileOutputStream(tmp)` —
|
||||
/// once per file, on its first chunk. Returns a shared handle for positioned writes.
|
||||
fn file_handle(&self, file_idx: usize) -> Result<Arc<File>, SinkError> {
|
||||
let file = &self.files[file_idx];
|
||||
let mut st = self
|
||||
@@ -385,9 +310,6 @@ impl<'a> GogSink<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// One verified chunk landed: count it; on the file's last chunk run the whole-file verify +
|
||||
/// rename and fire the per-file progress event (Java: doneCount++, totalBytes += df.totalSize,
|
||||
/// "Downloading: <name> <speed>").
|
||||
fn chunk_done(&self, file_idx: usize, chunk_bytes: u64) -> Result<(), SinkError> {
|
||||
let file = &self.files[file_idx];
|
||||
let mut st = self
|
||||
@@ -431,7 +353,6 @@ impl<'a> GogSink<'a> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `assembleDepotFile` tail: whole-file size + MD5 on the tmp, delete existing final, rename.
|
||||
fn finalize_file(&self, file_idx: usize, st: &mut FileState) -> Result<(), String> {
|
||||
let file = &self.files[file_idx];
|
||||
if let Some(handle) = st.handle.take() {
|
||||
@@ -469,7 +390,6 @@ impl<'a> GogSink<'a> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete every unfinished `.bhtmp` (Java: `tmpFile.delete()` on the failing/cancelled task).
|
||||
fn cleanup_partials(&self) {
|
||||
for st in &self.states {
|
||||
let Ok(mut st) = st.lock() else {
|
||||
@@ -496,8 +416,6 @@ impl<'a> GogSink<'a> {
|
||||
}
|
||||
|
||||
impl<'a> FetchSink for GogSink<'a> {
|
||||
/// Body mode: `fetchChunkVerified` on a whole buffer, then a positioned write of the inflated
|
||||
/// bytes (used by tests and available as a non-stream fallback).
|
||||
fn process(&self, item: &FetchItem, body: Vec<u8>) -> Result<u64, SinkError> {
|
||||
let Some(&(file_idx, chunk_idx)) = self.table.get(item.id as usize) else {
|
||||
return Err(SinkError::Fatal(format!("unknown item id {}", item.id)));
|
||||
@@ -508,7 +426,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
speed.record(body.len() as u64);
|
||||
}
|
||||
|
||||
// fetchChunkVerified, in order: compressed size → compressed MD5 → inflate → size → MD5.
|
||||
if chunk.compressed_size > 0 && body.len() as u64 != chunk.compressed_size {
|
||||
let msg = format!(
|
||||
"chunk compressed-size mismatch chunk={} exp={} got={}",
|
||||
@@ -528,7 +445,7 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
}
|
||||
let inflated = match inflate_zlib(&body) {
|
||||
Some(out) => out,
|
||||
None => body, // stored (non-zlib) chunk
|
||||
None => body,
|
||||
};
|
||||
if chunk.size > 0 && inflated.len() as u64 != chunk.size {
|
||||
let msg = format!(
|
||||
@@ -563,8 +480,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
Ok(credited)
|
||||
}
|
||||
|
||||
/// Stream mode: hash the compressed piece, inflate it straight into the chunk's byte range
|
||||
/// (or pass it through for a stored chunk), hashing + counting the decompressed output.
|
||||
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
let id = item.id as usize;
|
||||
let Some(&(file_idx, chunk_idx)) = self.table.get(id) else {
|
||||
@@ -581,7 +496,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
.and_then(|m| m.lock().ok())
|
||||
.ok_or_else(|| SinkError::Fatal("chunk stream poisoned".to_string()))?;
|
||||
if offset == 0 || slot.is_none() {
|
||||
// (Re)start of an attempt: fresh hashers + inflater; positioned writes re-cover the range.
|
||||
let handle = self.file_handle(file_idx)?;
|
||||
*slot = Some(ChunkStream::new(handle, chunk.offset, chunk.size));
|
||||
}
|
||||
@@ -601,7 +515,7 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
stream.mode = Some(if data[0] == 0x78 {
|
||||
ChunkMode::Zlib(flate2::write::ZlibDecoder::new(writer))
|
||||
} else {
|
||||
ChunkMode::Stored(writer) // stored (non-zlib) chunk — Java's `inflated = raw`
|
||||
ChunkMode::Stored(writer)
|
||||
});
|
||||
}
|
||||
let result = match stream.mode.as_mut() {
|
||||
@@ -616,7 +530,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
match result {
|
||||
Ok(()) => Ok(()),
|
||||
Err((_, Some(io))) => {
|
||||
// Disk, not stream: Java's `fos.write` exception → file fails → run aborts.
|
||||
if let Some(mut st) = self.states.get(file_idx).and_then(|m| m.lock().ok()) {
|
||||
st.failed = true;
|
||||
let _ = st.handle.take();
|
||||
@@ -627,7 +540,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
Err(SinkError::Fatal(msg))
|
||||
}
|
||||
Err((err, None)) => {
|
||||
// Corrupt zlib body. Java: inflate → null → raw → decompressed-size mismatch → retry.
|
||||
let msg = format!("chunk inflate failed chunk={} err={err}", chunk.hash);
|
||||
self.log(&msg);
|
||||
*slot = None;
|
||||
@@ -636,7 +548,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream mode: the body ended — run Java's checks in order and count the chunk.
|
||||
fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result<u64, SinkError> {
|
||||
let id = item.id as usize;
|
||||
let Some(&(file_idx, chunk_idx)) = self.table.get(id) else {
|
||||
@@ -652,7 +563,6 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
.take();
|
||||
let stream = match taken {
|
||||
Some(s) => s,
|
||||
// Empty body with no pieces: same checks on zero bytes.
|
||||
None => ChunkStream::new(self.file_handle(file_idx)?, chunk.offset, chunk.size),
|
||||
};
|
||||
|
||||
@@ -717,11 +627,8 @@ impl<'a> FetchSink for GogSink<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// How often the byte ticker reports while files are in flight.
|
||||
pub const BYTE_TICK: Duration = Duration::from_millis(250);
|
||||
|
||||
/// The CDN bases to fetch from: empty entries dropped, then one per distinct host (`host_key`),
|
||||
/// the first base seen for a host kept, order preserved.
|
||||
pub fn distinct_bases(bases: &[String]) -> Vec<String> {
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
bases
|
||||
@@ -732,10 +639,6 @@ pub fn distinct_bases(bases: &[String]) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Runs `fetch` with a reporter thread beside it that calls `on_bytes(bytes_done + partial,
|
||||
/// bytes_total)` every [`BYTE_TICK`] when the value moved. `on_file_done` only fires when a whole
|
||||
/// file is finished, so a game made of a few large files gave the UI a progress bar that sat
|
||||
/// still and then jumped; this fills the gaps. One last report is made when `fetch` returns.
|
||||
fn with_byte_ticker<T>(
|
||||
events: &dyn GogEvents,
|
||||
bytes_done: &AtomicU64,
|
||||
@@ -765,9 +668,6 @@ fn with_byte_ticker<T>(
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs one download loop. Blocking; returns when every pending file is finalized, on the first
|
||||
/// fatal file failure, or on cancel. Safe to call again with `skip_paths` extended by the files
|
||||
/// this run reported done (that is how the Java side re-runs after a secure-link refresh).
|
||||
pub fn run(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult {
|
||||
match req.kind {
|
||||
PlanKind::Gen2Chunks => run_gen2(req, cancel, events),
|
||||
@@ -808,8 +708,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
let files_verified = AtomicU32::new(0);
|
||||
|
||||
if files.is_empty() {
|
||||
// Java: the caller already returned "no depot files collected" before the loop; reaching
|
||||
// here means the two parsers disagree — surface it loudly rather than pretend success.
|
||||
return GogRunResult {
|
||||
success: false,
|
||||
error: "no files in plan (manifest parse mismatch)".to_string(),
|
||||
@@ -818,8 +716,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
};
|
||||
}
|
||||
|
||||
// Resume / repair pass — Java does this per task on the pool threads; here it runs up front
|
||||
// on `process_workers` threads so the fetch item list can exclude verified files.
|
||||
let pending: Vec<AtomicBool> = files.iter().map(|_| AtomicBool::new(false)).collect();
|
||||
let next = AtomicUsize::new(0);
|
||||
let verify_threads = process_workers.min(files.len().max(1));
|
||||
@@ -870,7 +766,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
};
|
||||
}
|
||||
|
||||
// Fetch items: one per chunk of every pending file, in plan order (chunks in manifest order).
|
||||
let mut table: Vec<(usize, usize)> = Vec::new();
|
||||
let mut items: Vec<FetchItem> = Vec::new();
|
||||
let mut states: Vec<Mutex<FileState>> = Vec::new();
|
||||
@@ -884,7 +779,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
handle: None,
|
||||
chunks_done: 0,
|
||||
partial: 0,
|
||||
// Files not pending (verified or skip-listed) are already final for this run.
|
||||
finalized: !is_pending,
|
||||
failed: false,
|
||||
tmp_path,
|
||||
@@ -899,7 +793,6 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
let path = build_cdn_path(&chunk.hash);
|
||||
items.push(FetchItem {
|
||||
id: table.len() as u64,
|
||||
// One URL per host, in `hosts` order: the core picks `urls[host_idx]`.
|
||||
urls: bases.iter().map(|b| build_chunk_url(b, &path)).collect(),
|
||||
reserve: chunk.compressed_size,
|
||||
range: None,
|
||||
@@ -943,15 +836,12 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
};
|
||||
let opts = FetchOptions {
|
||||
max_workers,
|
||||
// The ceiling split across the distinct CDN hosts (floor 6); one host gets all of it.
|
||||
per_host_cap,
|
||||
timeout: CHUNK_TIMEOUT,
|
||||
headers: vec![("User-Agent".to_string(), USER_AGENT.to_string())],
|
||||
ca_bundle_path: req.ca_bundle_path.clone(),
|
||||
process_workers,
|
||||
label: req.label.clone(),
|
||||
// Stream mode: pieces inflate straight to disk; no whole-body byte budget (the 24 MiB
|
||||
// body budget + largest-first big chunks left 1-3 requests in flight at window 16).
|
||||
stream: true,
|
||||
};
|
||||
let log = |line: &str| events.on_log(line);
|
||||
@@ -1010,23 +900,14 @@ fn run_gen2(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
// gen1: per-file HTTP Range GET streamed to disk (`runGen1` / `downloadRange`)
|
||||
// ─────────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
struct Gen1State {
|
||||
handle: Option<File>,
|
||||
/// Bytes this file's current attempt has written (credited to the sink's `partial`); a new
|
||||
/// attempt starts over, so a retry never counts its bytes twice.
|
||||
attempt_bytes: u64,
|
||||
out_path: PathBuf,
|
||||
finalized: bool,
|
||||
}
|
||||
|
||||
/// Stream-mode sink for gen1. Java's `downloadRange`: `outFile.delete()` before EVERY attempt,
|
||||
/// `new FileOutputStream(out)`, copy the Range body, no size/hash check; success → `doneG1++`,
|
||||
/// `totalBytesG1 += size`, `"Downloading: <name> <speed>"`. Pieces arrive in order with their
|
||||
/// offset; `offset == 0` marks the (re)start of an attempt, exactly where Java deletes the file.
|
||||
struct Gen1Sink<'a> {
|
||||
files: &'a [Gen1File],
|
||||
states: Vec<Mutex<Gen1State>>,
|
||||
@@ -1036,8 +917,6 @@ struct Gen1Sink<'a> {
|
||||
bytes_done: &'a AtomicU64,
|
||||
events: &'a dyn GogEvents,
|
||||
speed: Mutex<SpeedSampler>,
|
||||
/// Bytes of files still in flight (finished chunks; for gen1, the current attempt's pieces).
|
||||
/// `bytes_done + partial` is what the byte ticker reports between file completions.
|
||||
partial: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -1061,7 +940,6 @@ impl<'a> FetchSink for Gen1Sink<'a> {
|
||||
.lock()
|
||||
.map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?;
|
||||
if offset == 0 {
|
||||
// New attempt: Java `outFile.delete()` + `new FileOutputStream(out)`.
|
||||
self.partial
|
||||
.fetch_sub(std::mem::take(&mut st.attempt_bytes), Ordering::Relaxed);
|
||||
st.handle = None;
|
||||
@@ -1109,7 +987,7 @@ impl<'a> FetchSink for Gen1Sink<'a> {
|
||||
let mut st = state
|
||||
.lock()
|
||||
.map_err(|_| SinkError::Fatal("gen1 state poisoned".to_string()))?;
|
||||
st.handle = None; // close
|
||||
st.handle = None;
|
||||
st.finalized = true;
|
||||
let attempt = std::mem::take(&mut st.attempt_bytes);
|
||||
self.bytes_done.fetch_add(file.size, Ordering::Relaxed);
|
||||
@@ -1130,10 +1008,6 @@ impl<'a> FetchSink for Gen1Sink<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// gen1 loop. Resume = `exists && length == size` ("Resuming…", no bytes). Items = one Range GET
|
||||
/// per pending file; retries/back-off by the core (Java: 3 attempts, 1 s / 2 s). Failed or
|
||||
/// cancelled attempts leave whatever was written, like Java (a wrong-size partial fails the
|
||||
/// size-match resume and is re-pulled next time; a base-install cancel deletes the whole dir).
|
||||
fn run_gen1(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> GogRunResult {
|
||||
let mut files: Vec<Gen1File> = Vec::new();
|
||||
for manifest in &req.depot_manifests {
|
||||
@@ -1298,7 +1172,7 @@ fn run_gen1(req: &GogRequest, cancel: &AtomicBool, events: &dyn GogEvents) -> Go
|
||||
GogRunResult {
|
||||
success,
|
||||
cancelled,
|
||||
link_expiry: false, // gen1 has no secure-link refresh in Java
|
||||
link_expiry: false,
|
||||
error: if cancelled && error.is_empty() { "cancelled".to_string() } else { error },
|
||||
bytes_written: bytes_done.load(Ordering::Relaxed),
|
||||
files_done: files_done.load(Ordering::Relaxed),
|
||||
@@ -1412,8 +1286,6 @@ mod tests {
|
||||
assert!(peak >= 8.0, "2 MB across at most two buckets → ≥8 Mbps peak, got {peak}");
|
||||
}
|
||||
|
||||
/// Whole-file finalize through the sink with a synthetic body (no network): verifies the
|
||||
/// chunk → positioned write → whole-file MD5 → rename path and the progress event shape.
|
||||
#[test]
|
||||
fn sink_assembles_verifies_and_renames() {
|
||||
use flate2::write::ZlibEncoder;
|
||||
@@ -1468,10 +1340,8 @@ mod tests {
|
||||
};
|
||||
let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None };
|
||||
|
||||
// Second chunk first: positioned write must still produce the right bytes.
|
||||
assert_eq!(sink.process(&item(1), cb.clone()).unwrap(), part_b.len() as u64);
|
||||
assert!(tmp_path.exists() && !out_path.exists(), "still staging after 1 of 2 chunks");
|
||||
// A corrupt body is a retryable failure, not fatal.
|
||||
match sink.process(&item(0), b"\x78garbage".to_vec()) {
|
||||
Err(SinkError::Retry(msg)) => assert!(msg.contains("mismatch"), "{msg}"),
|
||||
Err(SinkError::Fatal(msg)) => panic!("expected Retry, got Fatal({msg})"),
|
||||
@@ -1580,16 +1450,14 @@ mod tests {
|
||||
assert_eq!(per_host_cap_for(0, 0), 6);
|
||||
}
|
||||
|
||||
/// Stream mode: zlib chunk split into pieces (with a poisoned first attempt restarting at
|
||||
/// offset 0), a stored chunk, corrupt data → Retry, then the whole-file finalize.
|
||||
#[test]
|
||||
fn stream_sink_inflates_pieces_verifies_and_finalizes() {
|
||||
use flate2::write::ZlibEncoder;
|
||||
use flate2::Compression;
|
||||
|
||||
let dir = temp_dir("stream");
|
||||
let part_a = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".repeat(50); // 2000 bytes, compresses well
|
||||
let part_b = b"\x01\x02stored bytes".to_vec(); // first byte != 0x78 → stored
|
||||
let part_a = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".repeat(50);
|
||||
let part_b = b"\x01\x02stored bytes".to_vec();
|
||||
let whole = [part_a.clone(), part_b.clone()].concat();
|
||||
let mut enc = ZlibEncoder::new(Vec::new(), Compression::default());
|
||||
enc.write_all(&part_a).unwrap();
|
||||
@@ -1632,7 +1500,6 @@ mod tests {
|
||||
};
|
||||
let item = |id: u64| FetchItem { id, urls: vec!["x".into()], reserve: 0, range: None };
|
||||
|
||||
// Attempt 1 of the zlib chunk: corrupt tail → Retry from on_chunk.
|
||||
let mut corrupt = ca.clone();
|
||||
for b in corrupt.iter_mut().skip(2) {
|
||||
*b = !*b;
|
||||
@@ -1642,7 +1509,6 @@ mod tests {
|
||||
Err(SinkError::Retry(m)) => assert!(m.contains("inflate failed"), "{m}"),
|
||||
Err(SinkError::Fatal(m)) => panic!("expected Retry, got Fatal({m})"),
|
||||
Ok(()) => {
|
||||
// Some corruptions only surface at finish: that must also be a Retry.
|
||||
match sink.on_finish(&item(0), corrupt.len() as u64) {
|
||||
Err(SinkError::Retry(_)) => {}
|
||||
other => panic!("expected Retry at finish, got {}", match other {
|
||||
@@ -1653,7 +1519,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Attempt 2: restart at offset 0 in three pieces.
|
||||
let cut1 = ca.len() / 3;
|
||||
let cut2 = 2 * ca.len() / 3;
|
||||
sink.on_chunk(&item(0), 0, &ca[..cut1]).unwrap();
|
||||
@@ -1661,7 +1526,6 @@ mod tests {
|
||||
sink.on_chunk(&item(0), cut2 as u64, &ca[cut2..]).unwrap();
|
||||
assert_eq!(sink.on_finish(&item(0), ca.len() as u64).unwrap(), 0);
|
||||
assert!(tmp_path.exists() && !out_path.exists());
|
||||
// Stored chunk, one piece; wrong length first (→ Retry), then the real one.
|
||||
sink.on_chunk(&item(1), 0, &part_b[..3]).unwrap();
|
||||
match sink.on_finish(&item(1), 3) {
|
||||
Err(SinkError::Retry(m)) => assert!(m.contains("compressed-size mismatch"), "{m}"),
|
||||
@@ -1705,7 +1569,6 @@ mod tests {
|
||||
partial: AtomicU64::new(0),
|
||||
};
|
||||
let item = FetchItem { id: 0, urls: vec!["x".into()], reserve: 8, range: Some((100, 107)) };
|
||||
// Attempt 1 writes a partial, attempt 2 restarts from offset 0 and truncates it.
|
||||
sink.on_chunk(&item, 0, b"ZZZZZZZZZZZZ").unwrap();
|
||||
sink.on_chunk(&item, 0, b"abcd").unwrap();
|
||||
sink.on_chunk(&item, 4, b"efgh").unwrap();
|
||||
@@ -1737,11 +1600,7 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
let rec = Recorder { events: StdMutex::new(Vec::new()) };
|
||||
// Cancel before the fetch so no network is touched: the resume pass still runs.
|
||||
let cancel = AtomicBool::new(false);
|
||||
// Use a pre-set cancel via a second request: run the resume pass, then expect the fetch
|
||||
// to be attempted for short.bin only. We can't reach the network here, so assert on the
|
||||
// plan log line instead of the outcome.
|
||||
cancel.store(true, Ordering::Relaxed);
|
||||
let res = run(&req, &cancel, &rec);
|
||||
assert!(res.cancelled, "{res:?}");
|
||||
|
||||
@@ -1,15 +1,3 @@
|
||||
//! JNI facade for the GOG Rust download engine — the native half of
|
||||
//! `com.droiddeck.launcher.stores.gog.GogNative` (Kotlin). Symbol names are bound to that
|
||||
//! package/class; the `@JvmStatic external` declarations there are the other side of this file.
|
||||
//!
|
||||
//! Shape mirrors the Steam download JNI (`jni.rs` `nativeDownloadApp`): `nativeStart` validates
|
||||
//! its inputs, takes a `GlobalRef` on the listener, spawns the worker thread and returns a handle
|
||||
//! immediately; callbacks are delivered from the worker / process-pool threads through a
|
||||
//! `JavaVM` attach; `nativeCancel` flips an `AtomicBool`; `nativeRelease` drops the Java side's
|
||||
//! reference to the handle (the worker keeps its own until it exits).
|
||||
//!
|
||||
//! Log tag `GogNative`: every engine log line goes to logcat here AND to the listener's `onLog`
|
||||
//! (which the Kotlin manager folds into its own debug log).
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
@@ -29,7 +17,6 @@ unsafe extern "C" {
|
||||
fn __android_log_write(prio: i32, tag: *const i8, text: *const i8) -> i32;
|
||||
}
|
||||
|
||||
/// `android.util.Log.i(LOG_TAG, message)`; a no-op on non-Android hosts (cargo test).
|
||||
pub fn android_log(message: &str) {
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
@@ -51,8 +38,6 @@ pub fn android_log(message: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// One in-flight run. The Java side owns a `Box<Arc<..>>` (the `jlong`); the worker thread owns a
|
||||
/// second `Arc` so `nativeRelease` is safe at any time after `nativeStart` returned.
|
||||
pub struct GogRunHandle {
|
||||
pub cancel: AtomicBool,
|
||||
}
|
||||
@@ -137,8 +122,6 @@ impl GogEvents for JniEvents {
|
||||
}
|
||||
|
||||
fn on_bytes(&self, bytes_done: u64, bytes_total: u64) {
|
||||
// A listener without `onBytes` (an app built against the older contract) makes this a
|
||||
// NoSuchMethodError, which clear_exception drops: the run is unaffected.
|
||||
self.with_env(|env| {
|
||||
let _ = env.call_method(
|
||||
self.listener.as_obj(),
|
||||
@@ -209,7 +192,6 @@ unsafe fn handle_ref(handle: jlong) -> Option<&'static Arc<GogRunHandle>> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Symbol-binding probe (see `GogNative.isAvailable`). Returns 1.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeProbe(
|
||||
_env: JNIEnv,
|
||||
@@ -218,9 +200,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativePr
|
||||
1
|
||||
}
|
||||
|
||||
/// Starts one download loop on a worker thread and returns a handle (0 = could not start; the
|
||||
/// listener then receives NO callbacks). Inputs are exactly what `GogDownloadManager` holds after
|
||||
/// manifest + secure-link resolution — see `GogRequest`.
|
||||
#[no_mangle]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeStart(
|
||||
@@ -290,8 +269,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeSt
|
||||
Box::into_raw(Box::new(handle)) as jlong
|
||||
}
|
||||
|
||||
/// Requests cancellation; the run stops at its next scheduling point and `onComplete` fires with
|
||||
/// `cancelled = true`. Idempotent.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCancel(
|
||||
_env: JNIEnv,
|
||||
@@ -303,8 +280,6 @@ pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeCa
|
||||
}
|
||||
}
|
||||
|
||||
/// Drops the Java side's reference. Call once, after `onComplete` (or after cancelling — the worker
|
||||
/// keeps the run alive until it exits either way).
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_gog_GogNative_nativeRelease(
|
||||
_env: JNIEnv,
|
||||
|
||||
@@ -1,50 +1,24 @@
|
||||
//! Depot-manifest JSON → download plan. Mirrors `GogDownloadManager.parseDepotManifest` rule for
|
||||
//! rule (see Bannerlator's `docs/RUST_GOG_PARITY.md` §3):
|
||||
//! - root `depot.items[]`; `path` with `\` → `/` and one leading `/` stripped;
|
||||
//! - a file is kept only when `path` is non-empty AND `chunks[]` is non-empty;
|
||||
//! - per chunk: CDN hash = `compressedMd5` when non-empty, else `md5`; a chunk with neither is
|
||||
//! dropped; `compressedSize` / `size` default 0;
|
||||
//! - file `md5` kept when non-empty; `total_size` = Σ chunk `size` (decompressed);
|
||||
//! - a file whose chunks all dropped is not added.
|
||||
//! `org.json` coercion (`optString` turns numbers/bools into their text, `optLong` parses numeric
|
||||
//! strings) is reproduced by [`opt_string`] / [`opt_u64`].
|
||||
//!
|
||||
//! Additionally each chunk carries `offset` = prefix-sum of the preceding chunks' decompressed
|
||||
//! sizes — Java appends sequentially so it never needs one; the Rust sink writes chunks at that
|
||||
//! offset as they arrive so the final bytes are identical.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ChunkRef {
|
||||
/// CDN-path hash (`compressedMd5` preferred, else `md5`).
|
||||
pub hash: String,
|
||||
/// Verify compressed bytes before inflate (may be empty).
|
||||
pub compressed_md5: String,
|
||||
/// Verify decompressed bytes after inflate (may be empty).
|
||||
pub md5: String,
|
||||
/// Expected compressed size, 0 if absent.
|
||||
pub compressed_size: u64,
|
||||
/// Expected decompressed size, 0 if absent.
|
||||
pub size: u64,
|
||||
/// Decompressed offset inside the file (prefix-sum of preceding `size`s).
|
||||
pub offset: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PlannedFile {
|
||||
/// Install-dir-relative path, forward slashes, no leading slash.
|
||||
pub relative_path: String,
|
||||
/// Whole-file MD5 from the manifest ("" when absent).
|
||||
pub md5: String,
|
||||
/// Σ chunk decompressed sizes (0 if unknown).
|
||||
pub total_size: u64,
|
||||
pub chunks: Vec<ChunkRef>,
|
||||
}
|
||||
|
||||
/// `JSONObject.optString(key, "")` semantics: strings as-is, numbers/bools rendered, null/absent
|
||||
/// → "". (`org.json` renders `JSONObject.NULL` as "null"; GOG manifests never carry JSON nulls in
|
||||
/// these fields, and treating one as absent is the only safe reading.)
|
||||
pub fn opt_string(obj: &Value, key: &str) -> String {
|
||||
match obj.get(key) {
|
||||
Some(Value::String(s)) => s.clone(),
|
||||
@@ -54,8 +28,6 @@ pub fn opt_string(obj: &Value, key: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// `JSONObject.optLong(key, 0)` semantics for a size field (numbers, or numeric strings; a
|
||||
/// negative/float value that Java would truncate is clamped to 0 — sizes are never negative).
|
||||
pub fn opt_u64(obj: &Value, key: &str) -> u64 {
|
||||
match obj.get(key) {
|
||||
Some(Value::Number(n)) => n
|
||||
@@ -68,8 +40,6 @@ pub fn opt_u64(obj: &Value, key: &str) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses one inflated gen2 depot manifest and appends its files to `out`
|
||||
/// (`GogDownloadManager.parseDepotManifest`). Malformed JSON adds nothing, like Java's catch-all.
|
||||
pub fn parse_depot_manifest(json: &str, out: &mut Vec<PlannedFile>) {
|
||||
let Ok(root) = serde_json::from_str::<Value>(json) else {
|
||||
return;
|
||||
@@ -137,20 +107,14 @@ pub fn parse_depot_manifest(json: &str, out: &mut Vec<PlannedFile>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// One gen1 file: a byte range of a (pre-signed) depot blob (`GogDownloadManager.Gen1File`).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Gen1File {
|
||||
/// Install-dir-relative path, used verbatim (Java's gen1 path does not normalise slashes).
|
||||
pub path: String,
|
||||
pub url: String,
|
||||
pub offset: u64,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// Parses a gen1 build manifest (`runGen1`): every `depot[]` entry that is not `support: true`,
|
||||
/// its `files[]` → (`path`, `url`, `offset` default 0, `size` default 0); an entry with
|
||||
/// `size == 0` is dropped (Java's `path == null || url == null` checks never fire — `optString`
|
||||
/// returns "" — so `size == 0` is the only effective filter, mirrored here).
|
||||
pub fn parse_gen1_manifest(json: &str, out: &mut Vec<Gen1File>) {
|
||||
let Ok(root) = serde_json::from_str::<Value>(json) else {
|
||||
return;
|
||||
@@ -184,10 +148,6 @@ pub fn parse_gen1_manifest(json: &str, out: &mut Vec<Gen1File>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses every manifest in order (the order Java fetched the depots) and optionally applies the
|
||||
/// base-install largest-first (LPT) ordering — a STABLE sort by descending `total_size`, exactly
|
||||
/// `Collections.sort(files, (a, b) -> Long.compare(b.totalSize, a.totalSize))`. The DLC and
|
||||
/// dependency paths in Java do not sort, so they pass `sort_largest_first = false`.
|
||||
pub fn build_plan(manifests: &[String], sort_largest_first: bool) -> Vec<PlannedFile> {
|
||||
let mut files = Vec::new();
|
||||
for manifest in manifests {
|
||||
@@ -199,8 +159,6 @@ pub fn build_plan(manifests: &[String], sort_largest_first: bool) -> Vec<Planned
|
||||
files
|
||||
}
|
||||
|
||||
/// `GogDownloadManager.buildCdnPath`: `"ab/cd/abcdef..."`. A hash shorter than 4 chars would
|
||||
/// throw in Java (and fail that file); here it is returned unchanged and fails at fetch time.
|
||||
pub fn build_cdn_path(hash: &str) -> String {
|
||||
if !hash.is_char_boundary(2) || !hash.is_char_boundary(4) || hash.len() < 4 {
|
||||
return hash.to_string();
|
||||
@@ -208,8 +166,6 @@ pub fn build_cdn_path(hash: &str) -> String {
|
||||
format!("{}/{}/{}", &hash[0..2], &hash[2..4], hash)
|
||||
}
|
||||
|
||||
/// `GogDownloadManager.buildChunkUrl`: appends the chunk path BEFORE any query string so the
|
||||
/// secure-link token survives.
|
||||
pub fn build_chunk_url(base: &str, chunk_path: &str) -> String {
|
||||
match base.find('?') {
|
||||
Some(q) => format!("{}/{}{}", &base[..q], chunk_path, &base[q..]),
|
||||
@@ -217,8 +173,6 @@ pub fn build_chunk_url(base: &str, chunk_path: &str) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-host key for the fetch core's host semaphores: `scheme://host[:port]` of `url` (lowercased
|
||||
/// host). Java has one resolved CDN base per run, so the host list has exactly one entry.
|
||||
pub fn host_key(url: &str) -> String {
|
||||
let (scheme, rest) = match url.find("://") {
|
||||
Some(i) => (&url[..i], &url[i + 3..]),
|
||||
|
||||
@@ -1,21 +1,4 @@
|
||||
//! DroidDeck's native store download engines (`libdroiddeckstores.so`).
|
||||
//!
|
||||
//! The GOG, Epic Games Store and Amazon Games download loops, ported from Bannerlator's
|
||||
//! `bl-steam-client` crate (GPL-3.0-or-later, itself derived from WinNative's `wn-steam-client`).
|
||||
//! Only the store engines and the fetch core they run on came across: DroidDeck's Steam client is
|
||||
//! the real one, running in the Linux runtime, so none of the Steam CM / depot code is here.
|
||||
//!
|
||||
//! Each store module owns its manifest → plan → `FetchItem` mapping, its `FetchSink` (inflate +
|
||||
//! hash + write in the exact output layout of the Kotlin manager that drives it) and its JNI facade.
|
||||
//! The JNI contract — classes, method names, signatures, threading — is written down in `JNI.md`
|
||||
//! next to `Cargo.toml`; the Kotlin side mirrors it exactly.
|
||||
//!
|
||||
//! What a run looks like: Kotlin hands over a manifest it already parsed (or, for Amazon, the
|
||||
//! plan it built from one), the resolved CDN base(s) and the install directory, calls `nativeStart`
|
||||
//! and gets a handle back at once. Everything after that happens on native threads: the fetch core
|
||||
//! keeps an adaptive window of HTTP requests in flight on one tokio runtime, hands bodies to a sync
|
||||
//! process pool that inflates, hashes and writes them, and reports progress through the listener
|
||||
//! object Kotlin passed in. `nativeCancel` flips a flag; `onComplete` fires exactly once.
|
||||
//! Ported from Bannerlator's bl-steam-client crate (GPL-3.0-or-later), itself derived from WinNative's wn-steam-client.
|
||||
|
||||
#![allow(clippy::missing_safety_doc, clippy::result_large_err)]
|
||||
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
//! Minimal MD5 (RFC 1321) for the store download engines (GOG chunk / file verification).
|
||||
//!
|
||||
//! In-crate on purpose: the crate is built `--locked` in CI with no way to add a dependency from
|
||||
//! this environment, and the algorithm is 100 lines. Not for anything security-sensitive — it only
|
||||
//! mirrors the checksum the Java managers already compare against.
|
||||
|
||||
/// One-shot digest.
|
||||
pub fn md5(data: &[u8]) -> [u8; 16] {
|
||||
let mut h = Md5::new();
|
||||
h.update(data);
|
||||
h.finalize()
|
||||
}
|
||||
|
||||
/// Lower-case hex of a 16-byte digest.
|
||||
pub fn md5_hex(digest: &[u8; 16]) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut out = String::with_capacity(32);
|
||||
@@ -22,7 +15,6 @@ pub fn md5_hex(digest: &[u8; 16]) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Streaming MD5: `update` any number of times, then `finalize`.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Md5 {
|
||||
state: [u32; 4],
|
||||
@@ -93,7 +85,6 @@ impl Md5 {
|
||||
let bit_len = self.total_len.wrapping_mul(8);
|
||||
let mut pad = [0u8; 72];
|
||||
pad[0] = 0x80;
|
||||
// Pad to 56 mod 64, then append the 64-bit little-endian bit length.
|
||||
let pad_len = if self.buffer_len < 56 {
|
||||
56 - self.buffer_len
|
||||
} else {
|
||||
@@ -189,7 +180,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn padding_edge_lengths() {
|
||||
// 55 / 56 / 64 bytes exercise the "pad fits in one block" vs "needs a second block" paths.
|
||||
assert_eq!(hex(&[b'a'; 55]), "ef1772b6dff9a122358552954ad0df65");
|
||||
assert_eq!(hex(&[b'a'; 56]), "3b0c8ac703f828b04c6c197006d17218");
|
||||
assert_eq!(hex(&[b'a'; 64]), "014842d480b571495a4a0363793f7367");
|
||||
|
||||
@@ -1,22 +1,5 @@
|
||||
//! Background priority for every thread the store engines create.
|
||||
//!
|
||||
//! A download keeps several threads busy at once (the run's worker, the tokio runtime and its
|
||||
//! blocking pool for DNS, the inflate/hash/write pool, the throughput reporter, GOG's verify pass,
|
||||
//! the Epic assembler). At the default nice 0 they compete as equals with the app's UI and input
|
||||
//! threads, and on the device the menus' animations and controller navigation slowed visibly for
|
||||
//! as long as a store download ran. At nice +10 the scheduler hands them only what the UI leaves,
|
||||
//! which for a download bound by network and storage costs next to nothing.
|
||||
//!
|
||||
//! Linux applies `setpriority(PRIO_PROCESS, tid, …)` to the one thread `tid`, not the process, and
|
||||
//! a new thread inherits its creator's nice value; every thread is still lowered explicitly at its
|
||||
//! start so the guarantee does not depend on who spawned it. Only threads this library creates are
|
||||
//! touched: the JNI calling thread belongs to the app and keeps its priority.
|
||||
|
||||
/// The nice value store-engine threads run at.
|
||||
pub const BACKGROUND_NICE: i32 = 10;
|
||||
|
||||
/// Lower the calling thread to [`BACKGROUND_NICE`]. Best effort: a refusal leaves the thread at
|
||||
/// its current priority and the download runs as before.
|
||||
pub fn background() {
|
||||
#[cfg(any(target_os = "android", target_os = "linux"))]
|
||||
unsafe {
|
||||
@@ -25,7 +8,6 @@ pub fn background() {
|
||||
}
|
||||
}
|
||||
|
||||
/// The calling thread's nice value as a log field, e.g. `priority=nice10`.
|
||||
pub fn log_field() -> String {
|
||||
#[cfg(any(target_os = "android", target_os = "linux"))]
|
||||
{
|
||||
|
||||
@@ -1,19 +1,10 @@
|
||||
//! The one export that is not a store engine: `com.droiddeck.launcher.stores.StoresNative`.
|
||||
//!
|
||||
//! `nativeVersion()` exists so the app can prove, at Setup time, that `libdroiddeckstores.so`
|
||||
//! loaded and its JNI exports bind — before a user is mid-download and an
|
||||
//! `UnsatisfiedLinkError` would surface as a failed install. It returns the crate version from
|
||||
//! `Cargo.toml`, which the app can show next to the other component versions.
|
||||
|
||||
use jni::objects::JClass;
|
||||
use jni::sys::jstring;
|
||||
use jni::JNIEnv;
|
||||
|
||||
/// The crate version, as `Cargo.toml` declares it.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// `StoresNative.nativeVersion(): String` — the crate version (`"0.1.0"`). A null return means the
|
||||
/// JVM could not allocate the string, which the caller treats like a failed load.
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_StoresNative_nativeVersion(
|
||||
env: JNIEnv,
|
||||
|
||||
Reference in new issue
Block a user