mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 12:56:06 +00:00
feat: EA store with sign-in, library, native downloads, updates and launch without the EA app (#2259)
* docs: research and plan for EA and Ubisoft stores * feat(ea): EA store with sign-in, library, native downloads, updates and launch without the EA app * perf(ea): fetch large archive records as parallel ranges * fix(ea): register game display names * fix(ea): license legacy EA Core games and harden the LSX endpoint * fix(ea): run legacy Origin SDK games with a stand-in Origin process * fix(ea): trust only the prepared game on LSX and stop stale library syncs * fix(ea): close LSX sessions on stop and drop clients that never hand-shake * fix(ea): require every hashed field before licensing or caching a probe * fix(ea): trust one LSX session per launch * fix(ea): serialize the library cache write with sign-out and rerun a skipped sync * fix(ea): hand off library sync requests under one lock * fix(ea): clear the sign-in before invalidating the library * fix(ea): serialize sign-out with token refresh and with the library cache * fix(ea): LSX trusts only verified peers, new launch closes old sessions, exit stops its own game * fix(ea): LSX hides profile and presence from unverified peers * fix(ea): LSX handshake has a fixed deadline and untrusted sessions yield their slot * fix(ea): LSX never registers a client after its listener was stopped * fix(ea): LSX evicts each untrusted session at most once * fix(ea): LSX admission and eviction share one lock and keep the cap * fix(ea): LSX trust promotion excludes eviction * fix(ea): LSX launch generation guards restarts and drops results after shutdown * fix(ea): LSX publishes launch state under the admission lock and tags accepts by generation * fix(ea): LSX restarts rebind its listener and promotion checks the launch generation * fix(ea): record why EA refuses a game licence in the store log * chore(ea): log owned offers and the full licence refusal * chore(ea): log the account's entitlements when no game licence is granted * chore(ea): log legacy entitlements when no game licence is granted * chore(ea): log only the refused game's own entitlements * fix(ea): skip trial launchers flagged 1 and license games whose LSX peer Android cannot name
This commit is contained in:
1 parent
9bf638a3c4
commit
d8ed02f2cf
46 files changed
+5284
-7
No files matched your search
@@ -0,0 +1,207 @@
|
||||
use std::panic::{catch_unwind, AssertUnwindSafe};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use jni::objects::{GlobalRef, JClass, JObject, JString, JValue};
|
||||
use jni::sys::{jlong, jstring, JNI_FALSE, JNI_TRUE};
|
||||
use jni::{JNIEnv, JavaVM};
|
||||
use serde_json::json;
|
||||
|
||||
use super::install;
|
||||
use crate::amazon::install::{Events, Stop};
|
||||
|
||||
fn read_string(env: &mut JNIEnv, value: &JString) -> String {
|
||||
if value.is_null() {
|
||||
return String::new();
|
||||
}
|
||||
env.get_string(value).map(|s| s.to_string_lossy().into_owned()).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn new_string(env: &mut JNIEnv, text: &str) -> jstring {
|
||||
env.new_string(text).map(|s| s.into_raw()).unwrap_or(std::ptr::null_mut())
|
||||
}
|
||||
|
||||
fn call(op: &str, request: &str) -> String {
|
||||
let parsed: serde_json::Value = serde_json::from_str(request).unwrap_or(serde_json::Value::Null);
|
||||
let url = parsed.get("url").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let answer = match op {
|
||||
"zip.size" => install::size(url).unwrap_or_else(|e| json!({ "error": e })),
|
||||
_ => json!({ "error": format!("unknown operation {op}") }),
|
||||
};
|
||||
answer.to_string()
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeCall(
|
||||
mut env: JNIEnv,
|
||||
_class: JClass,
|
||||
op: JString,
|
||||
request: JString,
|
||||
) -> jstring {
|
||||
let op = read_string(&mut env, &op);
|
||||
let request = read_string(&mut env, &request);
|
||||
match catch_unwind(AssertUnwindSafe(|| call(&op, &request))) {
|
||||
Ok(text) => new_string(&mut env, &text),
|
||||
Err(_) => std::ptr::null_mut(),
|
||||
}
|
||||
}
|
||||
|
||||
struct Listener {
|
||||
vm: JavaVM,
|
||||
target: GlobalRef,
|
||||
}
|
||||
|
||||
impl Listener {
|
||||
fn with_env<R>(&self, fallback: R, f: impl FnOnce(&mut JNIEnv) -> R) -> R {
|
||||
let Ok(mut env) = self.vm.attach_current_thread_as_daemon() else {
|
||||
return fallback;
|
||||
};
|
||||
let out = f(&mut env);
|
||||
if env.exception_check().unwrap_or(false) {
|
||||
let _ = env.exception_clear();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn text(&self, method: &str, text: &str) {
|
||||
self.with_env((), |env| {
|
||||
let Ok(s) = env.new_string(text) else { return };
|
||||
let s = JObject::from(s);
|
||||
let _ = env.call_method(self.target.as_obj(), method, "(Ljava/lang/String;)V", &[JValue::Object(&s)]);
|
||||
let _ = env.delete_local_ref(s);
|
||||
});
|
||||
}
|
||||
|
||||
fn counts(&self, method: &str, a: u64, b: u64, c: u32, d: u32) {
|
||||
self.with_env((), |env| {
|
||||
let _ = env.call_method(
|
||||
self.target.as_obj(),
|
||||
method,
|
||||
"(JJII)V",
|
||||
&[
|
||||
JValue::Long(a.min(i64::MAX as u64) as i64),
|
||||
JValue::Long(b.min(i64::MAX as u64) as i64),
|
||||
JValue::Int(c.min(i32::MAX as u32) as i32),
|
||||
JValue::Int(d.min(i32::MAX as u32) as i32),
|
||||
],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn complete(&self, success: bool, cancelled: bool, error: &str, result: &str) {
|
||||
self.with_env((), |env| {
|
||||
let (Ok(e), Ok(r)) = (env.new_string(error), env.new_string(result)) else { return };
|
||||
let (e, r) = (JObject::from(e), JObject::from(r));
|
||||
let _ = env.call_method(
|
||||
self.target.as_obj(),
|
||||
"onComplete",
|
||||
"(ZZLjava/lang/String;Ljava/lang/String;)V",
|
||||
&[
|
||||
JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }),
|
||||
JValue::Bool(if cancelled { JNI_TRUE } else { JNI_FALSE }),
|
||||
JValue::Object(&e),
|
||||
JValue::Object(&r),
|
||||
],
|
||||
);
|
||||
let _ = env.delete_local_ref(e);
|
||||
let _ = env.delete_local_ref(r);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl Events for Listener {
|
||||
fn log(&self, line: &str) {
|
||||
self.text("onLog", &crate::amazon::net::scrub_urls(line));
|
||||
}
|
||||
|
||||
fn status(&self, text: &str) {
|
||||
self.text("onStatus", text);
|
||||
}
|
||||
|
||||
fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32) {
|
||||
self.counts("onCheck", done_bytes, total_bytes, files, files_total);
|
||||
}
|
||||
|
||||
fn sizes(&self, pending_bytes: u64, total_bytes: u64) {
|
||||
self.with_env((), |env| {
|
||||
let _ = env.call_method(
|
||||
self.target.as_obj(),
|
||||
"onSizes",
|
||||
"(JJ)V",
|
||||
&[
|
||||
JValue::Long(pending_bytes.min(i64::MAX as u64) as i64),
|
||||
JValue::Long(total_bytes.min(i64::MAX as u64) as i64),
|
||||
],
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn progress(&self, done: u64, total: u64, files: u32, files_total: u32) {
|
||||
self.counts("onProgress", done, total, files, files_total);
|
||||
}
|
||||
|
||||
fn before_replace(&self) -> bool {
|
||||
self.with_env(false, |env| match env.call_method(self.target.as_obj(), "beforeReplace", "()Z", &[]) {
|
||||
Ok(v) => v.z().unwrap_or(false),
|
||||
Err(_) => false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeInstall(
|
||||
mut env: JNIEnv,
|
||||
_class: JClass,
|
||||
request: JString,
|
||||
listener: JObject,
|
||||
) -> jlong {
|
||||
if listener.is_null() {
|
||||
return 0;
|
||||
}
|
||||
let Ok(vm) = env.get_java_vm() else { return 0 };
|
||||
let Ok(target) = env.new_global_ref(&listener) else { return 0 };
|
||||
let listener = Listener { vm, target };
|
||||
let text = read_string(&mut env, &request);
|
||||
let parsed = serde_json::from_str::<serde_json::Value>(&text)
|
||||
.map_err(|_| "The install request could not be read".to_string())
|
||||
.and_then(|v| install::Request::from_json(&v));
|
||||
let request = match parsed {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
listener.complete(false, false, &e, "");
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
let cancel = Arc::new(AtomicBool::new(false));
|
||||
let worker_cancel = Arc::clone(&cancel);
|
||||
let spawned = std::thread::Builder::new().name("ea-install".into()).spawn(move || {
|
||||
crate::priority::background();
|
||||
let result = catch_unwind(AssertUnwindSafe(|| install::run(&request, &worker_cancel, &listener)))
|
||||
.unwrap_or_else(|_| Err(Stop::Failed("The EA install stopped on an internal error".into())));
|
||||
match result {
|
||||
Ok(done) => listener.complete(true, false, "", &done.to_json().to_string()),
|
||||
Err(Stop::Cancelled) => listener.complete(false, true, "cancelled", ""),
|
||||
Err(Stop::Failed(e)) => listener.complete(false, false, &e, ""),
|
||||
}
|
||||
});
|
||||
if spawned.is_err() {
|
||||
return 0;
|
||||
}
|
||||
Box::into_raw(Box::new(cancel)) as jlong
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeCancel(_env: JNIEnv, _class: JClass, handle: jlong) {
|
||||
let ptr = handle as *const Arc<AtomicBool>;
|
||||
if let Some(cancel) = unsafe { ptr.as_ref() } {
|
||||
cancel.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[no_mangle]
|
||||
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeRelease(_env: JNIEnv, _class: JClass, handle: jlong) {
|
||||
let ptr = handle as *mut Arc<AtomicBool>;
|
||||
if !ptr.is_null() {
|
||||
drop(unsafe { Box::from_raw(ptr) });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,850 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Installs or updates an EA game from its build archive: files whose size and CRC already match
|
||||
//! are kept, everything else is fetched entry by entry with ranged reads and inflated in place.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant, UNIX_EPOCH};
|
||||
|
||||
use flate2::{Crc, Decompress, FlushDecompress, Status};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::net;
|
||||
use super::zip::{self, Entry};
|
||||
use crate::amazon::install::check::{free_bytes, space_problem};
|
||||
use crate::amazon::install::{Events, ReplaceGate, Stop};
|
||||
use crate::fetch_core::{distinct_host_count, run_fetch, FetchItem, FetchOptions, FetchSink, SinkError};
|
||||
|
||||
pub const LEDGER: &str = ".droiddeck-ea-files";
|
||||
pub const IN_PROGRESS_MARKER: &str = ".droiddeck-ea-downloading";
|
||||
const TEMP_SUFFIX: &str = ".ddpart";
|
||||
const PROGRESS_EVERY_MS: u64 = 50;
|
||||
const CHECK_REPORT_EVERY: Duration = Duration::from_millis(200);
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
const INFLATE_BUFFER: usize = 256 * 1024;
|
||||
/// Records larger than this are fetched as parallel ranges into a scratch file, then inflated,
|
||||
/// so one big file does not ride a single connection.
|
||||
pub const PART_BYTES: u64 = 16 * 1024 * 1024;
|
||||
const SPLIT_SUFFIX: &str = ".ddzip";
|
||||
|
||||
struct Split {
|
||||
tmp: PathBuf,
|
||||
file: Mutex<Option<File>>,
|
||||
received: Vec<AtomicU64>,
|
||||
left: AtomicUsize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Work {
|
||||
Whole(usize),
|
||||
Part { job: usize, part: usize, start: u64, len: u64 },
|
||||
}
|
||||
|
||||
/// The fetches for each job: whole records, or byte ranges of one when it is large.
|
||||
fn works(jobs: &[&Planned], part_bytes: u64) -> Vec<Work> {
|
||||
let mut out = Vec::new();
|
||||
for (i, p) in jobs.iter().enumerate() {
|
||||
let span = p.range_end - p.entry.local_offset + 1;
|
||||
if span <= part_bytes * 2 {
|
||||
out.push(Work::Whole(i));
|
||||
continue;
|
||||
}
|
||||
let mut start = 0;
|
||||
let mut part = 0;
|
||||
while start < span {
|
||||
let len = part_bytes.min(span - start);
|
||||
out.push(Work::Part { job: i, part, start, len });
|
||||
start += len;
|
||||
part += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
pub const REPLACE_REFUSED: &str = "The install record could not be switched to updating; nothing was replaced";
|
||||
|
||||
pub struct Request {
|
||||
pub url: String,
|
||||
pub game_dir: PathBuf,
|
||||
pub max_workers: usize,
|
||||
pub process_workers: usize,
|
||||
pub ca_bundle: String,
|
||||
pub part_bytes: u64,
|
||||
}
|
||||
|
||||
impl Request {
|
||||
pub fn from_json(v: &Value) -> Result<Request, String> {
|
||||
let text = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string();
|
||||
let game_dir = text("gameDir");
|
||||
if game_dir.is_empty() || !game_dir.starts_with('/') {
|
||||
return Err("No install folder was given".into());
|
||||
}
|
||||
let url = text("url");
|
||||
if !url.starts_with("https://") && !url.starts_with("http://") {
|
||||
return Err("EA gave no download for this game".into());
|
||||
}
|
||||
Ok(Request {
|
||||
url,
|
||||
game_dir: PathBuf::from(game_dir),
|
||||
max_workers: v.get("maxWorkers").and_then(|n| n.as_u64()).unwrap_or(8) as usize,
|
||||
process_workers: v.get("processWorkers").and_then(|n| n.as_u64()).unwrap_or(2) as usize,
|
||||
ca_bundle: text("caBundle"),
|
||||
part_bytes: v.get("partBytes").and_then(|n| n.as_u64()).filter(|n| *n > 0).unwrap_or(PART_BYTES),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Finished {
|
||||
pub total_bytes: u64,
|
||||
pub download_bytes: u64,
|
||||
pub file_count: usize,
|
||||
pub files_written: usize,
|
||||
pub bytes_written: u64,
|
||||
pub removed: usize,
|
||||
}
|
||||
|
||||
impl Finished {
|
||||
pub fn to_json(&self) -> Value {
|
||||
json!({
|
||||
"totalBytes": self.total_bytes,
|
||||
"downloadBytes": self.download_bytes,
|
||||
"fileCount": self.file_count,
|
||||
"filesWritten": self.files_written,
|
||||
"bytesWritten": self.bytes_written,
|
||||
"removed": self.removed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct Known {
|
||||
crc: u32,
|
||||
size: u64,
|
||||
modified: u64,
|
||||
}
|
||||
|
||||
fn modified_ms(path: &Path) -> u64 {
|
||||
fs::metadata(path)
|
||||
.and_then(|m| m.modified())
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn read_ledger(dir: &Path) -> HashMap<String, Known> {
|
||||
let Ok(text) = fs::read_to_string(dir.join(LEDGER)) else { return HashMap::new() };
|
||||
let mut out = HashMap::new();
|
||||
for line in text.lines() {
|
||||
let mut parts = line.splitn(4, '\t');
|
||||
let (Some(crc), Some(size), Some(modified), Some(path)) = (parts.next(), parts.next(), parts.next(), parts.next()) else {
|
||||
continue;
|
||||
};
|
||||
let (Ok(crc), Ok(size), Ok(modified)) = (u32::from_str_radix(crc, 16), size.parse(), modified.parse()) else {
|
||||
continue;
|
||||
};
|
||||
out.insert(path.to_string(), Known { crc, size, modified });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn ledger_line(path: &str, k: Known) -> String {
|
||||
format!("{:08x}\t{}\t{}\t{path}\n", k.crc, k.size, k.modified)
|
||||
}
|
||||
|
||||
fn crc_of(path: &Path, cancel: &AtomicBool) -> Result<Option<u32>, ()> {
|
||||
let Ok(mut f) = File::open(path) else { return Ok(None) };
|
||||
let mut crc = Crc::new();
|
||||
let mut buf = vec![0u8; 1024 * 1024];
|
||||
loop {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err(());
|
||||
}
|
||||
match f.read(&mut buf) {
|
||||
Ok(0) => return Ok(Some(crc.sum())),
|
||||
Ok(n) => crc.update(&buf[..n]),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
|
||||
Err(_) => return Ok(None),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether two paths name one file, as they do for a case-only rename on case-insensitive storage.
|
||||
fn same_file(a: &Path, b: &Path) -> bool {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
match (fs::metadata(a), fs::metadata(b)) {
|
||||
(Ok(x), Ok(y)) => x.dev() == y.dev() && x.ino() == y.ino(),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn temp_path(dest: &Path) -> PathBuf {
|
||||
let mut name = dest.as_os_str().to_owned();
|
||||
name.push(TEMP_SUFFIX);
|
||||
PathBuf::from(name)
|
||||
}
|
||||
|
||||
pub struct Planned {
|
||||
pub entry: Entry,
|
||||
pub path: String,
|
||||
pub range_end: u64,
|
||||
}
|
||||
|
||||
/// Every file entry with its safe path and the last byte its local record can occupy.
|
||||
pub fn plan(entries: &[Entry], directory_offset: u64) -> Result<Vec<Planned>, String> {
|
||||
let mut offsets: Vec<u64> = entries.iter().map(|e| e.local_offset).collect();
|
||||
offsets.push(directory_offset);
|
||||
offsets.sort_unstable();
|
||||
offsets.dedup();
|
||||
let mut seen = HashSet::new();
|
||||
let mut out = Vec::new();
|
||||
for e in entries.iter().filter(|e| !e.is_dir()) {
|
||||
if e.method != zip::STORED && e.method != zip::DEFLATED {
|
||||
return Err(format!("{} uses an unsupported compression method ({})", e.name, e.method));
|
||||
}
|
||||
let path = zip::safe_path(&e.name).ok_or_else(|| format!("The archive names a file outside the game folder: {}", e.name))?;
|
||||
if !seen.insert(path.to_lowercase()) {
|
||||
continue;
|
||||
}
|
||||
let at = offsets.partition_point(|&o| o <= e.local_offset);
|
||||
let next = offsets.get(at).copied().unwrap_or(directory_offset);
|
||||
let minimum = e.local_offset + zip::LOCAL_FIXED as u64 + e.compressed;
|
||||
if next < minimum {
|
||||
return Err(format!("{} overlaps the next record in the archive", e.name));
|
||||
}
|
||||
out.push(Planned { entry: e.clone(), path, range_end: next - 1 });
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
struct Slot {
|
||||
header: Vec<u8>,
|
||||
payload_from: Option<usize>,
|
||||
consumed: u64,
|
||||
file: File,
|
||||
tmp: PathBuf,
|
||||
crc: Crc,
|
||||
written: u64,
|
||||
inflate: Option<Decompress>,
|
||||
buffer: Vec<u8>,
|
||||
}
|
||||
|
||||
struct Placer<'a> {
|
||||
jobs: &'a [&'a Planned],
|
||||
game_dir: &'a Path,
|
||||
slots: Vec<Mutex<Option<Slot>>>,
|
||||
splits: Vec<Option<Split>>,
|
||||
works: Vec<Work>,
|
||||
placed: Vec<AtomicBool>,
|
||||
ledger: Mutex<Option<File>>,
|
||||
events: &'a dyn Events,
|
||||
gate: &'a ReplaceGate,
|
||||
stop: &'a AtomicBool,
|
||||
failure: Mutex<Option<String>>,
|
||||
base_done: u64,
|
||||
base_files: u32,
|
||||
total: u64,
|
||||
total_files: u32,
|
||||
placed_bytes: AtomicU64,
|
||||
in_flight: AtomicU64,
|
||||
shown: AtomicU64,
|
||||
bytes_written: AtomicU64,
|
||||
files_placed: AtomicUsize,
|
||||
last_emit: AtomicU64,
|
||||
started: Instant,
|
||||
}
|
||||
|
||||
fn discard(slot: Slot) {
|
||||
drop(slot.file);
|
||||
let _ = fs::remove_file(&slot.tmp);
|
||||
}
|
||||
|
||||
impl Placer<'_> {
|
||||
fn fail(&self, message: String) -> String {
|
||||
let mut slot = self.failure.lock().expect("failure lock");
|
||||
if slot.is_none() {
|
||||
*slot = Some(message.clone());
|
||||
}
|
||||
drop(slot);
|
||||
self.stop.store(true, Ordering::SeqCst);
|
||||
message
|
||||
}
|
||||
|
||||
fn emit(&self, force: bool) {
|
||||
let now = self.started.elapsed().as_millis() as u64;
|
||||
let last = self.last_emit.load(Ordering::Relaxed);
|
||||
if !force && now < last + PROGRESS_EVERY_MS {
|
||||
return;
|
||||
}
|
||||
if !force && self.last_emit.compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed).is_err() {
|
||||
return;
|
||||
}
|
||||
self.last_emit.store(now, Ordering::Relaxed);
|
||||
let value = self.base_done + self.placed_bytes.load(Ordering::Relaxed) + self.in_flight.load(Ordering::Relaxed);
|
||||
let shown = self.shown.fetch_max(value, Ordering::Relaxed).max(value);
|
||||
self.events.progress(
|
||||
shown.min(self.total),
|
||||
self.total,
|
||||
self.base_files + self.files_placed.load(Ordering::Relaxed) as u32,
|
||||
self.total_files,
|
||||
);
|
||||
}
|
||||
|
||||
fn open(&self, job: &Planned) -> Result<Slot, String> {
|
||||
let dest = self.game_dir.join(&job.path);
|
||||
if let Some(parent) = dest.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| format!("Could not create the folder for {}: {e}", job.path))?;
|
||||
}
|
||||
let tmp = temp_path(&dest);
|
||||
let file = File::create(&tmp).map_err(|e| format!("Could not write {}: {e}", job.path))?;
|
||||
Ok(Slot {
|
||||
header: Vec::new(),
|
||||
payload_from: None,
|
||||
consumed: 0,
|
||||
file,
|
||||
tmp,
|
||||
crc: Crc::new(),
|
||||
written: 0,
|
||||
inflate: (job.entry.method == zip::DEFLATED).then(|| Decompress::new(false)),
|
||||
buffer: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
fn write_out(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
|
||||
if slot.written + data.len() as u64 > job.entry.size {
|
||||
return Err(SinkError::Retry(format!("{}: more data than the archive lists", job.path)));
|
||||
}
|
||||
slot.file
|
||||
.write_all(data)
|
||||
.map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?;
|
||||
slot.crc.update(data);
|
||||
slot.written += data.len() as u64;
|
||||
self.bytes_written.fetch_add(data.len() as u64, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn feed(&self, job: &Planned, slot: &mut Slot, payload: &[u8], finish: bool) -> Result<(), SinkError> {
|
||||
let Some(mut inflate) = slot.inflate.take() else {
|
||||
return self.write_out(job, slot, payload);
|
||||
};
|
||||
let mut out = std::mem::take(&mut slot.buffer);
|
||||
out.resize(INFLATE_BUFFER, 0);
|
||||
let result = self.inflate_into(job, slot, &mut inflate, &mut out, payload, finish);
|
||||
slot.inflate = Some(inflate);
|
||||
slot.buffer = out;
|
||||
result
|
||||
}
|
||||
|
||||
fn inflate_into(&self, job: &Planned, slot: &mut Slot, inflate: &mut Decompress, out: &mut [u8], mut input: &[u8], finish: bool) -> Result<(), SinkError> {
|
||||
let mode = if finish { FlushDecompress::Finish } else { FlushDecompress::None };
|
||||
loop {
|
||||
let (in_before, out_before) = (inflate.total_in(), inflate.total_out());
|
||||
let status = inflate
|
||||
.decompress(input, out, mode)
|
||||
.map_err(|e| SinkError::Retry(format!("{}: the compressed data is damaged ({e})", job.path)))?;
|
||||
let used = (inflate.total_in() - in_before) as usize;
|
||||
let produced = (inflate.total_out() - out_before) as usize;
|
||||
input = &input[used..];
|
||||
if produced > 0 {
|
||||
self.write_out(job, slot, &out[..produced])?;
|
||||
}
|
||||
if status == Status::StreamEnd || (used == 0 && produced == 0) {
|
||||
return Ok(());
|
||||
}
|
||||
if input.is_empty() && !finish && produced < out.len() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn piece(&self, i: usize, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
let job = self.jobs[i];
|
||||
let mut guard = self.slots[i].lock().expect("slot lock");
|
||||
if offset == 0 {
|
||||
if let Some(old) = guard.take() {
|
||||
self.in_flight.fetch_sub(old.consumed, Ordering::Relaxed);
|
||||
discard(old);
|
||||
}
|
||||
*guard = Some(self.open(job).map_err(|e| SinkError::Fatal(self.fail(e)))?);
|
||||
}
|
||||
let Some(slot) = guard.as_mut() else {
|
||||
return Err(SinkError::Retry(format!("{}: data arrived before the start of the record", job.path)));
|
||||
};
|
||||
self.consume(job, slot, data)
|
||||
}
|
||||
|
||||
fn consume(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
|
||||
if slot.payload_from.is_none() {
|
||||
slot.header.extend_from_slice(data);
|
||||
match zip::local_header_len(&slot.header) {
|
||||
None => return Ok(()),
|
||||
Some(Err(e)) => return Err(SinkError::Retry(format!("{}: {e}", job.path))),
|
||||
Some(Ok(len)) if slot.header.len() < len => return Ok(()),
|
||||
Some(Ok(len)) => {
|
||||
slot.payload_from = Some(len);
|
||||
let header = std::mem::take(&mut slot.header);
|
||||
let rest = header[len..].to_vec();
|
||||
return self.payload(job, slot, &rest);
|
||||
}
|
||||
}
|
||||
}
|
||||
self.payload(job, slot, data)
|
||||
}
|
||||
|
||||
fn payload(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
|
||||
let left = job.entry.compressed.saturating_sub(slot.consumed);
|
||||
let take = &data[..(left.min(data.len() as u64) as usize)];
|
||||
if take.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
slot.consumed += take.len() as u64;
|
||||
self.in_flight.fetch_add(take.len() as u64, Ordering::Relaxed);
|
||||
self.feed(job, slot, take, false)?;
|
||||
self.emit(false);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(&self, i: usize) -> Result<u64, SinkError> {
|
||||
let job = self.jobs[i];
|
||||
let taken = self.slots[i].lock().expect("slot lock").take();
|
||||
let Some(slot) = taken else {
|
||||
return Err(SinkError::Retry(format!("{}: the record arrived empty", job.path)));
|
||||
};
|
||||
self.complete(i, slot)
|
||||
}
|
||||
|
||||
fn complete(&self, i: usize, mut slot: Slot) -> Result<u64, SinkError> {
|
||||
let job = self.jobs[i];
|
||||
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
|
||||
if slot.consumed != job.entry.compressed {
|
||||
let got = slot.consumed;
|
||||
discard(slot);
|
||||
return Err(SinkError::Retry(format!("{}: {got} of {} compressed bytes arrived", job.path, job.entry.compressed)));
|
||||
}
|
||||
if slot.inflate.is_some() {
|
||||
if let Err(e) = self.feed(job, &mut slot, &[], true) {
|
||||
discard(slot);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
if slot.written != job.entry.size || slot.crc.sum() != job.entry.crc32 {
|
||||
let got = slot.written;
|
||||
discard(slot);
|
||||
return Err(SinkError::Retry(format!("{}: {got} bytes inflated with a CRC mismatch", job.path)));
|
||||
}
|
||||
let Slot { file, tmp, .. } = slot;
|
||||
if let Err(e) = file.sync_all() {
|
||||
drop(file);
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))));
|
||||
}
|
||||
drop(file);
|
||||
if self.stop.load(Ordering::SeqCst) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Ok(0);
|
||||
}
|
||||
if !self.gate.allow(self.events) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(SinkError::Fatal(self.fail(REPLACE_REFUSED.into())));
|
||||
}
|
||||
let dest = self.game_dir.join(&job.path);
|
||||
if let Err(e) = fs::rename(&tmp, &dest) {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
return Err(SinkError::Fatal(self.fail(format!("Could not put {} in place: {e}", job.path))));
|
||||
}
|
||||
self.record(job, &dest);
|
||||
self.placed[i].store(true, Ordering::SeqCst);
|
||||
self.placed_bytes.fetch_add(job.entry.compressed, Ordering::Relaxed);
|
||||
self.files_placed.fetch_add(1, Ordering::Relaxed);
|
||||
self.emit(true);
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
fn part_chunk(&self, i: usize, part: usize, start: u64, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
let job = self.jobs[i];
|
||||
let Some(split) = self.splits[i].as_ref() else {
|
||||
return Err(SinkError::Fatal(format!("{}: no split state", job.path)));
|
||||
};
|
||||
let mut file = split.file.lock().expect("split lock");
|
||||
if file.is_none() {
|
||||
let dest = self.game_dir.join(&job.path);
|
||||
if let Some(parent) = dest.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| SinkError::Fatal(self.fail(format!("Could not create the folder for {}: {e}", job.path))))?;
|
||||
}
|
||||
*file = Some(File::create(&split.tmp).map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?);
|
||||
}
|
||||
let handle = file.as_ref().expect("split file");
|
||||
if offset == 0 {
|
||||
let old = split.received[part].swap(0, Ordering::Relaxed);
|
||||
self.in_flight.fetch_sub(old, Ordering::Relaxed);
|
||||
}
|
||||
handle
|
||||
.write_all_at(data, start + offset)
|
||||
.map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?;
|
||||
drop(file);
|
||||
split.received[part].fetch_add(data.len() as u64, Ordering::Relaxed);
|
||||
self.in_flight.fetch_add(data.len() as u64, Ordering::Relaxed);
|
||||
self.emit(false);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn part_finish(&self, i: usize, part: usize, len: u64) -> Result<u64, SinkError> {
|
||||
let job = self.jobs[i];
|
||||
let Some(split) = self.splits[i].as_ref() else {
|
||||
return Err(SinkError::Fatal(format!("{}: no split state", job.path)));
|
||||
};
|
||||
let got = split.received[part].load(Ordering::Relaxed);
|
||||
if got != len {
|
||||
return Err(SinkError::Retry(format!("{}: part {part} got {got} of {len} bytes", job.path)));
|
||||
}
|
||||
if split.left.fetch_sub(1, Ordering::SeqCst) != 1 {
|
||||
return Ok(0);
|
||||
}
|
||||
let received: u64 = split.received.iter().map(|r| r.swap(0, Ordering::Relaxed)).sum();
|
||||
self.in_flight.fetch_sub(received, Ordering::Relaxed);
|
||||
let result = self.assemble(i, split);
|
||||
let _ = fs::remove_file(&split.tmp);
|
||||
*split.file.lock().expect("split lock") = None;
|
||||
result.map_err(|e| match e {
|
||||
SinkError::Retry(m) => SinkError::Fatal(self.fail(m)),
|
||||
fatal => fatal,
|
||||
})
|
||||
}
|
||||
|
||||
fn assemble(&self, i: usize, split: &Split) -> Result<u64, SinkError> {
|
||||
let job = self.jobs[i];
|
||||
let mut source = File::open(&split.tmp).map_err(|e| SinkError::Fatal(self.fail(format!("Could not read {}: {e}", job.path))))?;
|
||||
let mut slot = self.open(job).map_err(|e| SinkError::Fatal(self.fail(e)))?;
|
||||
let mut buf = vec![0u8; 1024 * 1024];
|
||||
loop {
|
||||
if self.stop.load(Ordering::Relaxed) {
|
||||
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
|
||||
discard(slot);
|
||||
return Ok(0);
|
||||
}
|
||||
let n = match source.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => {
|
||||
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
|
||||
discard(slot);
|
||||
return Err(SinkError::Fatal(self.fail(format!("Could not read {}: {e}", job.path))));
|
||||
}
|
||||
};
|
||||
if let Err(e) = self.consume(job, &mut slot, &buf[..n]) {
|
||||
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
|
||||
discard(slot);
|
||||
return Err(e);
|
||||
}
|
||||
if slot.consumed >= job.entry.compressed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.complete(i, slot)
|
||||
}
|
||||
|
||||
fn record(&self, job: &Planned, dest: &Path) {
|
||||
let known = Known { crc: job.entry.crc32, size: job.entry.size, modified: modified_ms(dest) };
|
||||
if let Some(file) = self.ledger.lock().expect("ledger lock").as_mut() {
|
||||
let _ = file.write_all(ledger_line(&job.path, known).as_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Sink<'p, 'a>(&'p Placer<'a>);
|
||||
|
||||
impl FetchSink for Sink<'_, '_> {
|
||||
fn process(&self, item: &FetchItem, _body: Vec<u8>) -> Result<u64, SinkError> {
|
||||
Err(SinkError::Fatal(format!("item {} arrived whole; streaming was expected", item.id)))
|
||||
}
|
||||
|
||||
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
|
||||
match self.0.works[item.id as usize] {
|
||||
Work::Whole(i) => self.0.piece(i, offset, data),
|
||||
Work::Part { job, part, start, .. } => self.0.part_chunk(job, part, start, offset, data),
|
||||
}
|
||||
}
|
||||
|
||||
fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result<u64, SinkError> {
|
||||
match self.0.works[item.id as usize] {
|
||||
Work::Whole(i) => self.0.finish(i),
|
||||
Work::Part { job, part, len, .. } => self.0.part_finish(job, part, len),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn origin(url: &str) -> String {
|
||||
let after = url.find("://").map(|i| i + 3).unwrap_or(0);
|
||||
let end = url[after..].find('/').map(|i| after + i).unwrap_or(url.len());
|
||||
url[..end].to_string()
|
||||
}
|
||||
|
||||
fn cancelled(cancel: &AtomicBool) -> Result<(), Stop> {
|
||||
if cancel.load(Ordering::SeqCst) {
|
||||
Err(Stop::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finished, Stop> {
|
||||
let failed = Stop::Failed;
|
||||
let started = Instant::now();
|
||||
fs::create_dir_all(&req.game_dir).map_err(|e| failed(format!("Could not create the install folder: {e}")))?;
|
||||
let _ = File::create(req.game_dir.join(IN_PROGRESS_MARKER));
|
||||
|
||||
ev.status("Reading the build…");
|
||||
let archive = net::read_archive(&req.url).map_err(|e| {
|
||||
ev.log(&format!("archive: {e}"));
|
||||
failed(format!("The EA build could not be read: {e}"))
|
||||
})?;
|
||||
cancelled(cancel)?;
|
||||
let planned = plan(&archive.entries, archive.directory.offset).map_err(failed)?;
|
||||
let total_size: u64 = planned.iter().map(|p| p.entry.size).sum();
|
||||
let total_compressed: u64 = planned.iter().map(|p| p.entry.compressed).sum();
|
||||
ev.log(&format!(
|
||||
"archive bytes={} entries={} files={} size={total_size} compressed={total_compressed}",
|
||||
archive.total,
|
||||
archive.entries.len(),
|
||||
planned.len()
|
||||
));
|
||||
|
||||
for p in &planned {
|
||||
let dest = req.game_dir.join(&p.path);
|
||||
let _ = fs::remove_file(temp_path(&dest));
|
||||
let mut split = dest.into_os_string();
|
||||
split.push(SPLIT_SUFFIX);
|
||||
let _ = fs::remove_file(split);
|
||||
}
|
||||
|
||||
ev.status("Checking files…");
|
||||
let mut ledger = read_ledger(&req.game_dir);
|
||||
let mut pending: Vec<&Planned> = Vec::new();
|
||||
let mut kept: Vec<(String, Known)> = Vec::new();
|
||||
let mut checked = 0u64;
|
||||
let mut last_report = Instant::now();
|
||||
for (n, p) in planned.iter().enumerate() {
|
||||
let dest = req.game_dir.join(&p.path);
|
||||
let size_ok = fs::symlink_metadata(&dest).map(|m| m.is_file() && m.len() == p.entry.size).unwrap_or(false);
|
||||
let mut complete = false;
|
||||
if size_ok {
|
||||
let modified = modified_ms(&dest);
|
||||
let known = ledger.get(&p.path).copied();
|
||||
if known == Some(Known { crc: p.entry.crc32, size: p.entry.size, modified }) {
|
||||
complete = true;
|
||||
} else {
|
||||
match crc_of(&dest, cancel) {
|
||||
Err(()) => return Err(Stop::Cancelled),
|
||||
Ok(Some(crc)) if crc == p.entry.crc32 => {
|
||||
complete = true;
|
||||
kept.push((p.path.clone(), Known { crc, size: p.entry.size, modified }));
|
||||
}
|
||||
Ok(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !complete {
|
||||
pending.push(p);
|
||||
}
|
||||
checked += p.entry.size;
|
||||
if last_report.elapsed() >= CHECK_REPORT_EVERY || n + 1 == planned.len() {
|
||||
ev.checking(checked, total_size, (n + 1) as u32, planned.len() as u32);
|
||||
last_report = Instant::now();
|
||||
}
|
||||
cancelled(cancel)?;
|
||||
}
|
||||
for (path, known) in kept {
|
||||
ledger.insert(path, known);
|
||||
}
|
||||
|
||||
let pending_size: u64 = pending.iter().map(|p| p.entry.size).sum();
|
||||
let pending_compressed: u64 = pending.iter().map(|p| p.entry.compressed).sum();
|
||||
ev.log(&format!(
|
||||
"resume kept_files={} pending_files={} pending_bytes={pending_size} pending_download={pending_compressed}",
|
||||
planned.len() - pending.len(),
|
||||
pending.len()
|
||||
));
|
||||
ev.sizes(pending_size, total_size);
|
||||
if let Some(message) = space_problem(pending_size, free_bytes(&req.game_dir)) {
|
||||
return Err(failed(message));
|
||||
}
|
||||
|
||||
let gate = ReplaceGate::new();
|
||||
let (empty, remote): (Vec<&Planned>, Vec<&Planned>) = pending.into_iter().partition(|p| p.entry.size == 0 && p.entry.crc32 == 0);
|
||||
for p in &empty {
|
||||
let dest = req.game_dir.join(&p.path);
|
||||
if let Some(parent) = dest.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| failed(format!("Could not create the folder for {}: {e}", p.path)))?;
|
||||
}
|
||||
if !gate.allow(ev) {
|
||||
return Err(failed(REPLACE_REFUSED.into()));
|
||||
}
|
||||
File::create(&dest).map_err(|e| failed(format!("Could not write {}: {e}", p.path)))?;
|
||||
ledger.insert(p.path.clone(), Known { crc: 0, size: 0, modified: modified_ms(&dest) });
|
||||
}
|
||||
|
||||
let mut text = String::new();
|
||||
for (path, known) in &ledger {
|
||||
text.push_str(&ledger_line(path, *known));
|
||||
}
|
||||
fs::write(req.game_dir.join(LEDGER), text).map_err(|e| failed(format!("Could not write the install record: {e}")))?;
|
||||
let ledger_file = OpenOptions::new().append(true).open(req.game_dir.join(LEDGER)).ok();
|
||||
|
||||
let host = origin(&req.url);
|
||||
let works = works(&remote, req.part_bytes);
|
||||
let splits: Vec<Option<Split>> = remote
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| {
|
||||
let parts = works.iter().filter(|w| matches!(w, Work::Part { job, .. } if *job == i)).count();
|
||||
(parts > 0).then(|| {
|
||||
let mut name = req.game_dir.join(&p.path).into_os_string();
|
||||
name.push(SPLIT_SUFFIX);
|
||||
Split {
|
||||
tmp: PathBuf::from(name),
|
||||
file: Mutex::new(None),
|
||||
received: (0..parts).map(|_| AtomicU64::new(0)).collect(),
|
||||
left: AtomicUsize::new(parts),
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let placer = Placer {
|
||||
jobs: &remote,
|
||||
game_dir: &req.game_dir,
|
||||
slots: remote.iter().map(|_| Mutex::new(None)).collect(),
|
||||
splits,
|
||||
works: works.clone(),
|
||||
placed: remote.iter().map(|_| AtomicBool::new(false)).collect(),
|
||||
ledger: Mutex::new(ledger_file),
|
||||
events: ev,
|
||||
gate: &gate,
|
||||
stop: cancel,
|
||||
failure: Mutex::new(None),
|
||||
base_done: total_compressed - pending_compressed,
|
||||
base_files: (planned.len() - remote.len()) as u32,
|
||||
total: total_compressed,
|
||||
total_files: planned.len() as u32,
|
||||
placed_bytes: AtomicU64::new(0),
|
||||
in_flight: AtomicU64::new(0),
|
||||
shown: AtomicU64::new(0),
|
||||
bytes_written: AtomicU64::new(0),
|
||||
files_placed: AtomicUsize::new(0),
|
||||
last_emit: AtomicU64::new(0),
|
||||
started: Instant::now(),
|
||||
};
|
||||
let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(2);
|
||||
let window = req.max_workers.clamp(1, 64);
|
||||
let hosts = vec![host.clone()];
|
||||
let options = FetchOptions {
|
||||
max_workers: window,
|
||||
per_host_cap: window,
|
||||
timeout: REQUEST_TIMEOUT,
|
||||
headers: vec![("User-Agent".to_string(), net::user_agent())],
|
||||
ca_bundle_path: req.ca_bundle.clone(),
|
||||
process_workers: req.process_workers.max(cores / 2).max(1),
|
||||
label: "ea".to_string(),
|
||||
stream: true,
|
||||
};
|
||||
let items: Vec<FetchItem> = works
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(n, w)| {
|
||||
let range = match *w {
|
||||
Work::Whole(i) => (remote[i].entry.local_offset, remote[i].range_end),
|
||||
Work::Part { job, start, len, .. } => {
|
||||
let at = remote[job].entry.local_offset + start;
|
||||
(at, at + len - 1)
|
||||
}
|
||||
};
|
||||
FetchItem { id: n as u64, urls: vec![req.url.clone()], reserve: 0, range: Some(range) }
|
||||
})
|
||||
.collect();
|
||||
ev.status("Downloading…");
|
||||
ev.log(&format!("engine files={} requests={} window={window} hosts={}", remote.len(), items.len(), distinct_host_count(&hosts)));
|
||||
placer.emit(true);
|
||||
let log = |line: &str| ev.log(line);
|
||||
let outcome = if items.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(run_fetch(items, &hosts, &options, &Sink(&placer), cancel, &|_: u64, _: u64| {}, &log))
|
||||
};
|
||||
for slot in &placer.slots {
|
||||
if let Some(active) = slot.lock().expect("slot lock").take() {
|
||||
discard(active);
|
||||
}
|
||||
}
|
||||
for split in placer.splits.iter().flatten() {
|
||||
*split.file.lock().expect("split lock") = None;
|
||||
let _ = fs::remove_file(&split.tmp);
|
||||
}
|
||||
placer.emit(true);
|
||||
let placed_all = placer.placed.iter().all(|p| p.load(Ordering::SeqCst));
|
||||
let bytes_written = placer.bytes_written.load(Ordering::Relaxed);
|
||||
let files_written = placer.files_placed.load(Ordering::Relaxed) + empty.len();
|
||||
let elapsed = started.elapsed().as_secs_f64().max(0.001);
|
||||
ev.log(&format!(
|
||||
"summary files_written={files_written} bytes_written={bytes_written} elapsed_s={elapsed:.1} avg_mbps={:.1}",
|
||||
bytes_written as f64 / 1_048_576.0 / elapsed
|
||||
));
|
||||
if let Some(message) = placer.failure.lock().expect("failure lock").take() {
|
||||
return Err(Stop::Failed(message));
|
||||
}
|
||||
if !placed_all {
|
||||
if let Some(error) = outcome.as_ref().and_then(|o| o.error.clone()) {
|
||||
ev.log(&format!("download failed: {error}"));
|
||||
return Err(Stop::Failed("A file failed to download".into()));
|
||||
}
|
||||
if cancel.load(Ordering::SeqCst) {
|
||||
return Err(Stop::Cancelled);
|
||||
}
|
||||
return Err(Stop::Failed("The download ended before every file was written".into()));
|
||||
}
|
||||
|
||||
let wanted: HashSet<&str> = planned.iter().map(|p| p.path.as_str()).collect();
|
||||
let by_case: HashMap<String, &str> = planned.iter().map(|p| (p.path.to_lowercase(), p.path.as_str())).collect();
|
||||
let mut removed = 0;
|
||||
let current = read_ledger(&req.game_dir);
|
||||
let mut text = String::new();
|
||||
for (path, known) in ¤t {
|
||||
if wanted.contains(path.as_str()) {
|
||||
text.push_str(&ledger_line(path, *known));
|
||||
} else if by_case.get(&path.to_lowercase()).is_some_and(|kept| same_file(&req.game_dir.join(kept), &req.game_dir.join(path))) {
|
||||
continue;
|
||||
} else if zip::safe_path(path).as_deref() == Some(path.as_str()) && fs::remove_file(req.game_dir.join(path)).is_ok() {
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
let _ = fs::write(req.game_dir.join(LEDGER), text);
|
||||
if removed > 0 {
|
||||
ev.log(&format!("removed {removed} files the new build no longer has"));
|
||||
}
|
||||
let _ = fs::remove_file(req.game_dir.join(IN_PROGRESS_MARKER));
|
||||
Ok(Finished {
|
||||
total_bytes: total_size,
|
||||
download_bytes: total_compressed,
|
||||
file_count: planned.len(),
|
||||
files_written,
|
||||
bytes_written,
|
||||
removed,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn size(url: &str) -> Result<Value, String> {
|
||||
let archive = net::read_archive(url)?;
|
||||
let planned = plan(&archive.entries, archive.directory.offset)?;
|
||||
Ok(json!({
|
||||
"totalBytes": planned.iter().map(|p| p.entry.size).sum::<u64>(),
|
||||
"downloadBytes": planned.iter().map(|p| p.entry.compressed).sum::<u64>(),
|
||||
"fileCount": planned.len(),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,258 @@
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use flate2::write::DeflateEncoder;
|
||||
use flate2::Compression;
|
||||
|
||||
use super::*;
|
||||
|
||||
struct Built {
|
||||
bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
fn build_zip(files: &[(&str, Vec<u8>, bool)]) -> Built {
|
||||
let mut out = Vec::new();
|
||||
let mut central = Vec::new();
|
||||
for (name, data, deflate) in files {
|
||||
let mut crc = Crc::new();
|
||||
crc.update(data);
|
||||
let body = if *deflate {
|
||||
let mut enc = DeflateEncoder::new(Vec::new(), Compression::default());
|
||||
enc.write_all(data).unwrap();
|
||||
enc.finish().unwrap()
|
||||
} else {
|
||||
data.clone()
|
||||
};
|
||||
let method: u16 = if *deflate { 8 } else { 0 };
|
||||
let offset = out.len() as u32;
|
||||
let extra = [0xAAu8, 0xBB, 2, 0, 1, 2];
|
||||
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[20, 0, 0, 0]);
|
||||
out.extend_from_slice(&method.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0, 0, 0]);
|
||||
out.extend_from_slice(&crc.sum().to_le_bytes());
|
||||
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(&(extra.len() as u16).to_le_bytes());
|
||||
out.extend_from_slice(name.as_bytes());
|
||||
out.extend_from_slice(&extra);
|
||||
out.extend_from_slice(&body);
|
||||
|
||||
central.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
|
||||
central.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
|
||||
central.extend_from_slice(&method.to_le_bytes());
|
||||
central.extend_from_slice(&[0, 0, 0, 0]);
|
||||
central.extend_from_slice(&crc.sum().to_le_bytes());
|
||||
central.extend_from_slice(&(body.len() as u32).to_le_bytes());
|
||||
central.extend_from_slice(&(data.len() as u32).to_le_bytes());
|
||||
central.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
central.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
central.extend_from_slice(&offset.to_le_bytes());
|
||||
central.extend_from_slice(name.as_bytes());
|
||||
}
|
||||
let cd_offset = out.len() as u32;
|
||||
let count = files.len() as u16;
|
||||
out.extend_from_slice(¢ral);
|
||||
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0, 0, 0]);
|
||||
out.extend_from_slice(&count.to_le_bytes());
|
||||
out.extend_from_slice(&count.to_le_bytes());
|
||||
out.extend_from_slice(&(central.len() as u32).to_le_bytes());
|
||||
out.extend_from_slice(&cd_offset.to_le_bytes());
|
||||
out.extend_from_slice(&[0, 0]);
|
||||
Built { bytes: out }
|
||||
}
|
||||
|
||||
struct Server {
|
||||
url: String,
|
||||
requests: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
fn serve(body: Vec<u8>) -> Server {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let body = Arc::new(body);
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let counter = Arc::clone(&requests);
|
||||
std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(stream) = stream else { continue };
|
||||
let body = Arc::clone(&body);
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
std::thread::spawn(move || answer(stream, &body));
|
||||
}
|
||||
});
|
||||
Server { url: format!("http://{addr}/build/game.zip?sig=abc"), requests }
|
||||
}
|
||||
|
||||
fn answer(mut stream: TcpStream, body: &[u8]) {
|
||||
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||
let mut first = String::new();
|
||||
if reader.read_line(&mut first).is_err() {
|
||||
return;
|
||||
}
|
||||
let mut range = None;
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
match reader.read_line(&mut line) {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(_) if line == "\r\n" => break,
|
||||
Ok(_) => {
|
||||
if let Some(v) = line.to_ascii_lowercase().strip_prefix("range: bytes=") {
|
||||
range = Some(v.trim().to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let total = body.len() as u64;
|
||||
let Some(range) = range else {
|
||||
let _ = stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {total}\r\nConnection: close\r\n\r\n").as_bytes());
|
||||
let _ = stream.write_all(body);
|
||||
return;
|
||||
};
|
||||
let (a, b) = range.split_once('-').unwrap();
|
||||
let (start, end) = if a.is_empty() {
|
||||
let n: u64 = b.parse().unwrap();
|
||||
(total.saturating_sub(n), total - 1)
|
||||
} else {
|
||||
let start: u64 = a.parse().unwrap();
|
||||
let end = if b.is_empty() { total - 1 } else { b.parse::<u64>().unwrap().min(total - 1) };
|
||||
(start, end)
|
||||
};
|
||||
let slice = &body[start as usize..=end as usize];
|
||||
let head = format!(
|
||||
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {start}-{end}/{total}\r\nConnection: close\r\n\r\n",
|
||||
slice.len()
|
||||
);
|
||||
let _ = stream.write_all(head.as_bytes());
|
||||
let _ = stream.write_all(slice);
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Recorder {
|
||||
logs: Mutex<Vec<String>>,
|
||||
gates: AtomicUsize,
|
||||
last: Mutex<(u64, u64)>,
|
||||
}
|
||||
|
||||
impl Events for Recorder {
|
||||
fn log(&self, line: &str) {
|
||||
self.logs.lock().unwrap().push(line.to_string());
|
||||
}
|
||||
fn status(&self, _text: &str) {}
|
||||
fn checking(&self, _d: u64, _t: u64, _f: u32, _ft: u32) {}
|
||||
fn sizes(&self, _pending: u64, _total: u64) {}
|
||||
fn progress(&self, done: u64, total: u64, _files: u32, _files_total: u32) {
|
||||
*self.last.lock().unwrap() = (done, total);
|
||||
}
|
||||
fn before_replace(&self) -> bool {
|
||||
self.gates.fetch_add(1, Ordering::SeqCst);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
fn sample() -> Vec<(&'static str, Vec<u8>, bool)> {
|
||||
let mut seed = 0x1234_5678u32;
|
||||
let big: Vec<u8> = (0..900_000u32).map(|_| { seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); (seed >> 16) as u8 }).collect();
|
||||
vec![
|
||||
("__Installer/installerdata.xml", b"<DiPManifest version=\"4.0\"></DiPManifest>".to_vec(), true),
|
||||
("Game.exe", big, true),
|
||||
("Data\\stored.bin", vec![9u8; 4096], false),
|
||||
("Data/", Vec::new(), false),
|
||||
("empty.txt", Vec::new(), false),
|
||||
]
|
||||
}
|
||||
|
||||
fn request(url: &str, dir: &Path) -> Request {
|
||||
Request { url: url.to_string(), game_dir: dir.to_path_buf(), max_workers: 4, process_workers: 2, ca_bundle: String::new(), part_bytes: 64 * 1024 }
|
||||
}
|
||||
|
||||
fn temp_dir(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("droiddeck-ea-{name}-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn installs_every_file_and_keeps_them_on_the_next_run() {
|
||||
let files = sample();
|
||||
let zip = build_zip(&files);
|
||||
let server = serve(zip.bytes);
|
||||
let dir = temp_dir("install");
|
||||
let ev = Recorder::default();
|
||||
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &ev).unwrap();
|
||||
assert_eq!(done.file_count, 4);
|
||||
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), files[1].1);
|
||||
assert_eq!(fs::read(dir.join("Data/stored.bin")).unwrap(), vec![9u8; 4096]);
|
||||
assert!(dir.join("empty.txt").is_file());
|
||||
assert!(dir.join("__Installer/installerdata.xml").is_file());
|
||||
assert!(!dir.join(IN_PROGRESS_MARKER).exists());
|
||||
let (shown, total) = *ev.last.lock().unwrap();
|
||||
assert_eq!(shown, total);
|
||||
|
||||
let before = server.requests.load(Ordering::SeqCst);
|
||||
let again = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
|
||||
assert_eq!(again.files_written, 0);
|
||||
assert!(server.requests.load(Ordering::SeqCst) - before <= 2);
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_update_replaces_changed_files_and_drops_removed_ones() {
|
||||
let dir = temp_dir("update");
|
||||
let first = serve(build_zip(&sample()).bytes);
|
||||
run(&request(&first.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
|
||||
|
||||
let mut next = sample();
|
||||
next[1].1[10] ^= 0xFF;
|
||||
next.retain(|f| f.0 != "Data\\stored.bin");
|
||||
let second = serve(build_zip(&next).bytes);
|
||||
let ev = Recorder::default();
|
||||
let done = run(&request(&second.url, &dir), &AtomicBool::new(false), &ev).unwrap();
|
||||
assert_eq!(done.files_written, 1);
|
||||
assert_eq!(done.removed, 1);
|
||||
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), next[1].1);
|
||||
assert!(!dir.join("Data/stored.bin").exists());
|
||||
assert_eq!(ev.gates.load(Ordering::SeqCst), 1);
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_file_on_disk_is_downloaded_again() {
|
||||
let dir = temp_dir("repair");
|
||||
let server = serve(build_zip(&sample()).bytes);
|
||||
run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
|
||||
let mut data = fs::read(dir.join("Game.exe")).unwrap();
|
||||
data[5] ^= 1;
|
||||
fs::write(dir.join("Game.exe"), &data).unwrap();
|
||||
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
|
||||
assert_eq!(done.files_written, 1);
|
||||
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), sample()[1].1);
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_the_archive_size() {
|
||||
let server = serve(build_zip(&sample()).bytes);
|
||||
let answer = size(&server.url).unwrap();
|
||||
assert_eq!(answer["fileCount"], 4);
|
||||
let expected: usize = sample().iter().map(|f| f.1.len()).sum();
|
||||
assert_eq!(answer["totalBytes"], expected as u64);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_records_arrive_in_parallel_parts() {
|
||||
let dir = temp_dir("parts");
|
||||
let server = serve(build_zip(&sample()).bytes);
|
||||
let before = server.requests.load(Ordering::SeqCst);
|
||||
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
|
||||
assert_eq!(done.files_written, 4);
|
||||
assert!(server.requests.load(Ordering::SeqCst) - before > 6);
|
||||
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), sample()[1].1);
|
||||
assert!(fs::read_dir(&dir).unwrap().all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".ddzip")));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod ffi;
|
||||
pub mod install;
|
||||
pub mod net;
|
||||
pub mod zip;
|
||||
@@ -0,0 +1,98 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Ranged reads of EA's build archives.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use super::zip::{self, Directory, Entry};
|
||||
|
||||
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const READ_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const MAX_DIRECTORY: u64 = 512 * 1024 * 1024;
|
||||
|
||||
pub fn user_agent() -> String {
|
||||
crate::amazon::net::user_agent()
|
||||
}
|
||||
|
||||
pub struct Ranged {
|
||||
pub body: Vec<u8>,
|
||||
pub start: u64,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
fn content_range(value: &str) -> Option<(u64, u64)> {
|
||||
let rest = value.trim().strip_prefix("bytes")?.trim_start();
|
||||
let (span, total) = rest.split_once('/')?;
|
||||
let start = span.split_once('-')?.0.trim().parse().ok()?;
|
||||
Some((start, total.trim().parse().ok()?))
|
||||
}
|
||||
|
||||
pub fn get_range(url: &str, range: &str) -> Result<Ranged, String> {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|e| format!("runtime: {e}"))?;
|
||||
runtime.block_on(async {
|
||||
let client = reqwest::Client::builder()
|
||||
.user_agent(user_agent())
|
||||
.connect_timeout(CONNECT_TIMEOUT)
|
||||
.read_timeout(READ_TIMEOUT)
|
||||
.build()
|
||||
.map_err(|e| format!("http client: {}", e.without_url()))?;
|
||||
let response = client
|
||||
.get(url)
|
||||
.header(reqwest::header::RANGE, format!("bytes={range}"))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("request failed: {}", e.without_url()))?;
|
||||
let status = response.status().as_u16();
|
||||
if status == 403 || status == 404 {
|
||||
return Err(format!("the download link was refused ({status})"));
|
||||
}
|
||||
if status != 206 {
|
||||
return Err(format!("the server did not answer a ranged read ({status})"));
|
||||
}
|
||||
let (start, total) = response
|
||||
.headers()
|
||||
.get(reqwest::header::CONTENT_RANGE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(content_range)
|
||||
.ok_or("the server gave no content range")?;
|
||||
let body = response.bytes().await.map_err(|e| format!("answer could not be read: {}", e.without_url()))?.to_vec();
|
||||
Ok(Ranged { body, start, total })
|
||||
})
|
||||
}
|
||||
|
||||
pub struct Archive {
|
||||
pub total: u64,
|
||||
pub directory: Directory,
|
||||
pub entries: Vec<Entry>,
|
||||
}
|
||||
|
||||
pub fn read_archive(url: &str) -> Result<Archive, String> {
|
||||
let tail = get_range(url, &format!("-{}", zip::TAIL_LEN))?;
|
||||
let directory = zip::locate(&tail.body, tail.start)?;
|
||||
if directory.size > MAX_DIRECTORY || directory.offset + directory.size > tail.total {
|
||||
return Err("The archive directory is out of bounds".into());
|
||||
}
|
||||
let bytes = if directory.offset >= tail.start {
|
||||
let at = (directory.offset - tail.start) as usize;
|
||||
let end = at + directory.size as usize;
|
||||
tail.body.get(at..end).ok_or("The archive directory is cut short")?.to_vec()
|
||||
} else {
|
||||
let end = directory.offset + directory.size - 1;
|
||||
get_range(url, &format!("{}-{end}", directory.offset))?.body
|
||||
};
|
||||
let entries = zip::parse(&bytes, directory.entries)?;
|
||||
Ok(Archive { total: tail.total, directory, entries })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::content_range;
|
||||
|
||||
#[test]
|
||||
fn reads_content_range() {
|
||||
assert_eq!(content_range("bytes 100-199/5000"), Some((100, 5000)));
|
||||
assert_eq!(content_range("bytes */5000"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The central directory of a remote ZIP archive, which is how EA publishes a game build.
|
||||
|
||||
const EOCD: u32 = 0x0605_4b50;
|
||||
const EOCD64: u32 = 0x0606_4b50;
|
||||
const EOCD64_LOCATOR: u32 = 0x0706_4b50;
|
||||
const CENTRAL: u32 = 0x0201_4b50;
|
||||
const LOCAL: u32 = 0x0403_4b50;
|
||||
const EOCD_LEN: usize = 22;
|
||||
const LOCATOR_LEN: usize = 20;
|
||||
const EOCD64_LEN: usize = 56;
|
||||
pub const LOCAL_FIXED: usize = 30;
|
||||
pub const TAIL_LEN: u64 = 128 * 1024;
|
||||
|
||||
pub const STORED: u16 = 0;
|
||||
pub const DEFLATED: u16 = 8;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Entry {
|
||||
pub name: String,
|
||||
pub crc32: u32,
|
||||
pub method: u16,
|
||||
pub compressed: u64,
|
||||
pub size: u64,
|
||||
pub local_offset: u64,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
pub fn is_dir(&self) -> bool {
|
||||
self.name.ends_with('/') || self.name.ends_with('\\')
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Directory {
|
||||
pub offset: u64,
|
||||
pub size: u64,
|
||||
pub entries: u64,
|
||||
}
|
||||
|
||||
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
|
||||
b.get(at..at + 2).map(|s| u16::from_le_bytes([s[0], s[1]]))
|
||||
}
|
||||
|
||||
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
|
||||
b.get(at..at + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))
|
||||
}
|
||||
|
||||
fn u64_at(b: &[u8], at: usize) -> Option<u64> {
|
||||
b.get(at..at + 8).map(|s| u64::from_le_bytes(s.try_into().unwrap_or([0; 8])))
|
||||
}
|
||||
|
||||
/// Finds the central directory from the last bytes of the archive. `tail_start` is where `tail`
|
||||
/// begins in the whole file.
|
||||
pub fn locate(tail: &[u8], tail_start: u64) -> Result<Directory, String> {
|
||||
if tail.len() < EOCD_LEN {
|
||||
return Err("The archive is too short to be a ZIP".into());
|
||||
}
|
||||
let mut at = tail.len() - EOCD_LEN;
|
||||
let found = loop {
|
||||
if u32_at(tail, at) == Some(EOCD) {
|
||||
let comment = u16_at(tail, at + 20).unwrap_or(0) as usize;
|
||||
if at + EOCD_LEN + comment <= tail.len() {
|
||||
break at;
|
||||
}
|
||||
}
|
||||
if at == 0 {
|
||||
return Err("The archive has no end of central directory".into());
|
||||
}
|
||||
at -= 1;
|
||||
};
|
||||
let entries = u16_at(tail, found + 10).unwrap_or(0) as u64;
|
||||
let size = u32_at(tail, found + 12).unwrap_or(0) as u64;
|
||||
let offset = u32_at(tail, found + 16).unwrap_or(0) as u64;
|
||||
let wide = entries == 0xFFFF || size == 0xFFFF_FFFF || offset == 0xFFFF_FFFF;
|
||||
if !wide {
|
||||
return Ok(Directory { offset, size, entries });
|
||||
}
|
||||
let locator = found.checked_sub(LOCATOR_LEN).filter(|&l| u32_at(tail, l) == Some(EOCD64_LOCATOR));
|
||||
let Some(locator) = locator else {
|
||||
return Err("The archive needs ZIP64 but has no ZIP64 locator".into());
|
||||
};
|
||||
let record_at = u64_at(tail, locator + 8).unwrap_or(0);
|
||||
let Some(rel) = record_at.checked_sub(tail_start).map(|r| r as usize) else {
|
||||
return Err("The ZIP64 directory record lies before the fetched tail".into());
|
||||
};
|
||||
if u32_at(tail, rel) != Some(EOCD64) || rel + EOCD64_LEN > tail.len() {
|
||||
return Err("The ZIP64 directory record is damaged".into());
|
||||
}
|
||||
Ok(Directory {
|
||||
entries: u64_at(tail, rel + 32).unwrap_or(0),
|
||||
size: u64_at(tail, rel + 40).unwrap_or(0),
|
||||
offset: u64_at(tail, rel + 48).unwrap_or(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn wide_values(extra: &[u8], need_size: bool, need_compressed: bool, need_offset: bool) -> (Option<u64>, Option<u64>, Option<u64>) {
|
||||
let mut at = 0;
|
||||
while at + 4 <= extra.len() {
|
||||
let id = u16_at(extra, at).unwrap_or(0);
|
||||
let len = u16_at(extra, at + 2).unwrap_or(0) as usize;
|
||||
let body = &extra[(at + 4).min(extra.len())..(at + 4 + len).min(extra.len())];
|
||||
if id == 0x0001 {
|
||||
let mut p = 0;
|
||||
let mut take = |want: bool| {
|
||||
if !want {
|
||||
return None;
|
||||
}
|
||||
let v = u64_at(body, p);
|
||||
p += 8;
|
||||
v
|
||||
};
|
||||
let size = take(need_size);
|
||||
let compressed = take(need_compressed);
|
||||
let offset = take(need_offset);
|
||||
return (size, compressed, offset);
|
||||
}
|
||||
at += 4 + len;
|
||||
}
|
||||
(None, None, None)
|
||||
}
|
||||
|
||||
pub fn parse(directory: &[u8], expected: u64) -> Result<Vec<Entry>, String> {
|
||||
let mut entries = Vec::with_capacity(expected.min(1 << 20) as usize);
|
||||
let mut at = 0usize;
|
||||
while at + 46 <= directory.len() {
|
||||
if u32_at(directory, at) != Some(CENTRAL) {
|
||||
break;
|
||||
}
|
||||
let flags = u16_at(directory, at + 8).unwrap_or(0);
|
||||
let method = u16_at(directory, at + 10).unwrap_or(0);
|
||||
let crc32 = u32_at(directory, at + 16).unwrap_or(0);
|
||||
let compressed32 = u32_at(directory, at + 20).unwrap_or(0);
|
||||
let size32 = u32_at(directory, at + 24).unwrap_or(0);
|
||||
let name_len = u16_at(directory, at + 28).unwrap_or(0) as usize;
|
||||
let extra_len = u16_at(directory, at + 30).unwrap_or(0) as usize;
|
||||
let comment_len = u16_at(directory, at + 32).unwrap_or(0) as usize;
|
||||
let offset32 = u32_at(directory, at + 42).unwrap_or(0);
|
||||
let name_at = at + 46;
|
||||
let end = name_at + name_len + extra_len + comment_len;
|
||||
if end > directory.len() {
|
||||
return Err("The archive directory is cut short".into());
|
||||
}
|
||||
let raw = &directory[name_at..name_at + name_len];
|
||||
let name = if flags & 0x0800 != 0 {
|
||||
String::from_utf8_lossy(raw).into_owned()
|
||||
} else {
|
||||
raw.iter().map(|&b| b as char).collect()
|
||||
};
|
||||
let extra = &directory[name_at + name_len..name_at + name_len + extra_len];
|
||||
let (size, compressed, offset) =
|
||||
wide_values(extra, size32 == 0xFFFF_FFFF, compressed32 == 0xFFFF_FFFF, offset32 == 0xFFFF_FFFF);
|
||||
if flags & 0x0001 != 0 {
|
||||
return Err(format!("{name} is encrypted"));
|
||||
}
|
||||
entries.push(Entry {
|
||||
name,
|
||||
crc32,
|
||||
method,
|
||||
compressed: compressed.unwrap_or(compressed32 as u64),
|
||||
size: size.unwrap_or(size32 as u64),
|
||||
local_offset: offset.unwrap_or(offset32 as u64),
|
||||
});
|
||||
at = end;
|
||||
}
|
||||
if expected != 0 && entries.len() as u64 != expected {
|
||||
return Err(format!("The archive directory lists {} of {expected} entries", entries.len()));
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Bytes before the compressed data of an entry, read from its local header.
|
||||
pub fn local_header_len(header: &[u8]) -> Option<Result<usize, String>> {
|
||||
if header.len() < LOCAL_FIXED {
|
||||
return None;
|
||||
}
|
||||
if u32_at(header, 0) != Some(LOCAL) {
|
||||
return Some(Err("An entry has no local header where the directory points".into()));
|
||||
}
|
||||
let name = u16_at(header, 26).unwrap_or(0) as usize;
|
||||
let extra = u16_at(header, 28).unwrap_or(0) as usize;
|
||||
Some(Ok(LOCAL_FIXED + name + extra))
|
||||
}
|
||||
|
||||
/// A relative path inside the install folder, or None for names that would leave it.
|
||||
pub fn safe_path(name: &str) -> Option<String> {
|
||||
let unified = name.replace('\\', "/");
|
||||
if unified.starts_with('/') || unified.contains(':') || unified.contains('\0') {
|
||||
return None;
|
||||
}
|
||||
let parts: Vec<&str> = unified.split('/').filter(|p| !p.is_empty() && *p != ".").collect();
|
||||
if parts.is_empty() || parts.iter().any(|p| *p == "..") {
|
||||
return None;
|
||||
}
|
||||
Some(parts.join("/"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn central(name: &str, method: u16, crc: u32, compressed: u32, size: u32, offset: u32, extra: &[u8]) -> Vec<u8> {
|
||||
let mut v = Vec::new();
|
||||
v.extend_from_slice(&CENTRAL.to_le_bytes());
|
||||
v.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
|
||||
v.extend_from_slice(&method.to_le_bytes());
|
||||
v.extend_from_slice(&[0, 0, 0, 0]);
|
||||
v.extend_from_slice(&crc.to_le_bytes());
|
||||
v.extend_from_slice(&compressed.to_le_bytes());
|
||||
v.extend_from_slice(&size.to_le_bytes());
|
||||
v.extend_from_slice(&(name.len() as u16).to_le_bytes());
|
||||
v.extend_from_slice(&(extra.len() as u16).to_le_bytes());
|
||||
v.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
v.extend_from_slice(&offset.to_le_bytes());
|
||||
v.extend_from_slice(name.as_bytes());
|
||||
v.extend_from_slice(extra);
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_plain_directory() {
|
||||
let mut dir = central("Game.exe", DEFLATED, 7, 10, 20, 0, &[]);
|
||||
dir.extend(central("__Installer/", STORED, 0, 0, 0, 40, &[]));
|
||||
let mut tail = vec![0u8; 5];
|
||||
tail.extend_from_slice(&dir);
|
||||
let cd_size = dir.len() as u32;
|
||||
tail.extend_from_slice(&EOCD.to_le_bytes());
|
||||
tail.extend_from_slice(&[0, 0, 0, 0, 2, 0, 2, 0]);
|
||||
tail.extend_from_slice(&cd_size.to_le_bytes());
|
||||
tail.extend_from_slice(&1000u32.to_le_bytes());
|
||||
tail.extend_from_slice(&[0, 0]);
|
||||
let found = locate(&tail, 995).unwrap();
|
||||
assert_eq!(found, Directory { offset: 1000, size: cd_size as u64, entries: 2 });
|
||||
let entries = parse(&dir, 2).unwrap();
|
||||
assert_eq!(entries[0].name, "Game.exe");
|
||||
assert_eq!(entries[0].size, 20);
|
||||
assert!(entries[1].is_dir());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_zip64_values() {
|
||||
let mut extra = vec![1, 0, 24, 0];
|
||||
extra.extend_from_slice(&6_000_000_000u64.to_le_bytes());
|
||||
extra.extend_from_slice(&5_000_000_000u64.to_le_bytes());
|
||||
extra.extend_from_slice(&7_000_000_000u64.to_le_bytes());
|
||||
let dir = central("big.pak", DEFLATED, 1, 0xFFFF_FFFF, 0xFFFF_FFFF, 0xFFFF_FFFF, &extra);
|
||||
let e = &parse(&dir, 1).unwrap()[0];
|
||||
assert_eq!((e.size, e.compressed, e.local_offset), (6_000_000_000, 5_000_000_000, 7_000_000_000));
|
||||
|
||||
let mut tail = Vec::new();
|
||||
let record_at = 9_000_000_000u64;
|
||||
tail.extend_from_slice(&EOCD64.to_le_bytes());
|
||||
tail.extend_from_slice(&44u64.to_le_bytes());
|
||||
tail.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
tail.extend_from_slice(&3u64.to_le_bytes());
|
||||
tail.extend_from_slice(&3u64.to_le_bytes());
|
||||
tail.extend_from_slice(&500u64.to_le_bytes());
|
||||
tail.extend_from_slice(&8_000_000_000u64.to_le_bytes());
|
||||
tail.extend_from_slice(&EOCD64_LOCATOR.to_le_bytes());
|
||||
tail.extend_from_slice(&0u32.to_le_bytes());
|
||||
tail.extend_from_slice(&record_at.to_le_bytes());
|
||||
tail.extend_from_slice(&1u32.to_le_bytes());
|
||||
tail.extend_from_slice(&EOCD.to_le_bytes());
|
||||
tail.extend_from_slice(&[0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0]);
|
||||
let found = locate(&tail, record_at).unwrap();
|
||||
assert_eq!(found, Directory { offset: 8_000_000_000, size: 500, entries: 3 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_paths_that_leave_the_folder() {
|
||||
assert_eq!(safe_path("Data\\Win32\\a.big").as_deref(), Some("Data/Win32/a.big"));
|
||||
assert_eq!(safe_path("./x//y").as_deref(), Some("x/y"));
|
||||
assert!(safe_path("../evil").is_none());
|
||||
assert!(safe_path("/etc/passwd").is_none());
|
||||
assert!(safe_path("C:/x").is_none());
|
||||
assert!(safe_path("a/../../b").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_header_length_counts_name_and_extra() {
|
||||
let mut h = LOCAL.to_le_bytes().to_vec();
|
||||
h.extend_from_slice(&[0u8; 22]);
|
||||
h.extend_from_slice(&5u16.to_le_bytes());
|
||||
h.extend_from_slice(&9u16.to_le_bytes());
|
||||
assert_eq!(local_header_len(&h), Some(Ok(44)));
|
||||
assert_eq!(local_header_len(&h[..10]), None);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
#![allow(clippy::missing_safety_doc, clippy::result_large_err)]
|
||||
|
||||
pub mod amazon;
|
||||
pub mod ea;
|
||||
pub mod epic;
|
||||
pub mod fetch_core;
|
||||
pub mod fetch_tuning;
|
||||
|
||||
Reference in new issue
Block a user