feat: EA store with sign-in, library, native downloads, updates and launch without the EA app (#2259)

* docs: research and plan for EA and Ubisoft stores

* feat(ea): EA store with sign-in, library, native downloads, updates and launch without the EA app

* perf(ea): fetch large archive records as parallel ranges

* fix(ea): register game display names

* fix(ea): license legacy EA Core games and harden the LSX endpoint

* fix(ea): run legacy Origin SDK games with a stand-in Origin process

* fix(ea): trust only the prepared game on LSX and stop stale library syncs

* fix(ea): close LSX sessions on stop and drop clients that never hand-shake

* fix(ea): require every hashed field before licensing or caching a probe

* fix(ea): trust one LSX session per launch

* fix(ea): serialize the library cache write with sign-out and rerun a skipped sync

* fix(ea): hand off library sync requests under one lock

* fix(ea): clear the sign-in before invalidating the library

* fix(ea): serialize sign-out with token refresh and with the library cache

* fix(ea): LSX trusts only verified peers, new launch closes old sessions, exit stops its own game

* fix(ea): LSX hides profile and presence from unverified peers

* fix(ea): LSX handshake has a fixed deadline and untrusted sessions yield their slot

* fix(ea): LSX never registers a client after its listener was stopped

* fix(ea): LSX evicts each untrusted session at most once

* fix(ea): LSX admission and eviction share one lock and keep the cap

* fix(ea): LSX trust promotion excludes eviction

* fix(ea): LSX launch generation guards restarts and drops results after shutdown

* fix(ea): LSX publishes launch state under the admission lock and tags accepts by generation

* fix(ea): LSX restarts rebind its listener and promotion checks the launch generation

* fix(ea): record why EA refuses a game licence in the store log

* chore(ea): log owned offers and the full licence refusal

* chore(ea): log the account's entitlements when no game licence is granted

* chore(ea): log legacy entitlements when no game licence is granted

* chore(ea): log only the refused game's own entitlements

* fix(ea): skip trial launchers flagged 1 and license games whose LSX peer Android cannot name
This commit is contained in:
Kurt Himebauch authored and GitHub committed 2026-11-10 14:56:29 -05:00
1 parent 9bf638a3c4
commit d8ed02f2cf
46 files changed
+5284 -7

No files matched your search

+207
View File
@@ -0,0 +1,207 @@
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use jni::objects::{GlobalRef, JClass, JObject, JString, JValue};
use jni::sys::{jlong, jstring, JNI_FALSE, JNI_TRUE};
use jni::{JNIEnv, JavaVM};
use serde_json::json;
use super::install;
use crate::amazon::install::{Events, Stop};
fn read_string(env: &mut JNIEnv, value: &JString) -> String {
if value.is_null() {
return String::new();
}
env.get_string(value).map(|s| s.to_string_lossy().into_owned()).unwrap_or_default()
}
fn new_string(env: &mut JNIEnv, text: &str) -> jstring {
env.new_string(text).map(|s| s.into_raw()).unwrap_or(std::ptr::null_mut())
}
fn call(op: &str, request: &str) -> String {
let parsed: serde_json::Value = serde_json::from_str(request).unwrap_or(serde_json::Value::Null);
let url = parsed.get("url").and_then(|v| v.as_str()).unwrap_or("");
let answer = match op {
"zip.size" => install::size(url).unwrap_or_else(|e| json!({ "error": e })),
_ => json!({ "error": format!("unknown operation {op}") }),
};
answer.to_string()
}
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeCall(
mut env: JNIEnv,
_class: JClass,
op: JString,
request: JString,
) -> jstring {
let op = read_string(&mut env, &op);
let request = read_string(&mut env, &request);
match catch_unwind(AssertUnwindSafe(|| call(&op, &request))) {
Ok(text) => new_string(&mut env, &text),
Err(_) => std::ptr::null_mut(),
}
}
struct Listener {
vm: JavaVM,
target: GlobalRef,
}
impl Listener {
fn with_env<R>(&self, fallback: R, f: impl FnOnce(&mut JNIEnv) -> R) -> R {
let Ok(mut env) = self.vm.attach_current_thread_as_daemon() else {
return fallback;
};
let out = f(&mut env);
if env.exception_check().unwrap_or(false) {
let _ = env.exception_clear();
}
out
}
fn text(&self, method: &str, text: &str) {
self.with_env((), |env| {
let Ok(s) = env.new_string(text) else { return };
let s = JObject::from(s);
let _ = env.call_method(self.target.as_obj(), method, "(Ljava/lang/String;)V", &[JValue::Object(&s)]);
let _ = env.delete_local_ref(s);
});
}
fn counts(&self, method: &str, a: u64, b: u64, c: u32, d: u32) {
self.with_env((), |env| {
let _ = env.call_method(
self.target.as_obj(),
method,
"(JJII)V",
&[
JValue::Long(a.min(i64::MAX as u64) as i64),
JValue::Long(b.min(i64::MAX as u64) as i64),
JValue::Int(c.min(i32::MAX as u32) as i32),
JValue::Int(d.min(i32::MAX as u32) as i32),
],
);
});
}
fn complete(&self, success: bool, cancelled: bool, error: &str, result: &str) {
self.with_env((), |env| {
let (Ok(e), Ok(r)) = (env.new_string(error), env.new_string(result)) else { return };
let (e, r) = (JObject::from(e), JObject::from(r));
let _ = env.call_method(
self.target.as_obj(),
"onComplete",
"(ZZLjava/lang/String;Ljava/lang/String;)V",
&[
JValue::Bool(if success { JNI_TRUE } else { JNI_FALSE }),
JValue::Bool(if cancelled { JNI_TRUE } else { JNI_FALSE }),
JValue::Object(&e),
JValue::Object(&r),
],
);
let _ = env.delete_local_ref(e);
let _ = env.delete_local_ref(r);
});
}
}
impl Events for Listener {
fn log(&self, line: &str) {
self.text("onLog", &crate::amazon::net::scrub_urls(line));
}
fn status(&self, text: &str) {
self.text("onStatus", text);
}
fn checking(&self, done_bytes: u64, total_bytes: u64, files: u32, files_total: u32) {
self.counts("onCheck", done_bytes, total_bytes, files, files_total);
}
fn sizes(&self, pending_bytes: u64, total_bytes: u64) {
self.with_env((), |env| {
let _ = env.call_method(
self.target.as_obj(),
"onSizes",
"(JJ)V",
&[
JValue::Long(pending_bytes.min(i64::MAX as u64) as i64),
JValue::Long(total_bytes.min(i64::MAX as u64) as i64),
],
);
});
}
fn progress(&self, done: u64, total: u64, files: u32, files_total: u32) {
self.counts("onProgress", done, total, files, files_total);
}
fn before_replace(&self) -> bool {
self.with_env(false, |env| match env.call_method(self.target.as_obj(), "beforeReplace", "()Z", &[]) {
Ok(v) => v.z().unwrap_or(false),
Err(_) => false,
})
}
}
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeInstall(
mut env: JNIEnv,
_class: JClass,
request: JString,
listener: JObject,
) -> jlong {
if listener.is_null() {
return 0;
}
let Ok(vm) = env.get_java_vm() else { return 0 };
let Ok(target) = env.new_global_ref(&listener) else { return 0 };
let listener = Listener { vm, target };
let text = read_string(&mut env, &request);
let parsed = serde_json::from_str::<serde_json::Value>(&text)
.map_err(|_| "The install request could not be read".to_string())
.and_then(|v| install::Request::from_json(&v));
let request = match parsed {
Ok(r) => r,
Err(e) => {
listener.complete(false, false, &e, "");
return 0;
}
};
let cancel = Arc::new(AtomicBool::new(false));
let worker_cancel = Arc::clone(&cancel);
let spawned = std::thread::Builder::new().name("ea-install".into()).spawn(move || {
crate::priority::background();
let result = catch_unwind(AssertUnwindSafe(|| install::run(&request, &worker_cancel, &listener)))
.unwrap_or_else(|_| Err(Stop::Failed("The EA install stopped on an internal error".into())));
match result {
Ok(done) => listener.complete(true, false, "", &done.to_json().to_string()),
Err(Stop::Cancelled) => listener.complete(false, true, "cancelled", ""),
Err(Stop::Failed(e)) => listener.complete(false, false, &e, ""),
}
});
if spawned.is_err() {
return 0;
}
Box::into_raw(Box::new(cancel)) as jlong
}
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeCancel(_env: JNIEnv, _class: JClass, handle: jlong) {
let ptr = handle as *const Arc<AtomicBool>;
if let Some(cancel) = unsafe { ptr.as_ref() } {
cancel.store(true, Ordering::SeqCst);
}
}
#[no_mangle]
pub extern "system" fn Java_com_droiddeck_launcher_stores_ea_EaNative_nativeRelease(_env: JNIEnv, _class: JClass, handle: jlong) {
let ptr = handle as *mut Arc<AtomicBool>;
if !ptr.is_null() {
drop(unsafe { Box::from_raw(ptr) });
}
}
+850
View File
@@ -0,0 +1,850 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! Installs or updates an EA game from its build archive: files whose size and CRC already match
//! are kept, everything else is fetched entry by entry with ranged reads and inflated in place.
use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::Mutex;
use std::time::{Duration, Instant, UNIX_EPOCH};
use flate2::{Crc, Decompress, FlushDecompress, Status};
use serde_json::{json, Value};
use super::net;
use super::zip::{self, Entry};
use crate::amazon::install::check::{free_bytes, space_problem};
use crate::amazon::install::{Events, ReplaceGate, Stop};
use crate::fetch_core::{distinct_host_count, run_fetch, FetchItem, FetchOptions, FetchSink, SinkError};
pub const LEDGER: &str = ".droiddeck-ea-files";
pub const IN_PROGRESS_MARKER: &str = ".droiddeck-ea-downloading";
const TEMP_SUFFIX: &str = ".ddpart";
const PROGRESS_EVERY_MS: u64 = 50;
const CHECK_REPORT_EVERY: Duration = Duration::from_millis(200);
const REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
const INFLATE_BUFFER: usize = 256 * 1024;
/// Records larger than this are fetched as parallel ranges into a scratch file, then inflated,
/// so one big file does not ride a single connection.
pub const PART_BYTES: u64 = 16 * 1024 * 1024;
const SPLIT_SUFFIX: &str = ".ddzip";
struct Split {
tmp: PathBuf,
file: Mutex<Option<File>>,
received: Vec<AtomicU64>,
left: AtomicUsize,
}
#[derive(Clone, Copy)]
enum Work {
Whole(usize),
Part { job: usize, part: usize, start: u64, len: u64 },
}
/// The fetches for each job: whole records, or byte ranges of one when it is large.
fn works(jobs: &[&Planned], part_bytes: u64) -> Vec<Work> {
let mut out = Vec::new();
for (i, p) in jobs.iter().enumerate() {
let span = p.range_end - p.entry.local_offset + 1;
if span <= part_bytes * 2 {
out.push(Work::Whole(i));
continue;
}
let mut start = 0;
let mut part = 0;
while start < span {
let len = part_bytes.min(span - start);
out.push(Work::Part { job: i, part, start, len });
start += len;
part += 1;
}
}
out
}
pub const REPLACE_REFUSED: &str = "The install record could not be switched to updating; nothing was replaced";
pub struct Request {
pub url: String,
pub game_dir: PathBuf,
pub max_workers: usize,
pub process_workers: usize,
pub ca_bundle: String,
pub part_bytes: u64,
}
impl Request {
pub fn from_json(v: &Value) -> Result<Request, String> {
let text = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string();
let game_dir = text("gameDir");
if game_dir.is_empty() || !game_dir.starts_with('/') {
return Err("No install folder was given".into());
}
let url = text("url");
if !url.starts_with("https://") && !url.starts_with("http://") {
return Err("EA gave no download for this game".into());
}
Ok(Request {
url,
game_dir: PathBuf::from(game_dir),
max_workers: v.get("maxWorkers").and_then(|n| n.as_u64()).unwrap_or(8) as usize,
process_workers: v.get("processWorkers").and_then(|n| n.as_u64()).unwrap_or(2) as usize,
ca_bundle: text("caBundle"),
part_bytes: v.get("partBytes").and_then(|n| n.as_u64()).filter(|n| *n > 0).unwrap_or(PART_BYTES),
})
}
}
#[derive(Debug, Default)]
pub struct Finished {
pub total_bytes: u64,
pub download_bytes: u64,
pub file_count: usize,
pub files_written: usize,
pub bytes_written: u64,
pub removed: usize,
}
impl Finished {
pub fn to_json(&self) -> Value {
json!({
"totalBytes": self.total_bytes,
"downloadBytes": self.download_bytes,
"fileCount": self.file_count,
"filesWritten": self.files_written,
"bytesWritten": self.bytes_written,
"removed": self.removed,
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct Known {
crc: u32,
size: u64,
modified: u64,
}
fn modified_ms(path: &Path) -> u64 {
fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
fn read_ledger(dir: &Path) -> HashMap<String, Known> {
let Ok(text) = fs::read_to_string(dir.join(LEDGER)) else { return HashMap::new() };
let mut out = HashMap::new();
for line in text.lines() {
let mut parts = line.splitn(4, '\t');
let (Some(crc), Some(size), Some(modified), Some(path)) = (parts.next(), parts.next(), parts.next(), parts.next()) else {
continue;
};
let (Ok(crc), Ok(size), Ok(modified)) = (u32::from_str_radix(crc, 16), size.parse(), modified.parse()) else {
continue;
};
out.insert(path.to_string(), Known { crc, size, modified });
}
out
}
fn ledger_line(path: &str, k: Known) -> String {
format!("{:08x}\t{}\t{}\t{path}\n", k.crc, k.size, k.modified)
}
fn crc_of(path: &Path, cancel: &AtomicBool) -> Result<Option<u32>, ()> {
let Ok(mut f) = File::open(path) else { return Ok(None) };
let mut crc = Crc::new();
let mut buf = vec![0u8; 1024 * 1024];
loop {
if cancel.load(Ordering::Relaxed) {
return Err(());
}
match f.read(&mut buf) {
Ok(0) => return Ok(Some(crc.sum())),
Ok(n) => crc.update(&buf[..n]),
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(_) => return Ok(None),
}
}
}
/// Whether two paths name one file, as they do for a case-only rename on case-insensitive storage.
fn same_file(a: &Path, b: &Path) -> bool {
use std::os::unix::fs::MetadataExt;
match (fs::metadata(a), fs::metadata(b)) {
(Ok(x), Ok(y)) => x.dev() == y.dev() && x.ino() == y.ino(),
_ => false,
}
}
fn temp_path(dest: &Path) -> PathBuf {
let mut name = dest.as_os_str().to_owned();
name.push(TEMP_SUFFIX);
PathBuf::from(name)
}
pub struct Planned {
pub entry: Entry,
pub path: String,
pub range_end: u64,
}
/// Every file entry with its safe path and the last byte its local record can occupy.
pub fn plan(entries: &[Entry], directory_offset: u64) -> Result<Vec<Planned>, String> {
let mut offsets: Vec<u64> = entries.iter().map(|e| e.local_offset).collect();
offsets.push(directory_offset);
offsets.sort_unstable();
offsets.dedup();
let mut seen = HashSet::new();
let mut out = Vec::new();
for e in entries.iter().filter(|e| !e.is_dir()) {
if e.method != zip::STORED && e.method != zip::DEFLATED {
return Err(format!("{} uses an unsupported compression method ({})", e.name, e.method));
}
let path = zip::safe_path(&e.name).ok_or_else(|| format!("The archive names a file outside the game folder: {}", e.name))?;
if !seen.insert(path.to_lowercase()) {
continue;
}
let at = offsets.partition_point(|&o| o <= e.local_offset);
let next = offsets.get(at).copied().unwrap_or(directory_offset);
let minimum = e.local_offset + zip::LOCAL_FIXED as u64 + e.compressed;
if next < minimum {
return Err(format!("{} overlaps the next record in the archive", e.name));
}
out.push(Planned { entry: e.clone(), path, range_end: next - 1 });
}
Ok(out)
}
struct Slot {
header: Vec<u8>,
payload_from: Option<usize>,
consumed: u64,
file: File,
tmp: PathBuf,
crc: Crc,
written: u64,
inflate: Option<Decompress>,
buffer: Vec<u8>,
}
struct Placer<'a> {
jobs: &'a [&'a Planned],
game_dir: &'a Path,
slots: Vec<Mutex<Option<Slot>>>,
splits: Vec<Option<Split>>,
works: Vec<Work>,
placed: Vec<AtomicBool>,
ledger: Mutex<Option<File>>,
events: &'a dyn Events,
gate: &'a ReplaceGate,
stop: &'a AtomicBool,
failure: Mutex<Option<String>>,
base_done: u64,
base_files: u32,
total: u64,
total_files: u32,
placed_bytes: AtomicU64,
in_flight: AtomicU64,
shown: AtomicU64,
bytes_written: AtomicU64,
files_placed: AtomicUsize,
last_emit: AtomicU64,
started: Instant,
}
fn discard(slot: Slot) {
drop(slot.file);
let _ = fs::remove_file(&slot.tmp);
}
impl Placer<'_> {
fn fail(&self, message: String) -> String {
let mut slot = self.failure.lock().expect("failure lock");
if slot.is_none() {
*slot = Some(message.clone());
}
drop(slot);
self.stop.store(true, Ordering::SeqCst);
message
}
fn emit(&self, force: bool) {
let now = self.started.elapsed().as_millis() as u64;
let last = self.last_emit.load(Ordering::Relaxed);
if !force && now < last + PROGRESS_EVERY_MS {
return;
}
if !force && self.last_emit.compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed).is_err() {
return;
}
self.last_emit.store(now, Ordering::Relaxed);
let value = self.base_done + self.placed_bytes.load(Ordering::Relaxed) + self.in_flight.load(Ordering::Relaxed);
let shown = self.shown.fetch_max(value, Ordering::Relaxed).max(value);
self.events.progress(
shown.min(self.total),
self.total,
self.base_files + self.files_placed.load(Ordering::Relaxed) as u32,
self.total_files,
);
}
fn open(&self, job: &Planned) -> Result<Slot, String> {
let dest = self.game_dir.join(&job.path);
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(|e| format!("Could not create the folder for {}: {e}", job.path))?;
}
let tmp = temp_path(&dest);
let file = File::create(&tmp).map_err(|e| format!("Could not write {}: {e}", job.path))?;
Ok(Slot {
header: Vec::new(),
payload_from: None,
consumed: 0,
file,
tmp,
crc: Crc::new(),
written: 0,
inflate: (job.entry.method == zip::DEFLATED).then(|| Decompress::new(false)),
buffer: Vec::new(),
})
}
fn write_out(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
if slot.written + data.len() as u64 > job.entry.size {
return Err(SinkError::Retry(format!("{}: more data than the archive lists", job.path)));
}
slot.file
.write_all(data)
.map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?;
slot.crc.update(data);
slot.written += data.len() as u64;
self.bytes_written.fetch_add(data.len() as u64, Ordering::Relaxed);
Ok(())
}
fn feed(&self, job: &Planned, slot: &mut Slot, payload: &[u8], finish: bool) -> Result<(), SinkError> {
let Some(mut inflate) = slot.inflate.take() else {
return self.write_out(job, slot, payload);
};
let mut out = std::mem::take(&mut slot.buffer);
out.resize(INFLATE_BUFFER, 0);
let result = self.inflate_into(job, slot, &mut inflate, &mut out, payload, finish);
slot.inflate = Some(inflate);
slot.buffer = out;
result
}
fn inflate_into(&self, job: &Planned, slot: &mut Slot, inflate: &mut Decompress, out: &mut [u8], mut input: &[u8], finish: bool) -> Result<(), SinkError> {
let mode = if finish { FlushDecompress::Finish } else { FlushDecompress::None };
loop {
let (in_before, out_before) = (inflate.total_in(), inflate.total_out());
let status = inflate
.decompress(input, out, mode)
.map_err(|e| SinkError::Retry(format!("{}: the compressed data is damaged ({e})", job.path)))?;
let used = (inflate.total_in() - in_before) as usize;
let produced = (inflate.total_out() - out_before) as usize;
input = &input[used..];
if produced > 0 {
self.write_out(job, slot, &out[..produced])?;
}
if status == Status::StreamEnd || (used == 0 && produced == 0) {
return Ok(());
}
if input.is_empty() && !finish && produced < out.len() {
return Ok(());
}
}
}
fn piece(&self, i: usize, offset: u64, data: &[u8]) -> Result<(), SinkError> {
let job = self.jobs[i];
let mut guard = self.slots[i].lock().expect("slot lock");
if offset == 0 {
if let Some(old) = guard.take() {
self.in_flight.fetch_sub(old.consumed, Ordering::Relaxed);
discard(old);
}
*guard = Some(self.open(job).map_err(|e| SinkError::Fatal(self.fail(e)))?);
}
let Some(slot) = guard.as_mut() else {
return Err(SinkError::Retry(format!("{}: data arrived before the start of the record", job.path)));
};
self.consume(job, slot, data)
}
fn consume(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
if slot.payload_from.is_none() {
slot.header.extend_from_slice(data);
match zip::local_header_len(&slot.header) {
None => return Ok(()),
Some(Err(e)) => return Err(SinkError::Retry(format!("{}: {e}", job.path))),
Some(Ok(len)) if slot.header.len() < len => return Ok(()),
Some(Ok(len)) => {
slot.payload_from = Some(len);
let header = std::mem::take(&mut slot.header);
let rest = header[len..].to_vec();
return self.payload(job, slot, &rest);
}
}
}
self.payload(job, slot, data)
}
fn payload(&self, job: &Planned, slot: &mut Slot, data: &[u8]) -> Result<(), SinkError> {
let left = job.entry.compressed.saturating_sub(slot.consumed);
let take = &data[..(left.min(data.len() as u64) as usize)];
if take.is_empty() {
return Ok(());
}
slot.consumed += take.len() as u64;
self.in_flight.fetch_add(take.len() as u64, Ordering::Relaxed);
self.feed(job, slot, take, false)?;
self.emit(false);
Ok(())
}
fn finish(&self, i: usize) -> Result<u64, SinkError> {
let job = self.jobs[i];
let taken = self.slots[i].lock().expect("slot lock").take();
let Some(slot) = taken else {
return Err(SinkError::Retry(format!("{}: the record arrived empty", job.path)));
};
self.complete(i, slot)
}
fn complete(&self, i: usize, mut slot: Slot) -> Result<u64, SinkError> {
let job = self.jobs[i];
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
if slot.consumed != job.entry.compressed {
let got = slot.consumed;
discard(slot);
return Err(SinkError::Retry(format!("{}: {got} of {} compressed bytes arrived", job.path, job.entry.compressed)));
}
if slot.inflate.is_some() {
if let Err(e) = self.feed(job, &mut slot, &[], true) {
discard(slot);
return Err(e);
}
}
if slot.written != job.entry.size || slot.crc.sum() != job.entry.crc32 {
let got = slot.written;
discard(slot);
return Err(SinkError::Retry(format!("{}: {got} bytes inflated with a CRC mismatch", job.path)));
}
let Slot { file, tmp, .. } = slot;
if let Err(e) = file.sync_all() {
drop(file);
let _ = fs::remove_file(&tmp);
return Err(SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))));
}
drop(file);
if self.stop.load(Ordering::SeqCst) {
let _ = fs::remove_file(&tmp);
return Ok(0);
}
if !self.gate.allow(self.events) {
let _ = fs::remove_file(&tmp);
return Err(SinkError::Fatal(self.fail(REPLACE_REFUSED.into())));
}
let dest = self.game_dir.join(&job.path);
if let Err(e) = fs::rename(&tmp, &dest) {
let _ = fs::remove_file(&tmp);
return Err(SinkError::Fatal(self.fail(format!("Could not put {} in place: {e}", job.path))));
}
self.record(job, &dest);
self.placed[i].store(true, Ordering::SeqCst);
self.placed_bytes.fetch_add(job.entry.compressed, Ordering::Relaxed);
self.files_placed.fetch_add(1, Ordering::Relaxed);
self.emit(true);
Ok(0)
}
fn part_chunk(&self, i: usize, part: usize, start: u64, offset: u64, data: &[u8]) -> Result<(), SinkError> {
let job = self.jobs[i];
let Some(split) = self.splits[i].as_ref() else {
return Err(SinkError::Fatal(format!("{}: no split state", job.path)));
};
let mut file = split.file.lock().expect("split lock");
if file.is_none() {
let dest = self.game_dir.join(&job.path);
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(|e| SinkError::Fatal(self.fail(format!("Could not create the folder for {}: {e}", job.path))))?;
}
*file = Some(File::create(&split.tmp).map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?);
}
let handle = file.as_ref().expect("split file");
if offset == 0 {
let old = split.received[part].swap(0, Ordering::Relaxed);
self.in_flight.fetch_sub(old, Ordering::Relaxed);
}
handle
.write_all_at(data, start + offset)
.map_err(|e| SinkError::Fatal(self.fail(format!("Could not write {}: {e}", job.path))))?;
drop(file);
split.received[part].fetch_add(data.len() as u64, Ordering::Relaxed);
self.in_flight.fetch_add(data.len() as u64, Ordering::Relaxed);
self.emit(false);
Ok(())
}
fn part_finish(&self, i: usize, part: usize, len: u64) -> Result<u64, SinkError> {
let job = self.jobs[i];
let Some(split) = self.splits[i].as_ref() else {
return Err(SinkError::Fatal(format!("{}: no split state", job.path)));
};
let got = split.received[part].load(Ordering::Relaxed);
if got != len {
return Err(SinkError::Retry(format!("{}: part {part} got {got} of {len} bytes", job.path)));
}
if split.left.fetch_sub(1, Ordering::SeqCst) != 1 {
return Ok(0);
}
let received: u64 = split.received.iter().map(|r| r.swap(0, Ordering::Relaxed)).sum();
self.in_flight.fetch_sub(received, Ordering::Relaxed);
let result = self.assemble(i, split);
let _ = fs::remove_file(&split.tmp);
*split.file.lock().expect("split lock") = None;
result.map_err(|e| match e {
SinkError::Retry(m) => SinkError::Fatal(self.fail(m)),
fatal => fatal,
})
}
fn assemble(&self, i: usize, split: &Split) -> Result<u64, SinkError> {
let job = self.jobs[i];
let mut source = File::open(&split.tmp).map_err(|e| SinkError::Fatal(self.fail(format!("Could not read {}: {e}", job.path))))?;
let mut slot = self.open(job).map_err(|e| SinkError::Fatal(self.fail(e)))?;
let mut buf = vec![0u8; 1024 * 1024];
loop {
if self.stop.load(Ordering::Relaxed) {
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
discard(slot);
return Ok(0);
}
let n = match source.read(&mut buf) {
Ok(0) => break,
Ok(n) => n,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => {
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
discard(slot);
return Err(SinkError::Fatal(self.fail(format!("Could not read {}: {e}", job.path))));
}
};
if let Err(e) = self.consume(job, &mut slot, &buf[..n]) {
self.in_flight.fetch_sub(slot.consumed, Ordering::Relaxed);
discard(slot);
return Err(e);
}
if slot.consumed >= job.entry.compressed {
break;
}
}
self.complete(i, slot)
}
fn record(&self, job: &Planned, dest: &Path) {
let known = Known { crc: job.entry.crc32, size: job.entry.size, modified: modified_ms(dest) };
if let Some(file) = self.ledger.lock().expect("ledger lock").as_mut() {
let _ = file.write_all(ledger_line(&job.path, known).as_bytes());
}
}
}
struct Sink<'p, 'a>(&'p Placer<'a>);
impl FetchSink for Sink<'_, '_> {
fn process(&self, item: &FetchItem, _body: Vec<u8>) -> Result<u64, SinkError> {
Err(SinkError::Fatal(format!("item {} arrived whole; streaming was expected", item.id)))
}
fn on_chunk(&self, item: &FetchItem, offset: u64, data: &[u8]) -> Result<(), SinkError> {
match self.0.works[item.id as usize] {
Work::Whole(i) => self.0.piece(i, offset, data),
Work::Part { job, part, start, .. } => self.0.part_chunk(job, part, start, offset, data),
}
}
fn on_finish(&self, item: &FetchItem, _total_len: u64) -> Result<u64, SinkError> {
match self.0.works[item.id as usize] {
Work::Whole(i) => self.0.finish(i),
Work::Part { job, part, len, .. } => self.0.part_finish(job, part, len),
}
}
}
fn origin(url: &str) -> String {
let after = url.find("://").map(|i| i + 3).unwrap_or(0);
let end = url[after..].find('/').map(|i| after + i).unwrap_or(url.len());
url[..end].to_string()
}
fn cancelled(cancel: &AtomicBool) -> Result<(), Stop> {
if cancel.load(Ordering::SeqCst) {
Err(Stop::Cancelled)
} else {
Ok(())
}
}
pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finished, Stop> {
let failed = Stop::Failed;
let started = Instant::now();
fs::create_dir_all(&req.game_dir).map_err(|e| failed(format!("Could not create the install folder: {e}")))?;
let _ = File::create(req.game_dir.join(IN_PROGRESS_MARKER));
ev.status("Reading the build…");
let archive = net::read_archive(&req.url).map_err(|e| {
ev.log(&format!("archive: {e}"));
failed(format!("The EA build could not be read: {e}"))
})?;
cancelled(cancel)?;
let planned = plan(&archive.entries, archive.directory.offset).map_err(failed)?;
let total_size: u64 = planned.iter().map(|p| p.entry.size).sum();
let total_compressed: u64 = planned.iter().map(|p| p.entry.compressed).sum();
ev.log(&format!(
"archive bytes={} entries={} files={} size={total_size} compressed={total_compressed}",
archive.total,
archive.entries.len(),
planned.len()
));
for p in &planned {
let dest = req.game_dir.join(&p.path);
let _ = fs::remove_file(temp_path(&dest));
let mut split = dest.into_os_string();
split.push(SPLIT_SUFFIX);
let _ = fs::remove_file(split);
}
ev.status("Checking files…");
let mut ledger = read_ledger(&req.game_dir);
let mut pending: Vec<&Planned> = Vec::new();
let mut kept: Vec<(String, Known)> = Vec::new();
let mut checked = 0u64;
let mut last_report = Instant::now();
for (n, p) in planned.iter().enumerate() {
let dest = req.game_dir.join(&p.path);
let size_ok = fs::symlink_metadata(&dest).map(|m| m.is_file() && m.len() == p.entry.size).unwrap_or(false);
let mut complete = false;
if size_ok {
let modified = modified_ms(&dest);
let known = ledger.get(&p.path).copied();
if known == Some(Known { crc: p.entry.crc32, size: p.entry.size, modified }) {
complete = true;
} else {
match crc_of(&dest, cancel) {
Err(()) => return Err(Stop::Cancelled),
Ok(Some(crc)) if crc == p.entry.crc32 => {
complete = true;
kept.push((p.path.clone(), Known { crc, size: p.entry.size, modified }));
}
Ok(_) => {}
}
}
}
if !complete {
pending.push(p);
}
checked += p.entry.size;
if last_report.elapsed() >= CHECK_REPORT_EVERY || n + 1 == planned.len() {
ev.checking(checked, total_size, (n + 1) as u32, planned.len() as u32);
last_report = Instant::now();
}
cancelled(cancel)?;
}
for (path, known) in kept {
ledger.insert(path, known);
}
let pending_size: u64 = pending.iter().map(|p| p.entry.size).sum();
let pending_compressed: u64 = pending.iter().map(|p| p.entry.compressed).sum();
ev.log(&format!(
"resume kept_files={} pending_files={} pending_bytes={pending_size} pending_download={pending_compressed}",
planned.len() - pending.len(),
pending.len()
));
ev.sizes(pending_size, total_size);
if let Some(message) = space_problem(pending_size, free_bytes(&req.game_dir)) {
return Err(failed(message));
}
let gate = ReplaceGate::new();
let (empty, remote): (Vec<&Planned>, Vec<&Planned>) = pending.into_iter().partition(|p| p.entry.size == 0 && p.entry.crc32 == 0);
for p in &empty {
let dest = req.game_dir.join(&p.path);
if let Some(parent) = dest.parent() {
fs::create_dir_all(parent).map_err(|e| failed(format!("Could not create the folder for {}: {e}", p.path)))?;
}
if !gate.allow(ev) {
return Err(failed(REPLACE_REFUSED.into()));
}
File::create(&dest).map_err(|e| failed(format!("Could not write {}: {e}", p.path)))?;
ledger.insert(p.path.clone(), Known { crc: 0, size: 0, modified: modified_ms(&dest) });
}
let mut text = String::new();
for (path, known) in &ledger {
text.push_str(&ledger_line(path, *known));
}
fs::write(req.game_dir.join(LEDGER), text).map_err(|e| failed(format!("Could not write the install record: {e}")))?;
let ledger_file = OpenOptions::new().append(true).open(req.game_dir.join(LEDGER)).ok();
let host = origin(&req.url);
let works = works(&remote, req.part_bytes);
let splits: Vec<Option<Split>> = remote
.iter()
.enumerate()
.map(|(i, p)| {
let parts = works.iter().filter(|w| matches!(w, Work::Part { job, .. } if *job == i)).count();
(parts > 0).then(|| {
let mut name = req.game_dir.join(&p.path).into_os_string();
name.push(SPLIT_SUFFIX);
Split {
tmp: PathBuf::from(name),
file: Mutex::new(None),
received: (0..parts).map(|_| AtomicU64::new(0)).collect(),
left: AtomicUsize::new(parts),
}
})
})
.collect();
let placer = Placer {
jobs: &remote,
game_dir: &req.game_dir,
slots: remote.iter().map(|_| Mutex::new(None)).collect(),
splits,
works: works.clone(),
placed: remote.iter().map(|_| AtomicBool::new(false)).collect(),
ledger: Mutex::new(ledger_file),
events: ev,
gate: &gate,
stop: cancel,
failure: Mutex::new(None),
base_done: total_compressed - pending_compressed,
base_files: (planned.len() - remote.len()) as u32,
total: total_compressed,
total_files: planned.len() as u32,
placed_bytes: AtomicU64::new(0),
in_flight: AtomicU64::new(0),
shown: AtomicU64::new(0),
bytes_written: AtomicU64::new(0),
files_placed: AtomicUsize::new(0),
last_emit: AtomicU64::new(0),
started: Instant::now(),
};
let cores = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(2);
let window = req.max_workers.clamp(1, 64);
let hosts = vec![host.clone()];
let options = FetchOptions {
max_workers: window,
per_host_cap: window,
timeout: REQUEST_TIMEOUT,
headers: vec![("User-Agent".to_string(), net::user_agent())],
ca_bundle_path: req.ca_bundle.clone(),
process_workers: req.process_workers.max(cores / 2).max(1),
label: "ea".to_string(),
stream: true,
};
let items: Vec<FetchItem> = works
.iter()
.enumerate()
.map(|(n, w)| {
let range = match *w {
Work::Whole(i) => (remote[i].entry.local_offset, remote[i].range_end),
Work::Part { job, start, len, .. } => {
let at = remote[job].entry.local_offset + start;
(at, at + len - 1)
}
};
FetchItem { id: n as u64, urls: vec![req.url.clone()], reserve: 0, range: Some(range) }
})
.collect();
ev.status("Downloading…");
ev.log(&format!("engine files={} requests={} window={window} hosts={}", remote.len(), items.len(), distinct_host_count(&hosts)));
placer.emit(true);
let log = |line: &str| ev.log(line);
let outcome = if items.is_empty() {
None
} else {
Some(run_fetch(items, &hosts, &options, &Sink(&placer), cancel, &|_: u64, _: u64| {}, &log))
};
for slot in &placer.slots {
if let Some(active) = slot.lock().expect("slot lock").take() {
discard(active);
}
}
for split in placer.splits.iter().flatten() {
*split.file.lock().expect("split lock") = None;
let _ = fs::remove_file(&split.tmp);
}
placer.emit(true);
let placed_all = placer.placed.iter().all(|p| p.load(Ordering::SeqCst));
let bytes_written = placer.bytes_written.load(Ordering::Relaxed);
let files_written = placer.files_placed.load(Ordering::Relaxed) + empty.len();
let elapsed = started.elapsed().as_secs_f64().max(0.001);
ev.log(&format!(
"summary files_written={files_written} bytes_written={bytes_written} elapsed_s={elapsed:.1} avg_mbps={:.1}",
bytes_written as f64 / 1_048_576.0 / elapsed
));
if let Some(message) = placer.failure.lock().expect("failure lock").take() {
return Err(Stop::Failed(message));
}
if !placed_all {
if let Some(error) = outcome.as_ref().and_then(|o| o.error.clone()) {
ev.log(&format!("download failed: {error}"));
return Err(Stop::Failed("A file failed to download".into()));
}
if cancel.load(Ordering::SeqCst) {
return Err(Stop::Cancelled);
}
return Err(Stop::Failed("The download ended before every file was written".into()));
}
let wanted: HashSet<&str> = planned.iter().map(|p| p.path.as_str()).collect();
let by_case: HashMap<String, &str> = planned.iter().map(|p| (p.path.to_lowercase(), p.path.as_str())).collect();
let mut removed = 0;
let current = read_ledger(&req.game_dir);
let mut text = String::new();
for (path, known) in &current {
if wanted.contains(path.as_str()) {
text.push_str(&ledger_line(path, *known));
} else if by_case.get(&path.to_lowercase()).is_some_and(|kept| same_file(&req.game_dir.join(kept), &req.game_dir.join(path))) {
continue;
} else if zip::safe_path(path).as_deref() == Some(path.as_str()) && fs::remove_file(req.game_dir.join(path)).is_ok() {
removed += 1;
}
}
let _ = fs::write(req.game_dir.join(LEDGER), text);
if removed > 0 {
ev.log(&format!("removed {removed} files the new build no longer has"));
}
let _ = fs::remove_file(req.game_dir.join(IN_PROGRESS_MARKER));
Ok(Finished {
total_bytes: total_size,
download_bytes: total_compressed,
file_count: planned.len(),
files_written,
bytes_written,
removed,
})
}
pub fn size(url: &str) -> Result<Value, String> {
let archive = net::read_archive(url)?;
let planned = plan(&archive.entries, archive.directory.offset)?;
Ok(json!({
"totalBytes": planned.iter().map(|p| p.entry.size).sum::<u64>(),
"downloadBytes": planned.iter().map(|p| p.entry.compressed).sum::<u64>(),
"fileCount": planned.len(),
}))
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,258 @@
use std::io::{BufRead, BufReader, Write};
use std::net::{TcpListener, TcpStream};
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::sync::{Arc, Mutex};
use flate2::write::DeflateEncoder;
use flate2::Compression;
use super::*;
struct Built {
bytes: Vec<u8>,
}
fn build_zip(files: &[(&str, Vec<u8>, bool)]) -> Built {
let mut out = Vec::new();
let mut central = Vec::new();
for (name, data, deflate) in files {
let mut crc = Crc::new();
crc.update(data);
let body = if *deflate {
let mut enc = DeflateEncoder::new(Vec::new(), Compression::default());
enc.write_all(data).unwrap();
enc.finish().unwrap()
} else {
data.clone()
};
let method: u16 = if *deflate { 8 } else { 0 };
let offset = out.len() as u32;
let extra = [0xAAu8, 0xBB, 2, 0, 1, 2];
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
out.extend_from_slice(&[20, 0, 0, 0]);
out.extend_from_slice(&method.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&crc.sum().to_le_bytes());
out.extend_from_slice(&(body.len() as u32).to_le_bytes());
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&(extra.len() as u16).to_le_bytes());
out.extend_from_slice(name.as_bytes());
out.extend_from_slice(&extra);
out.extend_from_slice(&body);
central.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
central.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
central.extend_from_slice(&method.to_le_bytes());
central.extend_from_slice(&[0, 0, 0, 0]);
central.extend_from_slice(&crc.sum().to_le_bytes());
central.extend_from_slice(&(body.len() as u32).to_le_bytes());
central.extend_from_slice(&(data.len() as u32).to_le_bytes());
central.extend_from_slice(&(name.len() as u16).to_le_bytes());
central.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
central.extend_from_slice(&offset.to_le_bytes());
central.extend_from_slice(name.as_bytes());
}
let cd_offset = out.len() as u32;
let count = files.len() as u16;
out.extend_from_slice(&central);
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&count.to_le_bytes());
out.extend_from_slice(&count.to_le_bytes());
out.extend_from_slice(&(central.len() as u32).to_le_bytes());
out.extend_from_slice(&cd_offset.to_le_bytes());
out.extend_from_slice(&[0, 0]);
Built { bytes: out }
}
struct Server {
url: String,
requests: Arc<AtomicUsize>,
}
fn serve(body: Vec<u8>) -> Server {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let body = Arc::new(body);
let requests = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&requests);
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let body = Arc::clone(&body);
counter.fetch_add(1, Ordering::SeqCst);
std::thread::spawn(move || answer(stream, &body));
}
});
Server { url: format!("http://{addr}/build/game.zip?sig=abc"), requests }
}
fn answer(mut stream: TcpStream, body: &[u8]) {
let mut reader = BufReader::new(stream.try_clone().unwrap());
let mut first = String::new();
if reader.read_line(&mut first).is_err() {
return;
}
let mut range = None;
loop {
let mut line = String::new();
match reader.read_line(&mut line) {
Ok(0) | Err(_) => break,
Ok(_) if line == "\r\n" => break,
Ok(_) => {
if let Some(v) = line.to_ascii_lowercase().strip_prefix("range: bytes=") {
range = Some(v.trim().to_string());
}
}
}
}
let total = body.len() as u64;
let Some(range) = range else {
let _ = stream.write_all(format!("HTTP/1.1 200 OK\r\nContent-Length: {total}\r\nConnection: close\r\n\r\n").as_bytes());
let _ = stream.write_all(body);
return;
};
let (a, b) = range.split_once('-').unwrap();
let (start, end) = if a.is_empty() {
let n: u64 = b.parse().unwrap();
(total.saturating_sub(n), total - 1)
} else {
let start: u64 = a.parse().unwrap();
let end = if b.is_empty() { total - 1 } else { b.parse::<u64>().unwrap().min(total - 1) };
(start, end)
};
let slice = &body[start as usize..=end as usize];
let head = format!(
"HTTP/1.1 206 Partial Content\r\nContent-Length: {}\r\nContent-Range: bytes {start}-{end}/{total}\r\nConnection: close\r\n\r\n",
slice.len()
);
let _ = stream.write_all(head.as_bytes());
let _ = stream.write_all(slice);
}
#[derive(Default)]
struct Recorder {
logs: Mutex<Vec<String>>,
gates: AtomicUsize,
last: Mutex<(u64, u64)>,
}
impl Events for Recorder {
fn log(&self, line: &str) {
self.logs.lock().unwrap().push(line.to_string());
}
fn status(&self, _text: &str) {}
fn checking(&self, _d: u64, _t: u64, _f: u32, _ft: u32) {}
fn sizes(&self, _pending: u64, _total: u64) {}
fn progress(&self, done: u64, total: u64, _files: u32, _files_total: u32) {
*self.last.lock().unwrap() = (done, total);
}
fn before_replace(&self) -> bool {
self.gates.fetch_add(1, Ordering::SeqCst);
true
}
}
fn sample() -> Vec<(&'static str, Vec<u8>, bool)> {
let mut seed = 0x1234_5678u32;
let big: Vec<u8> = (0..900_000u32).map(|_| { seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); (seed >> 16) as u8 }).collect();
vec![
("__Installer/installerdata.xml", b"<DiPManifest version=\"4.0\"></DiPManifest>".to_vec(), true),
("Game.exe", big, true),
("Data\\stored.bin", vec![9u8; 4096], false),
("Data/", Vec::new(), false),
("empty.txt", Vec::new(), false),
]
}
fn request(url: &str, dir: &Path) -> Request {
Request { url: url.to_string(), game_dir: dir.to_path_buf(), max_workers: 4, process_workers: 2, ca_bundle: String::new(), part_bytes: 64 * 1024 }
}
fn temp_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("droiddeck-ea-{name}-{}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn installs_every_file_and_keeps_them_on_the_next_run() {
let files = sample();
let zip = build_zip(&files);
let server = serve(zip.bytes);
let dir = temp_dir("install");
let ev = Recorder::default();
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &ev).unwrap();
assert_eq!(done.file_count, 4);
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), files[1].1);
assert_eq!(fs::read(dir.join("Data/stored.bin")).unwrap(), vec![9u8; 4096]);
assert!(dir.join("empty.txt").is_file());
assert!(dir.join("__Installer/installerdata.xml").is_file());
assert!(!dir.join(IN_PROGRESS_MARKER).exists());
let (shown, total) = *ev.last.lock().unwrap();
assert_eq!(shown, total);
let before = server.requests.load(Ordering::SeqCst);
let again = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
assert_eq!(again.files_written, 0);
assert!(server.requests.load(Ordering::SeqCst) - before <= 2);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn an_update_replaces_changed_files_and_drops_removed_ones() {
let dir = temp_dir("update");
let first = serve(build_zip(&sample()).bytes);
run(&request(&first.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
let mut next = sample();
next[1].1[10] ^= 0xFF;
next.retain(|f| f.0 != "Data\\stored.bin");
let second = serve(build_zip(&next).bytes);
let ev = Recorder::default();
let done = run(&request(&second.url, &dir), &AtomicBool::new(false), &ev).unwrap();
assert_eq!(done.files_written, 1);
assert_eq!(done.removed, 1);
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), next[1].1);
assert!(!dir.join("Data/stored.bin").exists());
assert_eq!(ev.gates.load(Ordering::SeqCst), 1);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn a_damaged_file_on_disk_is_downloaded_again() {
let dir = temp_dir("repair");
let server = serve(build_zip(&sample()).bytes);
run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
let mut data = fs::read(dir.join("Game.exe")).unwrap();
data[5] ^= 1;
fs::write(dir.join("Game.exe"), &data).unwrap();
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
assert_eq!(done.files_written, 1);
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), sample()[1].1);
let _ = fs::remove_dir_all(&dir);
}
#[test]
fn reports_the_archive_size() {
let server = serve(build_zip(&sample()).bytes);
let answer = size(&server.url).unwrap();
assert_eq!(answer["fileCount"], 4);
let expected: usize = sample().iter().map(|f| f.1.len()).sum();
assert_eq!(answer["totalBytes"], expected as u64);
}
#[test]
fn large_records_arrive_in_parallel_parts() {
let dir = temp_dir("parts");
let server = serve(build_zip(&sample()).bytes);
let before = server.requests.load(Ordering::SeqCst);
let done = run(&request(&server.url, &dir), &AtomicBool::new(false), &Recorder::default()).unwrap();
assert_eq!(done.files_written, 4);
assert!(server.requests.load(Ordering::SeqCst) - before > 6);
assert_eq!(fs::read(dir.join("Game.exe")).unwrap(), sample()[1].1);
assert!(fs::read_dir(&dir).unwrap().all(|e| !e.unwrap().file_name().to_string_lossy().ends_with(".ddzip")));
let _ = fs::remove_dir_all(&dir);
}
+4
View File
@@ -0,0 +1,4 @@
pub mod ffi;
pub mod install;
pub mod net;
pub mod zip;
+98
View File
@@ -0,0 +1,98 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! Ranged reads of EA's build archives.
use std::time::Duration;
use super::zip::{self, Directory, Entry};
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
const READ_TIMEOUT: Duration = Duration::from_secs(60);
const MAX_DIRECTORY: u64 = 512 * 1024 * 1024;
pub fn user_agent() -> String {
crate::amazon::net::user_agent()
}
pub struct Ranged {
pub body: Vec<u8>,
pub start: u64,
pub total: u64,
}
fn content_range(value: &str) -> Option<(u64, u64)> {
let rest = value.trim().strip_prefix("bytes")?.trim_start();
let (span, total) = rest.split_once('/')?;
let start = span.split_once('-')?.0.trim().parse().ok()?;
Some((start, total.trim().parse().ok()?))
}
pub fn get_range(url: &str, range: &str) -> Result<Ranged, String> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|e| format!("runtime: {e}"))?;
runtime.block_on(async {
let client = reqwest::Client::builder()
.user_agent(user_agent())
.connect_timeout(CONNECT_TIMEOUT)
.read_timeout(READ_TIMEOUT)
.build()
.map_err(|e| format!("http client: {}", e.without_url()))?;
let response = client
.get(url)
.header(reqwest::header::RANGE, format!("bytes={range}"))
.send()
.await
.map_err(|e| format!("request failed: {}", e.without_url()))?;
let status = response.status().as_u16();
if status == 403 || status == 404 {
return Err(format!("the download link was refused ({status})"));
}
if status != 206 {
return Err(format!("the server did not answer a ranged read ({status})"));
}
let (start, total) = response
.headers()
.get(reqwest::header::CONTENT_RANGE)
.and_then(|v| v.to_str().ok())
.and_then(content_range)
.ok_or("the server gave no content range")?;
let body = response.bytes().await.map_err(|e| format!("answer could not be read: {}", e.without_url()))?.to_vec();
Ok(Ranged { body, start, total })
})
}
pub struct Archive {
pub total: u64,
pub directory: Directory,
pub entries: Vec<Entry>,
}
pub fn read_archive(url: &str) -> Result<Archive, String> {
let tail = get_range(url, &format!("-{}", zip::TAIL_LEN))?;
let directory = zip::locate(&tail.body, tail.start)?;
if directory.size > MAX_DIRECTORY || directory.offset + directory.size > tail.total {
return Err("The archive directory is out of bounds".into());
}
let bytes = if directory.offset >= tail.start {
let at = (directory.offset - tail.start) as usize;
let end = at + directory.size as usize;
tail.body.get(at..end).ok_or("The archive directory is cut short")?.to_vec()
} else {
let end = directory.offset + directory.size - 1;
get_range(url, &format!("{}-{end}", directory.offset))?.body
};
let entries = zip::parse(&bytes, directory.entries)?;
Ok(Archive { total: tail.total, directory, entries })
}
#[cfg(test)]
mod tests {
use super::content_range;
#[test]
fn reads_content_range() {
assert_eq!(content_range("bytes 100-199/5000"), Some((100, 5000)));
assert_eq!(content_range("bytes */5000"), None);
}
}
+288
View File
@@ -0,0 +1,288 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! The central directory of a remote ZIP archive, which is how EA publishes a game build.
const EOCD: u32 = 0x0605_4b50;
const EOCD64: u32 = 0x0606_4b50;
const EOCD64_LOCATOR: u32 = 0x0706_4b50;
const CENTRAL: u32 = 0x0201_4b50;
const LOCAL: u32 = 0x0403_4b50;
const EOCD_LEN: usize = 22;
const LOCATOR_LEN: usize = 20;
const EOCD64_LEN: usize = 56;
pub const LOCAL_FIXED: usize = 30;
pub const TAIL_LEN: u64 = 128 * 1024;
pub const STORED: u16 = 0;
pub const DEFLATED: u16 = 8;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Entry {
pub name: String,
pub crc32: u32,
pub method: u16,
pub compressed: u64,
pub size: u64,
pub local_offset: u64,
}
impl Entry {
pub fn is_dir(&self) -> bool {
self.name.ends_with('/') || self.name.ends_with('\\')
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Directory {
pub offset: u64,
pub size: u64,
pub entries: u64,
}
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
b.get(at..at + 2).map(|s| u16::from_le_bytes([s[0], s[1]]))
}
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
b.get(at..at + 4).map(|s| u32::from_le_bytes([s[0], s[1], s[2], s[3]]))
}
fn u64_at(b: &[u8], at: usize) -> Option<u64> {
b.get(at..at + 8).map(|s| u64::from_le_bytes(s.try_into().unwrap_or([0; 8])))
}
/// Finds the central directory from the last bytes of the archive. `tail_start` is where `tail`
/// begins in the whole file.
pub fn locate(tail: &[u8], tail_start: u64) -> Result<Directory, String> {
if tail.len() < EOCD_LEN {
return Err("The archive is too short to be a ZIP".into());
}
let mut at = tail.len() - EOCD_LEN;
let found = loop {
if u32_at(tail, at) == Some(EOCD) {
let comment = u16_at(tail, at + 20).unwrap_or(0) as usize;
if at + EOCD_LEN + comment <= tail.len() {
break at;
}
}
if at == 0 {
return Err("The archive has no end of central directory".into());
}
at -= 1;
};
let entries = u16_at(tail, found + 10).unwrap_or(0) as u64;
let size = u32_at(tail, found + 12).unwrap_or(0) as u64;
let offset = u32_at(tail, found + 16).unwrap_or(0) as u64;
let wide = entries == 0xFFFF || size == 0xFFFF_FFFF || offset == 0xFFFF_FFFF;
if !wide {
return Ok(Directory { offset, size, entries });
}
let locator = found.checked_sub(LOCATOR_LEN).filter(|&l| u32_at(tail, l) == Some(EOCD64_LOCATOR));
let Some(locator) = locator else {
return Err("The archive needs ZIP64 but has no ZIP64 locator".into());
};
let record_at = u64_at(tail, locator + 8).unwrap_or(0);
let Some(rel) = record_at.checked_sub(tail_start).map(|r| r as usize) else {
return Err("The ZIP64 directory record lies before the fetched tail".into());
};
if u32_at(tail, rel) != Some(EOCD64) || rel + EOCD64_LEN > tail.len() {
return Err("The ZIP64 directory record is damaged".into());
}
Ok(Directory {
entries: u64_at(tail, rel + 32).unwrap_or(0),
size: u64_at(tail, rel + 40).unwrap_or(0),
offset: u64_at(tail, rel + 48).unwrap_or(0),
})
}
fn wide_values(extra: &[u8], need_size: bool, need_compressed: bool, need_offset: bool) -> (Option<u64>, Option<u64>, Option<u64>) {
let mut at = 0;
while at + 4 <= extra.len() {
let id = u16_at(extra, at).unwrap_or(0);
let len = u16_at(extra, at + 2).unwrap_or(0) as usize;
let body = &extra[(at + 4).min(extra.len())..(at + 4 + len).min(extra.len())];
if id == 0x0001 {
let mut p = 0;
let mut take = |want: bool| {
if !want {
return None;
}
let v = u64_at(body, p);
p += 8;
v
};
let size = take(need_size);
let compressed = take(need_compressed);
let offset = take(need_offset);
return (size, compressed, offset);
}
at += 4 + len;
}
(None, None, None)
}
pub fn parse(directory: &[u8], expected: u64) -> Result<Vec<Entry>, String> {
let mut entries = Vec::with_capacity(expected.min(1 << 20) as usize);
let mut at = 0usize;
while at + 46 <= directory.len() {
if u32_at(directory, at) != Some(CENTRAL) {
break;
}
let flags = u16_at(directory, at + 8).unwrap_or(0);
let method = u16_at(directory, at + 10).unwrap_or(0);
let crc32 = u32_at(directory, at + 16).unwrap_or(0);
let compressed32 = u32_at(directory, at + 20).unwrap_or(0);
let size32 = u32_at(directory, at + 24).unwrap_or(0);
let name_len = u16_at(directory, at + 28).unwrap_or(0) as usize;
let extra_len = u16_at(directory, at + 30).unwrap_or(0) as usize;
let comment_len = u16_at(directory, at + 32).unwrap_or(0) as usize;
let offset32 = u32_at(directory, at + 42).unwrap_or(0);
let name_at = at + 46;
let end = name_at + name_len + extra_len + comment_len;
if end > directory.len() {
return Err("The archive directory is cut short".into());
}
let raw = &directory[name_at..name_at + name_len];
let name = if flags & 0x0800 != 0 {
String::from_utf8_lossy(raw).into_owned()
} else {
raw.iter().map(|&b| b as char).collect()
};
let extra = &directory[name_at + name_len..name_at + name_len + extra_len];
let (size, compressed, offset) =
wide_values(extra, size32 == 0xFFFF_FFFF, compressed32 == 0xFFFF_FFFF, offset32 == 0xFFFF_FFFF);
if flags & 0x0001 != 0 {
return Err(format!("{name} is encrypted"));
}
entries.push(Entry {
name,
crc32,
method,
compressed: compressed.unwrap_or(compressed32 as u64),
size: size.unwrap_or(size32 as u64),
local_offset: offset.unwrap_or(offset32 as u64),
});
at = end;
}
if expected != 0 && entries.len() as u64 != expected {
return Err(format!("The archive directory lists {} of {expected} entries", entries.len()));
}
Ok(entries)
}
/// Bytes before the compressed data of an entry, read from its local header.
pub fn local_header_len(header: &[u8]) -> Option<Result<usize, String>> {
if header.len() < LOCAL_FIXED {
return None;
}
if u32_at(header, 0) != Some(LOCAL) {
return Some(Err("An entry has no local header where the directory points".into()));
}
let name = u16_at(header, 26).unwrap_or(0) as usize;
let extra = u16_at(header, 28).unwrap_or(0) as usize;
Some(Ok(LOCAL_FIXED + name + extra))
}
/// A relative path inside the install folder, or None for names that would leave it.
pub fn safe_path(name: &str) -> Option<String> {
let unified = name.replace('\\', "/");
if unified.starts_with('/') || unified.contains(':') || unified.contains('\0') {
return None;
}
let parts: Vec<&str> = unified.split('/').filter(|p| !p.is_empty() && *p != ".").collect();
if parts.is_empty() || parts.iter().any(|p| *p == "..") {
return None;
}
Some(parts.join("/"))
}
#[cfg(test)]
mod tests {
use super::*;
fn central(name: &str, method: u16, crc: u32, compressed: u32, size: u32, offset: u32, extra: &[u8]) -> Vec<u8> {
let mut v = Vec::new();
v.extend_from_slice(&CENTRAL.to_le_bytes());
v.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
v.extend_from_slice(&method.to_le_bytes());
v.extend_from_slice(&[0, 0, 0, 0]);
v.extend_from_slice(&crc.to_le_bytes());
v.extend_from_slice(&compressed.to_le_bytes());
v.extend_from_slice(&size.to_le_bytes());
v.extend_from_slice(&(name.len() as u16).to_le_bytes());
v.extend_from_slice(&(extra.len() as u16).to_le_bytes());
v.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
v.extend_from_slice(&offset.to_le_bytes());
v.extend_from_slice(name.as_bytes());
v.extend_from_slice(extra);
v
}
#[test]
fn reads_a_plain_directory() {
let mut dir = central("Game.exe", DEFLATED, 7, 10, 20, 0, &[]);
dir.extend(central("__Installer/", STORED, 0, 0, 0, 40, &[]));
let mut tail = vec![0u8; 5];
tail.extend_from_slice(&dir);
let cd_size = dir.len() as u32;
tail.extend_from_slice(&EOCD.to_le_bytes());
tail.extend_from_slice(&[0, 0, 0, 0, 2, 0, 2, 0]);
tail.extend_from_slice(&cd_size.to_le_bytes());
tail.extend_from_slice(&1000u32.to_le_bytes());
tail.extend_from_slice(&[0, 0]);
let found = locate(&tail, 995).unwrap();
assert_eq!(found, Directory { offset: 1000, size: cd_size as u64, entries: 2 });
let entries = parse(&dir, 2).unwrap();
assert_eq!(entries[0].name, "Game.exe");
assert_eq!(entries[0].size, 20);
assert!(entries[1].is_dir());
}
#[test]
fn reads_zip64_values() {
let mut extra = vec![1, 0, 24, 0];
extra.extend_from_slice(&6_000_000_000u64.to_le_bytes());
extra.extend_from_slice(&5_000_000_000u64.to_le_bytes());
extra.extend_from_slice(&7_000_000_000u64.to_le_bytes());
let dir = central("big.pak", DEFLATED, 1, 0xFFFF_FFFF, 0xFFFF_FFFF, 0xFFFF_FFFF, &extra);
let e = &parse(&dir, 1).unwrap()[0];
assert_eq!((e.size, e.compressed, e.local_offset), (6_000_000_000, 5_000_000_000, 7_000_000_000));
let mut tail = Vec::new();
let record_at = 9_000_000_000u64;
tail.extend_from_slice(&EOCD64.to_le_bytes());
tail.extend_from_slice(&44u64.to_le_bytes());
tail.extend_from_slice(&[45, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
tail.extend_from_slice(&3u64.to_le_bytes());
tail.extend_from_slice(&3u64.to_le_bytes());
tail.extend_from_slice(&500u64.to_le_bytes());
tail.extend_from_slice(&8_000_000_000u64.to_le_bytes());
tail.extend_from_slice(&EOCD64_LOCATOR.to_le_bytes());
tail.extend_from_slice(&0u32.to_le_bytes());
tail.extend_from_slice(&record_at.to_le_bytes());
tail.extend_from_slice(&1u32.to_le_bytes());
tail.extend_from_slice(&EOCD.to_le_bytes());
tail.extend_from_slice(&[0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0]);
let found = locate(&tail, record_at).unwrap();
assert_eq!(found, Directory { offset: 8_000_000_000, size: 500, entries: 3 });
}
#[test]
fn refuses_paths_that_leave_the_folder() {
assert_eq!(safe_path("Data\\Win32\\a.big").as_deref(), Some("Data/Win32/a.big"));
assert_eq!(safe_path("./x//y").as_deref(), Some("x/y"));
assert!(safe_path("../evil").is_none());
assert!(safe_path("/etc/passwd").is_none());
assert!(safe_path("C:/x").is_none());
assert!(safe_path("a/../../b").is_none());
}
#[test]
fn local_header_length_counts_name_and_extra() {
let mut h = LOCAL.to_le_bytes().to_vec();
h.extend_from_slice(&[0u8; 22]);
h.extend_from_slice(&5u16.to_le_bytes());
h.extend_from_slice(&9u16.to_le_bytes());
assert_eq!(local_header_len(&h), Some(Ok(44)));
assert_eq!(local_header_len(&h[..10]), None);
}
}
+1
View File
@@ -3,6 +3,7 @@
#![allow(clippy::missing_safety_doc, clippy::result_large_err)]
pub mod amazon;
pub mod ea;
pub mod epic;
pub mod fetch_core;
pub mod fetch_tuning;