fix(epic): a paused install keeps what it downloaded (#1504)

This commit is contained in:
MaxsTechReview authored and GitHub committed 2026-10-24 14:47:18 -04:00
1 parent 8156a9649d
commit 4dda1c1e52
7 files changed
+319 -25

No files matched your search

+1 -1
View File
@@ -198,7 +198,7 @@ completes.
On-disk protocol: game files are staged in `<gameDir>/.droiddeck-epic-staging/*.stage` (owner
marker `owner` = `DroidDeck Epic staging v1\n`) and renamed over their destination after size and
SHA-1 checks; reusable chunks live, decompressed, as `<GUID>` (extra copies `<GUID>.<n>`, writes via
SHA-1 checks; a paused run keeps each unfinished file there as `*.part` with its written parts in `*.part.list`; reusable chunks live, decompressed, as `<GUID>` (extra copies `<GUID>.<n>`, writes via
`.part`) in `<gameDir>/.chunks` or the scratch directory.
## `com.droiddeck.launcher.stores.amazon.AmazonNative`
@@ -5,7 +5,7 @@ pub mod space;
pub mod staging;
pub mod writer;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
@@ -140,6 +140,17 @@ pub fn fetch_manifest(info: &BuildInfo, log: &dyn Fn(&str)) -> Result<Manifest,
})
}
fn resume_from(stage: &Path, parts: &[PlanPart], size: u64) -> Option<HashSet<u64>> {
let meta = fs::symlink_metadata(stage).ok()?;
if !meta.is_file() || meta.len() > size {
return None;
}
let list = fs::read_to_string(staging::list_path(stage)).ok()?;
let kept: HashSet<u64> = list.lines().map(|l| l.parse().ok()).collect::<Option<_>>()?;
let fits = |o: &u64| parts.iter().any(|p| p.file_offset == *o && p.file_offset + p.size as u64 <= meta.len());
(!kept.is_empty() && kept.iter().all(fits)).then_some(kept)
}
pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finished, Stop> {
let failed = |m: String| Stop::Failed(m);
ev.status("Downloading the manifest…");
@@ -191,6 +202,7 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
}
let pending_bytes: u64 = pending.iter().map(|&(i, _)| manifest.files[i].size).sum();
ev.sizes(pending_bytes, disk_bytes);
let mut partials = staging::partials(&stage_dir);
let mut finished = Finished {
build_version: manifest.build_version.clone(),
@@ -211,6 +223,9 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
staging::remove_if_idle(&stage_dir);
};
if pending.is_empty() {
for stale in partials.values() {
staging::remove_partial(stale);
}
ev.log("every selected file is already up to date");
clean_caches();
ev.installing(disk_bytes, disk_bytes, total_selected, total_selected);
@@ -219,6 +234,8 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
let mut plan_files = Vec::with_capacity(pending.len());
let mut engine_files = Vec::with_capacity(pending.len());
let mut resumed_bytes = 0u64;
let mut kept_sizes = Vec::with_capacity(pending.len());
for &(i, old_size) in &pending {
let f = &manifest.files[i];
let mut parts = Vec::with_capacity(f.parts.len());
@@ -234,6 +251,22 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
parts.push(PlanPart { chunk: c as u32, chunk_offset: part.offset, size: part.size, file_offset: offset });
offset += part.size as u64;
}
let mut resume = None;
let mut kept_size = 0u64;
if let Some(stage) = partials.remove(&staging::partial_name(&f.path, f.size, f.sha1)) {
match resume_from(&stage, &parts, f.size).filter(|_| f.sha1.is_some()) {
Some(kept) => {
let bytes: u64 = parts.iter().filter(|p| kept.contains(&p.file_offset)).map(|p| p.size as u64).sum();
ev.log(&format!("resuming {} with {bytes} of {} bytes written", f.path, f.size));
resumed_bytes += bytes;
kept_size = bytes;
parts.retain(|p| !kept.contains(&p.file_offset));
resume = Some(writer::Resume { stage, kept: kept.into_iter().collect() });
}
None => staging::remove_partial(&stage),
}
}
kept_sizes.push(kept_size);
plan_files.push(PlanFile { size: f.size, parts: parts.clone() });
engine_files.push(writer::EngineFile {
path: f.path.clone(),
@@ -242,8 +275,14 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
sha1: f.sha1,
replacing: old_size > 0,
parts,
resume,
});
}
for stale in partials.values() {
staging::remove_partial(stale);
}
let fresh = pending.iter().all(|&(_, old_size)| old_size == 0);
let progress_base = resumed_bytes + if fresh { disk_bytes - pending_bytes } else { 0 };
let scratch = req
.scratch_dir
@@ -252,7 +291,8 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
.filter(|s| !space::same_volume(s, &req.game_dir));
let sizes: Vec<space::PendingSize> = pending
.iter()
.map(|&(i, old)| space::PendingSize { new_size: manifest.files[i].size, old_size: old })
.zip(&kept_sizes)
.map(|(&(i, old), kept)| space::PendingSize { new_size: manifest.files[i].size - kept, old_size: old })
.collect();
let free_game = space::free_bytes(&req.game_dir);
let scratch_space = scratch.as_ref().map(|s| space::Scratch { free: space::free_bytes(s) });
@@ -308,6 +348,8 @@ cache_budget_mib={} overlap_mib={} free_game={free_game} workers={} process_work
process_workers: req.process_workers,
ca_bundle: req.ca_bundle.clone(),
user_agent: net::user_agent(),
progress_base,
progress_total: progress_base + pending_bytes - resumed_bytes,
};
let stats = writer::run(&input, cancel, ev)?;
finished.bytes_written = stats.bytes_written;
@@ -1,12 +1,19 @@
use std::collections::HashMap;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use sha1::{Digest, Sha1};
use crate::epic::manifest::Sha1 as Hash;
pub const DIR_NAME: &str = ".droiddeck-epic-staging";
pub const OWNER_FILE: &str = "owner";
pub const OWNER_MARKER: &[u8] = b"DroidDeck Epic staging v1\n";
const SUFFIX: &str = ".stage";
pub const PARTIAL_SUFFIX: &str = ".part";
const LIST_SUFFIX: &str = ".part.list";
static SEQUENCE: AtomicU64 = AtomicU64::new(0);
@@ -15,6 +22,37 @@ pub fn conflicts(manifest_path: &str) -> bool {
first.eq_ignore_ascii_case(DIR_NAME)
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
pub fn partial_name(path: &str, size: u64, sha1: Option<Hash>) -> String {
let id: Hash = Sha1::digest(path.as_bytes()).into();
let content = sha1.map(|h| hex(&h)).unwrap_or_else(|| "0".into());
format!("{}-{content}-{size}{PARTIAL_SUFFIX}", hex(&id[..8]))
}
pub fn list_path(partial: &Path) -> PathBuf {
let mut name = partial.as_os_str().to_owned();
name.push(".list");
PathBuf::from(name)
}
pub fn remove_partial(partial: &Path) {
let _ = fs::remove_file(partial);
let _ = fs::remove_file(list_path(partial));
}
pub fn partials(dir: &Path) -> HashMap<String, PathBuf> {
let Ok(entries) = fs::read_dir(dir) else { return HashMap::new() };
entries
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.ends_with(PARTIAL_SUFFIX))
.map(|n| (n.clone(), dir.join(n)))
.collect()
}
pub fn unique_name() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
@@ -82,6 +120,8 @@ pub fn recover(game_dir: &Path) -> Result<PathBuf, String> {
} else if name.ends_with(SUFFIX) {
stale.push(path);
others += 1;
} else if name.ends_with(PARTIAL_SUFFIX) || name.ends_with(LIST_SUFFIX) {
others += 1;
} else {
return refuse("it holds an unexpected file");
}
@@ -130,8 +170,13 @@ mod tests {
fs::write(dir.join("file-1.stage"), b"x").unwrap();
fs::write(dir.join("stream-1-2-3.stage"), b"y").unwrap();
fs::write(game.join("keep.stage"), b"z").unwrap();
let partial = partial_name("Game/a.pak", 10, None);
fs::write(dir.join(&partial), b"p").unwrap();
fs::write(list_path(&dir.join(&partial)), b"0\n").unwrap();
recover(&game).unwrap();
assert_eq!(fs::read_dir(&dir).unwrap().count(), 1);
assert_eq!(fs::read_dir(&dir).unwrap().count(), 3);
assert_eq!(partials(&dir).into_keys().collect::<Vec<_>>(), vec![partial.clone()]);
remove_partial(&dir.join(&partial));
assert!(game.join("keep.stage").exists());
remove_if_idle(&dir);
assert!(!dir.exists());
@@ -170,5 +215,9 @@ mod tests {
let (a, b) = (unique_name(), unique_name());
assert_ne!(a, b);
assert!(a.ends_with(".stage"));
let p = partial_name("Game/a.pak", 10, Some([0xab; 20]));
assert!(p.ends_with(&format!("-{}-10.part", "ab".repeat(20))));
assert_ne!(p, partial_name("Game/b.pak", 10, Some([0xab; 20])));
assert_ne!(p, partial_name("Game/a.pak", 10, Some([0xac; 20])));
}
}
@@ -59,6 +59,7 @@ struct Recorder {
replaced: AtomicUsize,
refuse: bool,
last_progress: Mutex<(u64, u64)>,
first_progress: Mutex<Option<(u64, u64)>>,
sizes: Mutex<(u64, u64)>,
}
@@ -72,6 +73,7 @@ impl Events for Recorder {
*self.sizes.lock().unwrap() = (pending, disk);
}
fn progress(&self, written: u64, total: u64, _f: u32, _ft: u32) {
self.first_progress.lock().unwrap().get_or_insert((written, total));
let mut p = self.last_progress.lock().unwrap();
if written >= p.0 {
*p = (written, total);
@@ -262,6 +264,85 @@ fn a_cancelled_run_reports_cancel() {
let _ = fs::remove_dir_all(&game);
}
fn place_partial(fx: &Fixture, game: &Path, data: &[u8], list: &str) -> PathBuf {
let stage = staging::recover(game).unwrap();
let a = &fx.expected[0].1;
let partial = stage.join(staging::partial_name("a.bin", 4000, Some(Sha1::digest(a).into())));
fs::write(&partial, data).unwrap();
fs::write(staging::list_path(&partial), list).unwrap();
partial
}
fn place_done_c(fx: &Fixture, game: &Path) {
fs::create_dir_all(game.join("b")).unwrap();
fs::write(game.join("b/c.bin"), &fx.expected[1].1).unwrap();
}
#[test]
fn a_paused_install_resumes_from_its_written_parts() {
let fx = fixture();
let game = temp_game("resume");
place_done_c(&fx, &game);
place_partial(&fx, &game, &fx.expected[0].1[..3000], "0\n");
let stage = game.join(staging::DIR_NAME);
fs::write(stage.join(staging::partial_name("gone.bin", 9, None)), b"old").unwrap();
let events = Recorder::default();
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
for (path, bytes) in &fx.expected {
assert_eq!(&fs::read(game.join(path)).unwrap(), bytes, "{path}");
}
assert_eq!(fx.hits.load(Ordering::Relaxed), 2, "the manifest and the one chunk still missing");
assert_eq!(*events.first_progress.lock().unwrap(), Some((2096 + 3000, 6196)));
assert_eq!(*events.last_progress.lock().unwrap(), (6196, 6196));
assert!(events.logs.lock().unwrap().iter().any(|l| l == "resuming a.bin with 3000 of 4000 bytes written"));
assert!(!stage.exists());
let _ = fs::remove_dir_all(&game);
}
#[test]
fn a_part_written_out_of_order_is_kept_too() {
let fx = fixture();
let game = temp_game("resume-tail");
place_done_c(&fx, &game);
let mut data = vec![0u8; 3000];
data.extend_from_slice(&fx.expected[0].1[3000..]);
place_partial(&fx, &game, &data, "3000\n");
let events = Recorder::default();
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
assert_eq!(*events.first_progress.lock().unwrap(), Some((2096 + 1000, 6196)));
assert!(!game.join(staging::DIR_NAME).exists());
let _ = fs::remove_dir_all(&game);
}
#[test]
fn a_partial_that_no_longer_matches_is_dropped() {
let fx = fixture();
let game = temp_game("badpartial");
place_partial(&fx, &game, &fx.expected[0].1[..1234], "1234\n");
let events = Recorder::default();
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
assert_eq!(*events.first_progress.lock().unwrap(), Some((0, 6196)));
assert!(!game.join(staging::DIR_NAME).exists());
let _ = fs::remove_dir_all(&game);
}
#[test]
fn a_damaged_partial_fails_its_check_and_is_not_kept() {
let fx = fixture();
let game = temp_game("damagedpartial");
place_partial(&fx, &game, &[0u8; 3000], "0\n");
let err = run(&request(&fx, &game), &AtomicBool::new(false), &Recorder::default()).unwrap_err();
assert_eq!(err, Stop::Failed("SHA-1 mismatch for a.bin".into()));
assert!(staging::partials(&game.join(staging::DIR_NAME)).is_empty());
let retry = Recorder::default();
run(&request(&fx, &game), &AtomicBool::new(false), &retry).unwrap();
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
let _ = fs::remove_dir_all(&game);
}
#[test]
fn guid_names_are_cache_names() {
let c = chunk(5, 10);
@@ -1,6 +1,6 @@
use std::fs::{self, File, OpenOptions};
use std::os::unix::fs::FileExt;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex};
use std::time::{Duration, Instant};
@@ -25,6 +25,12 @@ pub struct EngineFile {
pub sha1: Option<Hash>,
pub replacing: bool,
pub parts: Vec<PlanPart>,
pub resume: Option<Resume>,
}
pub struct Resume {
pub stage: PathBuf,
pub kept: Vec<u64>,
}
pub struct EngineInput<'a> {
@@ -40,6 +46,8 @@ pub struct EngineInput<'a> {
pub process_workers: usize,
pub ca_bundle: String,
pub user_agent: String,
pub progress_base: u64,
pub progress_total: u64,
}
#[derive(Debug, Default)]
@@ -120,7 +128,7 @@ impl<'a> Engine<'a> {
return;
}
self.events.progress(
self.written.load(Ordering::Relaxed),
self.input.progress_base + self.written.load(Ordering::Relaxed),
self.total_bytes,
self.files_done.load(Ordering::Relaxed) as u32,
self.input.files.len() as u32,
@@ -232,10 +240,14 @@ impl<'a> Engine<'a> {
.sync_all()
.map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))?;
let len = handle.metadata().map(|m| m.len()).unwrap_or(u64::MAX);
drop(handle);
if len != file.size {
return Err(format!("Size mismatch for {}", file.path));
}
let hash = match (hash, &file.resume) {
(None, Some(_)) if file.sha1.is_some() => Some(hash_whole(&handle, file)?),
(hash, _) => hash,
};
drop(handle);
if let (Some(expected), Some(got)) = (file.sha1, hash) {
if expected != got {
return Err(format!("SHA-1 mismatch for {}", file.path));
@@ -260,6 +272,9 @@ impl<'a> Engine<'a> {
fs::rename(&stage, &file.dest)
.map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))
})();
if file.resume.is_some() {
let _ = fs::remove_file(staging::list_path(&stage));
}
if let Err(e) = result {
let _ = fs::remove_file(&stage);
return Err(e);
@@ -278,13 +293,16 @@ impl<'a> Engine<'a> {
}
fn open_stage(&self, f: usize) -> Result<(), String> {
let path = self.input.stage_dir.join(staging::unique_name());
let handle = OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&path)
.map_err(|e| format!("Could not stage {}: {e}", self.input.files[f].path))?;
let file = &self.input.files[f];
let (path, opened) = match &file.resume {
Some(r) => (r.stage.clone(), OpenOptions::new().read(true).write(true).open(&r.stage)),
None => {
let path = self.input.stage_dir.join(staging::unique_name());
let opened = OpenOptions::new().read(true).write(true).create_new(true).open(&path);
(path, opened)
}
};
let handle = opened.map_err(|e| format!("Could not stage {}: {e}", file.path))?;
let mut slot = self.slots[f].lock().expect("slot lock");
slot.file = Some(Arc::new(handle));
slot.stage = Some(path);
@@ -439,7 +457,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
stage: None,
written: vec![false; f.parts.len()],
next_hash: 0,
hasher: f.sha1.map(|_| Sha1::new()),
hasher: f.sha1.filter(|_| f.resume.is_none()).map(|_| Sha1::new()),
left: f.parts.len(),
done: false,
})
@@ -466,7 +484,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
started: Instant::now(),
replace_ok: Mutex::new(None),
events,
total_bytes: input.files.iter().map(|f| f.size).sum(),
total_bytes: input.progress_total,
};
for (i, p) in schedule.producers.iter().enumerate() {
if p.preloaded && remaining[i] == 0 {
@@ -503,7 +521,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
stream: false,
};
events.progress(0, engine.total_bytes, 0, input.files.len() as u32);
events.progress(input.progress_base, engine.total_bytes, 0, input.files.len() as u32);
let log = |line: &str| events.log(line);
let gate = |i: usize| -> bool {
engine.hint.fetch_max(i, Ordering::Relaxed);
@@ -546,12 +564,14 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
let complete = failure.is_none() && engine.files_done.load(Ordering::Relaxed) == input.files.len();
if !complete {
for slot in &engine.slots {
for (file, slot) in input.files.iter().zip(&engine.slots) {
let mut s = slot.lock().expect("slot lock");
if !s.done {
s.file.take();
if let Some(stage) = s.stage.take() {
let _ = fs::remove_file(stage);
if failure.is_some() || !keep_partial(file, &s.written, &stage, &input.stage_dir) {
let _ = fs::remove_file(stage);
}
}
}
}
@@ -588,6 +608,100 @@ fetches={} refetches={} peak_cache_mib={} hosts={}",
Err(Stop::Failed("The download ended before every file was written".into()))
}
fn hash_whole(handle: &File, file: &EngineFile) -> Result<Hash, String> {
let mut h = Sha1::new();
let mut buf = vec![0u8; 1024 * 1024];
let mut at = 0u64;
while at < file.size {
let n = buf.len().min((file.size - at) as usize);
handle
.read_exact_at(&mut buf[..n], at)
.map_err(|e| format!("Could not read back {}: {e}", file.path))?;
h.update(&buf[..n]);
at += n as u64;
}
Ok(h.finalize().into())
}
fn keep_partial(file: &EngineFile, written: &[bool], stage: &Path, dir: &Path) -> bool {
let mut kept: Vec<u64> = file.resume.as_ref().map(|r| r.kept.clone()).unwrap_or_default();
kept.extend(file.parts.iter().zip(written).filter(|(_, w)| **w).map(|(p, _)| p.file_offset));
if kept.is_empty() || file.sha1.is_none() {
return false;
}
kept.sort_unstable();
let partial = dir.join(staging::partial_name(&file.path, file.size, file.sha1));
let list: String = kept.iter().map(|o| format!("{o}\n")).collect();
let synced = OpenOptions::new().write(true).open(stage).and_then(|h| h.sync_all());
if synced.is_err() || fs::rename(stage, &partial).is_err() {
return false;
}
let resumed = file.resume.as_ref().is_some_and(|r| r.stage == partial);
let temp = dir.join(staging::unique_name());
let listed = fs::write(&temp, list).and_then(|_| fs::rename(&temp, staging::list_path(&partial))).is_ok();
if !listed {
let _ = fs::remove_file(&temp);
if !resumed {
let _ = fs::remove_file(&partial);
}
}
listed || resumed
}
fn tidy_fetch_error(message: &str) -> String {
message.strip_prefix("epic: ").unwrap_or(message).to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn part(file_offset: u64, size: u32) -> PlanPart {
PlanPart { chunk: 0, chunk_offset: 0, size, file_offset }
}
#[test]
fn a_pause_keeps_every_written_part_and_lists_them() {
let dir = std::env::temp_dir().join(format!("dd-epic-keep-{}", std::process::id()));
let _ = fs::remove_dir_all(&dir);
fs::create_dir_all(&dir).unwrap();
let file = EngineFile {
path: "Game/a.pak".into(),
dest: dir.join("a.pak"),
size: 30,
sha1: Some([1; 20]),
replacing: false,
parts: vec![part(0, 10), part(10, 10), part(20, 10)],
resume: None,
};
let kept = dir.join(staging::partial_name("Game/a.pak", 30, Some([1; 20])));
let stage = dir.join("x.stage");
fs::write(&stage, [7u8; 30]).unwrap();
assert!(!keep_partial(&file, &[false, false, false], &stage, &dir));
let unhashed = EngineFile {
path: file.path.clone(),
dest: file.dest.clone(),
size: 30,
sha1: None,
replacing: false,
parts: file.parts.clone(),
resume: None,
};
assert!(!keep_partial(&unhashed, &[true, true, true], &stage, &dir));
assert!(stage.exists());
assert!(keep_partial(&file, &[false, true, true], &stage, &dir));
assert_eq!(fs::read(&kept).unwrap(), [7u8; 30]);
assert_eq!(fs::read_to_string(staging::list_path(&kept)).unwrap(), "10\n20\n");
assert!(!stage.exists());
let resumed = EngineFile {
parts: vec![part(0, 10)],
resume: Some(Resume { stage: kept.clone(), kept: vec![20, 10] }),
..file
};
assert!(keep_partial(&resumed, &[false], &kept, &dir));
assert_eq!(fs::read_to_string(staging::list_path(&kept)).unwrap(), "10\n20\n");
let _ = fs::remove_dir_all(&dir);
}
}