mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-20 05:16:01 +00:00
fix(epic): a paused install keeps what it downloaded (#1504)
This commit is contained in:
1 parent
8156a9649d
commit
4dda1c1e52
7 files changed
+319
-25
No files matched your search
@@ -198,7 +198,7 @@ completes.
|
||||
|
||||
On-disk protocol: game files are staged in `<gameDir>/.droiddeck-epic-staging/*.stage` (owner
|
||||
marker `owner` = `DroidDeck Epic staging v1\n`) and renamed over their destination after size and
|
||||
SHA-1 checks; reusable chunks live, decompressed, as `<GUID>` (extra copies `<GUID>.<n>`, writes via
|
||||
SHA-1 checks; a paused run keeps each unfinished file there as `*.part` with its written parts in `*.part.list`; reusable chunks live, decompressed, as `<GUID>` (extra copies `<GUID>.<n>`, writes via
|
||||
`.part`) in `<gameDir>/.chunks` or the scratch directory.
|
||||
|
||||
## `com.droiddeck.launcher.stores.amazon.AmazonNative`
|
||||
|
||||
@@ -5,7 +5,7 @@ pub mod space;
|
||||
pub mod staging;
|
||||
pub mod writer;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
@@ -140,6 +140,17 @@ pub fn fetch_manifest(info: &BuildInfo, log: &dyn Fn(&str)) -> Result<Manifest,
|
||||
})
|
||||
}
|
||||
|
||||
fn resume_from(stage: &Path, parts: &[PlanPart], size: u64) -> Option<HashSet<u64>> {
|
||||
let meta = fs::symlink_metadata(stage).ok()?;
|
||||
if !meta.is_file() || meta.len() > size {
|
||||
return None;
|
||||
}
|
||||
let list = fs::read_to_string(staging::list_path(stage)).ok()?;
|
||||
let kept: HashSet<u64> = list.lines().map(|l| l.parse().ok()).collect::<Option<_>>()?;
|
||||
let fits = |o: &u64| parts.iter().any(|p| p.file_offset == *o && p.file_offset + p.size as u64 <= meta.len());
|
||||
(!kept.is_empty() && kept.iter().all(fits)).then_some(kept)
|
||||
}
|
||||
|
||||
pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finished, Stop> {
|
||||
let failed = |m: String| Stop::Failed(m);
|
||||
ev.status("Downloading the manifest…");
|
||||
@@ -191,6 +202,7 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
}
|
||||
let pending_bytes: u64 = pending.iter().map(|&(i, _)| manifest.files[i].size).sum();
|
||||
ev.sizes(pending_bytes, disk_bytes);
|
||||
let mut partials = staging::partials(&stage_dir);
|
||||
|
||||
let mut finished = Finished {
|
||||
build_version: manifest.build_version.clone(),
|
||||
@@ -211,6 +223,9 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
staging::remove_if_idle(&stage_dir);
|
||||
};
|
||||
if pending.is_empty() {
|
||||
for stale in partials.values() {
|
||||
staging::remove_partial(stale);
|
||||
}
|
||||
ev.log("every selected file is already up to date");
|
||||
clean_caches();
|
||||
ev.installing(disk_bytes, disk_bytes, total_selected, total_selected);
|
||||
@@ -219,6 +234,8 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
|
||||
let mut plan_files = Vec::with_capacity(pending.len());
|
||||
let mut engine_files = Vec::with_capacity(pending.len());
|
||||
let mut resumed_bytes = 0u64;
|
||||
let mut kept_sizes = Vec::with_capacity(pending.len());
|
||||
for &(i, old_size) in &pending {
|
||||
let f = &manifest.files[i];
|
||||
let mut parts = Vec::with_capacity(f.parts.len());
|
||||
@@ -234,6 +251,22 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
parts.push(PlanPart { chunk: c as u32, chunk_offset: part.offset, size: part.size, file_offset: offset });
|
||||
offset += part.size as u64;
|
||||
}
|
||||
let mut resume = None;
|
||||
let mut kept_size = 0u64;
|
||||
if let Some(stage) = partials.remove(&staging::partial_name(&f.path, f.size, f.sha1)) {
|
||||
match resume_from(&stage, &parts, f.size).filter(|_| f.sha1.is_some()) {
|
||||
Some(kept) => {
|
||||
let bytes: u64 = parts.iter().filter(|p| kept.contains(&p.file_offset)).map(|p| p.size as u64).sum();
|
||||
ev.log(&format!("resuming {} with {bytes} of {} bytes written", f.path, f.size));
|
||||
resumed_bytes += bytes;
|
||||
kept_size = bytes;
|
||||
parts.retain(|p| !kept.contains(&p.file_offset));
|
||||
resume = Some(writer::Resume { stage, kept: kept.into_iter().collect() });
|
||||
}
|
||||
None => staging::remove_partial(&stage),
|
||||
}
|
||||
}
|
||||
kept_sizes.push(kept_size);
|
||||
plan_files.push(PlanFile { size: f.size, parts: parts.clone() });
|
||||
engine_files.push(writer::EngineFile {
|
||||
path: f.path.clone(),
|
||||
@@ -242,8 +275,14 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
sha1: f.sha1,
|
||||
replacing: old_size > 0,
|
||||
parts,
|
||||
resume,
|
||||
});
|
||||
}
|
||||
for stale in partials.values() {
|
||||
staging::remove_partial(stale);
|
||||
}
|
||||
let fresh = pending.iter().all(|&(_, old_size)| old_size == 0);
|
||||
let progress_base = resumed_bytes + if fresh { disk_bytes - pending_bytes } else { 0 };
|
||||
|
||||
let scratch = req
|
||||
.scratch_dir
|
||||
@@ -252,7 +291,8 @@ pub fn run(req: &Request, cancel: &AtomicBool, ev: &dyn Events) -> Result<Finish
|
||||
.filter(|s| !space::same_volume(s, &req.game_dir));
|
||||
let sizes: Vec<space::PendingSize> = pending
|
||||
.iter()
|
||||
.map(|&(i, old)| space::PendingSize { new_size: manifest.files[i].size, old_size: old })
|
||||
.zip(&kept_sizes)
|
||||
.map(|(&(i, old), kept)| space::PendingSize { new_size: manifest.files[i].size - kept, old_size: old })
|
||||
.collect();
|
||||
let free_game = space::free_bytes(&req.game_dir);
|
||||
let scratch_space = scratch.as_ref().map(|s| space::Scratch { free: space::free_bytes(s) });
|
||||
@@ -308,6 +348,8 @@ cache_budget_mib={} overlap_mib={} free_game={free_game} workers={} process_work
|
||||
process_workers: req.process_workers,
|
||||
ca_bundle: req.ca_bundle.clone(),
|
||||
user_agent: net::user_agent(),
|
||||
progress_base,
|
||||
progress_total: progress_base + pending_bytes - resumed_bytes,
|
||||
};
|
||||
let stats = writer::run(&input, cancel, ev)?;
|
||||
finished.bytes_written = stats.bytes_written;
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
use crate::epic::manifest::Sha1 as Hash;
|
||||
|
||||
pub const DIR_NAME: &str = ".droiddeck-epic-staging";
|
||||
pub const OWNER_FILE: &str = "owner";
|
||||
pub const OWNER_MARKER: &[u8] = b"DroidDeck Epic staging v1\n";
|
||||
const SUFFIX: &str = ".stage";
|
||||
pub const PARTIAL_SUFFIX: &str = ".part";
|
||||
const LIST_SUFFIX: &str = ".part.list";
|
||||
|
||||
static SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
@@ -15,6 +22,37 @@ pub fn conflicts(manifest_path: &str) -> bool {
|
||||
first.eq_ignore_ascii_case(DIR_NAME)
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
pub fn partial_name(path: &str, size: u64, sha1: Option<Hash>) -> String {
|
||||
let id: Hash = Sha1::digest(path.as_bytes()).into();
|
||||
let content = sha1.map(|h| hex(&h)).unwrap_or_else(|| "0".into());
|
||||
format!("{}-{content}-{size}{PARTIAL_SUFFIX}", hex(&id[..8]))
|
||||
}
|
||||
|
||||
pub fn list_path(partial: &Path) -> PathBuf {
|
||||
let mut name = partial.as_os_str().to_owned();
|
||||
name.push(".list");
|
||||
PathBuf::from(name)
|
||||
}
|
||||
|
||||
pub fn remove_partial(partial: &Path) {
|
||||
let _ = fs::remove_file(partial);
|
||||
let _ = fs::remove_file(list_path(partial));
|
||||
}
|
||||
|
||||
pub fn partials(dir: &Path) -> HashMap<String, PathBuf> {
|
||||
let Ok(entries) = fs::read_dir(dir) else { return HashMap::new() };
|
||||
entries
|
||||
.flatten()
|
||||
.map(|e| e.file_name().to_string_lossy().into_owned())
|
||||
.filter(|n| n.ends_with(PARTIAL_SUFFIX))
|
||||
.map(|n| (n.clone(), dir.join(n)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn unique_name() -> String {
|
||||
let nanos = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -82,6 +120,8 @@ pub fn recover(game_dir: &Path) -> Result<PathBuf, String> {
|
||||
} else if name.ends_with(SUFFIX) {
|
||||
stale.push(path);
|
||||
others += 1;
|
||||
} else if name.ends_with(PARTIAL_SUFFIX) || name.ends_with(LIST_SUFFIX) {
|
||||
others += 1;
|
||||
} else {
|
||||
return refuse("it holds an unexpected file");
|
||||
}
|
||||
@@ -130,8 +170,13 @@ mod tests {
|
||||
fs::write(dir.join("file-1.stage"), b"x").unwrap();
|
||||
fs::write(dir.join("stream-1-2-3.stage"), b"y").unwrap();
|
||||
fs::write(game.join("keep.stage"), b"z").unwrap();
|
||||
let partial = partial_name("Game/a.pak", 10, None);
|
||||
fs::write(dir.join(&partial), b"p").unwrap();
|
||||
fs::write(list_path(&dir.join(&partial)), b"0\n").unwrap();
|
||||
recover(&game).unwrap();
|
||||
assert_eq!(fs::read_dir(&dir).unwrap().count(), 1);
|
||||
assert_eq!(fs::read_dir(&dir).unwrap().count(), 3);
|
||||
assert_eq!(partials(&dir).into_keys().collect::<Vec<_>>(), vec![partial.clone()]);
|
||||
remove_partial(&dir.join(&partial));
|
||||
assert!(game.join("keep.stage").exists());
|
||||
remove_if_idle(&dir);
|
||||
assert!(!dir.exists());
|
||||
@@ -170,5 +215,9 @@ mod tests {
|
||||
let (a, b) = (unique_name(), unique_name());
|
||||
assert_ne!(a, b);
|
||||
assert!(a.ends_with(".stage"));
|
||||
let p = partial_name("Game/a.pak", 10, Some([0xab; 20]));
|
||||
assert!(p.ends_with(&format!("-{}-10.part", "ab".repeat(20))));
|
||||
assert_ne!(p, partial_name("Game/b.pak", 10, Some([0xab; 20])));
|
||||
assert_ne!(p, partial_name("Game/a.pak", 10, Some([0xac; 20])));
|
||||
}
|
||||
}
|
||||
@@ -59,6 +59,7 @@ struct Recorder {
|
||||
replaced: AtomicUsize,
|
||||
refuse: bool,
|
||||
last_progress: Mutex<(u64, u64)>,
|
||||
first_progress: Mutex<Option<(u64, u64)>>,
|
||||
sizes: Mutex<(u64, u64)>,
|
||||
}
|
||||
|
||||
@@ -72,6 +73,7 @@ impl Events for Recorder {
|
||||
*self.sizes.lock().unwrap() = (pending, disk);
|
||||
}
|
||||
fn progress(&self, written: u64, total: u64, _f: u32, _ft: u32) {
|
||||
self.first_progress.lock().unwrap().get_or_insert((written, total));
|
||||
let mut p = self.last_progress.lock().unwrap();
|
||||
if written >= p.0 {
|
||||
*p = (written, total);
|
||||
@@ -262,6 +264,85 @@ fn a_cancelled_run_reports_cancel() {
|
||||
let _ = fs::remove_dir_all(&game);
|
||||
}
|
||||
|
||||
fn place_partial(fx: &Fixture, game: &Path, data: &[u8], list: &str) -> PathBuf {
|
||||
let stage = staging::recover(game).unwrap();
|
||||
let a = &fx.expected[0].1;
|
||||
let partial = stage.join(staging::partial_name("a.bin", 4000, Some(Sha1::digest(a).into())));
|
||||
fs::write(&partial, data).unwrap();
|
||||
fs::write(staging::list_path(&partial), list).unwrap();
|
||||
partial
|
||||
}
|
||||
|
||||
fn place_done_c(fx: &Fixture, game: &Path) {
|
||||
fs::create_dir_all(game.join("b")).unwrap();
|
||||
fs::write(game.join("b/c.bin"), &fx.expected[1].1).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_paused_install_resumes_from_its_written_parts() {
|
||||
let fx = fixture();
|
||||
let game = temp_game("resume");
|
||||
place_done_c(&fx, &game);
|
||||
place_partial(&fx, &game, &fx.expected[0].1[..3000], "0\n");
|
||||
let stage = game.join(staging::DIR_NAME);
|
||||
fs::write(stage.join(staging::partial_name("gone.bin", 9, None)), b"old").unwrap();
|
||||
|
||||
let events = Recorder::default();
|
||||
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
|
||||
for (path, bytes) in &fx.expected {
|
||||
assert_eq!(&fs::read(game.join(path)).unwrap(), bytes, "{path}");
|
||||
}
|
||||
assert_eq!(fx.hits.load(Ordering::Relaxed), 2, "the manifest and the one chunk still missing");
|
||||
assert_eq!(*events.first_progress.lock().unwrap(), Some((2096 + 3000, 6196)));
|
||||
assert_eq!(*events.last_progress.lock().unwrap(), (6196, 6196));
|
||||
assert!(events.logs.lock().unwrap().iter().any(|l| l == "resuming a.bin with 3000 of 4000 bytes written"));
|
||||
assert!(!stage.exists());
|
||||
let _ = fs::remove_dir_all(&game);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_part_written_out_of_order_is_kept_too() {
|
||||
let fx = fixture();
|
||||
let game = temp_game("resume-tail");
|
||||
place_done_c(&fx, &game);
|
||||
let mut data = vec![0u8; 3000];
|
||||
data.extend_from_slice(&fx.expected[0].1[3000..]);
|
||||
place_partial(&fx, &game, &data, "3000\n");
|
||||
let events = Recorder::default();
|
||||
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
|
||||
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
|
||||
assert_eq!(*events.first_progress.lock().unwrap(), Some((2096 + 1000, 6196)));
|
||||
assert!(!game.join(staging::DIR_NAME).exists());
|
||||
let _ = fs::remove_dir_all(&game);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partial_that_no_longer_matches_is_dropped() {
|
||||
let fx = fixture();
|
||||
let game = temp_game("badpartial");
|
||||
place_partial(&fx, &game, &fx.expected[0].1[..1234], "1234\n");
|
||||
let events = Recorder::default();
|
||||
run(&request(&fx, &game), &AtomicBool::new(false), &events).unwrap();
|
||||
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
|
||||
assert_eq!(*events.first_progress.lock().unwrap(), Some((0, 6196)));
|
||||
assert!(!game.join(staging::DIR_NAME).exists());
|
||||
let _ = fs::remove_dir_all(&game);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_partial_fails_its_check_and_is_not_kept() {
|
||||
let fx = fixture();
|
||||
let game = temp_game("damagedpartial");
|
||||
place_partial(&fx, &game, &[0u8; 3000], "0\n");
|
||||
let err = run(&request(&fx, &game), &AtomicBool::new(false), &Recorder::default()).unwrap_err();
|
||||
assert_eq!(err, Stop::Failed("SHA-1 mismatch for a.bin".into()));
|
||||
assert!(staging::partials(&game.join(staging::DIR_NAME)).is_empty());
|
||||
let retry = Recorder::default();
|
||||
run(&request(&fx, &game), &AtomicBool::new(false), &retry).unwrap();
|
||||
assert_eq!(fs::read(game.join("a.bin")).unwrap(), fx.expected[0].1);
|
||||
let _ = fs::remove_dir_all(&game);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guid_names_are_cache_names() {
|
||||
let c = chunk(5, 10);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -25,6 +25,12 @@ pub struct EngineFile {
|
||||
pub sha1: Option<Hash>,
|
||||
pub replacing: bool,
|
||||
pub parts: Vec<PlanPart>,
|
||||
pub resume: Option<Resume>,
|
||||
}
|
||||
|
||||
pub struct Resume {
|
||||
pub stage: PathBuf,
|
||||
pub kept: Vec<u64>,
|
||||
}
|
||||
|
||||
pub struct EngineInput<'a> {
|
||||
@@ -40,6 +46,8 @@ pub struct EngineInput<'a> {
|
||||
pub process_workers: usize,
|
||||
pub ca_bundle: String,
|
||||
pub user_agent: String,
|
||||
pub progress_base: u64,
|
||||
pub progress_total: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -120,7 +128,7 @@ impl<'a> Engine<'a> {
|
||||
return;
|
||||
}
|
||||
self.events.progress(
|
||||
self.written.load(Ordering::Relaxed),
|
||||
self.input.progress_base + self.written.load(Ordering::Relaxed),
|
||||
self.total_bytes,
|
||||
self.files_done.load(Ordering::Relaxed) as u32,
|
||||
self.input.files.len() as u32,
|
||||
@@ -232,10 +240,14 @@ impl<'a> Engine<'a> {
|
||||
.sync_all()
|
||||
.map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))?;
|
||||
let len = handle.metadata().map(|m| m.len()).unwrap_or(u64::MAX);
|
||||
drop(handle);
|
||||
if len != file.size {
|
||||
return Err(format!("Size mismatch for {}", file.path));
|
||||
}
|
||||
let hash = match (hash, &file.resume) {
|
||||
(None, Some(_)) if file.sha1.is_some() => Some(hash_whole(&handle, file)?),
|
||||
(hash, _) => hash,
|
||||
};
|
||||
drop(handle);
|
||||
if let (Some(expected), Some(got)) = (file.sha1, hash) {
|
||||
if expected != got {
|
||||
return Err(format!("SHA-1 mismatch for {}", file.path));
|
||||
@@ -260,6 +272,9 @@ impl<'a> Engine<'a> {
|
||||
fs::rename(&stage, &file.dest)
|
||||
.map_err(|e| format!("Could not assemble or atomically replace {}: {e}", file.path))
|
||||
})();
|
||||
if file.resume.is_some() {
|
||||
let _ = fs::remove_file(staging::list_path(&stage));
|
||||
}
|
||||
if let Err(e) = result {
|
||||
let _ = fs::remove_file(&stage);
|
||||
return Err(e);
|
||||
@@ -278,13 +293,16 @@ impl<'a> Engine<'a> {
|
||||
}
|
||||
|
||||
fn open_stage(&self, f: usize) -> Result<(), String> {
|
||||
let path = self.input.stage_dir.join(staging::unique_name());
|
||||
let handle = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|e| format!("Could not stage {}: {e}", self.input.files[f].path))?;
|
||||
let file = &self.input.files[f];
|
||||
let (path, opened) = match &file.resume {
|
||||
Some(r) => (r.stage.clone(), OpenOptions::new().read(true).write(true).open(&r.stage)),
|
||||
None => {
|
||||
let path = self.input.stage_dir.join(staging::unique_name());
|
||||
let opened = OpenOptions::new().read(true).write(true).create_new(true).open(&path);
|
||||
(path, opened)
|
||||
}
|
||||
};
|
||||
let handle = opened.map_err(|e| format!("Could not stage {}: {e}", file.path))?;
|
||||
let mut slot = self.slots[f].lock().expect("slot lock");
|
||||
slot.file = Some(Arc::new(handle));
|
||||
slot.stage = Some(path);
|
||||
@@ -439,7 +457,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
|
||||
stage: None,
|
||||
written: vec![false; f.parts.len()],
|
||||
next_hash: 0,
|
||||
hasher: f.sha1.map(|_| Sha1::new()),
|
||||
hasher: f.sha1.filter(|_| f.resume.is_none()).map(|_| Sha1::new()),
|
||||
left: f.parts.len(),
|
||||
done: false,
|
||||
})
|
||||
@@ -466,7 +484,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
|
||||
started: Instant::now(),
|
||||
replace_ok: Mutex::new(None),
|
||||
events,
|
||||
total_bytes: input.files.iter().map(|f| f.size).sum(),
|
||||
total_bytes: input.progress_total,
|
||||
};
|
||||
for (i, p) in schedule.producers.iter().enumerate() {
|
||||
if p.preloaded && remaining[i] == 0 {
|
||||
@@ -503,7 +521,7 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
|
||||
stream: false,
|
||||
};
|
||||
|
||||
events.progress(0, engine.total_bytes, 0, input.files.len() as u32);
|
||||
events.progress(input.progress_base, engine.total_bytes, 0, input.files.len() as u32);
|
||||
let log = |line: &str| events.log(line);
|
||||
let gate = |i: usize| -> bool {
|
||||
engine.hint.fetch_max(i, Ordering::Relaxed);
|
||||
@@ -546,12 +564,14 @@ pub fn run(input: &EngineInput<'_>, stop: &AtomicBool, events: &dyn Events) -> R
|
||||
let complete = failure.is_none() && engine.files_done.load(Ordering::Relaxed) == input.files.len();
|
||||
|
||||
if !complete {
|
||||
for slot in &engine.slots {
|
||||
for (file, slot) in input.files.iter().zip(&engine.slots) {
|
||||
let mut s = slot.lock().expect("slot lock");
|
||||
if !s.done {
|
||||
s.file.take();
|
||||
if let Some(stage) = s.stage.take() {
|
||||
let _ = fs::remove_file(stage);
|
||||
if failure.is_some() || !keep_partial(file, &s.written, &stage, &input.stage_dir) {
|
||||
let _ = fs::remove_file(stage);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -588,6 +608,100 @@ fetches={} refetches={} peak_cache_mib={} hosts={}",
|
||||
Err(Stop::Failed("The download ended before every file was written".into()))
|
||||
}
|
||||
|
||||
fn hash_whole(handle: &File, file: &EngineFile) -> Result<Hash, String> {
|
||||
let mut h = Sha1::new();
|
||||
let mut buf = vec![0u8; 1024 * 1024];
|
||||
let mut at = 0u64;
|
||||
while at < file.size {
|
||||
let n = buf.len().min((file.size - at) as usize);
|
||||
handle
|
||||
.read_exact_at(&mut buf[..n], at)
|
||||
.map_err(|e| format!("Could not read back {}: {e}", file.path))?;
|
||||
h.update(&buf[..n]);
|
||||
at += n as u64;
|
||||
}
|
||||
Ok(h.finalize().into())
|
||||
}
|
||||
|
||||
fn keep_partial(file: &EngineFile, written: &[bool], stage: &Path, dir: &Path) -> bool {
|
||||
let mut kept: Vec<u64> = file.resume.as_ref().map(|r| r.kept.clone()).unwrap_or_default();
|
||||
kept.extend(file.parts.iter().zip(written).filter(|(_, w)| **w).map(|(p, _)| p.file_offset));
|
||||
if kept.is_empty() || file.sha1.is_none() {
|
||||
return false;
|
||||
}
|
||||
kept.sort_unstable();
|
||||
let partial = dir.join(staging::partial_name(&file.path, file.size, file.sha1));
|
||||
let list: String = kept.iter().map(|o| format!("{o}\n")).collect();
|
||||
let synced = OpenOptions::new().write(true).open(stage).and_then(|h| h.sync_all());
|
||||
if synced.is_err() || fs::rename(stage, &partial).is_err() {
|
||||
return false;
|
||||
}
|
||||
let resumed = file.resume.as_ref().is_some_and(|r| r.stage == partial);
|
||||
let temp = dir.join(staging::unique_name());
|
||||
let listed = fs::write(&temp, list).and_then(|_| fs::rename(&temp, staging::list_path(&partial))).is_ok();
|
||||
if !listed {
|
||||
let _ = fs::remove_file(&temp);
|
||||
if !resumed {
|
||||
let _ = fs::remove_file(&partial);
|
||||
}
|
||||
}
|
||||
listed || resumed
|
||||
}
|
||||
|
||||
fn tidy_fetch_error(message: &str) -> String {
|
||||
message.strip_prefix("epic: ").unwrap_or(message).to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn part(file_offset: u64, size: u32) -> PlanPart {
|
||||
PlanPart { chunk: 0, chunk_offset: 0, size, file_offset }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pause_keeps_every_written_part_and_lists_them() {
|
||||
let dir = std::env::temp_dir().join(format!("dd-epic-keep-{}", std::process::id()));
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
let file = EngineFile {
|
||||
path: "Game/a.pak".into(),
|
||||
dest: dir.join("a.pak"),
|
||||
size: 30,
|
||||
sha1: Some([1; 20]),
|
||||
replacing: false,
|
||||
parts: vec![part(0, 10), part(10, 10), part(20, 10)],
|
||||
resume: None,
|
||||
};
|
||||
let kept = dir.join(staging::partial_name("Game/a.pak", 30, Some([1; 20])));
|
||||
let stage = dir.join("x.stage");
|
||||
|
||||
fs::write(&stage, [7u8; 30]).unwrap();
|
||||
assert!(!keep_partial(&file, &[false, false, false], &stage, &dir));
|
||||
let unhashed = EngineFile {
|
||||
path: file.path.clone(),
|
||||
dest: file.dest.clone(),
|
||||
size: 30,
|
||||
sha1: None,
|
||||
replacing: false,
|
||||
parts: file.parts.clone(),
|
||||
resume: None,
|
||||
};
|
||||
assert!(!keep_partial(&unhashed, &[true, true, true], &stage, &dir));
|
||||
assert!(stage.exists());
|
||||
assert!(keep_partial(&file, &[false, true, true], &stage, &dir));
|
||||
assert_eq!(fs::read(&kept).unwrap(), [7u8; 30]);
|
||||
assert_eq!(fs::read_to_string(staging::list_path(&kept)).unwrap(), "10\n20\n");
|
||||
assert!(!stage.exists());
|
||||
|
||||
let resumed = EngineFile {
|
||||
parts: vec![part(0, 10)],
|
||||
resume: Some(Resume { stage: kept.clone(), kept: vec![20, 10] }),
|
||||
..file
|
||||
};
|
||||
assert!(keep_partial(&resumed, &[false], &kept, &dir));
|
||||
assert_eq!(fs::read_to_string(staging::list_path(&kept)).unwrap(), "10\n20\n");
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user