//! Update transaction: Prepare -> Verify -> Snapshot -> Apply -> Migrate -> Validate -> Commit. //! //! Nothing is changed until the signed package is downloaded, verified and extracted into staging. Every file that is //! about to change is first copied into the transaction's `before/` folder, each operation is journaled, and a crash or //! failure at any point leaves the installation recoverable: `rollback` restores binaries and configuration exactly. //! Full recovery points protect database changes as well as files. Interrupted updates must be resolved first. use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::config::{merge, parser::ConfFile}; use crate::download::Cancel; use crate::error::{Error, Result}; use crate::fsx; use crate::manifest::{FileEntry, Kind, Manifest, ReplacePolicy}; use crate::migrations::ApplyReport; use crate::package; use crate::pkgsource::{fetch_manifest, fetch_parts, Source}; use crate::registry::{InstallMeta, MetaDir}; const STAGED_PREFIX: &str = "_migrations/"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum State { Prepared, Applying, /// Files replaced; waiting for migrations / health check. Applied, /// Files replaced but the new build did not become healthy; the owner decides (rollback is offered). NeedsDecision, Committed, RolledBack, Failed, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum Action { Create, Replace, MergeConfig, Skip, Conflict, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Op { pub path: String, pub action: Action, pub reason: Option, /// sha256 the file will have after the operation. pub new_sha256: String, pub had_previous: bool, pub started: bool, pub done: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum Resolution { Keep, Replace, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Txn { pub id: String, pub state: State, pub from_version: Option, pub to_version: String, pub recovery_point: Option, #[serde(default = "legacy_databases_started")] pub databases_started: bool, pub ops: Vec, pub message: Option, #[serde(default)] pub recovery_hashes: BTreeMap, } fn legacy_databases_started() -> bool { true } /// Everything the transaction needs from the outside world; tests provide a fake. pub trait Env { fn activate(&self) -> Result<()> { Ok(()) } fn recover_validation(&self) -> Result<()> { Ok(()) } fn automatic_rollback(&self) -> bool { false } fn validate_recovery(&self) -> Result<()> { self.validate() } fn ensure_stopped(&self) -> Result<()>; fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) } fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) } fn snapshot(&self) -> Result; fn restore_snapshot(&self, id: &str) -> Result<()>; fn migrate(&self, manifest: &Manifest, staged_migrations: &Path) -> Result; /// Start the server and confirm it becomes healthy. fn validate(&self) -> Result<()>; } fn updates_dir(meta: &Path) -> PathBuf { meta.join("updates") } fn txn_dir(meta: &Path, id: &str) -> Result { if id.is_empty() || !id .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) { return Err(Error::PathRejected(format!("bad update id {id:?}"))); } Ok(updates_dir(meta).join(id)) } fn save(meta: &Path, t: &Txn) -> Result<()> { fsx::atomic_write_json(&txn_dir(meta, &t.id)?.join("txn.json"), t) } pub fn load(meta: &Path, id: &str) -> Result { read_journal(&txn_dir(meta, id)?.join("txn.json")) } fn read_journal(path: &Path) -> Result { let mut txn: Txn = fsx::read_json(path).map_err(|e| { Error::Invalid(format!( "Update journal {} cannot be read: {e}. Available recovery copies were preserved.", path.display() )) })?; if path .parent() .and_then(Path::file_name) .and_then(|id| id.to_str()) != Some(txn.id.as_str()) { return Err(Error::Invalid( "Update journal identity is inconsistent.".into(), )); } txn_dir(path.parent().unwrap(), &txn.id)?; if let Some(id) = &txn.recovery_point { if id.is_empty() || !id .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) || id.contains("..") { return Err(Error::Invalid( "Update journal contains an invalid recovery point identity.".into(), )); } } let valid_hash = |hash: &str| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit()); let mut completed_legacy_repair = false; // Older successful repairs saved Committed without setting done. Recover only that // specific shape, and only if every repaired file still has its expected bytes. if txn.state == State::Committed && txn.id.contains("-repair-") && txn.message.as_deref() == Some("Repair is in progress.") && txn.from_version.as_deref() == Some(txn.to_version.as_str()) && txn.databases_started && txn.recovery_point.is_some() && txn.recovery_hashes.contains_key("@install") && txn.ops.iter().any(|op| !op.done) && txn.ops.iter().all(|op| { op.started && matches!( (op.action, op.had_previous), (Action::Create, false) | (Action::Replace, true) ) && valid_hash(&op.new_sha256) }) { let meta_dir = path .parent() .and_then(Path::parent) .and_then(Path::parent) .ok_or_else(|| Error::Invalid("Invalid repair journal location.".into()))?; let (_, installed) = MetaDir::open(meta_dir)?; let root = Path::new(&installed.server_path); if fsx::canonicalize_lenient(&crate::registry::metadata_dir_for(root)?)? != fsx::canonicalize_lenient(meta_dir)? { return Err(Error::Invalid( "The repair journal does not belong to its installed server.".into(), )); } complete_repair_ops(root, &mut txn)?; completed_legacy_repair = true; } let mut seen = std::collections::BTreeSet::new(); for op in &txn.ops { fsx::safe_join(path.parent().unwrap(), &op.path)?; if !seen.insert(op.path.replace('\\', "/").to_ascii_lowercase()) || !valid_hash(&op.new_sha256) || (op.done && !op.started && !matches!(op.action, Action::Skip | Action::Conflict)) || (txn.state == State::Prepared && op.started) || (txn.state == State::Committed && !op.done) { return Err(Error::Invalid( "Update journal contains inconsistent file operations.".into(), )); } } if txn.recovery_hashes.values().any(|hash| !valid_hash(hash)) { return Err(Error::Invalid( "Update journal contains invalid recovery hashes.".into(), )); } for key in txn .recovery_hashes .keys() .filter(|key| !matches!(key.as_str(), "@install" | "@point")) { fsx::safe_join(path.parent().unwrap(), key)?; } if completed_legacy_repair { // Persist the verified completion once: later legitimate updates can change // these files. Retain the original record for diagnostics and crash recovery. let original = path.with_file_name("txn-before-repair-completion.json"); if !original.exists() { fsx::atomic_write(&original, &fs::read(path)?)?; } fsx::atomic_write_json(path, &txn)?; } Ok(txn) } #[derive(Serialize, Deserialize)] struct CleanupMarker { id: String, state: State, } fn cleanup_marker(meta: &Path, id: &str) -> Result { txn_dir(meta, id)?; Ok(meta.join("update-cleanup").join(format!("{id}.json"))) } fn journals(meta: &Path) -> Result> { let entries = match fs::read_dir(updates_dir(meta)) { Ok(entries) => entries, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(vec![]), Err(e) => return Err(e.into()), }; let mut all = Vec::new(); for entry in entries { let entry = entry?; if entry.file_type()?.is_symlink() { return Err(Error::Invalid( "Linked update storage cannot be recovered safely.".into(), )); } if !entry.file_type()?.is_dir() { continue; } let dir = fsx::ensure_within(meta, &entry.path())?; let path = dir.join("txn.json"); if !path.is_file() { let id = entry.file_name().to_string_lossy().into_owned(); let marker = cleanup_marker(meta, &id)?; if fsx::read_json::(&marker).is_ok_and(|m| { m.id == id && matches!(m.state, State::Committed | State::RolledBack) }) { continue; } return Err(Error::Invalid(format!("Update {id} has no journal. Preserve its recovery copies and inspect it before starting or changing the server."))); } all.push(read_journal(&path)?); } all.sort_by(|a, b| b.id.cmp(&a.id)); Ok(all) } pub fn pending_checked(meta: &Path) -> Result> { Ok(journals(meta)?.into_iter().find(|t| { !matches!( t.state, State::Committed | State::RolledBack | State::Prepared ) && (t.state != State::Failed || t.databases_started || t.ops.iter().any(|o| o.started)) })) } /// A transaction that started but never reached a final state (crash, power loss). The UI offers rollback. pub fn unfinished(meta: &Path) -> Option { pending_checked(meta).ok().flatten() } pub fn ensure_recovered(meta: &Path) -> Result<()> { if crate::update_isolation::pending(meta) { return Err(Error::Invalid( "Interrupted startup validation must be recovered before starting the server.".into(), )); } if let Some(t) = pending_checked(meta)? { return Err(Error::Invalid(format!( "Resolve unfinished update {} before changing or starting the server.", t.id ))); } Ok(()) } fn sha_if_exists(p: &Path) -> Option { p.is_file().then(|| fsx::sha256_file(p).ok()).flatten() } #[derive(Debug, Serialize)] pub struct PlanItem { pub path: String, pub action: Action, pub reason: Option, } /// What an update would do to this installation. Read-only. pub fn plan( root: &Path, meta: &InstallMeta, manifest: &Manifest, resolutions: &BTreeMap, staged: Option<&Path>, ) -> Result> { let mut out = Vec::new(); // An update is decided by version. The same version has the same files, so a file the owner edited (or removed) // is not an update: Repair lists such files and restores them on request. let same_version = meta .core .version .as_deref() .is_some_and(|installed| installed == manifest.version); for f in manifest .files .iter() .filter(|f| !f.path.starts_with(STAGED_PREFIX)) { let target = fsx::ensure_within(root, &fsx::safe_join(root, &f.path)?)?; if same_version { out.push(PlanItem { path: f.path.clone(), action: Action::Skip, reason: Some("same version; Repair restores changed files".into()), }); continue; } let cur = sha_if_exists(&target); let recorded = meta.original_hashes.get(&f.path); let (action, reason) = match (f.policy, &cur) { (ReplacePolicy::NeverTouch, _) => (Action::Skip, Some("managed by you".to_string())), (_, None) => (Action::Create, None), (ReplacePolicy::CreateIfMissing, Some(_)) => { (Action::Skip, Some("kept your existing file".into())) } (ReplacePolicy::MergeConfig, Some(_)) => { let would_add = match staged { Some(tree) => { let (a, b) = ( fs::read(&target)?, fs::read(fsx::safe_join(tree, &f.path)?)?, ); !merge::plan(&ConfFile::parse_bytes(&a)?, &ConfFile::parse_bytes(&b)?) .added .is_empty() } None => true, }; if would_add { ( Action::MergeConfig, Some("new settings are added; your values are kept".into()), ) } else { (Action::Skip, Some("no new settings".into())) } } (_, Some(c)) if c.eq_ignore_ascii_case(&f.sha256) => { (Action::Skip, Some("already up to date".into())) } (_, Some(_)) if f.path == "Scripts/manage.py" && staged.is_some_and(|tree| { match (fs::read_to_string(tree.join(&f.path)), fs::read(&target)) { (Ok(signed), Ok(actual)) => { fsx::sha256_bytes(signed.as_bytes()) == f.sha256 && crate::driver::launcher_matches(&signed, &actual) } _ => false, } }) => { ( Action::Skip, Some("already up to date with Manager integration".into()), ) } (ReplacePolicy::Replace | ReplacePolicy::ReplaceIfPristine, Some(c)) => { let manager_launcher = f.path == "Scripts/manage.py" && recorded.is_some_and(|hash| { fs::read(&target).is_ok_and(|bytes| { crate::driver::launcher_matches_recorded(hash, &bytes) }) }); if f.path == "Core/worldserver.exe" && meta.kind == crate::registry::InstallKind::Imported && crate::squid::imported_repack(root) { match resolutions.get(&f.path) { Some(Resolution::Replace) => (Action::Replace, Some("Replacing this binary stops the original SquidBots launcher/updater from accepting its hash.".into())), Some(Resolution::Keep) => (Action::Skip, Some("kept the original SquidBots binary".into())), None => (Action::Conflict, Some("This SquidBots repack verifies its worldserver hash. Replacing it stops Start_All_Bots/coa_update from working. Choose explicitly whether to replace this binary.".into())), } } else if recorded.map(|r| r.eq_ignore_ascii_case(c)).unwrap_or(false) || manager_launcher { (Action::Replace, None) } else if f.policy == ReplacePolicy::ReplaceIfPristine { ( Action::Skip, Some("modified outside CoA Server Manager; kept".into()), ) } else { match resolutions.get(&f.path) { Some(Resolution::Replace) => ( Action::Replace, Some("you chose to replace your modified file".into()), ), Some(Resolution::Keep) => ( Action::Skip, Some("you chose to keep your modified file".into()), ), None => ( Action::Conflict, Some("modified outside CoA Server Manager".into()), ), } } } }; out.push(PlanItem { path: f.path.clone(), action, reason, }); } Ok(out) } #[derive(Debug, Serialize)] pub struct Preview { pub from_version: Option, pub to_version: String, #[serde(skip_serializing_if = "Option::is_none")] pub latest_version: Option, pub items: Vec, pub conflicts: Vec, pub migrations: usize, pub pending_migrations: usize, pub download_bytes: u64, /// False when the pending SQL was not counted (database stopped during a background check, or an error). pub database_checked: bool, /// Why the database could not be inspected. The check still answers about the files; applying the update starts /// the database itself and reports its own failure. #[serde(skip_serializing_if = "Option::is_none")] pub database_check_error: Option, } /// Whether looking for an update may start the database to count pending SQL. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DatabaseAccess { /// An explicit check: start the database if it is stopped (and stop it again afterwards). Start, /// The background check that repeats every few minutes must not start and stop MySQL on a stopped server. OnlyIfRunning, } /// The update packages are those of the Windows repack (executables, a bundled MySQL, a launcher): applying one to a Docker /// server would put Windows files in a Linux server folder. Updating a Docker server is separate work. fn refuse_for_docker(root: &Path) -> Result<()> { if crate::docker::is_docker(root) { return Err(Error::Invalid( "Updates are not available for Docker servers yet.".into(), )); } Ok(()) } /// Describe the update; a managed launcher may need a one-time authenticated payload check. pub fn preview( root: &Path, meta: &InstallMeta, source: &Source, trusted_key: &str, resolutions: &BTreeMap, ) -> Result { preview_with( root, meta, source, trusted_key, resolutions, DatabaseAccess::Start, ) } pub fn preview_with( root: &Path, meta: &InstallMeta, source: &Source, trusted_key: &str, resolutions: &BTreeMap, access: DatabaseAccess, ) -> Result { refuse_for_docker(root)?; let meta_dir = crate::registry::metadata_dir_for(root)?; let observed_version = MetaDir::open(&meta_dir)?.1.core.version; let (m, _) = fetch_manifest(source, trusted_key)?; let _lock = operation_lock_for(&meta_dir, "checking updates", true)?; ensure_recovered(&meta_dir)?; let (_, current_meta) = MetaDir::open(&meta_dir)?; let meta = if current_meta.core.version != observed_version { ¤t_meta } else { meta }; check_manifest(&m)?; reject_downgrade(meta, &m)?; let mut items = plan(root, meta, &m, resolutions, None)?; if items .iter() .any(|item| item.path == "Scripts/manage.py" && item.action != Action::Skip) && launcher_already_integrated(root, meta, &m)? { if let Some(item) = items .iter_mut() .find(|item| item.path == "Scripts/manage.py") { item.action = Action::Skip; item.reason = Some("already up to date with Manager integration".into()); } } // Counting pending SQL needs the database. If it cannot be started the files can still be compared, so report // that instead of failing the whole check. let (pending_migrations, database_checked, database_check_error) = match (RepackEnv { root, meta_dir: &meta_dir, }) .pending_migrations_with(&m, access) { Ok(Some(count)) => (count, true, None), Ok(None) => (m.migrations.len(), false, None), Err(error) => { tracing::warn!(%error, "update check: the database could not be inspected"); (m.migrations.len(), false, Some(error.to_string())) } }; Ok(Preview { from_version: meta.core.version.clone(), to_version: m.version.clone(), latest_version: None, conflicts: items .iter() .filter(|i| i.action == Action::Conflict) .map(|i| i.path.clone()) .collect(), items, migrations: m.migrations.len(), pending_migrations, download_bytes: m .archive .as_ref() .map(|a| a.parts.iter().map(|p| p.size).sum()) .unwrap_or(0), database_checked, database_check_error, }) } fn launcher_already_integrated( root: &Path, meta: &InstallMeta, manifest: &Manifest, ) -> Result { let Some(file) = manifest .files .iter() .find(|file| file.path == "Scripts/manage.py") else { return Ok(false); }; let Ok(actual) = fs::read(root.join(&file.path)) else { return Ok(false); }; if meta.original_hashes.get(&file.path) != Some(&fsx::sha256_bytes(&actual)) { return Ok(false); } Ok(crate::driver::launcher_matches_recorded( &file.sha256, &actual, )) } fn reject_downgrade(meta: &InstallMeta, manifest: &Manifest) -> Result<()> { if let Some(c) = &manifest.compatibility { let platform = if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" }; if c.platform != platform { return Err(Error::Invalid(format!( "This package targets {} rather than {platform}.", c.platform ))); } // The source list records release qualification, not an exclusive upgrade route. // Database history and target validation decide whether the update can complete. if meta .core .version .as_deref() .and_then(crate::manifest::parse_version) .is_none() { return Err(Error::Invalid("The installed server version is unknown. An imported repack needs a verified starting state before updating.".into())); } } if let Some(installed) = meta.core.version.as_deref() { if let (Some(have), Some(candidate)) = ( crate::manifest::parse_version(installed), crate::manifest::parse_version(&manifest.version), ) { if candidate < have { return Err(Error::Invalid(format!("Installed server {installed} is newer than package {}. Use its recovery point to undo an update; older packages cannot be applied.", manifest.version))); } } } Ok(()) } fn check_manifest(m: &Manifest) -> Result<()> { if m.kind != Kind::Update { return Err(Error::Invalid("This is not an update package.".into())); } if !m.compatible_with_manager(crate::MANAGER_VERSION) { return Err(Error::Invalid( "This update needs a newer version of CoA Server Manager.".into(), )); } Ok(()) } pub fn validate_candidate(meta: &InstallMeta, manifest: &Manifest) -> Result<()> { check_manifest(manifest)?; reject_downgrade(meta, manifest) } pub struct Params<'a> { pub root: &'a Path, pub meta_dir: &'a Path, pub source: Source, pub trusted_key: &'a str, pub cancel: Cancel, pub resolutions: BTreeMap, pub env: &'a dyn Env, /// Test hook: fail after this many completed file operations. pub fail_after_ops: Option, } #[derive(Debug, Serialize)] pub struct Outcome { pub txn: Txn, pub migrations: Option, } fn step(report: &dyn Fn(&str, u8), name: &str, pct: u8) { tracing::info!(step = name, percent = pct, "server update progress"); report(name, pct); } /// Run the whole update. On failure before the new build is validated the files are rolled back automatically; /// a build that is applied but unhealthy is left in `NeedsDecision` so the owner chooses. pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { refuse_for_docker(p.root)?; let (root, meta_dir) = (p.root, p.meta_dir); let _lock = operation_lock(meta_dir)?; ensure_recovered(meta_dir)?; let (md, mut meta) = MetaDir::open(meta_dir)?; let _ = md; step(report, "Checking the update", 2); let (m, manifest_bytes) = fetch_manifest(&p.source, p.trusted_key)?; tracing::info!(root = %root.display(), from_version = ?meta.core.version, to_version = %m.version, migrations = m.migrations.len(), "server update selected"); check_manifest(&m)?; reject_downgrade(&meta, &m)?; p.env.preflight(&m)?; let archive = m .archive .clone() .ok_or_else(|| Error::InvalidManifest("no archive".into()))?; let id = format!( "{}-{}-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), m.version.replace('.', "_"), uuid::Uuid::new_v4().simple() ); let tdir = txn_dir(meta_dir, &id)?; let (tree, before) = (tdir.join("tree"), tdir.join("before")); fs::create_dir_all(&before)?; let signature = crate::pkgsource::fetch_small(&p.source, "manifest.json.sig")?; crate::signing::verify( &manifest_bytes, &String::from_utf8_lossy(&signature), p.trusted_key, )?; fsx::atomic_write(&tdir.join("manifest.json"), &manifest_bytes)?; fsx::atomic_write(&tdir.join("manifest.json.sig"), &signature)?; fsx::atomic_write( &before.join("manager-install.json"), &fs::read(meta_dir.join("install.json"))?, )?; let mut txn = Txn { id: id.clone(), state: State::Prepared, from_version: meta.core.version.clone(), to_version: m.version.clone(), recovery_point: None, databases_started: false, ops: Vec::new(), message: None, recovery_hashes: BTreeMap::new(), }; txn.recovery_hashes.insert( "@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?, ); save(meta_dir, &txn)?; // Nothing has touched the installation yet; failures up to here just discard the staging area. let staged = (|| -> Result { step(report, "Downloading the update", 5); let parts = fetch_parts( &p.source, &m, &meta_dir.join("staging").join("download"), &p.cancel, &|f, _| step(report, "Downloading the update", 5 + (f * 35.0) as u8), )?; step(report, "Verifying the update", 42); package::extract(&parts, &m, &tree, &|_, _| {})?; crate::migrations::verify_files(&m.migrations, &tree.join("_migrations"))?; Ok(tree.clone()) })(); let tree = match staged { Ok(t) => t, Err(e) => { txn.state = State::Failed; txn.message = Some(e.to_string()); let _ = save(meta_dir, &txn); let _ = fs::remove_dir_all(&tree); return Err(e); } }; let items = plan(root, &meta, &m, &p.resolutions, Some(&tree))?; if let Some(c) = items.iter().find(|i| i.action == Action::Conflict) { txn.state = State::Failed; txn.message = Some(format!("{} needs a decision", c.path)); save(meta_dir, &txn)?; let _ = fs::remove_dir_all(&tree); return Err(Error::Invalid(format!("Some files were modified outside CoA Server Manager and need your decision (for example {}).", c.path))); } let need: u64 = m .files .iter() .filter(|f| { items.iter().any(|i| { i.path == f.path && matches!( i.action, Action::Replace | Action::Create | Action::MergeConfig ) }) }) .map(|f| f.size) .sum(); fsx::require_space(root, need)?; fsx::require_space(meta_dir, need)?; step(report, "Making sure the server is stopped", 48); p.env.ensure_stopped()?; step(report, "Saving a recovery point", 52); txn.recovery_point = Some(p.env.snapshot().map_err(|e| { Error::Invalid(format!( "The update was not applied because the safety backup failed: {e}" )) })?); let point_meta = crate::backup::point_json(meta_dir, txn.recovery_point.as_deref().unwrap())?; if point_meta.is_file() { txn.recovery_hashes .insert("@point".into(), fsx::sha256_file(&point_meta)?); } save(meta_dir, &txn)?; txn.ops = items .iter() .filter(|i| i.action != Action::Skip) .map(|i| { let f = m .files .iter() .find(|f| f.path == i.path) .expect("planned from manifest"); Op { path: i.path.clone(), action: i.action, reason: i.reason.clone(), new_sha256: f.sha256.clone(), had_previous: fsx::safe_join(root, &i.path) .map(|t| t.is_file()) .unwrap_or(false), started: false, done: false, } }) .collect(); txn.state = State::Applying; save(meta_dir, &txn)?; step(report, "Applying the update", 58); let applied = apply_ops( root, &tree, &before, &m.files, &mut txn, meta_dir, p.fail_after_ops, ); if let Err(e) = applied { step(report, "Undoing the update", 70); return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), None); } txn.state = State::Applied; save(meta_dir, &txn)?; // MySQL DDL can commit before a later statement fails. Restore the full recovery point on failure. let mut migrated = None; if !m.migrations.is_empty() || m.files .iter() .any(|file| file.path == crate::schema_check::CONTRACT) { step(report, "Updating the database", 75); txn.databases_started = true; save(meta_dir, &txn)?; match p.env.migrate(&m, &tree.join("_migrations")) { Ok(r) if r.failed.is_none() => migrated = Some(r), Ok(r) => { let (mid, why) = r.failed.clone().unwrap(); step(report, "Undoing the update", 76); return fail_after_apply( p, &mut txn, &before, &tree, format!("Database update {mid} failed: {why}"), Some(r), ); } Err(e) => { step(report, "Undoing the update", 76); return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), None); } } } step(report, "Starting the updated server", 85); // Core startup can apply SQL and write character data even when the package has no explicit migrations. txn.databases_started = true; save(meta_dir, &txn)?; if let Err(e) = p.env.validate() { if p.env.automatic_rollback() { return fail_after_apply( p, &mut txn, &before, &tree, format!("The updated server did not start correctly: {e}"), migrated, ); } txn.state = State::NeedsDecision; txn.message = Some(format!("The updated server did not start correctly: {e}")); save(meta_dir, &txn)?; return Ok(Outcome { txn, migrations: migrated, }); } finish(root, meta_dir, &mut meta, &m, &tree, &mut txn)?; p.env.activate()?; step(report, "Done", 100); let _ = archive; Ok(Outcome { txn, migrations: migrated, }) } fn finish( root: &Path, meta_dir: &Path, meta: &mut InstallMeta, m: &Manifest, tree: &Path, txn: &mut Txn, ) -> Result<()> { meta.core.version = Some(m.version.clone()); meta.core.commit = m.core.commit.clone().or(meta.core.commit.clone()); for op in &txn.ops { let hash = if op.path == "Scripts/manage.py" { match ( fs::read_to_string(tree.join(&op.path)), fs::read(root.join(&op.path)), ) { (Ok(signed), Ok(actual)) if crate::driver::launcher_matches(&signed, &actual) => { fsx::sha256_bytes(&actual) } _ => op.new_sha256.clone(), } } else { op.new_sha256.clone() }; meta.original_hashes.insert(op.path.clone(), hash); if !meta.managed_files.contains(&op.path) { meta.managed_files.push(op.path.clone()); } } for file in &m.files { if !txn.ops.iter().any(|op| op.path == file.path) { let path = fsx::safe_join(root, &file.path)?; if path.is_file() && fsx::sha256_file(&path)? == file.sha256 { meta.original_hashes .insert(file.path.clone(), file.sha256.clone()); } } } fsx::atomic_write_json(&meta_dir.join("install.json"), meta)?; fsx::atomic_write( &meta_dir .join("manifests") .join(format!("update-{}.json", m.version)), &serde_json::to_vec_pretty(m)?, )?; txn.state = State::Committed; txn.message = None; save(meta_dir, txn)?; let _ = fs::remove_dir_all(tree); prune(meta_dir, 3); Ok(()) } pub fn retry_validation( root: &Path, meta_dir: &Path, id: &str, fallback: &Source, trusted_key: &str, env: &dyn Env, ) -> Result { let _lock = operation_lock(meta_dir)?; let mut txn = load(meta_dir, id)?; if txn.state != State::NeedsDecision || unfinished(meta_dir).is_none_or(|t| t.id != id) { return Err(Error::Invalid( "Only an applied update awaiting startup validation can be retried.".into(), )); } let dir = txn_dir(meta_dir, id)?; let source = if dir.join("manifest.json").is_file() { Source::Dir(dir.clone()) } else { fallback.clone() }; let (manifest, _) = fetch_manifest(&source, trusted_key)?; check_manifest(&manifest)?; if manifest.version != txn.to_version || txn.ops.iter().any(|op| !op.done) { return Err(Error::Invalid( "The saved update does not match the applied transaction.".into(), )); } for op in txn .ops .iter() .filter(|op| matches!(op.action, Action::Create | Action::Replace)) { let path = fsx::safe_join(root, &op.path)?; let file = manifest .files .iter() .find(|f| f.path == op.path) .ok_or_else(|| { Error::Invalid( "The update journal contains a file absent from the signed package.".into(), ) })?; if matches!( file.policy, ReplacePolicy::CreateIfMissing | ReplacePolicy::MergeConfig | ReplacePolicy::NeverTouch ) { continue; } let bytes = fs::read(&path)?; let actual = fsx::sha256_bytes(&bytes); let launcher = op.path == "Scripts/manage.py" && fs::read_to_string(dir.join("tree").join(&op.path)) .ok() .filter(|s| fsx::sha256_bytes(s.as_bytes()) == file.sha256) .is_some_and(|s| crate::driver::launcher_matches(&s, &bytes)); if actual != file.sha256 && !launcher { return Err(Error::Invalid(format!( "{} changed since the update was applied; use repair or rollback.", op.path ))); } } env.ensure_stopped()?; if let Err(error) = env.validate() { txn.message = Some(format!( "The updated server did not start correctly: {error}" )); save(meta_dir, &txn)?; return Ok(txn); } let (_, mut meta) = MetaDir::open(meta_dir)?; finish( root, meta_dir, &mut meta, &manifest, &dir.join("tree"), &mut txn, )?; env.activate()?; Ok(txn) } fn fail_after_apply( p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: String, report: Option, ) -> Result { tracing::error!(transaction = %txn.id, databases_started = txn.databases_started, "server update failed; restoring recovery point"); let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env).and_then(|()| { if p.env.automatic_rollback() { p.env.validate_recovery() } else { Ok(()) } }); if restored.is_err() { let _ = p.env.ensure_stopped(); } tracing::info!(transaction = %txn.id, recovered = restored.is_ok(), "server update recovery finished"); txn.state = if restored.is_ok() { State::RolledBack } else { State::Failed }; txn.message = Some(match &restored { Ok(()) => why.clone(), Err(e) => format!("{why} Recovery failed: {e}"), }); save(p.meta_dir, txn)?; if restored.is_ok() { let _ = fs::remove_dir_all(tree); if p.env.automatic_rollback() { p.env.activate()?; } } let _ = report; Err(Error::Invalid(match restored { Ok(()) if txn.databases_started => { format!("{why} The server files and databases were restored to the recovery point.") } Ok(()) => format!("{why} The server files were restored."), Err(e) => format!( "{why} Recovery failed: {e}. Resolve the unfinished update before starting the server." ), })) } fn restore_transaction( root: &Path, meta: &Path, before: &Path, txn: &mut Txn, env: &dyn Env, ) -> Result<()> { for (rel, expected) in &txn.recovery_hashes { let saved = match rel.as_str() { "@install" => before.join("manager-install.json"), "@point" => crate::backup::point_json( meta, txn.recovery_point.as_deref().ok_or_else(|| { Error::Invalid("The recovery point identity is missing.".into()) })?, )?, _ => fsx::safe_join(before, rel)?, }; if fsx::sha256_file(&saved)? != *expected { return Err(Error::Invalid(format!( "Recovery copy {rel} is damaged; no restoration was started." ))); } } if let Some(id) = &txn.recovery_point { env.verify_snapshot(id)?; } for op in txn.ops.iter().filter(|op| op.started && op.had_previous) { if !fsx::safe_join(before, &op.path)?.is_file() { return Err(Error::Invalid(format!( "The recovery copy of {} is missing; no restoration was started.", op.path ))); } } if !before.join("manager-install.json").is_file() { return Err(Error::Invalid( "The saved installation metadata is missing; no restoration was started.".into(), )); } env.ensure_stopped()?; env.recover_validation()?; if txn.databases_started { let id = txn .recovery_point .as_deref() .ok_or_else(|| Error::Invalid("The database recovery point is missing.".into()))?; env.restore_snapshot(id)?; } restore_files(root, before, txn)?; let saved_meta = before.join("manager-install.json"); if saved_meta.exists() { fsx::atomic_write(&meta.join("install.json"), &fs::read(saved_meta)?)?; } Ok(()) } #[allow(clippy::explicit_counter_loop)] fn apply_ops( root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: &mut Txn, meta_dir: &Path, fail_after: Option, ) -> Result<()> { if fail_after == Some(0) && !txn.ops.is_empty() { return Err(Error::Invalid("simulated failure".into())); } let entries: BTreeMap<_, _> = files .iter() .map(|entry| (entry.path.as_str(), entry)) .collect(); // Prepare every recovery copy before changing any destination. One durable journal // can then cover the entire file phase: recovery restores old files even if their // replacement was not reached, and removes newly created files only by exact hash. // Rewriting the full journal twice per file makes large navigation updates quadratic. for op in &txn.ops { let target = fsx::ensure_within(root, &fsx::safe_join(root, &op.path)?)?; if op.had_previous { let keep = fsx::safe_join(before, &op.path)?; fs::create_dir_all(keep.parent().unwrap())?; fsx::atomic_write(&keep, &fs::read(&target)?)?; txn.recovery_hashes .insert(op.path.clone(), fsx::sha256_file(&keep)?); } } for op in &mut txn.ops { op.started = true; } save(meta_dir, txn)?; let mut completed = 0usize; for i in 0..txn.ops.len() { if fail_after == Some(completed) { return Err(Error::Invalid("simulated failure".into())); } let (path, action) = (txn.ops[i].path.clone(), txn.ops[i].action); let target = fsx::ensure_within(root, &fsx::safe_join(root, &path)?)?; let staged = fsx::safe_join(tree, &path)?; let entry = entries.get(path.as_str()).expect("listed"); fs::create_dir_all(target.parent().unwrap())?; match action { Action::Create | Action::Replace => { // Same-volume temp file + rename: the target is either the old or the new file, never half of one. let tmp = target.with_file_name(format!( "{}.coa-new", target.file_name().unwrap().to_string_lossy() )); fs::copy(&staged, &tmp)?; if let Err(e) = fs::rename(&tmp, &target) { let _ = fs::remove_file(&tmp); return Err(e.into()); } if !fsx::sha256_file(&target)?.eq_ignore_ascii_case(&entry.sha256) { return Err(Error::HashMismatch { path, expected: entry.sha256.clone(), actual: "different after writing".into(), }); } } Action::MergeConfig => { let mut conf = ConfFile::parse_bytes(&fs::read(&target)?)?; let incoming = ConfFile::parse_bytes(&fs::read(&staged)?)?; merge::apply(&mut conf, &incoming); fsx::atomic_write(&target, conf.to_text().as_bytes())?; txn.ops[i].new_sha256 = fsx::sha256_file(&target)?; } Action::Skip | Action::Conflict => {} } txn.ops[i].done = true; completed += 1; } save(meta_dir, txn) } /// Put every touched file back the way it was. Files created by the update (no previous version) are removed only if /// they still hold exactly what the update wrote. fn restore_files(root: &Path, before: &Path, txn: &mut Txn) -> Result<()> { for op in txn.ops.iter().rev().filter(|o| o.started) { let target = fsx::ensure_within(root, &fsx::safe_join(root, &op.path)?)?; let _ = fs::remove_file(target.with_file_name(format!( "{}.coa-new", target.file_name().unwrap().to_string_lossy() ))); if op.had_previous { let saved = fsx::safe_join(before, &op.path)?; if !saved.is_file() { return Err(Error::Invalid(format!( "the saved copy of {} is missing", op.path ))); } fsx::atomic_write(&target, &fs::read(saved)?)?; } else if sha_if_exists(&target) .map(|h| h.eq_ignore_ascii_case(&op.new_sha256)) .unwrap_or(false) { fs::remove_file(&target)?; } } Ok(()) } /// Restore the database recovery point first, then restore binaries and configuration. pub fn rollback(root: &Path, meta_dir: &Path, id: &str, env: &dyn Env) -> Result { refuse_for_docker(root)?; let _lock = operation_lock(meta_dir)?; let mut txn = load(meta_dir, id)?; if matches!(txn.state, State::Committed | State::RolledBack) { return Err(Error::Invalid("This update cannot be rolled back.".into())); } restore_transaction( root, meta_dir, &txn_dir(meta_dir, id)?.join("before"), &mut txn, env, )?; txn.state = State::RolledBack; txn.message = Some("rolled back".into()); save(meta_dir, &txn)?; let _ = fs::remove_dir_all(txn_dir(meta_dir, id)?.join("tree")); Ok(txn) } pub(crate) fn operation_lock(meta_dir: &Path) -> Result { operation_lock_for(meta_dir, "server maintenance", false) } pub(crate) fn operation_lock_for( meta_dir: &Path, operation: &str, wait_for_check: bool, ) -> Result { fs::create_dir_all(meta_dir)?; let lock = fs::OpenOptions::new() .read(true) .write(true) .create(true) .truncate(false) .open(meta_dir.join("update.lock"))?; let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); loop { if fs4::fs_std::FileExt::try_lock_exclusive(&lock)? { break; } let owner = fsx::read_json::(&meta_dir.join("update-lock-owner.json")).ok(); let checking = owner .as_ref() .and_then(|owner| owner.get("operation")) .and_then(|value| value.as_str()) == Some("checking updates"); if wait_for_check && checking && std::time::Instant::now() < deadline { std::thread::sleep(std::time::Duration::from_millis(100)); continue; } let details = owner .as_ref() .and_then(|owner| { Some(format!( " Operation: {}; Manager process: {}; started: {}.", owner.get("operation")?.as_str()?, owner.get("pid")?.as_u64()?, owner.get("started")?.as_str()? )) }) .unwrap_or_default(); return Err(Error::Invalid(format!("Another update, repair or recovery is in progress.{details} Wait for that operation to finish, then retry."))); } let record = serde_json::json!({"operation": operation, "pid": std::process::id(), "started": chrono::Utc::now().to_rfc3339()}); fsx::atomic_write_json(&meta_dir.join("update-lock-owner.json"), &record)?; Ok(lock) } /// Repair uses the same durable recovery journal as an update. Its caller holds operation_lock. pub(crate) fn begin_repair( root: &Path, meta_dir: &Path, version: &str, backup: &str, files: &[FileEntry], ) -> Result { let id = format!( "{}-repair-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), uuid::Uuid::new_v4().simple() ); let before = txn_dir(meta_dir, &id)?.join("before"); fsx::atomic_write( &before.join("manager-install.json"), &fs::read(meta_dir.join("install.json"))?, )?; let mut ops = Vec::new(); for file in files { let target = fsx::ensure_within(root, &fsx::safe_join(root, &file.path)?)?; let had_previous = target.is_file(); if had_previous { fsx::atomic_write(&fsx::safe_join(&before, &file.path)?, &fs::read(&target)?)?; } ops.push(Op { path: file.path.clone(), action: if had_previous { Action::Replace } else { Action::Create }, reason: None, new_sha256: file.sha256.clone(), had_previous, started: true, done: false, }); } let mut recovery_hashes = BTreeMap::new(); recovery_hashes.insert( "@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?, ); let point = crate::backup::point_json(meta_dir, backup)?; if point.is_file() { recovery_hashes.insert("@point".into(), fsx::sha256_file(&point)?); } for op in ops.iter().filter(|op| op.had_previous) { recovery_hashes.insert( op.path.clone(), fsx::sha256_file(&fsx::safe_join(&before, &op.path)?)?, ); } let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()), recovery_hashes, }; save(meta_dir, &txn)?; Ok(txn) } pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> { txn.state = State::Applied; txn.databases_started = true; save(meta_dir, txn) } pub(crate) fn finish_repair( root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env, ) -> Result<()> { if !success { if let Err(error) = restore_transaction( root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env, ) { txn.state = State::Failed; txn.message = Some(format!( "Repair recovery failed: {error}. Its recovery point was preserved." )); save(meta_dir, txn)?; return Err(error); } } if success { complete_repair_ops(root, txn)?; } txn.state = if success { State::Committed } else { State::RolledBack }; save(meta_dir, txn) } fn complete_repair_ops(root: &Path, txn: &mut Txn) -> Result<()> { for op in &txn.ops { let target = fsx::ensure_within(root, &fsx::safe_join(root, &op.path)?)?; if fsx::sha256_file(&target)? != op.new_sha256 { return Err(Error::Invalid(format!( "Repaired file {} failed verification. Its recovery record was preserved.", op.path ))); } } for op in &mut txn.ops { op.done = true; } Ok(()) } fn prune(meta_dir: &Path, keep: usize) { let Ok(rd) = fs::read_dir(updates_dir(meta_dir)) else { return; }; let mut dirs: Vec<_> = rd .flatten() .filter(|e| { read_journal(&e.path().join("txn.json")) .is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack)) }) .map(|e| e.path()) .collect(); dirs.sort(); while dirs.len() > keep { let dir = dirs.remove(0); if let Ok(txn) = read_journal(&dir.join("txn.json")) { if let Ok(marker) = cleanup_marker(meta_dir, &txn.id) { if fsx::atomic_write_json( &marker, &CleanupMarker { id: txn.id, state: txn.state, }, ) .is_ok() && fs::remove_dir_all(&dir).is_ok() { let _ = fs::remove_file(marker); } } } } } /// The real environment: a repack-shaped installation driven through its launcher. pub struct RepackEnv<'a> { pub root: &'a Path, pub meta_dir: &'a Path, } impl RepackEnv<'_> { pub fn pending_migrations(&self, manifest: &Manifest) -> Result { Ok(self .pending_migrations_with(manifest, DatabaseAccess::Start)? .unwrap_or(0)) } /// `None` when the database is not running and `access` forbids starting it. pub fn pending_migrations_with( &self, manifest: &Manifest, access: DatabaseAccess, ) -> Result> { if manifest.migrations.is_empty() { return Ok(Some(0)); } let count = |db: &crate::db::Db| -> Result { let realms = crate::realms::state(self.root)?; let modes = realms.modes(); let mut count = 0; for (index, mode) in modes.into_iter().enumerate() { let list: Vec<_> = manifest .migrations .iter() .filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)) .cloned() .collect(); count += crate::migrations::pending_count(&db.clone().for_realm(mode), &list)?; } Ok(count) }; match access { DatabaseAccess::Start => crate::backup::with_database(self.root, count).map(Some), DatabaseAccess::OnlyIfRunning => crate::backup::with_running_database(self.root, count), } } pub(crate) fn verify_recovery_point(&self, point: &crate::backup::RecoveryPoint) -> Result<()> { if point.kind != crate::backup::Kind::Full || !["characters", "auth", "world", "configs"] .iter() .all(|name| point.components.iter().any(|c| c.name == *name)) { return Err(Error::Invalid("The update recovery point does not contain every required database and configuration.".into())); } if !crate::backup::verify(self.meta_dir, &point.id)?.ok { return Err(Error::Invalid( "The recovery point failed verification.".into(), )); } if let Some(snapshot) = &point.mysql_snapshot { if fsx::sha256_file(&self.root.join("mysql/bin/mysqld.exe"))? != snapshot.server_sha256 { return Err(Error::Invalid( "The MySQL executable differs from the recovery point.".into(), )); } return Ok(()); } crate::backup::with_database(self.root, |db| { let db = db.clone().for_realm(crate::realms::Mode::Coa); let mut objects = Vec::new(); for component in point.components.iter().filter(|c| c.sha256.is_some()) { let schema = if component.name.contains('-') || component.name == "playerbots" { crate::db::schema_of(&component.name)? } else { point.realm.schema(&component.name)? }; objects.extend(db.recovery_objects(schema)?); } if !objects.is_empty() { let path = self .meta_dir .join("diagnostics") .join(format!("database-recovery-{}.json", uuid::Uuid::new_v4())); let report = serde_json::json!({ "schema": 1, "checkedAt": chrono::Utc::now().to_rfc3339(), "recoveryPoint": point.id, "objects": objects }); if let Err(error) = fsx::atomic_write_json(&path, &report) { tracing::warn!(%error, "Could not save database recovery compatibility report"); } let sample = objects .iter() .take(3) .map(|object| { format!( "{}.{} ({})", object["database"].as_str().unwrap_or("?"), object["name"].as_str().unwrap_or("?"), object["kind"].as_str().unwrap_or("?") ) }) .collect::>() .join(", "); return Err(Error::Invalid(format!("Automatic recovery does not support {} database objects: {sample}. Export diagnostics for the complete list. The update was not started.", objects.len()))); } Ok(()) }) } fn migration_snapshot(&self) -> Result { if let Some(id) = unfinished(self.meta_dir).and_then(|t| t.recovery_point) { if crate::backup::get(self.meta_dir, &id)?.kind == crate::backup::Kind::Full && crate::backup::verify(self.meta_dir, &id)?.ok { return Ok(id); } return Err(Error::Invalid( "The update recovery point is incomplete or damaged.".into(), )); } Ok(crate::backup::create( self.root, self.meta_dir, crate::backup::Kind::Full, crate::backup::Trigger::BeforeMigration, None, &|_| {}, )? .id) } } impl Env for RepackEnv<'_> { fn recover_validation(&self) -> Result<()> { crate::update_isolation::restore(self.root, self.meta_dir) } fn activate(&self) -> Result<()> { ensure_recovered(self.meta_dir)?; let outcome = crate::driver::validate_update(self.root)?; if outcome.ok { Ok(()) } else { Err(Error::Invalid(format!( "The update transaction finished, but the server could not start: {}", crate::driver::startup_failure(self.root, &outcome) ))) } } fn automatic_rollback(&self) -> bool { true } fn preflight(&self, manifest: &Manifest) -> Result<()> { crate::mysql_snapshot::stop_previous_rehearsals(self.meta_dir)?; if let Some(compatibility) = &manifest.compatibility { let (_, meta) = MetaDir::open(self.meta_dir)?; if let Some(source) = meta .core .version .as_ref() .and_then(|v| compatibility.source_databases.get(v)) { if let Some(expected) = &source.schema_sha256 { let contract = self.root.join("Scripts/database-schema.json"); if fsx::sha256_file(&contract)? != *expected { return Err(Error::Invalid("The installed database schema contract differs from the qualified upgrade source; repair the installation first.".into())); } } crate::backup::with_database(self.root, |db| { let modes = crate::realms::state(self.root)?.modes(); for mode in modes { if let Some(problem) = crate::schema_check::check_with_report( &db.clone().for_realm(mode), self.root, )? .first() { if crate::docker::is_docker(self.root) { return Err(Error::Invalid(format!("The installed {} database does not match its upgrade source: {}.{}: {}", mode.name(), problem.database, problem.table, problem.detail))); } tracing::warn!(realm = mode.name(), table = %problem.table, "Installed schema differs; pending migrations and target validation will run with a full recovery point"); } } Ok(()) })?; } } self.pending_migrations(manifest).map(|_| ()) } fn verify_snapshot(&self, id: &str) -> Result<()> { let point = crate::backup::get(self.meta_dir, id)?; if point.kind != crate::backup::Kind::Full || !["characters", "auth", "world", "configs"] .iter() .all(|name| point.components.iter().any(|c| c.name == *name)) { return Err(Error::Invalid( "The recovery point is incomplete; no restoration was started.".into(), )); } if !crate::backup::verify(self.meta_dir, id)?.ok { return Err(Error::Invalid( "The recovery point is damaged; available copies were preserved.".into(), )); } Ok(()) } fn ensure_stopped(&self) -> Result<()> { use crate::process::{observe, ServiceState}; let o = observe(self.root, &crate::layout::read_ports(self.root)); if o.world.state != ServiceState::Stopped || o.auth.state != ServiceState::Stopped || crate::multiworld::is_running(self.root) { return Err(Error::Invalid( "Stop the server before updating it; files cannot be replaced while it is running." .into(), )); } Ok(()) } fn snapshot(&self) -> Result { let label = Some("before update".to_string()); let point = crate::backup::create( self.root, self.meta_dir, crate::backup::Kind::Full, crate::backup::Trigger::BeforeUpdate, label, &|_| {}, )?; self.verify_recovery_point(&point)?; Ok(point.id) } fn restore_snapshot(&self, id: &str) -> Result<()> { let point = crate::backup::get(self.meta_dir, id)?; if point.kind != crate::backup::Kind::Full { return Err(Error::Invalid( "Database rollback requires a full recovery point.".into(), )); } if !["characters", "auth", "world", "configs"] .iter() .all(|name| point.components.iter().any(|c| c.name == *name)) || !crate::backup::verify(self.meta_dir, id)?.ok { return Err(Error::Invalid( "The full database recovery point is incomplete or damaged.".into(), )); } if let Some(snapshot) = &point.mysql_snapshot { crate::mysql_snapshot::stop(self.root)?; let point_dir = crate::backup::point_json(self.meta_dir, id)? .parent() .unwrap() .to_path_buf(); crate::mysql_snapshot::swap(self.root, self.meta_dir, &point_dir, snapshot, false)?; crate::backup::restore_configs(self.root, self.meta_dir, id)?; return Ok(()); } for component in point.components.iter().filter(|c| c.sha256.is_some()) { crate::backup::restore_database(self.root, self.meta_dir, id, &component.name)?; } crate::backup::restore_configs(self.root, self.meta_dir, id)?; Ok(()) } fn migrate(&self, manifest: &Manifest, staged: &Path) -> Result { crate::backup::with_database(self.root, |db| { let identities = database_identities(db)?; let result = migrate_databases(db, self.root, manifest, staged, &|| { self.migration_snapshot() })?; if result.failed.is_none() { verify_database_identities(db, &identities)?; } Ok(result) }) } fn validate(&self) -> Result<()> { self.validate_server(true) } fn validate_recovery(&self) -> Result<()> { self.validate_server(false) } } impl RepackEnv<'_> { fn validate_server(&self, check_schema: bool) -> Result<()> { crate::mysql_snapshot::stop(self.root)?; self.ensure_stopped()?; self.recover_validation()?; let result = (|| -> Result<()> { crate::update_isolation::begin(self.root, self.meta_dir)?; use crate::process::{observe, ServiceState}; let identities = crate::backup::with_database(self.root, |db| { if check_schema { let modes = crate::realms::state(self.root)?.modes(); for mode in modes { if let Some(p) = crate::schema_check::check_with_report( &db.clone().for_realm(mode), self.root, )? .first() { return Err(Error::Invalid(format!( "Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail ))); } } } database_identities(db) })?; // The check start is the first start of the new build: give the module configs the settings the update added // (as a normal start does), or the server logs a "missing property" line for every one of them. match crate::registry::MetaDir::open(self.meta_dir) { Ok((_, meta)) if meta.kind == crate::registry::InstallKind::New => { let _ = crate::config::materialize_module_configs(self.root); } // An imported server gets the files it lacks (nothing it has is changed). _ => { let _ = crate::config::create_missing_module_configs(self.root); } } let started = match crate::driver::validate_update(self.root) { Ok(out) => out, Err(e) => { let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); return Err(e); } }; let ports = crate::layout::read_ports(self.root); let healthy = started.ok && { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); let mut ready_since = None; loop { let o = observe(self.root, &ports); if o.mysql.state == ServiceState::Running && o.auth.state == ServiceState::Running && o.world.state == ServiceState::Running && o.secondary_world .as_ref() .is_none_or(|s| s.state == ServiceState::Running) { let since = ready_since.get_or_insert_with(std::time::Instant::now); if since.elapsed() >= std::time::Duration::from_secs(10) { break true; } } else { ready_since = None; } if std::time::Instant::now() > deadline { break false; } std::thread::sleep(std::time::Duration::from_secs(2)); } }; if healthy { crate::backup::with_database(self.root, |db| { verify_database_identities(db, &identities) })?; return Ok(()); } let cause = crate::driver::startup_failure(self.root, &started); // Leave the files unlocked so that a rollback can replace them. let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); Err(Error::Invalid(cause)) })(); crate::mysql_snapshot::stop(self.root)?; self.ensure_stopped()?; self.recover_validation()?; result } } /// One migration pass for both realms, with shared history checks and one recovery point. fn migrate_databases( db: &crate::db::Db, root: &Path, manifest: &Manifest, staged: &Path, snapshot: &dyn Fn() -> Result, ) -> Result { let realms = crate::realms::state(root)?; let modes = realms.modes(); // Reject a broken history in either realm before changing the first one. for (index, mode) in modes.iter().enumerate() { let list: Vec<_> = manifest .migrations .iter() .filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)) .cloned() .collect(); crate::migrations::preflight(&db.clone().for_realm(*mode), &list)?; } let contract = manifest .files .iter() .find(|file| file.path == crate::schema_check::CONTRACT) .map(|file| file.sha256.as_str()); let mut result = ApplyReport { applied: vec![], failed: None, snapshot: None, }; if contract.is_some() { result.snapshot = Some(snapshot()?); } for (index, mode) in modes.into_iter().enumerate() { let db = db.clone().for_realm(mode); let list: Vec<_> = manifest .migrations .iter() .filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)) .cloned() .collect(); crate::schema_check::repair_missing_defaults(&db, root, contract)?; let extra = crate::migrations::apply_pending(&db, &list, staged, &|| { if let Some(id) = &result.snapshot { Ok(id.clone()) } else { snapshot() } })?; result.applied.extend(extra.applied); result.failed = extra.failed; if result.snapshot.is_none() { result.snapshot = extra.snapshot; } if result.failed.is_some() { return Ok(result); } crate::schema_check::repair_missing_defaults(&db, root, contract)?; if let Some(p) = crate::schema_check::check_with_report(&db, root)?.first() { return Err(Error::Invalid(format!( "Database validation failed on {}: {}.{}: {}", mode.name(), p.table, p.column, p.detail ))); } } Ok(result) } fn database_identities( db: &crate::db::Db, ) -> Result>> { let db = db.clone().for_realm(crate::realms::Mode::Coa); let mut identities = BTreeMap::new(); for (schema, table, columns) in [ ("acore_characters", "characters", "guid,account,HEX(name)"), ("acore_characters_wildcard", "characters", "guid,account,HEX(name)"), ("acore_characters_reborn", "characters", "guid,account,HEX(name)"), ("acore_auth", "account", "id,HEX(username)"), ] { if !db.schema_exists(schema)? || !db.tables(schema)?.iter().any(|name| name == table) { continue; } let rows = db.query(&format!("SELECT {columns} FROM `{schema}`.`{table}`;"))?; identities.insert(schema.into(), rows.lines().map(str::to_string).collect()); } Ok(identities) } fn verify_database_identities( db: &crate::db::Db, before: &BTreeMap>, ) -> Result<()> { let after = database_identities(db)?; for (schema, expected) in before { if after .get(schema) .is_none_or(|actual| !expected.is_subset(actual)) { return Err(Error::Invalid(format!("The update changed or removed existing character or account identities in {schema}."))); } } Ok(()) } #[cfg(test)] mod tests { use super::*; use crate::package::{build, BuildOptions}; use crate::registry::{InstallKind, InstallMeta}; use base64::Engine; use ed25519_dalek::{Signer, SigningKey}; use std::cell::{Cell, RefCell}; struct Fake { activation_expected: RefCell>, activation_failed: Cell, activated: Cell, automatic_rollback: Cell, recovery_healthy: Cell, stopped: Cell, healthy: Cell, snapshot_ok: Cell, calls: RefCell>, migrate_fail: Cell, preflight_fail: Cell, restore_fail: Cell, } impl Fake { fn ok() -> Fake { Fake { activation_expected: RefCell::new(None), activation_failed: Cell::new(false), activated: Cell::new(false), automatic_rollback: Cell::new(false), recovery_healthy: Cell::new(true), stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false), } } } impl Env for Fake { fn activate(&self) -> Result<()> { if let Some((meta, version)) = self.activation_expected.borrow().as_ref() { ensure_recovered(meta)?; assert_eq!(MetaDir::open(meta)?.1.core.version.as_ref(), Some(version)); } self.activated.set(true); if self.activation_failed.get() { Err(Error::Invalid("Public startup failed".into())) } else { Ok(()) } } fn automatic_rollback(&self) -> bool { self.automatic_rollback.get() } fn validate_recovery(&self) -> Result<()> { self.calls.borrow_mut().push("validate-recovery"); if self.recovery_healthy.get() { Ok(()) } else { Err(Error::Invalid("Restored server is unhealthy".into())) } } fn preflight(&self, _: &Manifest) -> Result<()> { if self.preflight_fail.get() { Err(Error::Invalid("Conflicting migration history".into())) } else { Ok(()) } } fn ensure_stopped(&self) -> Result<()> { self.calls.borrow_mut().push("stop"); if self.stopped.get() { Ok(()) } else { Err(Error::Invalid("The server is running.".into())) } } fn snapshot(&self) -> Result { self.calls.borrow_mut().push("snapshot"); if self.snapshot_ok.get() { Ok("rp-1".into()) } else { Err(Error::Invalid("disk full".into())) } } fn migrate(&self, _m: &Manifest, _d: &Path) -> Result { self.calls.borrow_mut().push("migrate"); Ok(ApplyReport { applied: vec![], failed: self .migrate_fail .get() .then(|| ("m1".to_string(), "syntax".to_string())), snapshot: None, }) } fn restore_snapshot(&self, _id: &str) -> Result<()> { self.calls.borrow_mut().push("restore-databases"); if self.restore_fail.get() { Err(Error::Invalid("recovery disk unavailable".into())) } else { Ok(()) } } fn validate(&self) -> Result<()> { self.calls.borrow_mut().push("validate"); if self.healthy.get() { Ok(()) } else { Err(Error::Invalid("worldserver exited".into())) } } } struct World { _d: tempfile::TempDir, root: PathBuf, meta: PathBuf, pkg: PathBuf, key: String, } fn write(root: &Path, rel: &str, content: &[u8]) { let p = root.join(rel); fs::create_dir_all(p.parent().unwrap()).unwrap(); fs::write(p, content).unwrap(); } /// An installation at version 1 (files recorded as pristine) and a signed update package. fn world(extra_update_files: &[(&str, &[u8])], migrations: bool) -> World { let d = tempfile::tempdir().unwrap(); let root = d.path().join("srv"); let meta_dir = d.path().join("srv.manager"); let pkg = d.path().join("pkg"); let src = d.path().join("upd"); write(&root, "Core/worldserver.exe", b"world-v1"); write(&root, "Core/authserver.exe", b"auth-v1"); write( &root, "Settings/worldserver.conf.template", b"[worldserver]\nRate.XP.Kill = 3\n", ); write(&root, "user-notes.txt", b"mine"); fs::create_dir_all(&meta_dir).unwrap(); let mut meta = InstallMeta::new(InstallKind::New, &root); meta.core.version = Some("1.0.0".into()); for f in ["Core/worldserver.exe", "Core/authserver.exe"] { meta.original_hashes .insert(f.into(), fsx::sha256_file(&root.join(f)).unwrap()); } for sub in ["manifests", "backups", "updates", "staging"] { fs::create_dir_all(meta_dir.join(sub)).unwrap(); } fsx::atomic_write_json(&meta_dir.join("install.json"), &meta).unwrap(); write(&src, "Core/worldserver.exe", b"world-v2"); write( &src, "Settings/worldserver.conf.template", b"[worldserver]\nRate.XP.Kill = 1\nNew.Setting = 5\n", ); write(&src, "Core/newfile.dll", b"dll"); for (p, c) in extra_update_files { write(&src, p, c); } if migrations { write(&src, "_migrations/world/m1.sql", b"SELECT 1;"); } let mut m = build( &src, &pkg, &BuildOptions { kind: Kind::Update, version: "2.0.0".into(), core_commit: None, built_at: "x".into(), part_size: 1 << 20, bots_commit: None, migrations: vec![], }, &|_| {}, ) .unwrap(); // Settings/* are merge-config by the packager; make the migration entry visible to the runner if migrations { m.migrations.push(crate::manifest::Migration { compatible_sha256: vec![], id: "m1".into(), db: "world".into(), sha256: fsx::sha256_bytes(b"SELECT 1;"), destructive: false, }); fs::write( pkg.join("manifest.json"), serde_json::to_vec_pretty(&m).unwrap(), ) .unwrap(); } let sk = SigningKey::generate(&mut rand_core::OsRng); let bytes = fs::read(pkg.join("manifest.json")).unwrap(); fs::write( pkg.join("manifest.json.sig"), base64::engine::general_purpose::STANDARD.encode(sk.sign(&bytes).to_bytes()), ) .unwrap(); World { _d: d, root, meta: meta_dir, pkg, key: base64::engine::general_purpose::STANDARD.encode(sk.verifying_key().to_bytes()), } } const LEGACY_LAUNCHER: &str = "import sys\nROOT = Path(__file__).resolve().parents[1]\nfrom squid_playerbots import validate_bots\n(\"WorldDatabaseInfo\", \"acore_world\")\n(\"CharacterDatabaseInfo\", \"acore_characters\")\nSET name='AzerothCore',address=\nWHERE id=1;\nmysql(\"UPDATE acore_auth.realmlist SET flag=0\nif __name__ == \"__main__\":\n"; fn legacy_launcher_world(import_patch: bool) -> World { let w = world(&[("Scripts/manage.py", b"new release launcher")], false); let mut source = crate::realms::patch_launcher(LEGACY_LAUNCHER).unwrap(); if import_patch { source = crate::driver::patch_launcher_imports(&source).unwrap(); } write(&w.root, "Scripts/manage.py", source.as_bytes()); let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); // Older Managers retained the signed original hash after adding realm profiles. meta.original_hashes.insert( "Scripts/manage.py".into(), fsx::sha256_bytes(LEGACY_LAUNCHER.as_bytes()), ); fsx::atomic_write_json(&w.meta.join("install.json"), &meta).unwrap(); w } #[test] fn startup_waits_for_an_update_check_but_not_for_active_maintenance() { let temp = tempfile::tempdir().unwrap(); let checker = operation_lock_for(temp.path(), "checking updates", false).unwrap(); let root = temp.path().to_path_buf(); let release = std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(150)); drop(checker); }); let start = operation_lock_for(&root, "starting server", true).unwrap(); release.join().unwrap(); let error = operation_lock(&root).unwrap_err().to_string(); assert!(error.contains("starting server")); assert!(error.contains(&std::process::id().to_string())); drop(start); // Stale owner metadata is harmless once the OS lock is released. operation_lock(&root).unwrap(); } #[test] fn legacy_manager_realm_launcher_is_replaced_during_update() { let w = legacy_launcher_world(false); let outcome = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap(); assert_eq!(outcome.txn.state, State::Committed); assert_eq!(read(&w, "Scripts/manage.py"), b"new release launcher"); } #[test] fn legacy_realm_and_import_repairs_are_replaced_during_update() { let w = legacy_launcher_world(true); let outcome = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap(); assert_eq!(outcome.txn.state, State::Committed); assert_eq!(read(&w, "Scripts/manage.py"), b"new release launcher"); } #[test] fn genuine_launcher_edits_still_require_a_decision_and_can_be_kept() { let w = legacy_launcher_world(true); let mut custom = read(&w, "Scripts/manage.py"); custom.extend_from_slice(b"\n# owner's custom integration\n"); write(&w.root, "Scripts/manage.py", &custom); let (_, meta) = MetaDir::open(&w.meta).unwrap(); let manifest = Manifest::parse(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap(); let items = plan(&w.root, &meta, &manifest, &BTreeMap::new(), None).unwrap(); assert_eq!( items .iter() .find(|item| item.path == "Scripts/manage.py") .unwrap() .action, Action::Conflict ); let resolutions = BTreeMap::from([("Scripts/manage.py".into(), Resolution::Keep)]); let outcome = run(&w, &Fake::ok(), resolutions, None).unwrap(); assert_eq!(outcome.txn.state, State::Committed); assert_eq!(read(&w, "Scripts/manage.py"), custom); } fn run( w: &World, env: &Fake, res: BTreeMap, fail_after: Option, ) -> Result { apply( &Params { root: &w.root, meta_dir: &w.meta, source: Source::Dir(w.pkg.clone()), trusted_key: &w.key, cancel: Cancel::default(), resolutions: res, env, fail_after_ops: fail_after, }, &|_, _| {}, ) } fn read(w: &World, rel: &str) -> Vec { fs::read(w.root.join(rel)).unwrap() } #[test] fn imported_squid_repack_requires_a_decision_even_for_pristine_worldserver() { let w = world(&[], false); write(&w.root, "CoA-Bots/release.json", b"{}"); let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); meta.kind = InstallKind::Imported; let manifest: Manifest = serde_json::from_slice(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap(); let action = |resolutions: BTreeMap| { plan(&w.root, &meta, &manifest, &resolutions, None) .unwrap() .into_iter() .find(|item| item.path == "Core/worldserver.exe") .unwrap() .action }; assert_eq!(action(BTreeMap::new()), Action::Conflict); assert_eq!( action(BTreeMap::from([( "Core/worldserver.exe".into(), Resolution::Keep )])), Action::Skip ); assert_eq!( action(BTreeMap::from([( "Core/worldserver.exe".into(), Resolution::Replace )])), Action::Replace ); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn a_docker_server_is_never_given_the_repack_files() { // Found by trying the installer: the update check of a Docker server read the Windows channel and offered its files. let w = world(&[], false); write(&w.root, "Settings/docker.json", br#"{"project":"t1"}"#); let before = read(&w, "Core/worldserver.exe"); let meta = InstallMeta::new(InstallKind::New, &w.root); let err = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap_err(); assert!(err.to_string().contains("Docker"), "{err}"); let err = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap_err(); assert!(err.to_string().contains("Docker"), "{err}"); assert!(rollback(&w.root, &w.meta, "any", &Fake::ok()).is_err()); assert_eq!( read(&w, "Core/worldserver.exe"), before, "nothing was touched" ); assert!(!w.root.join("Core/newfile.dll").exists()); // The same package is applied to the same folder once it is not a Docker server. fs::remove_file(w.root.join("Settings/docker.json")).unwrap(); assert_eq!( run(&w, &Fake::ok(), BTreeMap::new(), None) .unwrap() .txn .state, State::Committed ); } #[test] fn happy_path_replaces_creates_merges_and_records_ownership() { let w = world(&[], false); let env = Fake::ok(); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); assert_eq!(out.txn.state, State::Committed); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2"); assert_eq!( read(&w, "Core/authserver.exe"), b"auth-v1", "files not in the update are untouched" ); assert_eq!(read(&w, "Core/newfile.dll"), b"dll"); assert_eq!(read(&w, "user-notes.txt"), b"mine"); let tpl = String::from_utf8(read(&w, "Settings/worldserver.conf.template")).unwrap(); assert!( tpl.contains("Rate.XP.Kill = 3"), "the user's value survives the merge: {tpl}" ); assert!(tpl.contains("New.Setting = 5"), "new keys are added"); let (_, meta) = MetaDir::open(&w.meta).unwrap(); assert_eq!(meta.core.version.as_deref(), Some("2.0.0")); assert_eq!( meta.original_hashes["Core/worldserver.exe"], fsx::sha256_bytes(b"world-v2") ); assert_eq!(*env.calls.borrow(), ["stop", "snapshot", "validate"]); assert!(!w.root.join("Core/worldserver.exe.coa-new").exists()); } #[test] fn public_startup_requires_a_completed_transaction_and_saved_version() { let w = world(&[], false); let env = Fake::ok(); *env.activation_expected.borrow_mut() = Some((w.meta.clone(), "2.0.0".into())); run(&w, &env, BTreeMap::new(), None).unwrap(); assert!(env.activated.get()); } #[test] fn startup_failure_after_commit_does_not_restore_a_live_database() { let w = world(&[], true); let env = Fake::ok(); env.activation_failed.set(true); assert!(run(&w, &env, BTreeMap::new(), None).is_err()); assert!(ensure_recovered(&w.meta).is_ok()); assert_eq!( MetaDir::open(&w.meta).unwrap().1.core.version.as_deref(), Some("2.0.0") ); assert!(!env.calls.borrow().contains(&"restore-databases")); } #[test] fn an_integrated_launcher_is_current_but_user_edits_and_new_package_bytes_are_not() { let source = b"import sys\nfrom pathlib import Path\nfrom squid_playerbots import configure\nROOT = Path(__file__).resolve().parents[1]\n"; let w = world(&[("Scripts/manage.py", source)], false); run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap(); let integrated = crate::driver::patch_launcher_imports(std::str::from_utf8(source).unwrap()).unwrap(); write(&w.root, "Scripts/manage.py", integrated.as_bytes()); let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); meta.original_hashes.insert( "Scripts/manage.py".into(), fsx::sha256_bytes(integrated.as_bytes()), ); fsx::atomic_write_json(&w.meta.join("install.json"), &meta).unwrap(); let original = fetch_manifest(&Source::Dir(w.pkg.clone()), &w.key) .unwrap() .0; for part in &original.archive.as_ref().unwrap().parts { fs::remove_file(w.pkg.join(&part.name)).unwrap(); } let p = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap(); assert_eq!( p.items .iter() .find(|item| item.path == "Scripts/manage.py") .unwrap() .action, Action::Skip ); let manifest = fetch_manifest(&Source::Dir(w.pkg.clone()), &w.key) .unwrap() .0; let applied_plan = plan( &w.root, &meta, &manifest, &BTreeMap::new(), Some(&w._d.path().join("upd")), ) .unwrap(); assert_eq!( applied_plan .iter() .find(|item| item.path == "Scripts/manage.py") .unwrap() .action, Action::Skip ); let mut changed = fetch_manifest(&Source::Dir(w.pkg.clone()), &w.key) .unwrap() .0; changed .files .iter_mut() .find(|file| file.path == "Scripts/manage.py") .unwrap() .sha256 = fsx::sha256_bytes(b"new launcher"); assert!(!launcher_already_integrated(&w.root, &meta, &changed).unwrap()); write( &w.root, "Scripts/manage.py", format!("{integrated}\n# user edit\n").as_bytes(), ); // Same version: the owner's edit is not an update (Repair lists and restores it). let p = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap(); assert!(p.items.iter().all(|item| item.action == Action::Skip) && p.conflicts.is_empty()); // An older installed build with the same edit does need a decision. meta.core.version = Some("1.0.0".into()); let p = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap(); assert_eq!( p.items .iter() .find(|item| item.path == "Scripts/manage.py") .unwrap() .action, Action::Conflict ); meta.original_hashes.insert( "Scripts/manage.py".into(), fsx::sha256_file(&w.root.join("Scripts/manage.py")).unwrap(), ); let p = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap(); assert_eq!( p.items .iter() .find(|item| item.path == "Scripts/manage.py") .unwrap() .action, Action::Replace, "Recorded user content must not be mistaken for a recognized Manager integration" ); } #[test] fn older_packages_are_rejected_before_preview_or_application() { let w = world(&[], false); let (_, mut installed) = MetaDir::open(&w.meta).unwrap(); installed.core.version = Some("3.0.0".into()); fsx::atomic_write_json(&w.meta.join("install.json"), &installed).unwrap(); let env = Fake::ok(); assert!(run(&w, &env, BTreeMap::new(), None) .unwrap_err() .to_string() .contains("newer than package")); assert!(preview( &w.root, &installed, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new() ) .unwrap_err() .to_string() .contains("newer than package")); assert!(env.calls.borrow().is_empty()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(pending_checked(&w.meta).unwrap().is_none()); } #[test] fn a_file_modified_by_the_user_is_a_conflict_until_resolved() { let w = world(&[], false); write(&w.root, "Core/worldserver.exe", b"my-custom-build"); let env = Fake::ok(); let e = run(&w, &env, BTreeMap::new(), None).unwrap_err(); assert!(e.to_string().contains("need your decision")); assert_eq!( read(&w, "Core/worldserver.exe"), b"my-custom-build", "nothing changed" ); assert!( env.calls.borrow().is_empty(), "the server was never touched or stopped" ); let mut keep = BTreeMap::new(); keep.insert("Core/worldserver.exe".to_string(), Resolution::Keep); run(&w, &env, keep, None).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"my-custom-build"); assert_eq!( read(&w, "Core/newfile.dll"), b"dll", "the rest of the update still applied" ); let w2 = world(&[], false); write(&w2.root, "Core/worldserver.exe", b"my-custom-build"); let mut rep = BTreeMap::new(); rep.insert("Core/worldserver.exe".to_string(), Resolution::Replace); run(&w2, &Fake::ok(), rep, None).unwrap(); assert_eq!(read(&w2, "Core/worldserver.exe"), b"world-v2"); let t = unfinished(&w2.meta); assert!(t.is_none()); let saved = fs::read_dir(w2.meta.join("updates")) .unwrap() .flatten() .next() .unwrap() .path() .join("before/Core/worldserver.exe"); assert_eq!( fs::read(saved).unwrap(), b"my-custom-build", "the replaced custom build is kept for rollback" ); } #[test] fn power_loss_during_bulk_files_restores_written_and_unwritten_files() { let d = tempfile::tempdir().unwrap(); let root = d.path().join("root"); let meta = d.path().join("meta"); let tree = d.path().join("tree"); let before = meta.join("updates/bulk/before"); let mut files = Vec::new(); let mut ops = Vec::new(); for (path, previous) in [ ("Data/maps/first.map", true), ("Data/maps/new.map", false), ("Data/maps/last.map", true), ] { if previous { write(&root, path, b"old"); } write(&tree, path, b"new"); let hash = fsx::sha256_file(&tree.join(path)).unwrap(); files.push(FileEntry { path: path.into(), sha256: hash.clone(), size: 3, policy: ReplacePolicy::Replace, owner: crate::manifest::Owner::Core, }); ops.push(Op { path: path.into(), action: if previous { Action::Replace } else { Action::Create }, reason: None, new_sha256: hash, had_previous: previous, started: false, done: false, }); } let mut txn = Txn { id: "bulk".into(), state: State::Applying, from_version: Some("1".into()), to_version: "2".into(), recovery_point: None, databases_started: false, ops, message: None, recovery_hashes: BTreeMap::new(), }; fs::create_dir_all(before.parent().unwrap()).unwrap(); assert!(apply_ops(&root, &tree, &before, &files, &mut txn, &meta, Some(2)).is_err()); // A power loss sees the pre-write checkpoint, not the in-memory completed flags. let mut durable = load(&meta, "bulk").unwrap(); assert!(durable.ops.iter().all(|op| op.started && !op.done)); assert_eq!(fs::read(root.join("Data/maps/last.map")).unwrap(), b"old"); restore_files(&root, &before, &mut durable).unwrap(); assert_eq!(fs::read(root.join("Data/maps/first.map")).unwrap(), b"old"); assert_eq!(fs::read(root.join("Data/maps/last.map")).unwrap(), b"old"); assert!(!root.join("Data/maps/new.map").exists()); // If preparing any recovery copy fails, no destination may be changed. for op in &mut durable.ops { op.started = false; op.done = false; } durable.recovery_hashes.clear(); save(&meta, &durable).unwrap(); fs::remove_file(root.join("Data/maps/last.map")).unwrap(); assert!(apply_ops(&root, &tree, &before, &files, &mut durable, &meta, None).is_err()); assert_eq!(fs::read(root.join("Data/maps/first.map")).unwrap(), b"old"); assert!(!root.join("Data/maps/new.map").exists()); assert!(load(&meta, "bulk") .unwrap() .ops .iter() .all(|op| !op.started)); } #[test] fn failure_in_the_middle_of_applying_restores_every_file_byte_for_byte() { for fail_after in [0usize, 1, 2] { let w = world(&[], false); let snap: Vec<(String, Vec)> = [ "Core/worldserver.exe", "Core/authserver.exe", "Settings/worldserver.conf.template", "user-notes.txt", ] .iter() .map(|p| (p.to_string(), read(&w, p))) .collect(); let e = run(&w, &Fake::ok(), BTreeMap::new(), Some(fail_after)).unwrap_err(); assert!(e.to_string().contains("simulated")); for (p, b) in &snap { assert_eq!(&read(&w, p), b, "{p} after failure at op {fail_after}"); } assert!( !w.root.join("Core/newfile.dll").exists(), "files created by the update are removed again" ); let (_, meta) = MetaDir::open(&w.meta).unwrap(); assert_eq!( meta.core.version.as_deref(), Some("1.0.0"), "version not bumped" ); } } #[test] fn unhealthy_new_build_is_kept_until_the_owner_rolls_back() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); assert_eq!(out.txn.state, State::NeedsDecision); assert_eq!( read(&w, "Core/worldserver.exe"), b"world-v2", "not undone automatically" ); assert_eq!(unfinished(&w.meta).unwrap().id, out.txn.id); let (_, meta) = MetaDir::open(&w.meta).unwrap(); assert_eq!(meta.core.version.as_deref(), Some("1.0.0"), "not committed"); rollback(&w.root, &w.meta, &out.txn.id, &env).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(!w.root.join("Core/newfile.dll").exists()); assert!(unfinished(&w.meta).is_none()); assert!( rollback(&w.root, &w.meta, &out.txn.id, &env).is_err(), "already rolled back" ); } #[test] fn automatic_health_failure_restores_databases_files_and_checks_recovery() { let w = world(&[], true); let env = Fake::ok(); env.automatic_rollback.set(true); env.healthy.set(false); assert!(run(&w, &env, BTreeMap::new(), None) .unwrap_err() .to_string() .contains("restored")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(!w.root.join("Core/newfile.dll").exists()); assert!(env.calls.borrow().contains(&"restore-databases")); assert!(env.calls.borrow().contains(&"validate-recovery")); assert!(unfinished(&w.meta).is_none()); } #[test] fn unlisted_source_is_allowed_but_unknown_versions_and_other_platforms_are_rejected() { let w = world(&[], false); let (_, meta) = MetaDir::open(&w.meta).unwrap(); let mut manifest = Manifest::parse(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap(); manifest.min_manager_version = "0.6.13".into(); manifest.core.commit = Some("a".repeat(40)); manifest.compatibility = Some(crate::manifest::Compatibility { schema: 1, platform: if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" } .into(), core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(), source_databases: Default::default(), client_patch_version: "1.5.1".into(), source_versions: vec!["1.0.0".into()], }); manifest.validate().unwrap(); validate_candidate(&meta, &manifest).unwrap(); manifest.compatibility.as_mut().unwrap().source_versions = vec!["0.9.0".into()]; validate_candidate(&meta, &manifest).unwrap(); let mut unknown = meta.clone(); unknown.core.version = None; assert!(validate_candidate(&unknown, &manifest) .unwrap_err() .to_string() .contains("unknown")); unknown.core.version = Some("unverified-repack".into()); assert!(validate_candidate(&unknown, &manifest).is_err()); let mut already_updated = meta.clone(); already_updated.core.version = Some(manifest.version.clone()); validate_candidate(&already_updated, &manifest).unwrap(); manifest.compatibility.as_mut().unwrap().source_versions = vec!["1.0.0".into()]; manifest.compatibility.as_mut().unwrap().platform = if cfg!(windows) { "linux-x86_64" } else { "windows-x86_64" } .into(); assert!(validate_candidate(&meta, &manifest) .unwrap_err() .to_string() .contains("targets")); assert!(validate_candidate(&already_updated, &manifest) .unwrap_err() .to_string() .contains("targets")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(unfinished(&w.meta).is_none()); } #[test] fn changed_installed_schema_contract_blocks_before_database_access() { let w = world(&[], false); write( &w.root, "Scripts/database-schema.json", b"modified schema contract", ); let mut manifest = Manifest::parse(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap(); manifest.compatibility = Some(crate::manifest::Compatibility { schema: 1, platform: "windows-x86_64".into(), core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(), client_patch_version: "1.5.1".into(), source_versions: vec!["1.0.0".into()], source_databases: [( "1.0.0".into(), crate::manifest::SourceDatabase { manifest_sha256: "a".repeat(64), schema_sha256: Some("b".repeat(64)), }, )] .into(), }); let env = RepackEnv { root: &w.root, meta_dir: &w.meta, }; let error = env.preflight(&manifest).unwrap_err().to_string(); assert!(error.contains("schema contract differs"), "{error}"); assert!(!w.root.join("Runtime/python/python.exe").exists()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(unfinished(&w.meta).is_none()); } #[test] fn unhealthy_restoration_preserves_recovery_and_blocks_startup() { let w = world(&[], true); let env = Fake::ok(); env.automatic_rollback.set(true); env.healthy.set(false); env.recovery_healthy.set(false); assert!(run(&w, &env, BTreeMap::new(), None) .unwrap_err() .to_string() .contains("Recovery failed")); let pending = unfinished(&w.meta).unwrap(); assert_eq!(pending.state, State::Failed); assert!(txn_dir(&w.meta, &pending.id) .unwrap() .join("before") .exists()); assert!(ensure_recovered(&w.meta).is_err()); } #[test] fn retrying_startup_commits_without_reapplying_sql_or_restoring_databases() { let w = world(&[], true); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); let calls = env.calls.borrow().len(); env.healthy.set(true); let done = retry_validation( &w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env, ) .unwrap(); assert_eq!(done.state, State::Committed); assert_eq!(done.recovery_point, out.txn.recovery_point); assert_eq!(&env.calls.borrow()[calls..], &["stop", "validate"]); assert!(unfinished(&w.meta).is_none()); assert_eq!( MetaDir::open(&w.meta).unwrap().1.core.version.as_deref(), Some("2.0.0") ); } #[test] fn legacy_retry_uses_the_exact_signed_release_and_preserves_failure() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); let dir = txn_dir(&w.meta, &out.txn.id).unwrap(); fs::remove_file(dir.join("manifest.json")).unwrap(); fs::remove_file(dir.join("manifest.json.sig")).unwrap(); let pending = retry_validation( &w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env, ) .unwrap(); assert_eq!(pending.state, State::NeedsDecision); assert_eq!(pending.recovery_point, out.txn.recovery_point); assert!(pending.message.unwrap().contains("worldserver exited")); rollback(&w.root, &w.meta, &out.txn.id, &env).unwrap(); assert!(retry_validation( &w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env ) .is_err()); } #[test] fn retry_rejects_modified_binaries_and_unsigned_manifests() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); write(&w.root, "Core/worldserver.exe", b"unexpected binary"); assert!(retry_validation( &w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env ) .is_err()); write(&w.root, "Core/worldserver.exe", b"world-v2"); fs::write( txn_dir(&w.meta, &out.txn.id) .unwrap() .join("manifest.json.sig"), "invalid", ) .unwrap(); assert!(retry_validation( &w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env ) .is_err()); assert_eq!(unfinished(&w.meta).unwrap().state, State::NeedsDecision); } #[test] fn crash_recovery_uses_the_journal() { let w = world(&[], false); // simulate a crash after two operations: run to failure, then reset the journal to a non-terminal state run(&w, &Fake::ok(), BTreeMap::new(), Some(2)).unwrap_err(); let id = fs::read_dir(w.meta.join("updates")) .unwrap() .flatten() .next() .unwrap() .file_name() .to_string_lossy() .into_owned(); // pretend the process died mid-apply: files are in the half-applied state again write(&w.root, "Core/worldserver.exe", b"world-v2"); let mut t = load(&w.meta, &id).unwrap(); t.state = State::Applying; for op in t.ops.iter_mut() { op.started = op.path == "Core/worldserver.exe"; op.done &= op.started; } save(&w.meta, &t).unwrap(); assert_eq!(unfinished(&w.meta).unwrap().id, id); rollback(&w.root, &w.meta, &id, &Fake::ok()).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn a_running_server_or_failed_backup_blocks_the_update_before_any_change() { let w = world(&[], false); let env = Fake::ok(); env.stopped.set(false); assert!(run(&w, &env, BTreeMap::new(), None).is_err()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); let env = Fake::ok(); env.snapshot_ok.set(false); let e = run(&w, &env, BTreeMap::new(), None).unwrap_err(); assert!(e.to_string().contains("safety backup failed")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn a_failed_migration_undoes_the_files_and_reports_it() { let w = world(&[], true); let env = Fake::ok(); env.migrate_fail.set(true); let e = run(&w, &env, BTreeMap::new(), None).unwrap_err(); assert!(e.to_string().contains("Database update m1 failed")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(env.calls.borrow().contains(&"restore-databases")); assert!( !env.calls.borrow().contains(&"validate"), "the new build is never started after a failed migration" ); let ok = Fake::ok(); let out = run(&w, &ok, BTreeMap::new(), None).unwrap(); assert_eq!(out.txn.state, State::Committed); assert_eq!( *ok.calls.borrow(), ["stop", "snapshot", "migrate", "validate"] ); } #[test] fn failed_database_recovery_keeps_new_files_and_blocks_retry_until_recovered() { let w = world(&[], true); let env = Fake::ok(); env.migrate_fail.set(true); env.restore_fail.set(true); let error = run(&w, &env, BTreeMap::new(), None).unwrap_err(); assert!(error.to_string().contains("Recovery failed")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2"); let pending = unfinished(&w.meta).unwrap(); assert_eq!(pending.state, State::Failed); assert!(pending.databases_started); assert!(txn_dir(&w.meta, &pending.id).unwrap().join("tree").exists()); assert!(run(&w, &Fake::ok(), BTreeMap::new(), None) .unwrap_err() .to_string() .contains("unfinished")); rollback(&w.root, &w.meta, &pending.id, &Fake::ok()).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(unfinished(&w.meta).is_none()); } #[test] fn interrupted_migration_is_restored_before_files_and_never_replayed() { let w = world(&[], true); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); let recovery = Fake::ok(); rollback(&w.root, &w.meta, &out.txn.id, &recovery).unwrap(); assert_eq!(*recovery.calls.borrow(), ["stop", "restore-databases"]); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn concurrent_updates_are_rejected_without_changes() { let w = world(&[], false); let _lock = operation_lock(&w.meta).unwrap(); assert!(run(&w, &Fake::ok(), BTreeMap::new(), None) .unwrap_err() .to_string() .contains("in progress")); assert!(crate::driver::run(&w.root, crate::driver::Verb::StartAll) .unwrap_err() .to_string() .contains("in progress")); assert!( crate::modules::set_enabled(&w.root, &w.meta, "companions", true) .unwrap_err() .to_string() .contains("in progress") ); assert!( crate::realms::select(&w.root, crate::realms::Mode::Wildcard) .unwrap_err() .to_string() .contains("in progress") ); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn legacy_completed_repair_is_accepted_only_with_verified_repaired_files() { let w = world(&[], false); let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); write(&w.root, "Core/worldserver.exe", b"repaired"); repair_migrating(&w.meta, &mut txn).unwrap(); txn.state = State::Committed; save(&w.meta, &txn).unwrap(); let original = fs::read(txn_dir(&w.meta, &txn.id).unwrap().join("txn.json")).unwrap(); write(&w.root, "Core/worldserver.exe", b"changed"); assert!(ensure_recovered(&w.meta).is_err()); write(&w.root, "Core/worldserver.exe", b"repaired"); ensure_recovered(&w.meta).unwrap(); assert!(load(&w.meta, &txn.id).unwrap().ops[0].done); assert_eq!( fs::read( txn_dir(&w.meta, &txn.id) .unwrap() .join("txn-before-repair-completion.json") ) .unwrap(), original ); write( &w.root, "Core/worldserver.exe", b"a later legitimate update", ); ensure_recovered(&w.meta).unwrap(); txn.ops[0].started = false; save(&w.meta, &txn).unwrap(); assert!(ensure_recovered(&w.meta).is_err()); } #[test] fn changed_repair_backup_inventory_blocks_recovery_before_stopping_or_restoring() { let w = world(&[], false); let point = crate::backup::point_json(&w.meta, "full-backup").unwrap(); fsx::atomic_write(&point, b"original recovery inventory").unwrap(); let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); write(&w.root, "Core/worldserver.exe", b"repaired"); repair_migrating(&w.meta, &mut txn).unwrap(); fsx::atomic_write(&point, b"changed recovery inventory").unwrap(); let env = Fake::ok(); assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err()); assert!(env.calls.borrow().is_empty()); assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired"); assert!(ensure_recovered(&w.meta).is_err()); } #[test] fn failed_repair_recovery_keeps_a_failed_journal_and_its_reason() { let w = world(&[], false); let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); write(&w.root, "Core/worldserver.exe", b"repaired"); repair_migrating(&w.meta, &mut txn).unwrap(); let env = Fake::ok(); env.restore_fail.set(true); assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err()); let saved = load(&w.meta, &txn.id).unwrap(); assert_eq!(saved.state, State::Failed); assert!(saved.message.unwrap().contains("Repair recovery failed")); assert_eq!(saved.recovery_point.as_deref(), Some("full-backup")); assert!(ensure_recovered(&w.meta).is_err()); } #[test] fn repair_cannot_commit_unverified_files() { let w = world(&[], false); let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); assert!(finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).is_err()); assert_eq!(load(&w.meta, &txn.id).unwrap().state, State::Applying); assert_eq!(pending_checked(&w.meta).unwrap().unwrap().id, txn.id); } #[test] fn successful_repair_leaves_a_readable_journal_and_allows_next_operation() { let w = world(&[], false); let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); write(&w.root, "Core/worldserver.exe", b"repaired"); repair_migrating(&w.meta, &mut txn).unwrap(); finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).unwrap(); let saved = load(&w.meta, &txn.id).unwrap(); assert_eq!(saved.state, State::Committed); assert!(saved.ops.iter().all(|op| op.started && op.done)); ensure_recovered(&w.meta).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired"); } #[test] fn interrupted_repair_restores_files_databases_and_ownership_metadata() { let w = world(&[], false); let files = vec![ FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }, FileEntry { path: "Core/repair-created.dll".into(), sha256: fsx::sha256_bytes(b"created"), size: 7, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace, }, ]; let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); write(&w.root, "Core/worldserver.exe", b"repaired"); write(&w.root, "Core/repair-created.dll", b"created"); repair_migrating(&w.meta, &mut txn).unwrap(); assert_eq!(unfinished(&w.meta).unwrap().id, txn.id); let env = Fake::ok(); rollback(&w.root, &w.meta, &txn.id, &env).unwrap(); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(!w.root.join("Core/repair-created.dll").exists()); assert_eq!(*env.calls.borrow(), ["stop", "restore-databases"]); assert!(unfinished(&w.meta).is_none()); } #[test] fn sql_manifest_mismatch_is_rejected_before_snapshot_or_replacement() { let w = world(&[], true); let manifest_path = w.pkg.join("manifest.json"); let mut manifest: Manifest = fsx::read_json(&manifest_path).unwrap(); manifest.migrations[0].sha256 = "a".repeat(64); let bytes = serde_json::to_vec_pretty(&manifest).unwrap(); // A publisher mistake: valid signed archive bytes disagree with the SQL metadata. let key = SigningKey::generate(&mut rand_core::OsRng); let trusted = base64::engine::general_purpose::STANDARD.encode(key.verifying_key().to_bytes()); fs::write(&manifest_path, &bytes).unwrap(); fs::write( w.pkg.join("manifest.json.sig"), base64::engine::general_purpose::STANDARD.encode(key.sign(&bytes).to_bytes()), ) .unwrap(); let env = Fake::ok(); let error = apply( &Params { root: &w.root, meta_dir: &w.meta, source: Source::Dir(w.pkg.clone()), trusted_key: &trusted, cancel: Default::default(), resolutions: Default::default(), env: &env, fail_after_ops: None, }, &|_, _| {}, ) .unwrap_err(); assert!( matches!(error, Error::HashMismatch { .. }), "Expected SQL metadata mismatch, received {error}" ); assert!(env.calls.borrow().is_empty()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(pending_checked(&w.meta).unwrap().is_none()); } #[test] fn preflight_rejection_precedes_snapshot_and_file_changes() { let w = world(&[], true); let env = Fake::ok(); env.preflight_fail.set(true); let original = fsx::sha256_file(&w.meta.join("install.json")).unwrap(); assert!(run(&w, &env, BTreeMap::new(), None) .unwrap_err() .to_string() .contains("Conflicting migration history")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert_eq!( fsx::sha256_file(&w.meta.join("install.json")).unwrap(), original ); assert!(env.calls.borrow().is_empty()); assert!(fs::read_dir(w.meta.join("updates")) .unwrap() .next() .is_none()); } #[test] fn update_saves_recovery_then_migrates_and_validates_once() { let w = world(&[], true); let env = Fake::ok(); let result = run(&w, &env, BTreeMap::new(), None).unwrap(); assert_eq!(result.txn.state, State::Committed); let calls = env.calls.borrow(); assert_eq!(calls.iter().filter(|call| **call == "snapshot").count(), 1); assert_eq!(calls.iter().filter(|call| **call == "migrate").count(), 1); assert_eq!(calls.iter().filter(|call| **call == "validate").count(), 1); assert!( calls.iter().position(|call| *call == "snapshot") < calls.iter().position(|call| *call == "migrate") ); assert!(!w.meta.join("rehearsals").exists()); } #[test] fn missing_and_semantically_corrupt_journals_block_operations() { let w = world(&[], false); let txn = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap().txn; let path = txn_dir(&w.meta, &txn.id).unwrap().join("txn.json"); let mut bad = txn.clone(); bad.id = "another-update".into(); fsx::atomic_write_json(&path, &bad).unwrap(); assert!(pending_checked(&w.meta).is_err()); bad = txn.clone(); bad.ops[0].path = "../outside".into(); fsx::atomic_write_json(&path, &bad).unwrap(); assert!(ensure_recovered(&w.meta).is_err()); bad = txn.clone(); bad.ops[0].started = false; fsx::atomic_write_json(&path, &bad).unwrap(); assert!(ensure_recovered(&w.meta).is_err()); fs::remove_file(&path).unwrap(); assert!(pending_checked(&w.meta).is_err()); let marker = cleanup_marker(&w.meta, &txn.id).unwrap(); fsx::atomic_write_json( &marker, &CleanupMarker { id: txn.id, state: State::Committed, }, ) .unwrap(); assert!(pending_checked(&w.meta).unwrap().is_none()); } #[test] fn damaged_saved_file_blocks_restore_before_any_environment_action() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); let txn = run(&w, &env, BTreeMap::new(), None).unwrap().txn; let before = txn_dir(&w.meta, &txn.id).unwrap().join("before"); write(&before, "Core/worldserver.exe", b"damaged original"); env.calls.borrow_mut().clear(); assert!(rollback(&w.root, &w.meta, &txn.id, &env) .unwrap_err() .to_string() .contains("damaged")); assert!(env.calls.borrow().is_empty()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2"); assert_eq!( pending_checked(&w.meta).unwrap().unwrap().state, State::NeedsDecision ); } #[test] fn unreadable_update_journal_blocks_changes_and_startup() { let w = world(&[], false); write(&w.meta, "updates/broken/txn.json", b"{truncated"); assert!(run(&w, &Fake::ok(), BTreeMap::new(), None) .unwrap_err() .to_string() .contains("cannot be read")); assert!(crate::driver::run(&w.root, crate::driver::Verb::StartAll) .unwrap_err() .to_string() .contains("cannot be read")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } #[test] fn crash_during_commit_restores_the_original_installation_metadata() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); let out = run(&w, &env, BTreeMap::new(), None).unwrap(); let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); meta.core.version = Some("2.0.0".into()); meta.core.commit = Some("a".repeat(40)); fsx::atomic_write_json(&w.meta.join("install.json"), &meta).unwrap(); rollback(&w.root, &w.meta, &out.txn.id, &Fake::ok()).unwrap(); let (_, restored) = MetaDir::open(&w.meta).unwrap(); assert_eq!(restored.core.version.as_deref(), Some("1.0.0")); assert_eq!(restored.core.commit, None); } #[test] fn bad_signature_and_wrong_package_kind_change_nothing() { let w = world(&[], false); let other = SigningKey::generate(&mut rand_core::OsRng); let wrong_key = base64::engine::general_purpose::STANDARD.encode(other.verifying_key().to_bytes()); let env = Fake::ok(); let r = apply( &Params { root: &w.root, meta_dir: &w.meta, source: Source::Dir(w.pkg.clone()), trusted_key: &wrong_key, cancel: Cancel::default(), resolutions: BTreeMap::new(), env: &env, fail_after_ops: None, }, &|_, _| {}, ); assert!(r.is_err()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); assert!(env.calls.borrow().is_empty()); let (_, meta) = MetaDir::open(&w.meta).unwrap(); let p = preview( &w.root, &meta, &Source::Dir(w.pkg.clone()), &w.key, &BTreeMap::new(), ) .unwrap(); assert_eq!(p.to_version, "2.0.0"); assert!(p .items .iter() .any(|i| i.path == "Core/newfile.dll" && i.action == Action::Create)); } }