diff --git a/changelog.d/20261004-recover-server-updates.md b/changelog.d/20261004-recover-server-updates.md new file mode 100644 index 0000000..9d0a9cc --- /dev/null +++ b/changelog.d/20261004-recover-server-updates.md @@ -0,0 +1,7 @@ +--- +area: manager +type: fixed +audience: admins +title: Failed server updates restore databases and files together +--- +Updates now save a full backup, detect interrupted SQL, and check required Wildcard tables. Unfinished updates block server startup until recovered. Update and repair recovery also restore the previous installation metadata. diff --git a/crates/coa-core/examples/update_recovery_e2e.rs b/crates/coa-core/examples/update_recovery_e2e.rs new file mode 100644 index 0000000..9cbf310 --- /dev/null +++ b/crates/coa-core/examples/update_recovery_e2e.rs @@ -0,0 +1,73 @@ +//! Failure injection on a NEW disposable fixture created by tools/make-fixture.ps1. +//! Uses real MySQL dumps, SQL execution, signed packages and update rollback. +use std::{collections::BTreeMap, path::{Path, PathBuf}}; +use coa_core::{backup, db::{Account, Db}, download::Cancel, driver::{self, Verb}, fsx, + manifest::{Kind, Migration}, package::{self, BuildOptions}, pkgsource::Source, + registry::{InstallKind, InstallMeta, MetaDir}, + update::{self, Env, RepackEnv}, Error, Result}; +use base64::Engine; +use ed25519_dalek::{Signer, SigningKey}; + +fn main() -> Result<()> { + let arg = std::env::args().nth(1).expect("usage: update_recovery_e2e "); + let root = PathBuf::from(arg); + assert!(root.file_name().unwrap().to_string_lossy().starts_with("coa-update-recovery-fixture-")); + assert!(!root.join("recovery-e2e-started").exists(), "Use a new fixture; never reuse an owned server"); + fsx::atomic_write(&root.join("recovery-e2e-started"), b"disposable fixture")?; + let result = exercise(&root); + let stopped = driver::run(&root, Verb::StopAll)?; + assert!(stopped.ok); + result +} + +fn exercise(root: &Path) -> Result<()> { + let meta = coa_core::registry::metadata_dir_for(root)?; + let mut installed = InstallMeta::new(InstallKind::New, root); + installed.core.version = Some("1.0.0".into()); + MetaDir::create(root, &installed)?; + backup::with_database(root, |db| { + db.query("CREATE TABLE acore_characters.manager_recovery_probe (id INT); INSERT INTO acore_characters.manager_recovery_probe VALUES (42); CREATE TABLE acore_world.manager_recovery_probe (id INT); INSERT INTO acore_world.manager_recovery_probe VALUES (43);")?; + Ok(()) + })?; + let source = root.join("fixture-update-source"); + fsx::atomic_write(&source.join("Core/recovery-probe.txt"), b"updated")?; + let mut migrations = Vec::new(); + for (db, id, sql) in [ + ("characters", "probe_chars", "UPDATE manager_recovery_probe SET id=999;"), + ("world", "probe_world", "DROP TABLE manager_recovery_probe; THIS IS INTENTIONALLY INVALID SQL;"), + ] { + fsx::atomic_write(&source.join(format!("_migrations/{db}/{id}.sql")), sql.as_bytes())?; + migrations.push(Migration { db: db.into(), id: id.into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); + } + let pkg = root.join("fixture-package"); + let manifest = package::build(&source, &pkg, &BuildOptions { + kind: Kind::Update, version: "2.0.0".into(), core_commit: None, bots_commit: None, + built_at: "fixture".into(), part_size: 1 << 20, migrations, + }, &|_| {})?; + let key = SigningKey::generate(&mut rand_core::OsRng); + let signature = base64::engine::general_purpose::STANDARD.encode(key.sign(&std::fs::read(pkg.join("manifest.json"))?).to_bytes()); + fsx::atomic_write(&pkg.join("manifest.json.sig"), signature.as_bytes())?; + let trusted = base64::engine::general_purpose::STANDARD.encode(key.verifying_key().to_bytes()); + let env = RepackEnv { root, meta_dir: &meta }; + let error = update::apply(&update::Params { + root, meta_dir: &meta, source: Source::Dir(pkg), trusted_key: &trusted, + cancel: Cancel::default(), resolutions: BTreeMap::new(), env: &env, fail_after_ops: None, + }, &|step, pct| println!("{pct}% {step}" )).unwrap_err(); + assert!(error.to_string().contains("probe_world"), "{error}"); + assert!(error.to_string().contains("were restored"), "{error}"); + assert!(!root.join("Core/recovery-probe.txt").exists()); + assert!(update::unfinished(&meta).is_none()); + assert_eq!(MetaDir::open(&meta)?.1.core.version.as_deref(), Some("1.0.0")); + backup::with_database(root, |db| { + assert_eq!(db.query("SELECT id FROM acore_characters.manager_recovery_probe;")?, "42"); + assert_eq!(db.query("SELECT id FROM acore_world.manager_recovery_probe;")?, "43"); + assert!(!coa_core::migrations::status(db, &manifest.migrations)?.iter().any(|m| m.status == coa_core::migrations::Status::Applied)); + Ok(()) + })?; + // A recovery point can be reused if the process died midway through restoring multiple schemas. + let point = backup::list(&meta).into_iter().find(|p| p.trigger == backup::Trigger::BeforeUpdate).ok_or_else(|| Error::Invalid("missing backup".into()))?; + env.restore_snapshot(&point.id)?; + assert_eq!(Db::from_repack(root, Account::Admin)?.realm(), coa_core::realms::Mode::Coa); + println!("PASS: failed SQL restored world, characters, ledger, files and version; repeated recovery succeeded"); + Ok(()) +} diff --git a/crates/coa-core/src/backup.rs b/crates/coa-core/src/backup.rs index e612dc0..5fcad0d 100644 --- a/crates/coa-core/src/backup.rs +++ b/crates/coa-core/src/backup.rs @@ -179,7 +179,7 @@ fn wanted_databases(kind: Kind) -> &'static [&'static str] { /// Create a recovery point. The server may be running: dumps are consistent snapshots (`--single-transaction`). pub fn create(root: &Path, meta: &Path, kind: Kind, trigger: Trigger, label: Option, progress: &dyn Fn(&str)) -> Result { - let id = format!("{}-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), match trigger { + let id = format!("{}-{}-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), match trigger { Trigger::Manual => "manual", Trigger::Automatic => "auto", Trigger::BeforeUpdate => "before-update", @@ -188,7 +188,7 @@ pub fn create(root: &Path, meta: &Path, kind: Kind, trigger: Trigger, label: Opt Trigger::BeforeBots => "before-bots", Trigger::BeforeRepair => "before-repair", Trigger::BeforeDangerousChange => "before-change", - }); + }, uuid::Uuid::new_v4().simple()); let final_dir = point_dir(meta, &id)?; let partial = backups_dir(meta).join(format!("{id}.partial")); fs::create_dir_all(&partial)?; @@ -296,6 +296,9 @@ pub fn verify(meta: &Path, id: &str) -> Result { /// Delete one recovery point (its own folder only). pub fn delete(meta: &Path, id: &str) -> Result<()> { + if crate::update::unfinished(meta).is_some_and(|t| t.recovery_point.as_deref() == Some(id)) { + return Err(Error::Invalid("This recovery point is required by an unfinished update and cannot be deleted.".into())); + } let dir = point_dir(meta, id)?; get(meta, id)?; // must be a real recovery point fs::remove_dir_all(dir)?; @@ -347,7 +350,7 @@ pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Resul let live = if name.contains('-') || name == "playerbots" { db::schema_of(name)? } else { point.realm.schema(name)? }; let expected_tables = comp.tables.unwrap_or(0); let dump = point_dir(meta, id)?.join(&comp.path); - let stamp = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); + let stamp = uuid::Uuid::new_v4().simple().to_string()[..16].to_string(); // 1. Safety copy of the current state, so even a wrong restore is reversible. let safety = create(root, meta, Kind::Database, Trigger::BeforeRestore, Some(format!("before restoring {name} from {id}")), &|_| {})?; @@ -364,7 +367,7 @@ pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Resul db.import_from(&staging, &dump)?; // 3. Sanity checks. let staged = db.tables(&staging)?; - if staged.is_empty() || (expected_tables > 0 && staged.len() != expected_tables) { + if comp.tables.is_some_and(|expected| staged.len() != expected) || (comp.tables.is_none() && staged.is_empty()) { return Err(Error::Invalid(format!("The restored copy looks incomplete ({} of {expected_tables} tables); nothing was changed.", staged.len()))); } if db.extra_objects(&staging)? > 0 || db.extra_objects(live)? > 0 { @@ -380,7 +383,7 @@ pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Resul for t in &staged { renames.push(format!("`{staging}`.`{t}` TO `{live}`.`{t}`")); } - db.query(&format!("RENAME TABLE {};", renames.join(", ")))?; + if !renames.is_empty() { db.query(&format!("RENAME TABLE {};", renames.join(", ")))?; } Ok(DbRestore { previous_schema: old, safety_backup: safety.id.clone(), tables_restored: staged.len() }) }) } diff --git a/crates/coa-core/src/driver.rs b/crates/coa-core/src/driver.rs index 7ab1073..6385678 100644 --- a/crates/coa-core/src/driver.rs +++ b/crates/coa-core/src/driver.rs @@ -81,7 +81,21 @@ fn launcher(root: &Path) -> Result<(PathBuf, PathBuf)> { /// Run a launcher verb to completion (blocking; call from a worker thread). pub fn run(root: &Path, verb: Verb) -> Result { + run_inner(root, verb, false) +} + +pub(crate) fn validate_update(root: &Path) -> Result { + run_inner(root, Verb::StartAll, true) +} + +fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result { let root = fsx::canonicalize_lenient(root)?; + let _update_lock = if !validating_update && matches!(verb, Verb::StartAll | Verb::StartWorld) { + Some(crate::update::operation_lock(&crate::registry::metadata_dir_for(&root)?)?) + } else { None }; + if !validating_update && matches!(verb, Verb::StartAll | Verb::StartWorld) { + crate::update::ensure_recovered(&crate::registry::metadata_dir_for(&root)?)?; + } if matches!(verb, Verb::StartAll | Verb::StartWorld) { crate::modules::ensure_bot_exclusivity(&root)?; } if crate::docker::is_docker(&root) { return crate::docker::run(&root, verb); diff --git a/crates/coa-core/src/migrations.rs b/crates/coa-core/src/migrations.rs index 00e0be1..6a53449 100644 --- a/crates/coa-core/src/migrations.rs +++ b/crates/coa-core/src/migrations.rs @@ -19,6 +19,7 @@ pub enum Status { Applied, Pending, Failed, + Running, } #[derive(Debug, Clone)] @@ -55,7 +56,8 @@ impl Store for Db { let mut rows = Vec::new(); // Files the core's own updater (or whoever prepared this database) already recorded count as applied. for (kind, schema) in db::SCHEMAS { - if let Ok(names) = self.query(&format!("SELECT name FROM `{schema}`.`updates`;")) { + if self.tables(schema)?.iter().any(|name| name == "updates") { + let names = self.query(&format!("SELECT name FROM `{schema}`.`updates`;"))?; for name in names.lines().filter(|n| n.ends_with(".sql")) { rows.push(LedgerRow { db: kind.into(), id: name.trim_end_matches(".sql").into(), sha256: "0".repeat(64), status: Status::Applied, error: None, baseline: true }); } @@ -64,14 +66,17 @@ impl Store for Db { for line in out.lines().filter(|l| !l.is_empty()) { let f: Vec<&str> = line.split('\t').collect(); if f.len() < 5 { - continue; + return Err(Error::Invalid("The database migration history is malformed; no SQL was replayed.".into())); + } + if !matches!(f[3], "applied" | "failed" | "running") || !ident_ok(f[1]) || f[2].len() != 64 || !f[2].chars().all(|c| c.is_ascii_hexdigit()) { + return Err(Error::Invalid("The database migration history is invalid; no SQL was replayed.".into())); } let err = f.get(5).filter(|h| !h.is_empty()).and_then(|h| hex::decode(h).ok()).map(|b| String::from_utf8_lossy(&b).into_owned()); rows.push(LedgerRow { db: f[0].into(), id: f[1].into(), sha256: f[2].into(), - status: if f[3] == "applied" { Status::Applied } else { Status::Failed }, + status: match f[3] { "applied" => Status::Applied, "running" => Status::Running, _ => Status::Failed }, baseline: f[4] == "1", error: err, }); @@ -83,7 +88,7 @@ impl Store for Db { if !ident_ok(&r.id) || !ident_ok(&r.db) || r.sha256.len() != 64 || !r.sha256.chars().all(|c| c.is_ascii_hexdigit()) { return Err(Error::Invalid("migration record has an invalid identifier".into())); } - let status = if r.status == Status::Applied { "applied" } else { "failed" }; + let status = match r.status { Status::Applied => "applied", Status::Running => "running", _ => "failed" }; let err = r.error.as_deref().map(|e| format!("UNHEX('{}')", hex_of(e))).unwrap_or_else(|| "NULL".into()); self.query(&format!( "REPLACE INTO `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db`,`id`,`sha256`,`status`,`applied_at`,`error`,`baseline`) VALUES ('{}','{}','{}','{status}',NOW(),{err},{});", @@ -129,7 +134,7 @@ pub fn status(store: &dyn Store, list: &[Migration]) -> Result> { Item { id: m.id.clone(), db: m.db.clone(), - status: if changed { Status::Failed } else { row.map(|r| r.status).unwrap_or(Status::Pending) }, + status: if changed || row.is_some_and(|r| r.status == Status::Running) { Status::Failed } else { row.map(|r| r.status).unwrap_or(Status::Pending) }, destructive: m.destructive, baseline: row.map(|r| r.baseline).unwrap_or(false), error: if changed { Some("An already applied SQL file changed. Publish a new migration instead of replaying it.".into()) } else { row.and_then(|r| r.error.clone()) }, @@ -174,6 +179,9 @@ pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot let rows = store.load()?; // The list order is the release's order of application (released updates, then pending, then modules). for m in list { + if recorded(&rows, m).is_some_and(|r| matches!(r.status, Status::Running | Status::Failed)) { + return Err(Error::Invalid(format!("Database update {} ({}) failed or was interrupted and may be partially applied. Restore its recovery point before retrying.", m.id, m.db))); + } if recorded(&rows, m).is_some_and(|r| hash_changed(r, m)) { return Err(Error::Invalid(format!("Applied database update {} ({}) has a different checksum. A new migration is required; it was not replayed.", m.id, m.db))); } @@ -206,6 +214,7 @@ pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot report.snapshot = Some(snapshot().map_err(|e| Error::Invalid(format!("No database update was applied because the safety backup failed: {e}")))?); } for (m, schema, path) in files { + store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Running, error: Some("Interrupted SQL must be recovered before replay.".into()), baseline: false })?; match store.run_file(schema, &path) { Ok(()) => { store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Applied, error: None, baseline: false })?; @@ -277,7 +286,8 @@ mod tests { let failed = LedgerRow { sha256: list[0].sha256.clone(), status: Status::Failed, error: Some("previous failure".into()), baseline: false, ..old.clone() }; store.rows.borrow_mut().extend([old, failed]); assert_eq!(status(&store, &list).unwrap()[0].status, Status::Failed); - assert_eq!(apply_pending(&store, &list, d.path(), &no_snapshot).unwrap().applied, ["same_id"]); + assert!(apply_pending(&store, &list, d.path(), &no_snapshot).unwrap_err().to_string().contains("partially applied")); + assert!(store.ran.borrow().is_empty()); } #[test] @@ -340,6 +350,16 @@ mod tests { assert!(apply_pending(&store, &list[..1], d.path(), &no_snapshot).is_err()); } + #[test] + fn interrupted_sql_is_reported_and_never_automatically_replayed() { + let (d, list) = setup(&[("world", "partial", "DROP TABLE valuable;")]); + let store = Mem::default(); + store.rows.borrow_mut().push(LedgerRow { db: "world".into(), id: "partial".into(), sha256: list[0].sha256.clone(), status: Status::Running, error: Some("interrupted".into()), baseline: false }); + assert_eq!(status(&store, &list).unwrap()[0].status, Status::Failed); + assert!(apply_pending(&store, &list, d.path(), &no_snapshot).unwrap_err().to_string().contains("interrupted")); + assert!(store.ran.borrow().is_empty()); + } + #[test] fn baseline_marks_without_running_and_hostile_ids_are_rejected() { let (d, list) = setup(&[("world", "b1", "SELECT 1;")]); diff --git a/crates/coa-core/src/modules.rs b/crates/coa-core/src/modules.rs index a18f228..f5df2eb 100644 --- a/crates/coa-core/src/modules.rs +++ b/crates/coa-core/src/modules.rs @@ -185,11 +185,15 @@ fn require_bot_server_stopped(root: &Path) -> Result<()> { } pub fn set_enabled(root: &Path, meta: &Path, id: &str, on: bool) -> Result<()> { + let _update_lock = crate::update::operation_lock(meta)?; + crate::update::ensure_recovered(meta)?; let _lock = if matches!(id, "companions" | "playerbots") { require_bot_server_stopped(root)?; fs::create_dir_all(root.join(".state"))?; let lock = fs::OpenOptions::new().read(true).write(true).create(true).truncate(false).open(root.join(".state/control.lock"))?; - fs4::fs_std::FileExt::try_lock_exclusive(&lock).map_err(|_| Error::Invalid("Another start/stop action is in progress.".into()))?; + if !fs4::fs_std::FileExt::try_lock_exclusive(&lock)? { + return Err(Error::Invalid("Another start/stop action is in progress.".into())); + } require_bot_server_stopped(root)?; Some(lock) } else { None }; diff --git a/crates/coa-core/src/realms.rs b/crates/coa-core/src/realms.rs index a02d3e9..1255072 100644 --- a/crates/coa-core/src/realms.rs +++ b/crates/coa-core/src/realms.rs @@ -179,13 +179,12 @@ fn create_databases(root: &Path) -> Result<()> { return Err(Error::Invalid("Realm creation does not support databases with routines, triggers or views.".into())); } } - for (schema, tables) in [ - ("acore_world", &["ascension_wildcard_boss_marks", "ascension_wildcard_tooltip_links"][..]), - ("acore_characters", &["coa_wildcard_skill_card", "coa_wildcard_skill_card_pending", "coa_wildcard_skill_card_account", "coa_wildcard_skill_card_purchase", "coa_wildcard_specialization_cache"][..]), - ] { + for (kind, tables) in crate::schema_check::WILDCARD_TABLES { + let schema = crate::db::schema_of(kind)?; let present = db.tables(schema)?; - if tables.iter().any(|table| !present.iter().any(|p| p == table)) { - return Err(Error::Invalid("Wildcard database migrations are missing. Update the server database before creating this realm.".into())); + let missing: Vec<_> = tables.iter().filter(|table| !present.iter().any(|p| p == **table)).copied().collect(); + if !missing.is_empty() { + return Err(Error::Invalid(format!("Cannot create Wildcard: {schema} is missing tables: {}. Run Check files and database and include its report when requesting support.", missing.join(", ")))); } } let cache = root.join("Settings/realm-profiles/staging"); @@ -226,6 +225,9 @@ fn create_databases(root: &Path) -> Result<()> { } pub fn select(root: &Path, mode: Mode) -> Result { + let meta = crate::registry::metadata_dir_for(root)?; + let _update_lock = crate::update::operation_lock(&meta)?; + crate::update::ensure_recovered(&meta)?; require_world_stopped(root)?; recover(root)?; let mut s = state(root)?; @@ -247,7 +249,9 @@ pub fn select(root: &Path, mode: Mode) -> Result { // The launcher and batch files share this lock; recheck after the potentially long database copy. fs::create_dir_all(root.join(".state"))?; let lock = fs::OpenOptions::new().read(true).write(true).create(true).truncate(false).open(root.join(".state/control.lock"))?; - fs4::fs_std::FileExt::try_lock_exclusive(&lock).map_err(|_| Error::Invalid("Another start/stop action is in progress.".into()))?; + if !fs4::fs_std::FileExt::try_lock_exclusive(&lock)? { + return Err(Error::Invalid("Another start/stop action is in progress.".into())); + } require_world_stopped(root)?; let original = state(root)?; fsx::atomic_write_json(&root.join(JOURNAL), &Journal { before: original, files: before })?; diff --git a/crates/coa-core/src/repair.rs b/crates/coa-core/src/repair.rs index 23fe1a7..d97672e 100644 --- a/crates/coa-core/src/repair.rs +++ b/crates/coa-core/src/repair.rs @@ -90,9 +90,10 @@ fn restore_files(root: &Path, work: &Path, before: &Path, broken: &[(&FileEntry, } pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source, key: &str, progress: &dyn Fn(&str, u8)) -> Result { + let _lock = crate::update::operation_lock(dir)?; let env = RepackEnv { root, meta_dir: dir }; env.ensure_stopped()?; - if crate::update::unfinished(dir).is_some() { return Err(Error::Invalid("Resolve the unfinished update before repairing the server.".into())); } + crate::update::ensure_recovered(dir)?; let (_, mut meta) = MetaDir::open(dir)?; progress("Checking signed packages", 2); let (update, _) = pkgsource::fetch_manifest(update_source, key)?; @@ -128,6 +129,9 @@ pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source } progress("Backing up before repair", 65); let backup = crate::backup::create(root, dir, crate::backup::Kind::Full, crate::backup::Trigger::Manual, Some("before repair".into()), &|_| {})?.id; + env.verify_recovery_point(&crate::backup::get(dir, &backup)?)?; + let repair_files: Vec<_> = broken.iter().map(|(file, _)| (*file).clone()).collect(); + let mut txn = crate::update::begin_repair(root, dir, &update.version, &backup, &repair_files)?; let before = dir.join("repairs").join(&backup); fs::create_dir_all(&before)?; fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?; @@ -137,15 +141,23 @@ pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source } fsx::atomic_write_json(&dir.join("install.json"), &meta)?; progress("Repairing pending database updates", 80); + crate::update::repair_migrating(dir, &mut txn)?; let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) { Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))), Err(e) => (Vec::new(), Some(e.to_string())), }; + if error.is_some() { + crate::update::finish_repair(root, dir, &mut txn, false, &env)?; + let report = Report { restored: Vec::new(), applied: Vec::new(), backup, database: check(root, dir)?, error }; + fsx::atomic_write_json(&before.join("report.json"), &report)?; + return Ok(report); + } crate::config::create_missing_module_configs(root)?; progress("Checking database after repair", 95); let database = check(root, dir)?; let report = Report { restored, applied, backup, database, error }; fsx::atomic_write_json(&before.join("report.json"), &report)?; + crate::update::finish_repair(root, dir, &mut txn, true, &env)?; progress("Done", 100); Ok(report) })(); diff --git a/crates/coa-core/src/schema_check.rs b/crates/coa-core/src/schema_check.rs index dec3d23..5b74496 100644 --- a/crates/coa-core/src/schema_check.rs +++ b/crates/coa-core/src/schema_check.rs @@ -5,6 +5,19 @@ use serde::{Deserialize, Serialize}; use crate::{db::Db, error::Result, fsx}; pub const CONTRACT: &str = "Scripts/database-schema.json"; +pub const WILDCARD_TABLES: &[(&str, &[&str])] = &[ + ("world", &["ascension_wildcard_boss_marks", "ascension_wildcard_tooltip_links"]), + ("characters", &["coa_wildcard_skill_card", "coa_wildcard_skill_card_pending", "coa_wildcard_skill_card_account", "coa_wildcard_skill_card_purchase", "coa_wildcard_specialization_cache"]), +]; +const WILDCARD_COLUMNS: &[(&str, &str, &[&str])] = &[ + ("world", "ascension_wildcard_boss_marks", &["CreatureEntry", "Amount"]), + ("world", "ascension_wildcard_tooltip_links", &["Talent", "Ability"]), + ("characters", "coa_wildcard_skill_card", &["account", "card", "progress"]), + ("characters", "coa_wildcard_skill_card_pending", &["account", "id", "card"]), + ("characters", "coa_wildcard_skill_card_account", &["account", "bonus_progress"]), + ("characters", "coa_wildcard_skill_card_purchase", &["account", "type", "count"]), + ("characters", "coa_wildcard_specialization_cache", &["account", "claimed_at"]), +]; type Columns = BTreeMap>>; #[derive(Debug, Serialize, Deserialize)] @@ -44,6 +57,26 @@ pub fn check(db: &Db, root: &Path) -> Result> { Contract { schema: 1, columns: BTreeMap::from([("characters".into(), tables.into_iter().map(|(t, cols)| (t, cols.into_iter().map(|c| (c, String::new())).collect())).collect())]) } }; let mut problems = Vec::new(); + // Older releases omit a schema contract. Still verify the feature's actual tables, + // independently of its migration history, before declaring an update healthy. + if std::fs::read(root.join("Core/worldserver.exe")).is_ok_and(|bytes| bytes.windows(b"Wildcard synergy settings".len()).any(|w| w == b"Wildcard synergy settings")) { + for (kind, tables) in WILDCARD_TABLES { + let actual = read_columns(db, kind)?; + for table in *tables { + if !actual.contains_key(*table) { + problems.push(Problem { database: (*kind).into(), table: (*table).into(), column: String::new(), detail: "Missing required Wildcard table; migration history is not proof of its presence".into() }); + } else { + for (_, _, columns) in WILDCARD_COLUMNS.iter().filter(|(k, t, _)| k == kind && t == table) { + for column in *columns { + if !actual[*table].contains_key(*column) { + problems.push(Problem { database: (*kind).into(), table: (*table).into(), column: (*column).into(), detail: "Missing required Wildcard column".into() }); + } + } + } + } + } + } + } for (kind, expected) in &contract.columns { let actual = read_columns(db, kind)?; for (table, cols) in expected { diff --git a/crates/coa-core/src/update.rs b/crates/coa-core/src/update.rs index 531278d..c45e92a 100644 --- a/crates/coa-core/src/update.rs +++ b/crates/coa-core/src/update.rs @@ -3,7 +3,7 @@ //! Nothing is changed until the signed package is downloaded, verified and extracted into staging. Every file that is //! about to change is first copied into the transaction's `before/` folder, each operation is journaled, and a crash or //! failure at any point leaves the installation recoverable: `rollback` restores binaries and configuration exactly. -//! Databases are never rolled back automatically (players may have progressed); restoring one is a separate action. +//! Full recovery points protect database changes as well as files. Interrupted updates must be resolved first. use std::collections::BTreeMap; use std::fs; @@ -73,15 +73,20 @@ pub struct Txn { pub from_version: Option, pub to_version: String, pub recovery_point: Option, + #[serde(default = "legacy_databases_started")] + pub databases_started: bool, pub ops: Vec, pub message: Option, } +fn legacy_databases_started() -> bool { true } + /// Everything the transaction needs from the outside world; tests provide a fake. pub trait Env { fn ensure_stopped(&self) -> Result<()>; /// Create a recovery point for the databases and configuration; returns its id. fn snapshot(&self) -> Result; + fn restore_snapshot(&self, id: &str) -> Result<()>; fn migrate(&self, manifest: &Manifest, staged_migrations: &Path) -> Result; /// Start the server and confirm it becomes healthy. fn validate(&self) -> Result<()>; @@ -111,7 +116,27 @@ pub fn unfinished(meta: &Path) -> Option { let rd = fs::read_dir(updates_dir(meta)).ok()?; let mut all: Vec = rd.flatten().filter_map(|e| fsx::read_json::(&e.path().join("txn.json")).ok()).collect(); all.sort_by(|a, b| b.id.cmp(&a.id)); - all.into_iter().find(|t| !matches!(t.state, State::Committed | State::RolledBack | State::Failed | State::Prepared)) + all.into_iter().find(|t| !matches!(t.state, State::Committed | State::RolledBack | State::Prepared) && (t.state != State::Failed || t.databases_started || t.ops.iter().any(|o| o.started))) +} + +pub fn ensure_recovered(meta: &Path) -> Result<()> { + match fs::read_dir(updates_dir(meta)) { + Ok(entries) => { + for entry in entries { + let entry = entry?; + let journal = entry.path().join("txn.json"); + if journal.exists() { + fsx::read_json::(&journal).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Recover it before changing or starting the server.", journal.display())))?; + } + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.into()), + } + if let Some(t) = unfinished(meta) { + return Err(Error::Invalid(format!("Resolve unfinished update {} before changing or starting the server.", t.id))); + } + Ok(()) } fn sha_if_exists(p: &Path) -> Option { @@ -227,6 +252,8 @@ fn step(report: &dyn Fn(&str, u8), name: &str, pct: u8) { /// a build that is applied but unhealthy is left in `NeedsDecision` so the owner chooses. pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let (root, meta_dir) = (p.root, p.meta_dir); + let _lock = operation_lock(meta_dir)?; + ensure_recovered(meta_dir)?; let (md, mut meta) = MetaDir::open(meta_dir)?; let _ = md; @@ -234,12 +261,13 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let (m, _) = fetch_manifest(&p.source, p.trusted_key)?; check_manifest(&m)?; let archive = m.archive.clone().ok_or_else(|| Error::InvalidManifest("no archive".into()))?; - let id = format!("{}-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), m.version.replace('.', "_")); + let id = format!("{}-{}-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), m.version.replace('.', "_"), uuid::Uuid::new_v4().simple()); let tdir = txn_dir(meta_dir, &id)?; let (tree, before) = (tdir.join("tree"), tdir.join("before")); fs::create_dir_all(&before)?; + fsx::atomic_write(&before.join("manager-install.json"), &fs::read(meta_dir.join("install.json"))?)?; - let mut txn = Txn { id: id.clone(), state: State::Prepared, from_version: meta.core.version.clone(), to_version: m.version.clone(), recovery_point: None, ops: Vec::new(), message: None }; + let mut txn = Txn { id: id.clone(), state: State::Prepared, from_version: meta.core.version.clone(), to_version: m.version.clone(), recovery_point: None, databases_started: false, ops: Vec::new(), message: None }; save(meta_dir, &txn)?; // Nothing has touched the installation yet; failures up to here just discard the staging area. @@ -293,20 +321,17 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let applied = apply_ops(root, &tree, &before, &m.files, &mut txn, meta_dir, p.fail_after_ops); if let Err(e) = applied { step(report, "Undoing the update", 70); - let restore = restore_files(root, &before, &mut txn); - txn.state = if restore.is_ok() { State::RolledBack } else { State::Failed }; - txn.message = Some(e.to_string()); - save(meta_dir, &txn)?; - let _ = fs::remove_dir_all(&tree); - return Err(e); + return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), None); } txn.state = State::Applied; save(meta_dir, &txn)?; - // Database migrations: a failure here undoes the file changes; the databases are left as they are. + // MySQL DDL can commit before a later statement fails. Restore the full recovery point on failure. let mut migrated = None; if !m.migrations.is_empty() { step(report, "Updating the database", 75); + txn.databases_started = true; + save(meta_dir, &txn)?; match p.env.migrate(&m, &tree.join("_migrations")) { Ok(r) if r.failed.is_none() => migrated = Some(r), Ok(r) => { @@ -359,13 +384,31 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { } fn fail_after_apply(p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: String, report: Option) -> Result { - let restored = restore_files(p.root, before, txn); + let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env); txn.state = if restored.is_ok() { State::RolledBack } else { State::Failed }; - txn.message = Some(why.clone()); + txn.message = Some(match &restored { Ok(()) => why.clone(), Err(e) => format!("{why} Recovery failed: {e}") }); save(p.meta_dir, txn)?; - let _ = fs::remove_dir_all(tree); + if restored.is_ok() { let _ = fs::remove_dir_all(tree); } let _ = report; - Err(Error::Invalid(format!("{why} Your files were restored and your databases were not touched by the restore."))) + Err(Error::Invalid(match restored { + Ok(()) if txn.databases_started => format!("{why} The server files and databases were restored to the recovery point."), + Ok(()) => format!("{why} The server files were restored."), + Err(e) => format!("{why} Recovery failed: {e}. Resolve the unfinished update before starting the server."), + })) +} + +fn restore_transaction(root: &Path, meta: &Path, before: &Path, txn: &mut Txn, env: &dyn Env) -> Result<()> { + env.ensure_stopped()?; + if txn.databases_started { + let id = txn.recovery_point.as_deref().ok_or_else(|| Error::Invalid("The database recovery point is missing.".into()))?; + env.restore_snapshot(id)?; + } + restore_files(root, before, txn)?; + let saved_meta = before.join("manager-install.json"); + if saved_meta.exists() { + fsx::atomic_write(&meta.join("install.json"), &fs::read(saved_meta)?)?; + } + Ok(()) } fn apply_ops(root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: &mut Txn, meta_dir: &Path, fail_after: Option) -> Result<()> { @@ -378,14 +421,15 @@ fn apply_ops(root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: let target = fsx::ensure_within(root, &fsx::safe_join(root, &path)?)?; let staged = fsx::safe_join(tree, &path)?; let entry = files.iter().find(|f| f.path == path).expect("listed"); - txn.ops[i].started = true; - save(meta_dir, txn)?; - if target.is_file() { let keep = fsx::safe_join(before, &path)?; fs::create_dir_all(keep.parent().unwrap())?; - fs::copy(&target, &keep)?; + fsx::atomic_write(&keep, &fs::read(&target)?)?; } + // Journal only after the old bytes are safely saved; a crash during backup must not + // turn an untouched file into a rollback operation with a missing saved copy. + txn.ops[i].started = true; + save(meta_dir, txn)?; fs::create_dir_all(target.parent().unwrap())?; match action { Action::Create | Action::Replace => { @@ -435,14 +479,14 @@ fn restore_files(root: &Path, before: &Path, txn: &mut Txn) -> Result<()> { Ok(()) } -/// Roll a transaction back (binaries and configuration only; databases are never touched). +/// Restore the database recovery point first, then restore binaries and configuration. pub fn rollback(root: &Path, meta_dir: &Path, id: &str, env: &dyn Env) -> Result { + let _lock = operation_lock(meta_dir)?; let mut txn = load(meta_dir, id)?; if matches!(txn.state, State::Committed | State::RolledBack) { return Err(Error::Invalid("This update cannot be rolled back.".into())); } - env.ensure_stopped()?; - restore_files(root, &txn_dir(meta_dir, id)?.join("before"), &mut txn)?; + restore_transaction(root, meta_dir, &txn_dir(meta_dir, id)?.join("before"), &mut txn, env)?; txn.state = State::RolledBack; txn.message = Some("rolled back".into()); save(meta_dir, &txn)?; @@ -450,9 +494,47 @@ pub fn rollback(root: &Path, meta_dir: &Path, id: &str, env: &dyn Env) -> Result Ok(txn) } +pub(crate) fn operation_lock(meta_dir: &Path) -> Result { + fs::create_dir_all(meta_dir)?; + let lock = fs::OpenOptions::new().read(true).write(true).create(true).truncate(false).open(meta_dir.join("update.lock"))?; + if !fs4::fs_std::FileExt::try_lock_exclusive(&lock)? { + return Err(Error::Invalid("Another update, repair or recovery is in progress.".into())); + } + Ok(lock) +} + +/// Repair uses the same durable recovery journal as an update. Its caller holds operation_lock. +pub(crate) fn begin_repair(root: &Path, meta_dir: &Path, version: &str, backup: &str, files: &[FileEntry]) -> Result { + let id = format!("{}-repair-{}", chrono::Utc::now().format("%Y%m%d-%H%M%S"), uuid::Uuid::new_v4().simple()); + let before = txn_dir(meta_dir, &id)?.join("before"); + fsx::atomic_write(&before.join("manager-install.json"), &fs::read(meta_dir.join("install.json"))?)?; + let mut ops = Vec::new(); + for file in files { + let target = fsx::ensure_within(root, &fsx::safe_join(root, &file.path)?)?; + let had_previous = target.is_file(); + if had_previous { fsx::atomic_write(&fsx::safe_join(&before, &file.path)?, &fs::read(&target)?)?; } + ops.push(Op { path: file.path.clone(), action: if had_previous { Action::Replace } else { Action::Create }, reason: None, new_sha256: file.sha256.clone(), had_previous, started: true, done: false }); + } + let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()) }; + save(meta_dir, &txn)?; + Ok(txn) +} + +pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> { + txn.state = State::Applied; + txn.databases_started = true; + save(meta_dir, txn) +} + +pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> { + if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; } + txn.state = if success { State::Committed } else { State::RolledBack }; + save(meta_dir, txn) +} + fn prune(meta_dir: &Path, keep: usize) { let Ok(rd) = fs::read_dir(updates_dir(meta_dir)) else { return }; - let mut dirs: Vec<_> = rd.flatten().filter(|e| e.path().join("txn.json").is_file()).map(|e| e.path()).collect(); + let mut dirs: Vec<_> = rd.flatten().filter(|e| fsx::read_json::(&e.path().join("txn.json")).is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack))).map(|e| e.path()).collect(); dirs.sort(); while dirs.len() > keep { let _ = fs::remove_dir_all(dirs.remove(0)); @@ -465,6 +547,35 @@ pub struct RepackEnv<'a> { pub meta_dir: &'a Path, } +impl RepackEnv<'_> { + pub(crate) fn verify_recovery_point(&self, point: &crate::backup::RecoveryPoint) -> Result<()> { + if point.kind != crate::backup::Kind::Full || !["characters", "auth", "world", "configs"].iter().all(|name| point.components.iter().any(|c| c.name == *name)) { + return Err(Error::Invalid("The update recovery point does not contain every required database and configuration.".into())); + } + if !crate::backup::verify(self.meta_dir, &point.id)?.ok { return Err(Error::Invalid("The recovery point failed verification.".into())); } + crate::backup::with_database(self.root, |db| { + let db = db.clone().for_realm(crate::realms::Mode::Coa); + for component in point.components.iter().filter(|c| c.sha256.is_some()) { + let schema = if component.name.contains('-') || component.name == "playerbots" { crate::db::schema_of(&component.name)? } else { point.realm.schema(&component.name)? }; + if db.extra_objects(schema)? != 0 { + return Err(Error::Invalid(format!("Database {schema} has routines, triggers or views that automatic recovery cannot restore. The update was not started."))); + } + } + Ok(()) + }) + } + + fn migration_snapshot(&self) -> Result { + if let Some(id) = unfinished(self.meta_dir).and_then(|t| t.recovery_point) { + if crate::backup::get(self.meta_dir, &id)?.kind == crate::backup::Kind::Full && crate::backup::verify(self.meta_dir, &id)?.ok { + return Ok(id); + } + return Err(Error::Invalid("The update recovery point is incomplete or damaged.".into())); + } + Ok(crate::backup::create(self.root, self.meta_dir, crate::backup::Kind::Full, crate::backup::Trigger::BeforeMigration, None, &|_| {})?.id) + } +} + impl Env for RepackEnv<'_> { fn ensure_stopped(&self) -> Result<()> { use crate::process::{observe, ServiceState}; @@ -477,15 +588,30 @@ impl Env for RepackEnv<'_> { fn snapshot(&self) -> Result { let label = Some("before update".to_string()); - Ok(crate::backup::create(self.root, self.meta_dir, crate::backup::Kind::Quick, crate::backup::Trigger::BeforeUpdate, label, &|_| {})?.id) + let point = crate::backup::create(self.root, self.meta_dir, crate::backup::Kind::Full, crate::backup::Trigger::BeforeUpdate, label, &|_| {})?; + self.verify_recovery_point(&point)?; + Ok(point.id) + } + + fn restore_snapshot(&self, id: &str) -> Result<()> { + let point = crate::backup::get(self.meta_dir, id)?; + if point.kind != crate::backup::Kind::Full { return Err(Error::Invalid("Database rollback requires a full recovery point.".into())); } + if !["characters", "auth", "world", "configs"].iter().all(|name| point.components.iter().any(|c| c.name == *name)) || !crate::backup::verify(self.meta_dir, id)?.ok { + return Err(Error::Invalid("The full database recovery point is incomplete or damaged.".into())); + } + for component in point.components.iter().filter(|c| c.sha256.is_some()) { + crate::backup::restore_database(self.root, self.meta_dir, id, &component.name)?; + } + crate::backup::restore_configs(self.root, self.meta_dir, id)?; + Ok(()) } fn migrate(&self, manifest: &Manifest, staged: &Path) -> Result { - let (root, meta) = (self.root, self.meta_dir); + let root = self.root; crate::backup::with_database(root, |db| { let realms = crate::realms::state(root)?; let mut result = crate::migrations::apply_pending(db, &manifest.migrations, staged, &|| { - Ok(crate::backup::create(root, meta, crate::backup::Kind::Full, crate::backup::Trigger::BeforeMigration, None, &|_| {})?.id) + self.migration_snapshot() })?; if realms.wildcard_created && result.failed.is_none() { let other = if realms.active == crate::realms::Mode::Coa { crate::realms::Mode::Wildcard } else { crate::realms::Mode::Coa }; @@ -494,7 +620,7 @@ impl Env for RepackEnv<'_> { let migrations: Vec<_> = manifest.migrations.iter().filter(|m| m.db != "auth").cloned().collect(); let extra = crate::migrations::apply_pending(&other_db, &migrations, staged, &|| { if let Some(id) = &shared_snapshot { return Ok(id.clone()); } - Ok(crate::backup::create(root, meta, crate::backup::Kind::Full, crate::backup::Trigger::BeforeMigration, None, &|_| {})?.id) + self.migration_snapshot() })?; result.applied.extend(extra.applied); result.failed = extra.failed; @@ -517,6 +643,16 @@ impl Env for RepackEnv<'_> { fn validate(&self) -> Result<()> { use crate::process::{observe, ServiceState}; + crate::backup::with_database(self.root, |db| { + let mut modes = vec![crate::realms::Mode::Coa]; + if crate::realms::state(self.root)?.wildcard_created { modes.push(crate::realms::Mode::Wildcard); } + for mode in modes { + if let Some(p) = crate::schema_check::check(&db.clone().for_realm(mode), self.root)?.first() { + return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail))); + } + } + Ok(()) + })?; // The check start is the first start of the new build: give the module configs the settings the update added // (as a normal start does), or the server logs a "missing property" line for every one of them. match crate::registry::MetaDir::open(self.meta_dir) { @@ -528,14 +664,24 @@ impl Env for RepackEnv<'_> { let _ = crate::config::create_missing_module_configs(self.root); } } - let started = crate::driver::run(self.root, crate::driver::Verb::StartAll)?; + let started = match crate::driver::validate_update(self.root) { + Ok(out) => out, + Err(e) => { + let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); + return Err(e); + } + }; let ports = crate::layout::read_ports(self.root); let healthy = started.ok && { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); + let mut ready_since = None; loop { let o = observe(self.root, &ports); if o.mysql.state == ServiceState::Running && o.auth.state == ServiceState::Running && o.world.state == ServiceState::Running && o.secondary_world.as_ref().is_none_or(|s| s.state == ServiceState::Running) { - break true; + let since = ready_since.get_or_insert_with(std::time::Instant::now); + if since.elapsed() >= std::time::Duration::from_secs(10) { break true; } + } else { + ready_since = None; } if std::time::Instant::now() > deadline { break false; @@ -568,11 +714,12 @@ mod tests { snapshot_ok: Cell, calls: RefCell>, migrate_fail: Cell, + restore_fail: Cell, } impl Fake { fn ok() -> Fake { - Fake { stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false) } + Fake { stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), restore_fail: Cell::new(false) } } } @@ -589,6 +736,10 @@ mod tests { self.calls.borrow_mut().push("migrate"); Ok(ApplyReport { applied: vec![], failed: self.migrate_fail.get().then(|| ("m1".to_string(), "syntax".to_string())), snapshot: None }) } + fn restore_snapshot(&self, _id: &str) -> Result<()> { + self.calls.borrow_mut().push("restore-databases"); + if self.restore_fail.get() { Err(Error::Invalid("recovery disk unavailable".into())) } else { Ok(()) } + } fn validate(&self) -> Result<()> { self.calls.borrow_mut().push("validate"); if self.healthy.get() { Ok(()) } else { Err(Error::Invalid("worldserver exited".into())) } @@ -787,6 +938,7 @@ mod tests { let e = run(&w, &env, BTreeMap::new(), None).unwrap_err(); assert!(e.to_string().contains("Database update m1 failed")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + assert!(env.calls.borrow().contains(&"restore-databases")); assert!(!env.calls.borrow().contains(&"validate"), "the new build is never started after a failed migration"); let ok = Fake::ok(); let out = run(&w, &ok, BTreeMap::new(), None).unwrap(); @@ -794,6 +946,93 @@ mod tests { assert_eq!(*ok.calls.borrow(), ["stop", "snapshot", "migrate", "validate"]); } + #[test] + fn failed_database_recovery_keeps_new_files_and_blocks_retry_until_recovered() { + let w = world(&[], true); + let env = Fake::ok(); + env.migrate_fail.set(true); + env.restore_fail.set(true); + let error = run(&w, &env, BTreeMap::new(), None).unwrap_err(); + assert!(error.to_string().contains("Recovery failed")); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2"); + let pending = unfinished(&w.meta).unwrap(); + assert_eq!(pending.state, State::Failed); + assert!(pending.databases_started); + assert!(txn_dir(&w.meta, &pending.id).unwrap().join("tree").exists()); + assert!(run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap_err().to_string().contains("unfinished")); + rollback(&w.root, &w.meta, &pending.id, &Fake::ok()).unwrap(); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + assert!(unfinished(&w.meta).is_none()); + } + + #[test] + fn interrupted_migration_is_restored_before_files_and_never_replayed() { + let w = world(&[], true); + let env = Fake::ok(); + env.healthy.set(false); + let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let recovery = Fake::ok(); + rollback(&w.root, &w.meta, &out.txn.id, &recovery).unwrap(); + assert_eq!(*recovery.calls.borrow(), ["stop", "restore-databases"]); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + } + + #[test] + fn concurrent_updates_are_rejected_without_changes() { + let w = world(&[], false); + let _lock = operation_lock(&w.meta).unwrap(); + assert!(run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap_err().to_string().contains("in progress")); + assert!(crate::driver::run(&w.root, crate::driver::Verb::StartAll).unwrap_err().to_string().contains("in progress")); + assert!(crate::modules::set_enabled(&w.root, &w.meta, "companions", true).unwrap_err().to_string().contains("in progress")); + assert!(crate::realms::select(&w.root, crate::realms::Mode::Wildcard).unwrap_err().to_string().contains("in progress")); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + } + + #[test] + fn interrupted_repair_restores_files_databases_and_ownership_metadata() { + let w = world(&[], false); + let files = vec![ + FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }, + FileEntry { path: "Core/repair-created.dll".into(), sha256: fsx::sha256_bytes(b"created"), size: 7, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }, + ]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + write(&w.root, "Core/worldserver.exe", b"repaired"); + write(&w.root, "Core/repair-created.dll", b"created"); + repair_migrating(&w.meta, &mut txn).unwrap(); + assert_eq!(unfinished(&w.meta).unwrap().id, txn.id); + let env = Fake::ok(); + rollback(&w.root, &w.meta, &txn.id, &env).unwrap(); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + assert!(!w.root.join("Core/repair-created.dll").exists()); + assert_eq!(*env.calls.borrow(), ["stop", "restore-databases"]); + assert!(unfinished(&w.meta).is_none()); + } + + #[test] + fn unreadable_update_journal_blocks_changes_and_startup() { + let w = world(&[], false); + write(&w.meta, "updates/broken/txn.json", b"{truncated"); + assert!(run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap_err().to_string().contains("cannot be read")); + assert!(crate::driver::run(&w.root, crate::driver::Verb::StartAll).unwrap_err().to_string().contains("cannot be read")); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + } + + #[test] + fn crash_during_commit_restores_the_original_installation_metadata() { + let w = world(&[], false); + let env = Fake::ok(); + env.healthy.set(false); + let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); + meta.core.version = Some("2.0.0".into()); + meta.core.commit = Some("a".repeat(40)); + fsx::atomic_write_json(&w.meta.join("install.json"), &meta).unwrap(); + rollback(&w.root, &w.meta, &out.txn.id, &Fake::ok()).unwrap(); + let (_, restored) = MetaDir::open(&w.meta).unwrap(); + assert_eq!(restored.core.version.as_deref(), Some("1.0.0")); + assert_eq!(restored.core.commit, None); + } + #[test] fn bad_signature_and_wrong_package_kind_change_nothing() { let w = world(&[], false); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ed7b288..c83fc52 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -233,18 +233,21 @@ hash**: current == recorded → safe to replace; current ≠ recorded → *modif 2. **Download** to `staging\\dl\` with `.part` files, HTTP Range resume, retry/backoff, cancel, progress. 3. **Verify** — SHA-256 of every artefact; nothing is executed or extracted into the live tree before this. 4. **Snapshot** — server must be stopped (never copy over a running worldserver); create recovery point: - replaced binaries, changed managed configs, module state, `characters`+`auth` dump (world optional); + replaced binaries, changed managed configs, module state, full dumps of every managed realm database; write `txn.json {state: Snapshotted}`. 5. **Apply** into `staging\\tree\`, resolve ownership conflicts, merge configs, then swap file-by-file with same-volume atomic renames, journaling each step (`txn.json` lists done/undone operations). -6. **Migrate** — run pending SQL through the tracked runner (D7); failure ⇒ stop, keep DB backup, offer restore. -7. **Validate** — start server, health check (D3) with timeout. +6. **Migrate** — durably mark SQL as running before execution. Failure restores the full database recovery point + before restoring files. Interrupted SQL is never automatically replayed. +7. **Validate** — check database structure, start server, require ten seconds of continuous readiness with timeout. 8. **Commit** — ownership table + `install.json` version bump, mark txn `Committed`, prune staging. Any crash before Commit leaves `txn.json` in a non-terminal state; on next launch the Manager detects it and offers *Roll forward* or *Roll back* — the install is "recoverable" by construction. -Rollback = binaries + managed config state + module state only. Database restore is a **separate explicit -action** (spec §19–20). +Rollback restores files, configuration and installation metadata. Once migrations started, it also restores all +databases from the full pre-update recovery point. Failed recovery remains unfinished and blocks startup and another +update. The same journal protects Repair. Old transactions without a full recovery point cannot roll back database +changes automatically. Independent database restore remains available for other recovery points. ### D7. Database strategy (`db` module) @@ -253,14 +256,15 @@ action** (spec §19–20). native Rust client (`mysql_async`) is used for health pings and migration ledger reads. * Ledger: table `acore_world.coa_manager_migrations(id, db, sha256, applied_at, status, error)` (created by the Manager, in a dedicated table — no core schema is changed) mirrored to `.manager\migrations\`. - Statuses `Applied | Pending | Failed`. Each file runs in a single connection; DDL is not transactional, so + Statuses `Applied | Pending | Failed | Running`. Each file runs in a single connection; DDL is not transactional, so "destructive" migrations (flag in manifest, or heuristic scan for `DROP|TRUNCATE|DELETE|ALTER ... DROP`) force a fresh DB backup first and require explicit confirmation. * Backups: `mysqldump --single-transaction --routines --quick` per DB. Quick = characters + auth + configs + metadata; Full adds world (large); Config-only; DB-only. Compressed with zstd. Restore is always to a staging schema first (`…_restore`), swapped only after a row-count sanity check, with the current DB dumped beforehand. -* Never: `DROP DATABASE`, resetting characters, overwriting DBs, or restoring DB on a binary rollback. Delete/ +* Never: `DROP DATABASE` or resetting characters. Update recovery restores databases into staging schemas and + keeps the replaced tables in separate schemas. Delete/ reset lives only in **Settings → Advanced → Danger Zone**, deletes tracked files only, and refuses to remove a directory that still contains untracked files. * Clean install: extract `mysql\data.7z` (packaged data dir), then **rotate** root/app passwords to fresh random @@ -392,8 +396,11 @@ files the running server itself changes, diffed before/after. disk-full at write n, network cut at byte n, migration returns error, new worldserver exits non-zero, port taken, exe quarantined (file vanishes), folder moved, user-modified managed file, unknown config keys. -### 3.4 Rollback — restore recovery point's binaries/configs/module state; database untouched unless the user -picks "Restore database" separately (shows dump timestamp and warns that progress since then is lost). +### 3.4 Rollback — restore the pre-update databases when migrations started, followed by files and metadata + +The UI explains that progress after the recovery point is lost. SQL failures before startup recover automatically; +failed health checks keep the installation blocked until the owner chooses recovery. Every recovery point and staging +schema has a unique identifier, and corrupt update journals fail closed. --------------------------------------------------------------------------------------------------------- diff --git a/src/i18n/locales/de.ts b/src/i18n/locales/de.ts index 215a7fe..cef95ac 100644 --- a/src/i18n/locales/de.ts +++ b/src/i18n/locales/de.ts @@ -349,7 +349,7 @@ const de: Partial> = { "upd.pendingBad": "Der aktualisierte Server ist nicht korrekt gestartet", "upd.pendingUnfinished": "Ein früheres Update wurde nicht abgeschlossen", "upd.safeBack": "Du kannst sicher zur vorherigen Version zurückkehren.", - "upd.backNote": "Das Zurückkehren stellt Programmdateien und Einstellungen wieder her. Charaktere und Konten werden nie automatisch zurückgesetzt.", + "upd.backNote": "Die Rückkehr stellt Dateien und Einstellungen wieder her. Wurden Datenbankupdates begonnen, werden auch alle Datenbanken aus der Sicherung vor dem Update wiederhergestellt. Späterer Fortschritt geht verloren.", "upd.goBackTo": "Zu Version {v} zurückkehren", "upd.goBackBefore": "Zur Version vor dem Update zurückkehren", "upd.title": "Server-Updates", diff --git a/src/i18n/locales/en.ts b/src/i18n/locales/en.ts index 728e6c8..19228b5 100644 --- a/src/i18n/locales/en.ts +++ b/src/i18n/locales/en.ts @@ -382,7 +382,7 @@ const en = { "upd.pendingBad": "The updated server did not start correctly", "upd.pendingUnfinished": "An earlier update did not finish", "upd.safeBack": "You can safely go back to the previous version.", - "upd.backNote": "Going back restores program files and settings. Characters and accounts are never rolled back automatically.", + "upd.backNote": "Going back restores files and settings. If database updates started, it also restores all databases to the backup taken before the update. Later progress will be lost.", "upd.goBackTo": "Go back to version {v}", "upd.goBackBefore": "Go back to the version before the update", "upd.title": "Server updates", diff --git a/src/i18n/locales/es.ts b/src/i18n/locales/es.ts index 26b97d6..e06b4af 100644 --- a/src/i18n/locales/es.ts +++ b/src/i18n/locales/es.ts @@ -349,7 +349,7 @@ const es: Partial> = { "upd.pendingBad": "El servidor actualizado no se inició correctamente", "upd.pendingUnfinished": "Una actualización anterior no terminó", "upd.safeBack": "Puedes volver con seguridad a la versión anterior.", - "upd.backNote": "Volver restaura los archivos del programa y los ajustes. Los personajes y cuentas nunca se revierten automáticamente.", + "upd.backNote": "Volver restaura los archivos y los ajustes. Si comenzaron las actualizaciones de las bases de datos, también se restauran todas las bases desde la copia anterior a la actualización. Se perderá el progreso posterior.", "upd.goBackTo": "Volver a la versión {v}", "upd.goBackBefore": "Volver a la versión anterior a la actualización", "upd.title": "Actualizaciones del servidor", diff --git a/src/i18n/locales/fr.ts b/src/i18n/locales/fr.ts index 6cccf22..1701abc 100644 --- a/src/i18n/locales/fr.ts +++ b/src/i18n/locales/fr.ts @@ -349,7 +349,7 @@ const fr: Partial> = { "upd.pendingBad": "Le serveur mis à jour n’a pas démarré correctement", "upd.pendingUnfinished": "Une mise à jour précédente ne s’est pas terminée", "upd.safeBack": "Vous pouvez revenir en toute sécurité à la version précédente.", - "upd.backNote": "Le retour restaure les fichiers du programme et les paramètres. Les personnages et comptes ne sont jamais restaurés automatiquement.", + "upd.backNote": "Le retour restaure les fichiers et les paramètres. Si la mise à jour des bases a commencé, toutes les bases sont aussi restaurées depuis la sauvegarde précédant la mise à jour. La progression ultérieure sera perdue.", "upd.goBackTo": "Revenir à la version {v}", "upd.goBackBefore": "Revenir à la version d’avant la mise à jour", "upd.title": "Mises à jour du serveur", diff --git a/src/i18n/locales/ru.ts b/src/i18n/locales/ru.ts index 3328d9d..ce6f0f3 100644 --- a/src/i18n/locales/ru.ts +++ b/src/i18n/locales/ru.ts @@ -380,7 +380,7 @@ const ru: Partial> = { "upd.pendingBad": "Обновлённый сервер запустился некорректно", "upd.pendingUnfinished": "Предыдущее обновление не завершилось", "upd.safeBack": "Можно безопасно вернуться к предыдущей версии.", - "upd.backNote": "Возврат восстанавливает файлы программы и настройки. Персонажи и учётные записи автоматически не откатываются никогда.", + "upd.backNote": "Возврат восстанавливает файлы и настройки. Если обновление баз уже началось, все базы также восстанавливаются из копии перед обновлением. Более поздний прогресс будет потерян.", "upd.goBackTo": "Вернуться к версии {v}", "upd.goBackBefore": "Вернуться к версии до обновления", "upd.title": "Обновления сервера", diff --git a/src/i18n/locales/zh.ts b/src/i18n/locales/zh.ts index 54e0e80..13a3d62 100644 --- a/src/i18n/locales/zh.ts +++ b/src/i18n/locales/zh.ts @@ -361,7 +361,7 @@ const zh = { "upd.pendingBad": "更新后的服务器未能正常启动", "upd.pendingUnfinished": "先前的更新尚未完成", "upd.safeBack": "你可以安全回滚至先前的稳定版本。", - "upd.backNote": "回滚操作将恢复程序文件与配置文件。角色和账号数据绝不会被自动回滚。", + "upd.backNote": "回滚将恢复文件和设置。如果数据库更新已开始,所有数据库也将从更新前的备份恢复。之后的进度将丢失。", "upd.goBackTo": "回滚至版本 {v}", "upd.goBackBefore": "回滚至更新前的版本", "upd.title": "服务器更新", diff --git a/src/lib/api.ts b/src/lib/api.ts index 06efb6b..450c320 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -274,6 +274,7 @@ export interface UpdateTxn { from_version: string | null; to_version: string; recovery_point: string | null; + databases_started: boolean; message: string | null; }