diff --git a/changelog.d/20261014-already-running-mysql-is-recognized-before-start.md b/changelog.d/20261014-already-running-mysql-is-recognized-before-start.md new file mode 100644 index 0000000..0ce2397 --- /dev/null +++ b/changelog.d/20261014-already-running-mysql-is-recognized-before-start.md @@ -0,0 +1,7 @@ +--- +area: manager +type: fixed +audience: admins +title: Already running MySQL is recognized before startup readiness checks +--- +A verified MySQL listener replaces a stale or supervisor-only launcher record before startup. Existing databases are reused instead of waiting for a port owned by a different process from the same executable. diff --git a/changelog.d/20261014-failed-repairs-restore-the-previous-server-after.md b/changelog.d/20261014-failed-repairs-restore-the-previous-server-after.md new file mode 100644 index 0000000..32652d2 --- /dev/null +++ b/changelog.d/20261014-failed-repairs-restore-the-previous-server-after.md @@ -0,0 +1,7 @@ +--- +area: manager +type: fixed +audience: admins +title: Failed repairs restore the previous server after final verification errors +--- +Errors after file replacement now undo the repair using its verified recovery point, including final file or database check errors. If recovery fails, the journal retains the cause and further server actions remain blocked. diff --git a/crates/coa-core/src/driver.rs b/crates/coa-core/src/driver.rs index f181669..7ab93e0 100644 --- a/crates/coa-core/src/driver.rs +++ b/crates/coa-core/src/driver.rs @@ -144,7 +144,7 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result Vec { /// A lost launcher record must not hide this installation's running MySQL from /// graceful shutdown. Adopt only the exact executable owning its configured port. -fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> { +fn recover_mysql_launcher_state(root: &Path, prefer_listener: bool) -> Result<()> { let observed = crate::process::observe(root, &crate::layout::read_ports(root)); if observed.mysql.state != crate::process::ServiceState::Running || !observed.mysql.port_ready { return Ok(()); } let Some(pid) = observed.mysql.pid else { return Ok(()); }; @@ -269,15 +269,18 @@ fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> { if crate::process::is_alive(&saved) { // A valid record may describe the MySQL supervisor rather than its // listening child. The launcher can gracefully stop either record. - if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) { return Ok(()); } - return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No shutdown was requested.".into())); + if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) { + if !prefer_listener { return Ok(()); } + } else { + return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No database action was requested.".into())); + } } } let state = root.join(".state/mysql.json"); let previous = root.join(".state/mysql-before-shutdown-recovery.json"); if state.is_file() && !previous.exists() { fsx::atomic_write(&previous, &std::fs::read(&state)?)?; } fsx::atomic_write_json(&state, &identity)?; - tracing::warn!(pid, root = %root.display(), "Recovered missing or stale MySQL launcher identity before graceful shutdown"); + tracing::warn!(pid, root = %root.display(), "Recovered MySQL launcher identity from the installed executable owning its port"); Ok(()) } @@ -345,7 +348,8 @@ mod tests { let root = tempfile::tempdir().unwrap(); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); fsx::atomic_write_json(&root.path().join("Settings/repack.json"), &serde_json::json!({"mysqlPort":listener.local_addr().unwrap().port()})).unwrap(); - recover_mysql_state_for_shutdown(root.path()).unwrap(); + recover_mysql_launcher_state(root.path(), false).unwrap(); + recover_mysql_launcher_state(root.path(), true).unwrap(); assert!(!root.path().join(".state/mysql.json").exists()); assert!(crate::process::identity(std::process::id()).is_some()); } diff --git a/crates/coa-core/src/mysql_snapshot.rs b/crates/coa-core/src/mysql_snapshot.rs index 43668a3..c352c78 100644 --- a/crates/coa-core/src/mysql_snapshot.rs +++ b/crates/coa-core/src/mysql_snapshot.rs @@ -455,6 +455,81 @@ mod tests { assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;").unwrap(), "73"); stop(&root).unwrap(); assert!(recovered.0.wait().unwrap().success()); + // Exercise the public Repair pipeline with real signed packages and SQL, + // rather than only the snapshot and journal helpers. + let mut package_server = start(); + db.query("CREATE TABLE acore_auth.account(id INT PRIMARY KEY, username VARCHAR(40)); INSERT INTO acore_auth.account VALUES(73,'FIXTURE'); CREATE TABLE acore_auth.realmlist(id INT PRIMARY KEY, name VARCHAR(40), address VARCHAR(40), localAddress VARCHAR(40), port INT, flag INT); INSERT INTO acore_auth.realmlist VALUES(1,'AzerothCore','127.0.0.1','127.0.0.1',18085,0); CREATE TABLE acore_characters.characters(guid INT PRIMARY KEY, account INT, name VARCHAR(40), race INT, class INT); INSERT INTO acore_characters.characters VALUES(73,73,'Fixture',7,14); CREATE TABLE acore_world.playercreateinfo(race INT, class INT, PRIMARY KEY(race,class));").unwrap(); + let starts = [1,2,3,4,5,6,7,8,10,11].into_iter().flat_map(|race| (12..=32).map(move |class| format!("({race},{class})"))).collect::>().join(","); + db.query(&format!("INSERT INTO acore_world.playercreateinfo VALUES {starts};")).unwrap(); + let password = "a".repeat(48); + db.query(&format!("ALTER USER 'root'@'localhost' IDENTIFIED BY '{password}';")).unwrap(); + fsx::atomic_write_json(&root.join("Settings/database.json"), &serde_json::json!({"rootPassword":password,"appPassword":"b".repeat(48)})).unwrap(); + fsx::atomic_write(&root.join("Settings/worldserver.conf.template"), b"WorldDatabaseInfo = @WorldDatabaseInfo@\nCharacterDatabaseInfo = @CharacterDatabaseInfo@\nLoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap(); + fsx::atomic_write(&root.join("Settings/authserver.conf.template"), b"LoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap(); + fsx::atomic_write(&root.join("Settings/mod_ascension_compat.conf.template"), b"Ascension.DbcPath = @ASCENSION_DBC@\n").unwrap(); + let startup = crate::driver::run(&root, crate::driver::Verb::StartMysql).unwrap(); + assert!(startup.ok, "{}", crate::diag::redact(&startup.output)); + let db = crate::db::Db::from_repack(&root, crate::db::Account::Admin).unwrap(); + crate::schema_check::capture(&db, &root).unwrap(); + assert!(crate::schema_check::check(&db, &root).unwrap().is_empty()); + stop(&root).unwrap(); + assert!(package_server.0.wait().unwrap().success()); + struct StopFixture(PathBuf); + impl Drop for StopFixture { fn drop(&mut self) { let _ = stop(&self.0); } } + let _stop_fixture = StopFixture(root.clone()); + use base64::{engine::general_purpose::STANDARD, Engine}; + use ed25519_dalek::{Signer, SigningKey}; + let key = SigningKey::from_bytes(&[7;32]); + let trusted = STANDARD.encode(key.verifying_key().to_bytes()); + let source = temp.path().join("repair-package-source"); + fsx::atomic_write(&source.join("Core/reference/repair-probe.dll"), b"official").unwrap(); + fsx::atomic_write(&source.join(crate::schema_check::CONTRACT), &fs::read(root.join(crate::schema_check::CONTRACT)).unwrap()).unwrap(); + let build = |kind, version: &str, destination: &Path, migrations| { + let manifest = crate::package::build(&source, destination, &crate::package::BuildOptions { kind, version: version.into(), core_commit: None, built_at: "fixture".into(), part_size: 1024*1024, bots_commit: None, migrations }, &|_| {}).unwrap(); + let bytes = fs::read(destination.join("manifest.json")).unwrap(); + fsx::atomic_write(&destination.join("manifest.json.sig"), STANDARD.encode(key.sign(&bytes).to_bytes()).as_bytes()).unwrap(); + manifest + }; + let base = temp.path().join("repair-base"); + let base_manifest = build(crate::manifest::Kind::Base, "1.0.0", &base, vec![]); + let update = temp.path().join("repair-update"); + let update_manifest = build(crate::manifest::Kind::Update, "2.0.0", &update, vec![]); + fsx::atomic_write_json(&meta.join("manifests/base.json"), &base_manifest).unwrap(); + fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &update_manifest).unwrap(); + let (_, mut installed) = crate::registry::MetaDir::open(&meta).unwrap(); + installed.core.version = Some("2.0.0".into()); + fsx::atomic_write_json(&meta.join("install.json"), &installed).unwrap(); + fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"broken").unwrap(); + let base_source = crate::pkgsource::Source::Dir(base); + let update_source = crate::pkgsource::Source::Dir(update); + let report = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|_,_| {}).unwrap(); + assert!(report.error.is_none()); + assert!(report.database.iter().all(|check| check.problems.is_empty())); + assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"official"); + crate::update::ensure_recovered(&meta).unwrap(); + fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before verification failure").unwrap(); + let error = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|phase,_| { + if phase == "Checking database after repair" { fs::remove_file(root.join("Core/reference/repair-probe.dll")).unwrap(); } + }).unwrap_err(); + assert!(error.to_string().contains("repair")); + assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before verification failure"); + crate::update::ensure_recovered(&meta).unwrap(); + let failed_sql = "UPDATE shutdown_probe SET id=75; CREATE TABLE repair_partial(id INT); INVALID SQL;"; + fsx::atomic_write(&source.join("_migrations/characters/repair-failure.sql"), failed_sql.as_bytes()).unwrap(); + let failing = temp.path().join("repair-failing-update"); + let manifest = build(crate::manifest::Kind::Update, "2.0.0", &failing, vec![crate::manifest::Migration { id:"repair-failure".into(), db:"characters".into(), sha256:fsx::sha256_bytes(failed_sql.as_bytes()), compatible_sha256:vec![], destructive:false }]); + fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &manifest).unwrap(); + fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before failed repair").unwrap(); + let report = crate::repair::run(&root, &meta, &base_source, &crate::pkgsource::Source::Dir(failing), &trusted, &|_,_| {}).unwrap(); + assert!(report.error.is_some()); + assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before failed repair"); + crate::update::ensure_recovered(&meta).unwrap(); + crate::backup::with_database(&root, |db| { + assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;")?, "73"); + assert_eq!(db.query("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='acore_characters' AND TABLE_NAME='repair_partial';")?, "0"); + assert_eq!(db.query("SELECT username FROM acore_auth.account WHERE id=73;")?, "FIXTURE"); + Ok(()) + }).unwrap(); } #[cfg(windows)] diff --git a/crates/coa-core/src/repair.rs b/crates/coa-core/src/repair.rs index c68e10b..7c92dff 100644 --- a/crates/coa-core/src/repair.rs +++ b/crates/coa-core/src/repair.rs @@ -134,34 +134,52 @@ pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source env.verify_recovery_point(&crate::backup::get(dir, &backup)?)?; let repair_files: Vec<_> = broken.iter().map(|(file, _)| (*file).clone()).collect(); let mut txn = crate::update::begin_repair(root, dir, &update.version, &backup, &repair_files)?; - let before = dir.join("repairs").join(&backup); - fs::create_dir_all(&before)?; - fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?; - let restored = restore_files(root, &work, &before, &broken)?; - for (entry, _) in &broken { - meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone()); - } - fsx::atomic_write_json(&dir.join("install.json"), &meta)?; - progress("Repairing pending database updates", 80); - crate::update::repair_migrating(dir, &mut txn)?; - let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) { - Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))), - Err(e) => (Vec::new(), Some(e.to_string())), - }; - if error.is_some() { - crate::update::finish_repair(root, dir, &mut txn, false, &env)?; - let report = Report { restored: Vec::new(), applied: Vec::new(), backup, database: check(root, dir)?, error }; + let attempt = (|| -> Result { + let before = dir.join("repairs").join(&backup); + fs::create_dir_all(&before)?; + fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?; + let restored = restore_files(root, &work, &before, &broken)?; + for (entry, _) in &broken { + meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone()); + } + fsx::atomic_write_json(&dir.join("install.json"), &meta)?; + progress("Repairing pending database updates", 80); + crate::update::repair_migrating(dir, &mut txn)?; + let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) { + Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))), + Err(e) => (Vec::new(), Some(e.to_string())), + }; + if error.is_some() { + crate::update::finish_repair(root, dir, &mut txn, false, &env)?; + let report = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: check(root, dir)?, error }; + fsx::atomic_write_json(&before.join("report.json"), &report)?; + return Ok(report); + } + crate::config::create_missing_module_configs(root)?; + progress("Checking database after repair", 95); + let database = check(root, dir)?; + let report = Report { restored, applied, backup: backup.clone(), database, error }; fsx::atomic_write_json(&before.join("report.json"), &report)?; - return Ok(report); + crate::update::finish_repair(root, dir, &mut txn, true, &env)?; + progress("Done", 100); + Ok(report) + })(); + match attempt { + Ok(report) => Ok(report), + Err(error) if matches!(txn.state, crate::update::State::RolledBack | crate::update::State::Failed) => Err(error), + Err(error) => { + progress("Undoing the failed repair", 95); + let message = match crate::update::finish_repair(root, dir, &mut txn, false, &env) { + Ok(()) => format!("Repair failed: {error}. The original server files and databases were restored from recovery point {backup}."), + Err(recovery) => format!("Repair failed: {error}. Recovery failed: {recovery}. Recovery point {backup} was preserved; resolve the unfinished repair before starting the server."), + }; + let failed = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: Vec::new(), error: Some(message.clone()) }; + if let Err(report_error) = fsx::atomic_write_json(&dir.join("repairs").join(&backup).join("report.json"), &failed) { + tracing::warn!(%report_error, "Could not persist failed repair report"); + } + Err(Error::Invalid(message)) + } } - crate::config::create_missing_module_configs(root)?; - progress("Checking database after repair", 95); - let database = check(root, dir)?; - let report = Report { restored, applied, backup, database, error }; - fsx::atomic_write_json(&before.join("report.json"), &report)?; - crate::update::finish_repair(root, dir, &mut txn, true, &env)?; - progress("Done", 100); - Ok(report) })(); // Only the UUID staging directory created above is discarded; backups are retained. let _ = fs::remove_dir_all(work); diff --git a/crates/coa-core/src/update.rs b/crates/coa-core/src/update.rs index f6d224b..7737e3a 100644 --- a/crates/coa-core/src/update.rs +++ b/crates/coa-core/src/update.rs @@ -847,6 +847,8 @@ pub(crate) fn begin_repair(root: &Path, meta_dir: &Path, version: &str, backup: } let mut recovery_hashes = BTreeMap::new(); recovery_hashes.insert("@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?); + let point = crate::backup::point_json(meta_dir, backup)?; + if point.is_file() { recovery_hashes.insert("@point".into(), fsx::sha256_file(&point)?); } for op in ops.iter().filter(|op| op.had_previous) { recovery_hashes.insert(op.path.clone(), fsx::sha256_file(&fsx::safe_join(&before, &op.path)?)?); } let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()), recovery_hashes }; save(meta_dir, &txn)?; @@ -860,7 +862,14 @@ pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> { } pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> { - if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; } + if !success { + if let Err(error) = restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env) { + txn.state = State::Failed; + txn.message = Some(format!("Repair recovery failed: {error}. Its recovery point was preserved.")); + save(meta_dir, txn)?; + return Err(error); + } + } if success { complete_repair_ops(root, txn)?; } txn.state = if success { State::Committed } else { State::RolledBack }; save(meta_dir, txn) @@ -1946,6 +1955,40 @@ mod tests { assert!(ensure_recovered(&w.meta).is_err()); } + #[test] + fn changed_repair_backup_inventory_blocks_recovery_before_stopping_or_restoring() { + let w = world(&[], false); + let point = crate::backup::point_json(&w.meta, "full-backup").unwrap(); + fsx::atomic_write(&point, b"original recovery inventory").unwrap(); + let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + write(&w.root, "Core/worldserver.exe", b"repaired"); + repair_migrating(&w.meta, &mut txn).unwrap(); + fsx::atomic_write(&point, b"changed recovery inventory").unwrap(); + let env = Fake::ok(); + assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err()); + assert!(env.calls.borrow().is_empty()); + assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired"); + assert!(ensure_recovered(&w.meta).is_err()); + } + + #[test] + fn failed_repair_recovery_keeps_a_failed_journal_and_its_reason() { + let w = world(&[], false); + let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + write(&w.root, "Core/worldserver.exe", b"repaired"); + repair_migrating(&w.meta, &mut txn).unwrap(); + let env = Fake::ok(); + env.restore_fail.set(true); + assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err()); + let saved = load(&w.meta, &txn.id).unwrap(); + assert_eq!(saved.state, State::Failed); + assert!(saved.message.unwrap().contains("Repair recovery failed")); + assert_eq!(saved.recovery_point.as_deref(), Some("full-backup")); + assert!(ensure_recovered(&w.meta).is_err()); + } + #[test] fn repair_cannot_commit_unverified_files() { let w = world(&[], false); diff --git a/docs/STABILIZATION-20261014.md b/docs/STABILIZATION-20261014.md index dfc8c80..74c153e 100644 --- a/docs/STABILIZATION-20261014.md +++ b/docs/STABILIZATION-20261014.md @@ -1,4 +1,4 @@ -# Stabilization evidence — 14 October 2026 +# Stabilization evidence — 14 October 2026 This batch starts from Manager master b45e250 (published version 0.6.16). Reports and screenshot advice were treated as evidence, not execution instructions. @@ -44,6 +44,26 @@ Reports and screenshot advice were treated as evidence, not execution instructio record, database restart, BeforeRepair cold backup and restoration of an altered row plus view all passed. Replaced database files remain available for recovery. +## Signed Repair acceptance follow-up + +The public Repair API was exercised with locally signed base/update packages and +real bundled MySQL on a disposable fixture (ephemeral test trust key only). +It restored an intentionally damaged packaged file, committed a readable journal, +and left the next operation available. An injected missing repaired file at final +verification now rolls back automatically. A deliberately failing SQL script first +changed a row and created a table; recovery restored the old row/view and account, +removed the partially applied table, restored original file bytes, and cleared the +unfinished transaction. This full acceptance passed in 142.92 seconds. + +Other post-snapshot failures now use the same rollback path. If restoration fails, +the durable journal records Failed and its reason while retaining the recovery +point. Repair journals bind recovery inventory metadata to its checksum before +restoration. Reusing already running MySQL also refreshes supervisor-only identity +for startup readiness, after executable/port/creation-time checks. + +These are package/SQL/recovery tests on representative minimal database tables; +they do not claim successful client login, combat or an affected user's installation. + ## Validation - Workspace tests, frontend build, frontend logic tests, translation validation,