mirror of
https://github.com/Corfirean/coa-server-manager.git
synced 2026-11-22 11:25:56 +00:00
Roll back late repair failures and verify signed repair recovery
This commit is contained in:
1 parent
4a9760b864
commit
24d593d787
7 files changed
+208
-34
No files matched your search
@@ -144,7 +144,7 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result<DriverO
|
||||
}
|
||||
}
|
||||
discard_damaged_launcher_state(&root);
|
||||
if verb == Verb::StopAll { recover_mysql_state_for_shutdown(&root)?; }
|
||||
if matches!(verb, Verb::StopAll | Verb::StartMysql | Verb::StartAll) { recover_mysql_launcher_state(&root, verb != Verb::StopAll)?; }
|
||||
let mut cmd = Command::new(&python);
|
||||
cmd.arg("-B").arg("-c").arg(LAUNCH_SCRIPT).arg(&script).arg(verb.arg())
|
||||
.current_dir(&root).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||
@@ -256,7 +256,7 @@ pub(crate) fn discard_damaged_launcher_state(root: &Path) -> Vec<PathBuf> {
|
||||
|
||||
/// A lost launcher record must not hide this installation's running MySQL from
|
||||
/// graceful shutdown. Adopt only the exact executable owning its configured port.
|
||||
fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> {
|
||||
fn recover_mysql_launcher_state(root: &Path, prefer_listener: bool) -> Result<()> {
|
||||
let observed = crate::process::observe(root, &crate::layout::read_ports(root));
|
||||
if observed.mysql.state != crate::process::ServiceState::Running || !observed.mysql.port_ready { return Ok(()); }
|
||||
let Some(pid) = observed.mysql.pid else { return Ok(()); };
|
||||
@@ -269,15 +269,18 @@ fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> {
|
||||
if crate::process::is_alive(&saved) {
|
||||
// A valid record may describe the MySQL supervisor rather than its
|
||||
// listening child. The launcher can gracefully stop either record.
|
||||
if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) { return Ok(()); }
|
||||
return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No shutdown was requested.".into()));
|
||||
if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) {
|
||||
if !prefer_listener { return Ok(()); }
|
||||
} else {
|
||||
return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No database action was requested.".into()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = root.join(".state/mysql.json");
|
||||
let previous = root.join(".state/mysql-before-shutdown-recovery.json");
|
||||
if state.is_file() && !previous.exists() { fsx::atomic_write(&previous, &std::fs::read(&state)?)?; }
|
||||
fsx::atomic_write_json(&state, &identity)?;
|
||||
tracing::warn!(pid, root = %root.display(), "Recovered missing or stale MySQL launcher identity before graceful shutdown");
|
||||
tracing::warn!(pid, root = %root.display(), "Recovered MySQL launcher identity from the installed executable owning its port");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -345,7 +348,8 @@ mod tests {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
fsx::atomic_write_json(&root.path().join("Settings/repack.json"), &serde_json::json!({"mysqlPort":listener.local_addr().unwrap().port()})).unwrap();
|
||||
recover_mysql_state_for_shutdown(root.path()).unwrap();
|
||||
recover_mysql_launcher_state(root.path(), false).unwrap();
|
||||
recover_mysql_launcher_state(root.path(), true).unwrap();
|
||||
assert!(!root.path().join(".state/mysql.json").exists());
|
||||
assert!(crate::process::identity(std::process::id()).is_some());
|
||||
}
|
||||
|
||||
@@ -455,6 +455,81 @@ mod tests {
|
||||
assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;").unwrap(), "73");
|
||||
stop(&root).unwrap();
|
||||
assert!(recovered.0.wait().unwrap().success());
|
||||
// Exercise the public Repair pipeline with real signed packages and SQL,
|
||||
// rather than only the snapshot and journal helpers.
|
||||
let mut package_server = start();
|
||||
db.query("CREATE TABLE acore_auth.account(id INT PRIMARY KEY, username VARCHAR(40)); INSERT INTO acore_auth.account VALUES(73,'FIXTURE'); CREATE TABLE acore_auth.realmlist(id INT PRIMARY KEY, name VARCHAR(40), address VARCHAR(40), localAddress VARCHAR(40), port INT, flag INT); INSERT INTO acore_auth.realmlist VALUES(1,'AzerothCore','127.0.0.1','127.0.0.1',18085,0); CREATE TABLE acore_characters.characters(guid INT PRIMARY KEY, account INT, name VARCHAR(40), race INT, class INT); INSERT INTO acore_characters.characters VALUES(73,73,'Fixture',7,14); CREATE TABLE acore_world.playercreateinfo(race INT, class INT, PRIMARY KEY(race,class));").unwrap();
|
||||
let starts = [1,2,3,4,5,6,7,8,10,11].into_iter().flat_map(|race| (12..=32).map(move |class| format!("({race},{class})"))).collect::<Vec<_>>().join(",");
|
||||
db.query(&format!("INSERT INTO acore_world.playercreateinfo VALUES {starts};")).unwrap();
|
||||
let password = "a".repeat(48);
|
||||
db.query(&format!("ALTER USER 'root'@'localhost' IDENTIFIED BY '{password}';")).unwrap();
|
||||
fsx::atomic_write_json(&root.join("Settings/database.json"), &serde_json::json!({"rootPassword":password,"appPassword":"b".repeat(48)})).unwrap();
|
||||
fsx::atomic_write(&root.join("Settings/worldserver.conf.template"), b"WorldDatabaseInfo = @WorldDatabaseInfo@\nCharacterDatabaseInfo = @CharacterDatabaseInfo@\nLoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap();
|
||||
fsx::atomic_write(&root.join("Settings/authserver.conf.template"), b"LoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap();
|
||||
fsx::atomic_write(&root.join("Settings/mod_ascension_compat.conf.template"), b"Ascension.DbcPath = @ASCENSION_DBC@\n").unwrap();
|
||||
let startup = crate::driver::run(&root, crate::driver::Verb::StartMysql).unwrap();
|
||||
assert!(startup.ok, "{}", crate::diag::redact(&startup.output));
|
||||
let db = crate::db::Db::from_repack(&root, crate::db::Account::Admin).unwrap();
|
||||
crate::schema_check::capture(&db, &root).unwrap();
|
||||
assert!(crate::schema_check::check(&db, &root).unwrap().is_empty());
|
||||
stop(&root).unwrap();
|
||||
assert!(package_server.0.wait().unwrap().success());
|
||||
struct StopFixture(PathBuf);
|
||||
impl Drop for StopFixture { fn drop(&mut self) { let _ = stop(&self.0); } }
|
||||
let _stop_fixture = StopFixture(root.clone());
|
||||
use base64::{engine::general_purpose::STANDARD, Engine};
|
||||
use ed25519_dalek::{Signer, SigningKey};
|
||||
let key = SigningKey::from_bytes(&[7;32]);
|
||||
let trusted = STANDARD.encode(key.verifying_key().to_bytes());
|
||||
let source = temp.path().join("repair-package-source");
|
||||
fsx::atomic_write(&source.join("Core/reference/repair-probe.dll"), b"official").unwrap();
|
||||
fsx::atomic_write(&source.join(crate::schema_check::CONTRACT), &fs::read(root.join(crate::schema_check::CONTRACT)).unwrap()).unwrap();
|
||||
let build = |kind, version: &str, destination: &Path, migrations| {
|
||||
let manifest = crate::package::build(&source, destination, &crate::package::BuildOptions { kind, version: version.into(), core_commit: None, built_at: "fixture".into(), part_size: 1024*1024, bots_commit: None, migrations }, &|_| {}).unwrap();
|
||||
let bytes = fs::read(destination.join("manifest.json")).unwrap();
|
||||
fsx::atomic_write(&destination.join("manifest.json.sig"), STANDARD.encode(key.sign(&bytes).to_bytes()).as_bytes()).unwrap();
|
||||
manifest
|
||||
};
|
||||
let base = temp.path().join("repair-base");
|
||||
let base_manifest = build(crate::manifest::Kind::Base, "1.0.0", &base, vec![]);
|
||||
let update = temp.path().join("repair-update");
|
||||
let update_manifest = build(crate::manifest::Kind::Update, "2.0.0", &update, vec![]);
|
||||
fsx::atomic_write_json(&meta.join("manifests/base.json"), &base_manifest).unwrap();
|
||||
fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &update_manifest).unwrap();
|
||||
let (_, mut installed) = crate::registry::MetaDir::open(&meta).unwrap();
|
||||
installed.core.version = Some("2.0.0".into());
|
||||
fsx::atomic_write_json(&meta.join("install.json"), &installed).unwrap();
|
||||
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"broken").unwrap();
|
||||
let base_source = crate::pkgsource::Source::Dir(base);
|
||||
let update_source = crate::pkgsource::Source::Dir(update);
|
||||
let report = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|_,_| {}).unwrap();
|
||||
assert!(report.error.is_none());
|
||||
assert!(report.database.iter().all(|check| check.problems.is_empty()));
|
||||
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"official");
|
||||
crate::update::ensure_recovered(&meta).unwrap();
|
||||
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before verification failure").unwrap();
|
||||
let error = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|phase,_| {
|
||||
if phase == "Checking database after repair" { fs::remove_file(root.join("Core/reference/repair-probe.dll")).unwrap(); }
|
||||
}).unwrap_err();
|
||||
assert!(error.to_string().contains("repair"));
|
||||
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before verification failure");
|
||||
crate::update::ensure_recovered(&meta).unwrap();
|
||||
let failed_sql = "UPDATE shutdown_probe SET id=75; CREATE TABLE repair_partial(id INT); INVALID SQL;";
|
||||
fsx::atomic_write(&source.join("_migrations/characters/repair-failure.sql"), failed_sql.as_bytes()).unwrap();
|
||||
let failing = temp.path().join("repair-failing-update");
|
||||
let manifest = build(crate::manifest::Kind::Update, "2.0.0", &failing, vec![crate::manifest::Migration { id:"repair-failure".into(), db:"characters".into(), sha256:fsx::sha256_bytes(failed_sql.as_bytes()), compatible_sha256:vec![], destructive:false }]);
|
||||
fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &manifest).unwrap();
|
||||
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before failed repair").unwrap();
|
||||
let report = crate::repair::run(&root, &meta, &base_source, &crate::pkgsource::Source::Dir(failing), &trusted, &|_,_| {}).unwrap();
|
||||
assert!(report.error.is_some());
|
||||
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before failed repair");
|
||||
crate::update::ensure_recovered(&meta).unwrap();
|
||||
crate::backup::with_database(&root, |db| {
|
||||
assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;")?, "73");
|
||||
assert_eq!(db.query("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='acore_characters' AND TABLE_NAME='repair_partial';")?, "0");
|
||||
assert_eq!(db.query("SELECT username FROM acore_auth.account WHERE id=73;")?, "FIXTURE");
|
||||
Ok(())
|
||||
}).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
|
||||
@@ -134,34 +134,52 @@ pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source
|
||||
env.verify_recovery_point(&crate::backup::get(dir, &backup)?)?;
|
||||
let repair_files: Vec<_> = broken.iter().map(|(file, _)| (*file).clone()).collect();
|
||||
let mut txn = crate::update::begin_repair(root, dir, &update.version, &backup, &repair_files)?;
|
||||
let before = dir.join("repairs").join(&backup);
|
||||
fs::create_dir_all(&before)?;
|
||||
fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?;
|
||||
let restored = restore_files(root, &work, &before, &broken)?;
|
||||
for (entry, _) in &broken {
|
||||
meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone());
|
||||
}
|
||||
fsx::atomic_write_json(&dir.join("install.json"), &meta)?;
|
||||
progress("Repairing pending database updates", 80);
|
||||
crate::update::repair_migrating(dir, &mut txn)?;
|
||||
let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) {
|
||||
Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))),
|
||||
Err(e) => (Vec::new(), Some(e.to_string())),
|
||||
};
|
||||
if error.is_some() {
|
||||
crate::update::finish_repair(root, dir, &mut txn, false, &env)?;
|
||||
let report = Report { restored: Vec::new(), applied: Vec::new(), backup, database: check(root, dir)?, error };
|
||||
let attempt = (|| -> Result<Report> {
|
||||
let before = dir.join("repairs").join(&backup);
|
||||
fs::create_dir_all(&before)?;
|
||||
fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?;
|
||||
let restored = restore_files(root, &work, &before, &broken)?;
|
||||
for (entry, _) in &broken {
|
||||
meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone());
|
||||
}
|
||||
fsx::atomic_write_json(&dir.join("install.json"), &meta)?;
|
||||
progress("Repairing pending database updates", 80);
|
||||
crate::update::repair_migrating(dir, &mut txn)?;
|
||||
let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) {
|
||||
Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))),
|
||||
Err(e) => (Vec::new(), Some(e.to_string())),
|
||||
};
|
||||
if error.is_some() {
|
||||
crate::update::finish_repair(root, dir, &mut txn, false, &env)?;
|
||||
let report = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: check(root, dir)?, error };
|
||||
fsx::atomic_write_json(&before.join("report.json"), &report)?;
|
||||
return Ok(report);
|
||||
}
|
||||
crate::config::create_missing_module_configs(root)?;
|
||||
progress("Checking database after repair", 95);
|
||||
let database = check(root, dir)?;
|
||||
let report = Report { restored, applied, backup: backup.clone(), database, error };
|
||||
fsx::atomic_write_json(&before.join("report.json"), &report)?;
|
||||
return Ok(report);
|
||||
crate::update::finish_repair(root, dir, &mut txn, true, &env)?;
|
||||
progress("Done", 100);
|
||||
Ok(report)
|
||||
})();
|
||||
match attempt {
|
||||
Ok(report) => Ok(report),
|
||||
Err(error) if matches!(txn.state, crate::update::State::RolledBack | crate::update::State::Failed) => Err(error),
|
||||
Err(error) => {
|
||||
progress("Undoing the failed repair", 95);
|
||||
let message = match crate::update::finish_repair(root, dir, &mut txn, false, &env) {
|
||||
Ok(()) => format!("Repair failed: {error}. The original server files and databases were restored from recovery point {backup}."),
|
||||
Err(recovery) => format!("Repair failed: {error}. Recovery failed: {recovery}. Recovery point {backup} was preserved; resolve the unfinished repair before starting the server."),
|
||||
};
|
||||
let failed = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: Vec::new(), error: Some(message.clone()) };
|
||||
if let Err(report_error) = fsx::atomic_write_json(&dir.join("repairs").join(&backup).join("report.json"), &failed) {
|
||||
tracing::warn!(%report_error, "Could not persist failed repair report");
|
||||
}
|
||||
Err(Error::Invalid(message))
|
||||
}
|
||||
}
|
||||
crate::config::create_missing_module_configs(root)?;
|
||||
progress("Checking database after repair", 95);
|
||||
let database = check(root, dir)?;
|
||||
let report = Report { restored, applied, backup, database, error };
|
||||
fsx::atomic_write_json(&before.join("report.json"), &report)?;
|
||||
crate::update::finish_repair(root, dir, &mut txn, true, &env)?;
|
||||
progress("Done", 100);
|
||||
Ok(report)
|
||||
})();
|
||||
// Only the UUID staging directory created above is discarded; backups are retained.
|
||||
let _ = fs::remove_dir_all(work);
|
||||
|
||||
@@ -847,6 +847,8 @@ pub(crate) fn begin_repair(root: &Path, meta_dir: &Path, version: &str, backup:
|
||||
}
|
||||
let mut recovery_hashes = BTreeMap::new();
|
||||
recovery_hashes.insert("@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?);
|
||||
let point = crate::backup::point_json(meta_dir, backup)?;
|
||||
if point.is_file() { recovery_hashes.insert("@point".into(), fsx::sha256_file(&point)?); }
|
||||
for op in ops.iter().filter(|op| op.had_previous) { recovery_hashes.insert(op.path.clone(), fsx::sha256_file(&fsx::safe_join(&before, &op.path)?)?); }
|
||||
let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()), recovery_hashes };
|
||||
save(meta_dir, &txn)?;
|
||||
@@ -860,7 +862,14 @@ pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> {
|
||||
}
|
||||
|
||||
pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> {
|
||||
if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; }
|
||||
if !success {
|
||||
if let Err(error) = restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env) {
|
||||
txn.state = State::Failed;
|
||||
txn.message = Some(format!("Repair recovery failed: {error}. Its recovery point was preserved."));
|
||||
save(meta_dir, txn)?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
if success { complete_repair_ops(root, txn)?; }
|
||||
txn.state = if success { State::Committed } else { State::RolledBack };
|
||||
save(meta_dir, txn)
|
||||
@@ -1946,6 +1955,40 @@ mod tests {
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn changed_repair_backup_inventory_blocks_recovery_before_stopping_or_restoring() {
|
||||
let w = world(&[], false);
|
||||
let point = crate::backup::point_json(&w.meta, "full-backup").unwrap();
|
||||
fsx::atomic_write(&point, b"original recovery inventory").unwrap();
|
||||
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
|
||||
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
|
||||
write(&w.root, "Core/worldserver.exe", b"repaired");
|
||||
repair_migrating(&w.meta, &mut txn).unwrap();
|
||||
fsx::atomic_write(&point, b"changed recovery inventory").unwrap();
|
||||
let env = Fake::ok();
|
||||
assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err());
|
||||
assert!(env.calls.borrow().is_empty());
|
||||
assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired");
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_repair_recovery_keeps_a_failed_journal_and_its_reason() {
|
||||
let w = world(&[], false);
|
||||
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
|
||||
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
|
||||
write(&w.root, "Core/worldserver.exe", b"repaired");
|
||||
repair_migrating(&w.meta, &mut txn).unwrap();
|
||||
let env = Fake::ok();
|
||||
env.restore_fail.set(true);
|
||||
assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err());
|
||||
let saved = load(&w.meta, &txn.id).unwrap();
|
||||
assert_eq!(saved.state, State::Failed);
|
||||
assert!(saved.message.unwrap().contains("Repair recovery failed"));
|
||||
assert_eq!(saved.recovery_point.as_deref(), Some("full-backup"));
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repair_cannot_commit_unverified_files() {
|
||||
let w = world(&[], false);
|
||||
|
||||
Reference in new issue
Block a user