Roll back late repair failures and verify signed repair recovery

This commit is contained in:
Дмитрий committed 2026-10-14 07:25:56 +05:00
1 parent 4a9760b864
commit 24d593d787
7 files changed
+208 -34

No files matched your search

+10 -6
View File
@@ -144,7 +144,7 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result<DriverO
}
}
discard_damaged_launcher_state(&root);
if verb == Verb::StopAll { recover_mysql_state_for_shutdown(&root)?; }
if matches!(verb, Verb::StopAll | Verb::StartMysql | Verb::StartAll) { recover_mysql_launcher_state(&root, verb != Verb::StopAll)?; }
let mut cmd = Command::new(&python);
cmd.arg("-B").arg("-c").arg(LAUNCH_SCRIPT).arg(&script).arg(verb.arg())
.current_dir(&root).stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped());
@@ -256,7 +256,7 @@ pub(crate) fn discard_damaged_launcher_state(root: &Path) -> Vec<PathBuf> {
/// A lost launcher record must not hide this installation's running MySQL from
/// graceful shutdown. Adopt only the exact executable owning its configured port.
fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> {
fn recover_mysql_launcher_state(root: &Path, prefer_listener: bool) -> Result<()> {
let observed = crate::process::observe(root, &crate::layout::read_ports(root));
if observed.mysql.state != crate::process::ServiceState::Running || !observed.mysql.port_ready { return Ok(()); }
let Some(pid) = observed.mysql.pid else { return Ok(()); };
@@ -269,15 +269,18 @@ fn recover_mysql_state_for_shutdown(root: &Path) -> Result<()> {
if crate::process::is_alive(&saved) {
// A valid record may describe the MySQL supervisor rather than its
// listening child. The launcher can gracefully stop either record.
if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) { return Ok(()); }
return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No shutdown was requested.".into()));
if crate::process::find_by_exe(&executable).iter().any(|candidate| candidate.same_as(&saved)) {
if !prefer_listener { return Ok(()); }
} else {
return Err(Error::Invalid("The MySQL launcher record identifies a different live process. No database action was requested.".into()));
}
}
}
let state = root.join(".state/mysql.json");
let previous = root.join(".state/mysql-before-shutdown-recovery.json");
if state.is_file() && !previous.exists() { fsx::atomic_write(&previous, &std::fs::read(&state)?)?; }
fsx::atomic_write_json(&state, &identity)?;
tracing::warn!(pid, root = %root.display(), "Recovered missing or stale MySQL launcher identity before graceful shutdown");
tracing::warn!(pid, root = %root.display(), "Recovered MySQL launcher identity from the installed executable owning its port");
Ok(())
}
@@ -345,7 +348,8 @@ mod tests {
let root = tempfile::tempdir().unwrap();
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
fsx::atomic_write_json(&root.path().join("Settings/repack.json"), &serde_json::json!({"mysqlPort":listener.local_addr().unwrap().port()})).unwrap();
recover_mysql_state_for_shutdown(root.path()).unwrap();
recover_mysql_launcher_state(root.path(), false).unwrap();
recover_mysql_launcher_state(root.path(), true).unwrap();
assert!(!root.path().join(".state/mysql.json").exists());
assert!(crate::process::identity(std::process::id()).is_some());
}
+75
View File
@@ -455,6 +455,81 @@ mod tests {
assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;").unwrap(), "73");
stop(&root).unwrap();
assert!(recovered.0.wait().unwrap().success());
// Exercise the public Repair pipeline with real signed packages and SQL,
// rather than only the snapshot and journal helpers.
let mut package_server = start();
db.query("CREATE TABLE acore_auth.account(id INT PRIMARY KEY, username VARCHAR(40)); INSERT INTO acore_auth.account VALUES(73,'FIXTURE'); CREATE TABLE acore_auth.realmlist(id INT PRIMARY KEY, name VARCHAR(40), address VARCHAR(40), localAddress VARCHAR(40), port INT, flag INT); INSERT INTO acore_auth.realmlist VALUES(1,'AzerothCore','127.0.0.1','127.0.0.1',18085,0); CREATE TABLE acore_characters.characters(guid INT PRIMARY KEY, account INT, name VARCHAR(40), race INT, class INT); INSERT INTO acore_characters.characters VALUES(73,73,'Fixture',7,14); CREATE TABLE acore_world.playercreateinfo(race INT, class INT, PRIMARY KEY(race,class));").unwrap();
let starts = [1,2,3,4,5,6,7,8,10,11].into_iter().flat_map(|race| (12..=32).map(move |class| format!("({race},{class})"))).collect::<Vec<_>>().join(",");
db.query(&format!("INSERT INTO acore_world.playercreateinfo VALUES {starts};")).unwrap();
let password = "a".repeat(48);
db.query(&format!("ALTER USER 'root'@'localhost' IDENTIFIED BY '{password}';")).unwrap();
fsx::atomic_write_json(&root.join("Settings/database.json"), &serde_json::json!({"rootPassword":password,"appPassword":"b".repeat(48)})).unwrap();
fsx::atomic_write(&root.join("Settings/worldserver.conf.template"), b"WorldDatabaseInfo = @WorldDatabaseInfo@\nCharacterDatabaseInfo = @CharacterDatabaseInfo@\nLoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap();
fsx::atomic_write(&root.join("Settings/authserver.conf.template"), b"LoginDatabaseInfo = @LoginDatabaseInfo@\n").unwrap();
fsx::atomic_write(&root.join("Settings/mod_ascension_compat.conf.template"), b"Ascension.DbcPath = @ASCENSION_DBC@\n").unwrap();
let startup = crate::driver::run(&root, crate::driver::Verb::StartMysql).unwrap();
assert!(startup.ok, "{}", crate::diag::redact(&startup.output));
let db = crate::db::Db::from_repack(&root, crate::db::Account::Admin).unwrap();
crate::schema_check::capture(&db, &root).unwrap();
assert!(crate::schema_check::check(&db, &root).unwrap().is_empty());
stop(&root).unwrap();
assert!(package_server.0.wait().unwrap().success());
struct StopFixture(PathBuf);
impl Drop for StopFixture { fn drop(&mut self) { let _ = stop(&self.0); } }
let _stop_fixture = StopFixture(root.clone());
use base64::{engine::general_purpose::STANDARD, Engine};
use ed25519_dalek::{Signer, SigningKey};
let key = SigningKey::from_bytes(&[7;32]);
let trusted = STANDARD.encode(key.verifying_key().to_bytes());
let source = temp.path().join("repair-package-source");
fsx::atomic_write(&source.join("Core/reference/repair-probe.dll"), b"official").unwrap();
fsx::atomic_write(&source.join(crate::schema_check::CONTRACT), &fs::read(root.join(crate::schema_check::CONTRACT)).unwrap()).unwrap();
let build = |kind, version: &str, destination: &Path, migrations| {
let manifest = crate::package::build(&source, destination, &crate::package::BuildOptions { kind, version: version.into(), core_commit: None, built_at: "fixture".into(), part_size: 1024*1024, bots_commit: None, migrations }, &|_| {}).unwrap();
let bytes = fs::read(destination.join("manifest.json")).unwrap();
fsx::atomic_write(&destination.join("manifest.json.sig"), STANDARD.encode(key.sign(&bytes).to_bytes()).as_bytes()).unwrap();
manifest
};
let base = temp.path().join("repair-base");
let base_manifest = build(crate::manifest::Kind::Base, "1.0.0", &base, vec![]);
let update = temp.path().join("repair-update");
let update_manifest = build(crate::manifest::Kind::Update, "2.0.0", &update, vec![]);
fsx::atomic_write_json(&meta.join("manifests/base.json"), &base_manifest).unwrap();
fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &update_manifest).unwrap();
let (_, mut installed) = crate::registry::MetaDir::open(&meta).unwrap();
installed.core.version = Some("2.0.0".into());
fsx::atomic_write_json(&meta.join("install.json"), &installed).unwrap();
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"broken").unwrap();
let base_source = crate::pkgsource::Source::Dir(base);
let update_source = crate::pkgsource::Source::Dir(update);
let report = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|_,_| {}).unwrap();
assert!(report.error.is_none());
assert!(report.database.iter().all(|check| check.problems.is_empty()));
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"official");
crate::update::ensure_recovered(&meta).unwrap();
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before verification failure").unwrap();
let error = crate::repair::run(&root, &meta, &base_source, &update_source, &trusted, &|phase,_| {
if phase == "Checking database after repair" { fs::remove_file(root.join("Core/reference/repair-probe.dll")).unwrap(); }
}).unwrap_err();
assert!(error.to_string().contains("repair"));
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before verification failure");
crate::update::ensure_recovered(&meta).unwrap();
let failed_sql = "UPDATE shutdown_probe SET id=75; CREATE TABLE repair_partial(id INT); INVALID SQL;";
fsx::atomic_write(&source.join("_migrations/characters/repair-failure.sql"), failed_sql.as_bytes()).unwrap();
let failing = temp.path().join("repair-failing-update");
let manifest = build(crate::manifest::Kind::Update, "2.0.0", &failing, vec![crate::manifest::Migration { id:"repair-failure".into(), db:"characters".into(), sha256:fsx::sha256_bytes(failed_sql.as_bytes()), compatible_sha256:vec![], destructive:false }]);
fsx::atomic_write_json(&meta.join("manifests/update-2.0.0.json"), &manifest).unwrap();
fsx::atomic_write(&root.join("Core/reference/repair-probe.dll"), b"before failed repair").unwrap();
let report = crate::repair::run(&root, &meta, &base_source, &crate::pkgsource::Source::Dir(failing), &trusted, &|_,_| {}).unwrap();
assert!(report.error.is_some());
assert_eq!(fs::read(root.join("Core/reference/repair-probe.dll")).unwrap(), b"before failed repair");
crate::update::ensure_recovered(&meta).unwrap();
crate::backup::with_database(&root, |db| {
assert_eq!(db.query("SELECT id FROM acore_characters.shutdown_view;")?, "73");
assert_eq!(db.query("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA='acore_characters' AND TABLE_NAME='repair_partial';")?, "0");
assert_eq!(db.query("SELECT username FROM acore_auth.account WHERE id=73;")?, "FIXTURE");
Ok(())
}).unwrap();
}
#[cfg(windows)]
+44 -26
View File
@@ -134,34 +134,52 @@ pub fn run(root: &Path, dir: &Path, base_source: &Source, update_source: &Source
env.verify_recovery_point(&crate::backup::get(dir, &backup)?)?;
let repair_files: Vec<_> = broken.iter().map(|(file, _)| (*file).clone()).collect();
let mut txn = crate::update::begin_repair(root, dir, &update.version, &backup, &repair_files)?;
let before = dir.join("repairs").join(&backup);
fs::create_dir_all(&before)?;
fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?;
let restored = restore_files(root, &work, &before, &broken)?;
for (entry, _) in &broken {
meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone());
}
fsx::atomic_write_json(&dir.join("install.json"), &meta)?;
progress("Repairing pending database updates", 80);
crate::update::repair_migrating(dir, &mut txn)?;
let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) {
Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))),
Err(e) => (Vec::new(), Some(e.to_string())),
};
if error.is_some() {
crate::update::finish_repair(root, dir, &mut txn, false, &env)?;
let report = Report { restored: Vec::new(), applied: Vec::new(), backup, database: check(root, dir)?, error };
let attempt = (|| -> Result<Report> {
let before = dir.join("repairs").join(&backup);
fs::create_dir_all(&before)?;
fsx::require_space(root, broken.iter().map(|(e,_)| e.size.saturating_mul(2)).sum())?;
let restored = restore_files(root, &work, &before, &broken)?;
for (entry, _) in &broken {
meta.original_hashes.insert(entry.path.clone(), entry.sha256.clone());
}
fsx::atomic_write_json(&dir.join("install.json"), &meta)?;
progress("Repairing pending database updates", 80);
crate::update::repair_migrating(dir, &mut txn)?;
let (applied, error) = match env.migrate(&update, &work.join("update/_migrations")) {
Ok(r) => (r.applied, r.failed.map(|(id,why)| format!("{id}: {why}"))),
Err(e) => (Vec::new(), Some(e.to_string())),
};
if error.is_some() {
crate::update::finish_repair(root, dir, &mut txn, false, &env)?;
let report = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: check(root, dir)?, error };
fsx::atomic_write_json(&before.join("report.json"), &report)?;
return Ok(report);
}
crate::config::create_missing_module_configs(root)?;
progress("Checking database after repair", 95);
let database = check(root, dir)?;
let report = Report { restored, applied, backup: backup.clone(), database, error };
fsx::atomic_write_json(&before.join("report.json"), &report)?;
return Ok(report);
crate::update::finish_repair(root, dir, &mut txn, true, &env)?;
progress("Done", 100);
Ok(report)
})();
match attempt {
Ok(report) => Ok(report),
Err(error) if matches!(txn.state, crate::update::State::RolledBack | crate::update::State::Failed) => Err(error),
Err(error) => {
progress("Undoing the failed repair", 95);
let message = match crate::update::finish_repair(root, dir, &mut txn, false, &env) {
Ok(()) => format!("Repair failed: {error}. The original server files and databases were restored from recovery point {backup}."),
Err(recovery) => format!("Repair failed: {error}. Recovery failed: {recovery}. Recovery point {backup} was preserved; resolve the unfinished repair before starting the server."),
};
let failed = Report { restored: Vec::new(), applied: Vec::new(), backup: backup.clone(), database: Vec::new(), error: Some(message.clone()) };
if let Err(report_error) = fsx::atomic_write_json(&dir.join("repairs").join(&backup).join("report.json"), &failed) {
tracing::warn!(%report_error, "Could not persist failed repair report");
}
Err(Error::Invalid(message))
}
}
crate::config::create_missing_module_configs(root)?;
progress("Checking database after repair", 95);
let database = check(root, dir)?;
let report = Report { restored, applied, backup, database, error };
fsx::atomic_write_json(&before.join("report.json"), &report)?;
crate::update::finish_repair(root, dir, &mut txn, true, &env)?;
progress("Done", 100);
Ok(report)
})();
// Only the UUID staging directory created above is discarded; backups are retained.
let _ = fs::remove_dir_all(work);
+44 -1
View File
@@ -847,6 +847,8 @@ pub(crate) fn begin_repair(root: &Path, meta_dir: &Path, version: &str, backup:
}
let mut recovery_hashes = BTreeMap::new();
recovery_hashes.insert("@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?);
let point = crate::backup::point_json(meta_dir, backup)?;
if point.is_file() { recovery_hashes.insert("@point".into(), fsx::sha256_file(&point)?); }
for op in ops.iter().filter(|op| op.had_previous) { recovery_hashes.insert(op.path.clone(), fsx::sha256_file(&fsx::safe_join(&before, &op.path)?)?); }
let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()), recovery_hashes };
save(meta_dir, &txn)?;
@@ -860,7 +862,14 @@ pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> {
}
pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> {
if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; }
if !success {
if let Err(error) = restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env) {
txn.state = State::Failed;
txn.message = Some(format!("Repair recovery failed: {error}. Its recovery point was preserved."));
save(meta_dir, txn)?;
return Err(error);
}
}
if success { complete_repair_ops(root, txn)?; }
txn.state = if success { State::Committed } else { State::RolledBack };
save(meta_dir, txn)
@@ -1946,6 +1955,40 @@ mod tests {
assert!(ensure_recovered(&w.meta).is_err());
}
#[test]
fn changed_repair_backup_inventory_blocks_recovery_before_stopping_or_restoring() {
let w = world(&[], false);
let point = crate::backup::point_json(&w.meta, "full-backup").unwrap();
fsx::atomic_write(&point, b"original recovery inventory").unwrap();
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
write(&w.root, "Core/worldserver.exe", b"repaired");
repair_migrating(&w.meta, &mut txn).unwrap();
fsx::atomic_write(&point, b"changed recovery inventory").unwrap();
let env = Fake::ok();
assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err());
assert!(env.calls.borrow().is_empty());
assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired");
assert!(ensure_recovered(&w.meta).is_err());
}
#[test]
fn failed_repair_recovery_keeps_a_failed_journal_and_its_reason() {
let w = world(&[], false);
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
write(&w.root, "Core/worldserver.exe", b"repaired");
repair_migrating(&w.meta, &mut txn).unwrap();
let env = Fake::ok();
env.restore_fail.set(true);
assert!(finish_repair(&w.root, &w.meta, &mut txn, false, &env).is_err());
let saved = load(&w.meta, &txn.id).unwrap();
assert_eq!(saved.state, State::Failed);
assert!(saved.message.unwrap().contains("Repair recovery failed"));
assert_eq!(saved.recovery_point.as_deref(), Some("full-backup"));
assert!(ensure_recovered(&w.meta).is_err());
}
#[test]
fn repair_cannot_commit_unverified_files() {
let w = world(&[], false);