mirror of
https://github.com/Corfirean/coa-server-manager.git
synced 2026-11-19 11:25:56 +00:00
Fix repair completion and expose SQUID and rehearsal failures
This commit is contained in:
1 parent
b45e2502ec
commit
482bbfcdfd
11 files changed
+318
-13
No files matched your search
@@ -180,7 +180,7 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result<DriverO
|
||||
};
|
||||
let mut output = out_t.join().unwrap_or_default();
|
||||
output.push_str(&err_t.join().unwrap_or_default());
|
||||
let output = output.trim().to_string();
|
||||
let mut output = output.trim().to_string();
|
||||
let (ok, exit_code) = match status {
|
||||
Some(st) => (st.success(), st.code()),
|
||||
None => (false, None),
|
||||
@@ -195,6 +195,14 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result<DriverO
|
||||
other => Some(other),
|
||||
}
|
||||
};
|
||||
if code == Some(ErrorCode::PortInUse) {
|
||||
let observed = crate::process::observe(&root, &crate::layout::read_ports(&root));
|
||||
for service in [&observed.mysql, &observed.auth, &observed.world].into_iter().chain(observed.secondary_world.iter()) {
|
||||
if let Some(conflict) = &service.conflict {
|
||||
output.push_str(&format!("\nPort {} is used by process {} ({}).", conflict.port, conflict.pid, conflict.exe.as_deref().unwrap_or("executable unavailable")));
|
||||
}
|
||||
}
|
||||
}
|
||||
tracing::info!(?verb, ok, ?code, "driver: launcher verb finished");
|
||||
if !ok && !output.is_empty() {
|
||||
let redacted = crate::diag::redact(&output);
|
||||
|
||||
@@ -129,6 +129,10 @@ pub(crate) fn inventory(root: &Path) -> Result<BTreeMap<String, String>> {
|
||||
}
|
||||
|
||||
pub(crate) fn copy_verified(source: &Path, target: &Path, files: &BTreeMap<String, String>) -> Result<()> {
|
||||
copy_verified_with_progress(source, target, files, &|_| {})
|
||||
}
|
||||
|
||||
fn copy_verified_with_progress(source: &Path, target: &Path, files: &BTreeMap<String, String>, report: &dyn Fn(&str)) -> Result<()> {
|
||||
if target.exists() { return Err(Error::Invalid("Database snapshot destination already exists.".into())); }
|
||||
let bytes = files.keys().try_fold(0u64, |total, name| -> Result<u64> {
|
||||
let path = fsx::ensure_within(source, &fsx::safe_join(source, name)?)?;
|
||||
@@ -138,13 +142,15 @@ pub(crate) fn copy_verified(source: &Path, target: &Path, files: &BTreeMap<Strin
|
||||
fs::create_dir_all(parent)?;
|
||||
fsx::require_space(parent, bytes)?;
|
||||
fs::create_dir(target)?;
|
||||
for (relative, expected) in files {
|
||||
for (index, (relative, expected)) in files.iter().enumerate() {
|
||||
if index % 250 == 0 { report(&format!("Copying private server files: {} of {}", index, files.len())); }
|
||||
let source_file = fsx::ensure_within(source, &fsx::safe_join(source, relative)?)?;
|
||||
let target_file = fsx::ensure_within(target, &fsx::safe_join(target, relative)?)?;
|
||||
fs::create_dir_all(target_file.parent().unwrap())?;
|
||||
fs::copy(&source_file, &target_file)?;
|
||||
if fsx::sha256_file(&target_file)? != *expected { return Err(Error::Invalid(format!("Database snapshot file {relative} failed verification."))); }
|
||||
}
|
||||
report("Verifying the complete private file copy");
|
||||
if inventory(target)? != *files { return Err(Error::Invalid("Database snapshot inventory differs.".into())); }
|
||||
Ok(())
|
||||
}
|
||||
@@ -153,15 +159,33 @@ pub(crate) fn stop(root: &Path) -> Result<()> {
|
||||
let observed = crate::process::observe(root, &crate::layout::read_ports(root));
|
||||
if [observed.mysql.state, observed.auth.state, observed.world.state].iter().all(|state| *state == crate::process::ServiceState::Stopped)
|
||||
&& !crate::multiworld::is_running(root) { return Ok(()); }
|
||||
let outcome = crate::driver::run(root, crate::driver::Verb::StopAll)?;
|
||||
let outcome = crate::driver::run(root, crate::driver::Verb::StopAll)
|
||||
.map_err(|error| Error::Invalid(format!("Could not stop server {} before copying or restoring MySQL: {error}", root.display())))?;
|
||||
let status = crate::process::observe(root, &crate::layout::read_ports(root));
|
||||
if !outcome.ok || [status.mysql.state, status.auth.state, status.world.state].iter().any(|state| *state != crate::process::ServiceState::Stopped)
|
||||
|| crate::multiworld::is_running(root) {
|
||||
return Err(Error::Invalid("The complete server must be stopped before copying or restoring MySQL.".into()));
|
||||
return Err(Error::Invalid(stop_failure(root, &status, &outcome)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stop_failure(root: &Path, status: &crate::process::Observed, outcome: &crate::driver::DriverOutcome) -> String {
|
||||
let mut details = vec![format!("The complete server must be stopped before copying or restoring MySQL. Server: {}.", root.display())];
|
||||
for service in [&status.mysql, &status.auth, &status.world].into_iter().chain(status.secondary_world.iter()) {
|
||||
if service.state != crate::process::ServiceState::Stopped {
|
||||
details.push(format!("{}: {:?}; PID: {}; port: {}.", service.name, service.state, service.pid.map(|pid| pid.to_string()).unwrap_or_else(|| "unknown".into()), service.port));
|
||||
}
|
||||
if let Some(conflict) = &service.conflict {
|
||||
details.push(format!("Port {} is used by process {} ({}).", conflict.port, conflict.pid, conflict.exe.as_deref().unwrap_or("executable unavailable")));
|
||||
}
|
||||
}
|
||||
let output = crate::diag::redact(&outcome.output);
|
||||
let lines: Vec<_> = output.lines().rev().take(12).collect();
|
||||
if !lines.is_empty() { details.push(lines.into_iter().rev().collect::<Vec<_>>().join("\n")); }
|
||||
else if !outcome.ok { details.push(format!("The stop command failed (exit code: {:?}).", outcome.exit_code)); }
|
||||
details.join("\n")
|
||||
}
|
||||
|
||||
pub(crate) fn stop_previous_rehearsals(meta: &Path) -> Result<()> {
|
||||
let location = meta.join("rehearsals");
|
||||
if !location.exists() { return Ok(()); }
|
||||
@@ -226,7 +250,8 @@ pub(crate) fn verify(point: &Path, snapshot: &Snapshot) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn rehearsal_copy(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot) -> Result<PathBuf> {
|
||||
pub(crate) fn rehearsal_copy_with_progress(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot, report: &dyn Fn(&str)) -> Result<PathBuf> {
|
||||
report("Verifying the database recovery copy before rehearsal");
|
||||
verify(point, snapshot)?;
|
||||
let canonical = fsx::canonicalize_lenient(root)?;
|
||||
let root = canonical.as_path();
|
||||
@@ -249,11 +274,15 @@ pub(crate) fn rehearsal_copy(root: &Path, meta: &Path, point: &Path, snapshot: &
|
||||
reject_link(&metadata)?;
|
||||
let path = fsx::ensure_within(root, &raw)?;
|
||||
if metadata.is_dir() { directories.push(path); }
|
||||
else if metadata.is_file() { files.insert(relative, fsx::sha256_file(&path)?); }
|
||||
else if metadata.is_file() {
|
||||
files.insert(relative, fsx::sha256_file(&path)?);
|
||||
if files.len() % 250 == 0 { report(&format!("Preparing the private copy: {} files checked", files.len())); }
|
||||
}
|
||||
else { return Err(Error::Invalid("Unsupported rehearsal file.".into())); }
|
||||
}
|
||||
}
|
||||
copy_verified(root, &destination, &files)?;
|
||||
copy_verified_with_progress(root, &destination, &files, report)?;
|
||||
report("Restoring MySQL into the private server copy");
|
||||
copy_snapshot(point, snapshot, &destination.join("mysql/data"))?;
|
||||
let (_, mut installed) = crate::registry::MetaDir::open(meta)?;
|
||||
installed.id = uuid::Uuid::new_v4().to_string();
|
||||
@@ -338,6 +367,17 @@ pub(crate) fn swap(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot,
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn failed_stop_identifies_the_server_process_and_retains_redacted_launcher_reason() {
|
||||
use crate::process::{Observed, ServiceStatus, ServiceState};
|
||||
let stopped = |name| ServiceStatus { name, state: ServiceState::Stopped, pid: None, port: 0, port_ready: false, conflict: None, uptime_secs: None };
|
||||
let status = Observed { mysql: ServiceStatus { state: ServiceState::Stopping, pid: Some(42), port: 3307, ..stopped("mysql") }, auth: stopped("auth"), world: stopped("world"), secondary_world: None };
|
||||
let outcome = crate::driver::DriverOutcome { ok: false, exit_code: Some(1), code: None, human: None, output: "MySQL is still stopping; wait before copying the repack.\nrootPassword=hunter2".into() };
|
||||
let message = stop_failure(Path::new("private-rehearsal/server"), &status, &outcome);
|
||||
assert!(message.contains("private-rehearsal") && message.contains("PID: 42") && message.contains("3307"));
|
||||
assert!(message.contains("MySQL is still stopping"));
|
||||
assert!(!message.contains("hunter2"));
|
||||
}
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
#[ignore = "requires COA_MYSQL_TEST_BIN pointing to the bundled MySQL tools"]
|
||||
@@ -373,6 +413,14 @@ mod tests {
|
||||
let mut server = start();
|
||||
db.query("CREATE DATABASE acore_characters; CREATE DATABASE acore_world; CREATE DATABASE acore_auth; CREATE TABLE acore_characters.probe(id INT PRIMARY KEY, note VARCHAR(40)); CREATE TABLE acore_world.probe(id INT NOT NULL DEFAULT 0); INSERT INTO acore_world.probe VALUES(43); CREATE TABLE acore_auth.probe(id INT); INSERT INTO acore_auth.probe VALUES(44); CREATE USER 'fixture_user'@'localhost' IDENTIFIED BY 'fixture'; CREATE TRIGGER acore_characters.probe_insert BEFORE INSERT ON acore_characters.probe FOR EACH ROW SET NEW.note='triggered'; INSERT INTO acore_characters.probe VALUES(42,'original'); CREATE VIEW acore_characters.probe_view AS SELECT c.id,w.id AS world_id FROM acore_characters.probe c CROSS JOIN acore_world.probe w; CREATE PROCEDURE acore_characters.probe_proc() SELECT id FROM acore_characters.probe; CREATE FUNCTION acore_characters.probe_func() RETURNS INT DETERMINISTIC RETURN 73;").unwrap();
|
||||
assert_eq!(db.recovery_objects("acore_characters").unwrap().len(), 4);
|
||||
db.query("CREATE TABLE acore_characters.coa_squid_pending(path VARCHAR(255), sha256 CHAR(64)); INSERT INTO acore_characters.coa_squid_pending VALUES('playerbots/base/ai_playerbot_texts.sql', REPEAT('a',64));").unwrap();
|
||||
let squid_problems = crate::squid::database_problems(&root, crate::realms::Mode::Coa, &|sql| {
|
||||
assert!(sql.starts_with("SELECT "));
|
||||
db.query(sql)
|
||||
}).unwrap();
|
||||
assert_eq!(squid_problems.len(), 1);
|
||||
assert!(squid_problems[0].detail.contains("interrupted"));
|
||||
assert_eq!(db.query("SELECT COUNT(*) FROM acore_characters.coa_squid_pending;").unwrap(), "1");
|
||||
crate::schema_check::capture(&db, &root).unwrap();
|
||||
let hash = fsx::sha256_file(&root.join(crate::schema_check::CONTRACT)).unwrap();
|
||||
db.query("ALTER TABLE acore_world.probe ALTER COLUMN id DROP DEFAULT;").unwrap();
|
||||
@@ -486,8 +534,13 @@ mod tests {
|
||||
fsx::atomic_write(&root.join("Core/worldserver.exe"), b"exe").unwrap();
|
||||
fsx::atomic_write(&root.join("Core/configs/worldserver.conf"), b"conf").unwrap();
|
||||
fsx::atomic_write(&root.join("Settings/repack.json"), b"{}").unwrap();
|
||||
let dest = rehearsal_copy(&root, &meta, &point, &snapshot).unwrap();
|
||||
let phases = std::cell::RefCell::new(Vec::new());
|
||||
let dest = rehearsal_copy_with_progress(&root, &meta, &point, &snapshot, &|phase| phases.borrow_mut().push(phase.to_string())).unwrap();
|
||||
assert_eq!(fs::read(dest.join("Core/configs/worldserver.conf")).unwrap(), b"conf");
|
||||
assert_eq!(fs::read(dest.join("mysql/data/characters.ibd")).unwrap(), b"saved");
|
||||
let phases = phases.into_inner();
|
||||
assert!(phases[0].contains("Verifying the database"));
|
||||
assert!(phases.iter().any(|phase| phase.contains("Copying private server files")));
|
||||
assert!(phases.last().unwrap().contains("Restoring MySQL"));
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ impl crate::update::Env for QualificationEnv<'_> {
|
||||
fn snapshot(&self) -> Result<String> { self.0.snapshot() }
|
||||
fn verify_snapshot(&self, id: &str) -> Result<()> { self.0.verify_snapshot(id) }
|
||||
fn rehearse(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str) -> Result<()> { self.0.rehearse(manifest, tree, items, point) }
|
||||
fn rehearse_with_progress(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str, report: &dyn Fn(&str, u8)) -> Result<()> { self.0.rehearse_with_progress(manifest, tree, items, point, report) }
|
||||
fn restore_snapshot(&self, id: &str) -> Result<()> { self.0.restore_snapshot(id) }
|
||||
fn migrate(&self, manifest: &crate::manifest::Manifest, staged: &Path) -> Result<crate::migrations::ApplyReport> { self.0.migrate(manifest, staged) }
|
||||
fn validate(&self) -> Result<()> { self.0.validate() }
|
||||
|
||||
@@ -30,7 +30,9 @@ pub fn check(root: &Path, dir: &Path) -> Result<Vec<DatabaseCheck>> {
|
||||
for mode in modes {
|
||||
let db = db.clone().for_realm(mode);
|
||||
let migrations: Vec<_> = m.migrations.iter().filter(|m| mode == crate::realms::Mode::Coa || m.db != "auth").cloned().collect();
|
||||
checks.push(DatabaseCheck { realm: mode, migrations: crate::migrations::status(&db, &migrations)?, problems: crate::schema_check::check(&db, root)?, full_schema: root.join(crate::schema_check::CONTRACT).exists() });
|
||||
let mut problems = crate::schema_check::check(&db, root)?;
|
||||
problems.extend(crate::squid::database_problems(root, mode, &|sql| db.query(sql))?);
|
||||
checks.push(DatabaseCheck { realm: mode, migrations: crate::migrations::status(&db, &migrations)?, problems, full_schema: root.join(crate::schema_check::CONTRACT).exists() });
|
||||
}
|
||||
Ok(checks)
|
||||
})?;
|
||||
|
||||
@@ -1,6 +1,35 @@
|
||||
use std::path::Path;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// SQUID has its own import ledger, outside the signed package migration ledger.
|
||||
/// Keep this read-only: a pending base import can contain player data and must not
|
||||
/// be replayed or cleared merely because the normal schema contract passes.
|
||||
pub(crate) fn database_problems(root: &Path, mode: crate::realms::Mode, query: &impl Fn(&str) -> crate::Result<String>) -> crate::Result<Vec<crate::schema_check::Problem>> {
|
||||
let active = root.join("Core/configs/modules/playerbots.conf");
|
||||
let config = if active.is_file() { active } else { root.join("Core/configs/modules/playerbots.conf.dist") };
|
||||
let enabled = if config.is_file() {
|
||||
let conf = crate::config::parser::ConfFile::parse_bytes(&std::fs::read(config)?)?;
|
||||
!matches!(conf.get("AiPlayerbot.Enabled").unwrap_or("1").trim().trim_matches('"').to_ascii_lowercase().as_str(), "0" | "false" | "no" | "off")
|
||||
} else { false };
|
||||
let mut problems = Vec::new();
|
||||
for schema in [mode.schema("characters")?, mode.schema("world")?, "acore_playerbots"] {
|
||||
let tables: std::collections::BTreeSet<_> = query(&format!("SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA='{schema}';"))?.lines().map(str::to_owned).collect();
|
||||
if tables.contains("coa_squid_pending") {
|
||||
for pending in query(&format!("SELECT path FROM `{schema}`.coa_squid_pending;"))?.lines().filter(|path| !path.is_empty()) {
|
||||
problems.push(crate::schema_check::Problem { database: schema.into(), table: "coa_squid_pending".into(), column: String::new(), detail: format!("SQUID base import was interrupted: {pending}. File repair and package SQL updates cannot complete this import. Restore a verified database backup from before the import; do not delete the pending record or replay base SQL.") });
|
||||
}
|
||||
}
|
||||
if enabled && schema == mode.schema("characters")? {
|
||||
for table in ["playerbots_names", "playerbots_guild_names", "playerbots_arena_team_names"] {
|
||||
if !tables.contains(table) {
|
||||
problems.push(crate::schema_check::Problem { database: schema.into(), table: table.into(), column: String::new(), detail: "Required SQUID character table is missing. A normal server start provisions it when no interrupted import is present; an interrupted import requires database recovery first.".into() });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(problems)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
pub struct Release {
|
||||
pub tag: Option<String>,
|
||||
@@ -129,6 +158,33 @@ pub fn validate(field: &Field, value: &str) -> crate::Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
#[test]
|
||||
fn database_check_reports_pending_squid_imports_in_each_realm_without_writes() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for mode in [crate::realms::Mode::Coa, crate::realms::Mode::Wildcard] {
|
||||
let problems = database_problems(root.path(), mode, &|sql| {
|
||||
assert!(sql.starts_with("SELECT "));
|
||||
if sql.contains("information_schema.TABLES") { Ok("coa_squid_pending".into()) }
|
||||
else { Ok("playerbots/base/ai_playerbot_texts.sql".into()) }
|
||||
}).unwrap();
|
||||
assert_eq!(problems.len(), 3);
|
||||
assert_eq!(problems[0].database, mode.schema("characters").unwrap());
|
||||
assert!(problems.iter().all(|p| p.detail.contains("interrupted")));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enabled_squid_requires_character_tables_but_disabled_squid_does_not() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let config = root.path().join("Core/configs/modules/playerbots.conf");
|
||||
crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 1\n").unwrap();
|
||||
let query = |_: &str| Ok(String::new());
|
||||
assert_eq!(database_problems(root.path(), crate::realms::Mode::Wildcard, &query).unwrap().len(), 3);
|
||||
crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 0\n").unwrap();
|
||||
assert!(database_problems(root.path(), crate::realms::Mode::Wildcard, &query).unwrap().is_empty());
|
||||
crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 1\n").unwrap();
|
||||
assert!(database_problems(root.path(), crate::realms::Mode::Coa, &|_| Ok("playerbots_names\nplayerbots_guild_names\nplayerbots_arena_team_names".into())).unwrap().is_empty());
|
||||
}
|
||||
#[test]
|
||||
fn a_setting_with_choices_only_accepts_one_of_them() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
crate::fsx::atomic_write(&root.path().join("Core/configs/modules/playerbots.conf.settings.json"), br#"{"format":1,"settings":[
|
||||
|
||||
@@ -93,6 +93,7 @@ pub trait Env {
|
||||
fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) }
|
||||
fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
fn rehearse(&self, _manifest: &Manifest, _tree: &Path, _items: &[PlanItem], _point: &str) -> Result<()> { Ok(()) }
|
||||
fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], point: &str, _report: &dyn Fn(&str, u8)) -> Result<()> { self.rehearse(manifest, tree, items, point) }
|
||||
/// Create a recovery point for the databases and configuration; returns its id.
|
||||
fn snapshot(&self) -> Result<String>;
|
||||
fn restore_snapshot(&self, id: &str) -> Result<()>;
|
||||
@@ -121,13 +122,32 @@ pub fn load(meta: &Path, id: &str) -> Result<Txn> {
|
||||
}
|
||||
|
||||
fn read_journal(path: &Path) -> Result<Txn> {
|
||||
let txn: Txn = fsx::read_json(path).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Available recovery copies were preserved.", path.display())))?;
|
||||
let mut txn: Txn = fsx::read_json(path).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Available recovery copies were preserved.", path.display())))?;
|
||||
if path.parent().and_then(Path::file_name).and_then(|id| id.to_str()) != Some(txn.id.as_str()) { return Err(Error::Invalid("Update journal identity is inconsistent.".into())); }
|
||||
txn_dir(path.parent().unwrap(), &txn.id)?;
|
||||
if let Some(id) = &txn.recovery_point {
|
||||
if id.is_empty() || !id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) || id.contains("..") { return Err(Error::Invalid("Update journal contains an invalid recovery point identity.".into())); }
|
||||
}
|
||||
let valid_hash = |hash: &str| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit());
|
||||
let mut completed_legacy_repair = false;
|
||||
// Older successful repairs saved Committed without setting done. Recover only that
|
||||
// specific shape, and only if every repaired file still has its expected bytes.
|
||||
if txn.state == State::Committed && txn.id.contains("-repair-")
|
||||
&& txn.message.as_deref() == Some("Repair is in progress.")
|
||||
&& txn.from_version.as_deref() == Some(txn.to_version.as_str())
|
||||
&& txn.databases_started && txn.recovery_point.is_some() && txn.recovery_hashes.contains_key("@install")
|
||||
&& txn.ops.iter().any(|op| !op.done)
|
||||
&& txn.ops.iter().all(|op| op.started && matches!((op.action, op.had_previous), (Action::Create, false) | (Action::Replace, true)) && valid_hash(&op.new_sha256)) {
|
||||
let meta_dir = path.parent().and_then(Path::parent).and_then(Path::parent)
|
||||
.ok_or_else(|| Error::Invalid("Invalid repair journal location.".into()))?;
|
||||
let (_, installed) = MetaDir::open(meta_dir)?;
|
||||
let root = Path::new(&installed.server_path);
|
||||
if fsx::canonicalize_lenient(&crate::registry::metadata_dir_for(root)?)? != fsx::canonicalize_lenient(meta_dir)? {
|
||||
return Err(Error::Invalid("The repair journal does not belong to its installed server.".into()));
|
||||
}
|
||||
complete_repair_ops(root, &mut txn)?;
|
||||
completed_legacy_repair = true;
|
||||
}
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
for op in &txn.ops {
|
||||
fsx::safe_join(path.parent().unwrap(), &op.path)?;
|
||||
@@ -140,6 +160,13 @@ fn read_journal(path: &Path) -> Result<Txn> {
|
||||
}
|
||||
if txn.recovery_hashes.values().any(|hash| !valid_hash(hash)) { return Err(Error::Invalid("Update journal contains invalid recovery hashes.".into())); }
|
||||
for key in txn.recovery_hashes.keys().filter(|key| !matches!(key.as_str(), "@install" | "@point")) { fsx::safe_join(path.parent().unwrap(), key)?; }
|
||||
if completed_legacy_repair {
|
||||
// Persist the verified completion once: later legitimate updates can change
|
||||
// these files. Retain the original record for diagnostics and crash recovery.
|
||||
let original = path.with_file_name("txn-before-repair-completion.json");
|
||||
if !original.exists() { fsx::atomic_write(&original, &fs::read(path)?)?; }
|
||||
fsx::atomic_write_json(path, &txn)?;
|
||||
}
|
||||
Ok(txn)
|
||||
}
|
||||
|
||||
@@ -499,7 +526,7 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result<Outcome> {
|
||||
if point_meta.is_file() { txn.recovery_hashes.insert("@point".into(), fsx::sha256_file(&point_meta)?); }
|
||||
save(meta_dir, &txn)?;
|
||||
step(report, "Testing the update on a private copy of your server", 54);
|
||||
if let Err(error) = p.env.rehearse(&m, &tree, &items, txn.recovery_point.as_deref().unwrap()) {
|
||||
if let Err(error) = p.env.rehearse_with_progress(&m, &tree, &items, txn.recovery_point.as_deref().unwrap(), report) {
|
||||
txn.state = State::Failed;
|
||||
txn.message = Some(format!("The private update rehearsal failed: {error}. The installed server was not updated."));
|
||||
save(meta_dir, &txn)?;
|
||||
@@ -834,10 +861,22 @@ pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> {
|
||||
|
||||
pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> {
|
||||
if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; }
|
||||
if success { complete_repair_ops(root, txn)?; }
|
||||
txn.state = if success { State::Committed } else { State::RolledBack };
|
||||
save(meta_dir, txn)
|
||||
}
|
||||
|
||||
fn complete_repair_ops(root: &Path, txn: &mut Txn) -> Result<()> {
|
||||
for op in &txn.ops {
|
||||
let target = fsx::ensure_within(root, &fsx::safe_join(root, &op.path)?)?;
|
||||
if fsx::sha256_file(&target)? != op.new_sha256 {
|
||||
return Err(Error::Invalid(format!("Repaired file {} failed verification. Its recovery record was preserved.", op.path)));
|
||||
}
|
||||
}
|
||||
for op in &mut txn.ops { op.done = true; }
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prune(meta_dir: &Path, keep: usize) {
|
||||
let Ok(rd) = fs::read_dir(updates_dir(meta_dir)) else { return };
|
||||
let mut dirs: Vec<_> = rd.flatten().filter(|e| read_journal(&e.path().join("txn.json")).is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack))).map(|e| e.path()).collect();
|
||||
@@ -939,6 +978,10 @@ impl Env for RepackEnv<'_> {
|
||||
|
||||
fn automatic_rollback(&self) -> bool { true }
|
||||
fn rehearse(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str) -> Result<()> {
|
||||
self.rehearse_with_progress(manifest, tree, items, id, &|_, _| {})
|
||||
}
|
||||
|
||||
fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str, report: &dyn Fn(&str, u8)) -> Result<()> {
|
||||
if items.iter().any(|item| item.action != Action::Skip && item.path.to_ascii_lowercase().starts_with("mysql/")) {
|
||||
return Err(Error::Invalid("A server update cannot replace MySQL while using a cold recovery copy; a separate database engine upgrade is required.".into()));
|
||||
}
|
||||
@@ -951,7 +994,7 @@ impl Env for RepackEnv<'_> {
|
||||
crate::mysql_snapshot::validate_connections(self.root, self.root)?;
|
||||
crate::mysql_snapshot::stop(self.root)?;
|
||||
let point_dir = crate::backup::point_json(self.meta_dir, id)?.parent().unwrap().to_path_buf();
|
||||
let fixture = crate::mysql_snapshot::rehearsal_copy(self.root, self.meta_dir, &point_dir, snapshot)?;
|
||||
let fixture = crate::mysql_snapshot::rehearsal_copy_with_progress(self.root, self.meta_dir, &point_dir, snapshot, &|phase| step(report, phase, 54))?;
|
||||
let metadata = crate::registry::metadata_dir_for(&fixture)?;
|
||||
let env = RepackEnv { root: &fixture, meta_dir: &metadata };
|
||||
let result = crate::install::with_scratch_ports(&fixture, || {
|
||||
@@ -964,9 +1007,11 @@ impl Env for RepackEnv<'_> {
|
||||
}).collect(), message:None, recovery_hashes:BTreeMap::new() };
|
||||
let before = txn_dir(&metadata, &txn.id)?.join("before");
|
||||
fs::create_dir_all(&before)?;
|
||||
step(report, "Applying update files to the private server copy", 55);
|
||||
apply_ops(&fixture, tree, &before, &manifest.files, &mut txn, &metadata, None)?;
|
||||
crate::mysql_snapshot::validate_connections(&fixture, self.root)?;
|
||||
crate::mysql_snapshot::isolate_connections(&fixture)?;
|
||||
step(report, "Testing database updates on the private server copy", 56);
|
||||
let identities = crate::backup::with_database(&fixture, |db| {
|
||||
let identities = database_identities(db)?;
|
||||
let realms = crate::realms::state(&fixture)?;
|
||||
@@ -983,10 +1028,12 @@ impl Env for RepackEnv<'_> {
|
||||
verify_database_identities(db, &identities)?;
|
||||
Ok(identities)
|
||||
})?;
|
||||
step(report, "Testing startup of the private server copy", 57);
|
||||
env.validate()?;
|
||||
crate::backup::with_database(&fixture, |db| verify_database_identities(db, &identities))?;
|
||||
env.ensure_stopped()
|
||||
});
|
||||
step(report, "Stopping the private server copy and verifying the installed database", 57);
|
||||
let stopped = crate::mysql_snapshot::stop(&fixture);
|
||||
let live_state = if result.is_ok() { crate::mysql_snapshot::stop(self.root).and_then(|()| crate::mysql_snapshot::unchanged(self.root, snapshot)) } else { Ok(()) };
|
||||
let logs: BTreeMap<_, _> = ["Core/Logs/auth-console.log", "Core/Logs/world-console.log", "Core/Logs/supervisor.log", "mysql/logs/mysql-error.log"].into_iter()
|
||||
@@ -1876,6 +1923,54 @@ mod tests {
|
||||
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_completed_repair_is_accepted_only_with_verified_repaired_files() {
|
||||
let w = world(&[], false);
|
||||
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
|
||||
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
|
||||
write(&w.root, "Core/worldserver.exe", b"repaired");
|
||||
repair_migrating(&w.meta, &mut txn).unwrap();
|
||||
txn.state = State::Committed;
|
||||
save(&w.meta, &txn).unwrap();
|
||||
let original = fs::read(txn_dir(&w.meta, &txn.id).unwrap().join("txn.json")).unwrap();
|
||||
write(&w.root, "Core/worldserver.exe", b"changed");
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
write(&w.root, "Core/worldserver.exe", b"repaired");
|
||||
ensure_recovered(&w.meta).unwrap();
|
||||
assert!(load(&w.meta, &txn.id).unwrap().ops[0].done);
|
||||
assert_eq!(fs::read(txn_dir(&w.meta, &txn.id).unwrap().join("txn-before-repair-completion.json")).unwrap(), original);
|
||||
write(&w.root, "Core/worldserver.exe", b"a later legitimate update");
|
||||
ensure_recovered(&w.meta).unwrap();
|
||||
txn.ops[0].started = false;
|
||||
save(&w.meta, &txn).unwrap();
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repair_cannot_commit_unverified_files() {
|
||||
let w = world(&[], false);
|
||||
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
|
||||
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
|
||||
assert!(finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).is_err());
|
||||
assert_eq!(load(&w.meta, &txn.id).unwrap().state, State::Applying);
|
||||
assert_eq!(pending_checked(&w.meta).unwrap().unwrap().id, txn.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn successful_repair_leaves_a_readable_journal_and_allows_next_operation() {
|
||||
let w = world(&[], false);
|
||||
let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }];
|
||||
let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap();
|
||||
write(&w.root, "Core/worldserver.exe", b"repaired");
|
||||
repair_migrating(&w.meta, &mut txn).unwrap();
|
||||
finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).unwrap();
|
||||
let saved = load(&w.meta, &txn.id).unwrap();
|
||||
assert_eq!(saved.state, State::Committed);
|
||||
assert!(saved.ops.iter().all(|op| op.started && op.done));
|
||||
ensure_recovered(&w.meta).unwrap();
|
||||
assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interrupted_repair_restores_files_databases_and_ownership_metadata() {
|
||||
let w = world(&[], false);
|
||||
|
||||
Reference in new issue
Block a user