From 482bbfcdfd9e5cf24ebeb8a23fa9b1a02b434a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Wed, 14 Oct 2026 06:52:24 +0500 Subject: [PATCH] Fix repair completion and expose SQUID and rehearsal failures --- ...pdate-checks-show-preparation-stages-an.md | 7 ++ ...hecks-report-interrupted-squid-database.md | 7 ++ ...ul-server-repairs-no-longer-block-later.md | 7 ++ crates/coa-core/src/driver.rs | 10 +- crates/coa-core/src/mysql_snapshot.rs | 67 ++++++++++-- crates/coa-core/src/release_schema.rs | 1 + crates/coa-core/src/repair.rs | 4 +- crates/coa-core/src/squid.rs | 56 ++++++++++ crates/coa-core/src/update.rs | 101 +++++++++++++++++- docs/STABILIZATION-20261014.md | 53 +++++++++ tools/server-update-refresh.test.mjs | 18 +++- 11 files changed, 318 insertions(+), 13 deletions(-) create mode 100644 changelog.d/20261014-private-update-checks-show-preparation-stages-an.md create mode 100644 changelog.d/20261014-server-checks-report-interrupted-squid-database.md create mode 100644 changelog.d/20261014-successful-server-repairs-no-longer-block-later.md create mode 100644 docs/STABILIZATION-20261014.md diff --git a/changelog.d/20261014-private-update-checks-show-preparation-stages-an.md b/changelog.d/20261014-private-update-checks-show-preparation-stages-an.md new file mode 100644 index 0000000..71cef60 --- /dev/null +++ b/changelog.d/20261014-private-update-checks-show-preparation-stages-an.md @@ -0,0 +1,7 @@ +--- +area: manager +type: changed +audience: admins +title: Private update checks show preparation stages and stop failure details +--- +Private-copy preparation reports checked and copied file counts, database tests and startup stages. Failed shutdown reports the server folder, remaining processes and launcher reason. Port conflicts identify the owning process and executable. diff --git a/changelog.d/20261014-server-checks-report-interrupted-squid-database.md b/changelog.d/20261014-server-checks-report-interrupted-squid-database.md new file mode 100644 index 0000000..77e0e07 --- /dev/null +++ b/changelog.d/20261014-server-checks-report-interrupted-squid-database.md @@ -0,0 +1,7 @@ +--- +area: manager +type: fixed +audience: admins +title: Server checks report interrupted SQUID database imports +--- +Check and Repair now report SQUID pending imports and missing bot character tables separately from package migrations. Interrupted imports retain their records and require verified database recovery. diff --git a/changelog.d/20261014-successful-server-repairs-no-longer-block-later.md b/changelog.d/20261014-successful-server-repairs-no-longer-block-later.md new file mode 100644 index 0000000..24bca4c --- /dev/null +++ b/changelog.d/20261014-successful-server-repairs-no-longer-block-later.md @@ -0,0 +1,7 @@ +--- +area: manager +type: fixed +audience: admins +title: Successful server repairs no longer block later server actions +--- +Repair verifies restored files before completing its recovery record. Older completed repair records are accepted only when every repaired file still matches its expected content. diff --git a/crates/coa-core/src/driver.rs b/crates/coa-core/src/driver.rs index 636ecfa..c5540c4 100644 --- a/crates/coa-core/src/driver.rs +++ b/crates/coa-core/src/driver.rs @@ -180,7 +180,7 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result (st.success(), st.code()), None => (false, None), @@ -195,6 +195,14 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result Some(other), } }; + if code == Some(ErrorCode::PortInUse) { + let observed = crate::process::observe(&root, &crate::layout::read_ports(&root)); + for service in [&observed.mysql, &observed.auth, &observed.world].into_iter().chain(observed.secondary_world.iter()) { + if let Some(conflict) = &service.conflict { + output.push_str(&format!("\nPort {} is used by process {} ({}).", conflict.port, conflict.pid, conflict.exe.as_deref().unwrap_or("executable unavailable"))); + } + } + } tracing::info!(?verb, ok, ?code, "driver: launcher verb finished"); if !ok && !output.is_empty() { let redacted = crate::diag::redact(&output); diff --git a/crates/coa-core/src/mysql_snapshot.rs b/crates/coa-core/src/mysql_snapshot.rs index ea14db9..14b8cae 100644 --- a/crates/coa-core/src/mysql_snapshot.rs +++ b/crates/coa-core/src/mysql_snapshot.rs @@ -129,6 +129,10 @@ pub(crate) fn inventory(root: &Path) -> Result> { } pub(crate) fn copy_verified(source: &Path, target: &Path, files: &BTreeMap) -> Result<()> { + copy_verified_with_progress(source, target, files, &|_| {}) +} + +fn copy_verified_with_progress(source: &Path, target: &Path, files: &BTreeMap, report: &dyn Fn(&str)) -> Result<()> { if target.exists() { return Err(Error::Invalid("Database snapshot destination already exists.".into())); } let bytes = files.keys().try_fold(0u64, |total, name| -> Result { let path = fsx::ensure_within(source, &fsx::safe_join(source, name)?)?; @@ -138,13 +142,15 @@ pub(crate) fn copy_verified(source: &Path, target: &Path, files: &BTreeMap Result<()> { let observed = crate::process::observe(root, &crate::layout::read_ports(root)); if [observed.mysql.state, observed.auth.state, observed.world.state].iter().all(|state| *state == crate::process::ServiceState::Stopped) && !crate::multiworld::is_running(root) { return Ok(()); } - let outcome = crate::driver::run(root, crate::driver::Verb::StopAll)?; + let outcome = crate::driver::run(root, crate::driver::Verb::StopAll) + .map_err(|error| Error::Invalid(format!("Could not stop server {} before copying or restoring MySQL: {error}", root.display())))?; let status = crate::process::observe(root, &crate::layout::read_ports(root)); if !outcome.ok || [status.mysql.state, status.auth.state, status.world.state].iter().any(|state| *state != crate::process::ServiceState::Stopped) || crate::multiworld::is_running(root) { - return Err(Error::Invalid("The complete server must be stopped before copying or restoring MySQL.".into())); + return Err(Error::Invalid(stop_failure(root, &status, &outcome))); } Ok(()) } +fn stop_failure(root: &Path, status: &crate::process::Observed, outcome: &crate::driver::DriverOutcome) -> String { + let mut details = vec![format!("The complete server must be stopped before copying or restoring MySQL. Server: {}.", root.display())]; + for service in [&status.mysql, &status.auth, &status.world].into_iter().chain(status.secondary_world.iter()) { + if service.state != crate::process::ServiceState::Stopped { + details.push(format!("{}: {:?}; PID: {}; port: {}.", service.name, service.state, service.pid.map(|pid| pid.to_string()).unwrap_or_else(|| "unknown".into()), service.port)); + } + if let Some(conflict) = &service.conflict { + details.push(format!("Port {} is used by process {} ({}).", conflict.port, conflict.pid, conflict.exe.as_deref().unwrap_or("executable unavailable"))); + } + } + let output = crate::diag::redact(&outcome.output); + let lines: Vec<_> = output.lines().rev().take(12).collect(); + if !lines.is_empty() { details.push(lines.into_iter().rev().collect::>().join("\n")); } + else if !outcome.ok { details.push(format!("The stop command failed (exit code: {:?}).", outcome.exit_code)); } + details.join("\n") +} + pub(crate) fn stop_previous_rehearsals(meta: &Path) -> Result<()> { let location = meta.join("rehearsals"); if !location.exists() { return Ok(()); } @@ -226,7 +250,8 @@ pub(crate) fn verify(point: &Path, snapshot: &Snapshot) -> Result<()> { Ok(()) } -pub(crate) fn rehearsal_copy(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot) -> Result { +pub(crate) fn rehearsal_copy_with_progress(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot, report: &dyn Fn(&str)) -> Result { + report("Verifying the database recovery copy before rehearsal"); verify(point, snapshot)?; let canonical = fsx::canonicalize_lenient(root)?; let root = canonical.as_path(); @@ -249,11 +274,15 @@ pub(crate) fn rehearsal_copy(root: &Path, meta: &Path, point: &Path, snapshot: & reject_link(&metadata)?; let path = fsx::ensure_within(root, &raw)?; if metadata.is_dir() { directories.push(path); } - else if metadata.is_file() { files.insert(relative, fsx::sha256_file(&path)?); } + else if metadata.is_file() { + files.insert(relative, fsx::sha256_file(&path)?); + if files.len() % 250 == 0 { report(&format!("Preparing the private copy: {} files checked", files.len())); } + } else { return Err(Error::Invalid("Unsupported rehearsal file.".into())); } } } - copy_verified(root, &destination, &files)?; + copy_verified_with_progress(root, &destination, &files, report)?; + report("Restoring MySQL into the private server copy"); copy_snapshot(point, snapshot, &destination.join("mysql/data"))?; let (_, mut installed) = crate::registry::MetaDir::open(meta)?; installed.id = uuid::Uuid::new_v4().to_string(); @@ -338,6 +367,17 @@ pub(crate) fn swap(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot, #[cfg(test)] mod tests { use super::*; + #[test] + fn failed_stop_identifies_the_server_process_and_retains_redacted_launcher_reason() { + use crate::process::{Observed, ServiceStatus, ServiceState}; + let stopped = |name| ServiceStatus { name, state: ServiceState::Stopped, pid: None, port: 0, port_ready: false, conflict: None, uptime_secs: None }; + let status = Observed { mysql: ServiceStatus { state: ServiceState::Stopping, pid: Some(42), port: 3307, ..stopped("mysql") }, auth: stopped("auth"), world: stopped("world"), secondary_world: None }; + let outcome = crate::driver::DriverOutcome { ok: false, exit_code: Some(1), code: None, human: None, output: "MySQL is still stopping; wait before copying the repack.\nrootPassword=hunter2".into() }; + let message = stop_failure(Path::new("private-rehearsal/server"), &status, &outcome); + assert!(message.contains("private-rehearsal") && message.contains("PID: 42") && message.contains("3307")); + assert!(message.contains("MySQL is still stopping")); + assert!(!message.contains("hunter2")); + } #[cfg(windows)] #[test] #[ignore = "requires COA_MYSQL_TEST_BIN pointing to the bundled MySQL tools"] @@ -373,6 +413,14 @@ mod tests { let mut server = start(); db.query("CREATE DATABASE acore_characters; CREATE DATABASE acore_world; CREATE DATABASE acore_auth; CREATE TABLE acore_characters.probe(id INT PRIMARY KEY, note VARCHAR(40)); CREATE TABLE acore_world.probe(id INT NOT NULL DEFAULT 0); INSERT INTO acore_world.probe VALUES(43); CREATE TABLE acore_auth.probe(id INT); INSERT INTO acore_auth.probe VALUES(44); CREATE USER 'fixture_user'@'localhost' IDENTIFIED BY 'fixture'; CREATE TRIGGER acore_characters.probe_insert BEFORE INSERT ON acore_characters.probe FOR EACH ROW SET NEW.note='triggered'; INSERT INTO acore_characters.probe VALUES(42,'original'); CREATE VIEW acore_characters.probe_view AS SELECT c.id,w.id AS world_id FROM acore_characters.probe c CROSS JOIN acore_world.probe w; CREATE PROCEDURE acore_characters.probe_proc() SELECT id FROM acore_characters.probe; CREATE FUNCTION acore_characters.probe_func() RETURNS INT DETERMINISTIC RETURN 73;").unwrap(); assert_eq!(db.recovery_objects("acore_characters").unwrap().len(), 4); + db.query("CREATE TABLE acore_characters.coa_squid_pending(path VARCHAR(255), sha256 CHAR(64)); INSERT INTO acore_characters.coa_squid_pending VALUES('playerbots/base/ai_playerbot_texts.sql', REPEAT('a',64));").unwrap(); + let squid_problems = crate::squid::database_problems(&root, crate::realms::Mode::Coa, &|sql| { + assert!(sql.starts_with("SELECT ")); + db.query(sql) + }).unwrap(); + assert_eq!(squid_problems.len(), 1); + assert!(squid_problems[0].detail.contains("interrupted")); + assert_eq!(db.query("SELECT COUNT(*) FROM acore_characters.coa_squid_pending;").unwrap(), "1"); crate::schema_check::capture(&db, &root).unwrap(); let hash = fsx::sha256_file(&root.join(crate::schema_check::CONTRACT)).unwrap(); db.query("ALTER TABLE acore_world.probe ALTER COLUMN id DROP DEFAULT;").unwrap(); @@ -486,8 +534,13 @@ mod tests { fsx::atomic_write(&root.join("Core/worldserver.exe"), b"exe").unwrap(); fsx::atomic_write(&root.join("Core/configs/worldserver.conf"), b"conf").unwrap(); fsx::atomic_write(&root.join("Settings/repack.json"), b"{}").unwrap(); - let dest = rehearsal_copy(&root, &meta, &point, &snapshot).unwrap(); + let phases = std::cell::RefCell::new(Vec::new()); + let dest = rehearsal_copy_with_progress(&root, &meta, &point, &snapshot, &|phase| phases.borrow_mut().push(phase.to_string())).unwrap(); assert_eq!(fs::read(dest.join("Core/configs/worldserver.conf")).unwrap(), b"conf"); assert_eq!(fs::read(dest.join("mysql/data/characters.ibd")).unwrap(), b"saved"); + let phases = phases.into_inner(); + assert!(phases[0].contains("Verifying the database")); + assert!(phases.iter().any(|phase| phase.contains("Copying private server files"))); + assert!(phases.last().unwrap().contains("Restoring MySQL")); } } diff --git a/crates/coa-core/src/release_schema.rs b/crates/coa-core/src/release_schema.rs index 28f1994..a9bde6d 100644 --- a/crates/coa-core/src/release_schema.rs +++ b/crates/coa-core/src/release_schema.rs @@ -14,6 +14,7 @@ impl crate::update::Env for QualificationEnv<'_> { fn snapshot(&self) -> Result { self.0.snapshot() } fn verify_snapshot(&self, id: &str) -> Result<()> { self.0.verify_snapshot(id) } fn rehearse(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str) -> Result<()> { self.0.rehearse(manifest, tree, items, point) } + fn rehearse_with_progress(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str, report: &dyn Fn(&str, u8)) -> Result<()> { self.0.rehearse_with_progress(manifest, tree, items, point, report) } fn restore_snapshot(&self, id: &str) -> Result<()> { self.0.restore_snapshot(id) } fn migrate(&self, manifest: &crate::manifest::Manifest, staged: &Path) -> Result { self.0.migrate(manifest, staged) } fn validate(&self) -> Result<()> { self.0.validate() } diff --git a/crates/coa-core/src/repair.rs b/crates/coa-core/src/repair.rs index d97672e..9ca4a0e 100644 --- a/crates/coa-core/src/repair.rs +++ b/crates/coa-core/src/repair.rs @@ -30,7 +30,9 @@ pub fn check(root: &Path, dir: &Path) -> Result> { for mode in modes { let db = db.clone().for_realm(mode); let migrations: Vec<_> = m.migrations.iter().filter(|m| mode == crate::realms::Mode::Coa || m.db != "auth").cloned().collect(); - checks.push(DatabaseCheck { realm: mode, migrations: crate::migrations::status(&db, &migrations)?, problems: crate::schema_check::check(&db, root)?, full_schema: root.join(crate::schema_check::CONTRACT).exists() }); + let mut problems = crate::schema_check::check(&db, root)?; + problems.extend(crate::squid::database_problems(root, mode, &|sql| db.query(sql))?); + checks.push(DatabaseCheck { realm: mode, migrations: crate::migrations::status(&db, &migrations)?, problems, full_schema: root.join(crate::schema_check::CONTRACT).exists() }); } Ok(checks) })?; diff --git a/crates/coa-core/src/squid.rs b/crates/coa-core/src/squid.rs index 67222b0..0c3f09d 100644 --- a/crates/coa-core/src/squid.rs +++ b/crates/coa-core/src/squid.rs @@ -1,6 +1,35 @@ use std::path::Path; use serde::{Deserialize, Serialize}; +/// SQUID has its own import ledger, outside the signed package migration ledger. +/// Keep this read-only: a pending base import can contain player data and must not +/// be replayed or cleared merely because the normal schema contract passes. +pub(crate) fn database_problems(root: &Path, mode: crate::realms::Mode, query: &impl Fn(&str) -> crate::Result) -> crate::Result> { + let active = root.join("Core/configs/modules/playerbots.conf"); + let config = if active.is_file() { active } else { root.join("Core/configs/modules/playerbots.conf.dist") }; + let enabled = if config.is_file() { + let conf = crate::config::parser::ConfFile::parse_bytes(&std::fs::read(config)?)?; + !matches!(conf.get("AiPlayerbot.Enabled").unwrap_or("1").trim().trim_matches('"').to_ascii_lowercase().as_str(), "0" | "false" | "no" | "off") + } else { false }; + let mut problems = Vec::new(); + for schema in [mode.schema("characters")?, mode.schema("world")?, "acore_playerbots"] { + let tables: std::collections::BTreeSet<_> = query(&format!("SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA='{schema}';"))?.lines().map(str::to_owned).collect(); + if tables.contains("coa_squid_pending") { + for pending in query(&format!("SELECT path FROM `{schema}`.coa_squid_pending;"))?.lines().filter(|path| !path.is_empty()) { + problems.push(crate::schema_check::Problem { database: schema.into(), table: "coa_squid_pending".into(), column: String::new(), detail: format!("SQUID base import was interrupted: {pending}. File repair and package SQL updates cannot complete this import. Restore a verified database backup from before the import; do not delete the pending record or replay base SQL.") }); + } + } + if enabled && schema == mode.schema("characters")? { + for table in ["playerbots_names", "playerbots_guild_names", "playerbots_arena_team_names"] { + if !tables.contains(table) { + problems.push(crate::schema_check::Problem { database: schema.into(), table: table.into(), column: String::new(), detail: "Required SQUID character table is missing. A normal server start provisions it when no interrupted import is present; an interrupted import requires database recovery first.".into() }); + } + } + } + } + Ok(problems) +} + #[derive(Debug, Serialize, PartialEq, Eq)] pub struct Release { pub tag: Option, @@ -129,6 +158,33 @@ pub fn validate(field: &Field, value: &str) -> crate::Result<()> { mod tests { use super::*; #[test] + fn database_check_reports_pending_squid_imports_in_each_realm_without_writes() { + let root = tempfile::tempdir().unwrap(); + for mode in [crate::realms::Mode::Coa, crate::realms::Mode::Wildcard] { + let problems = database_problems(root.path(), mode, &|sql| { + assert!(sql.starts_with("SELECT ")); + if sql.contains("information_schema.TABLES") { Ok("coa_squid_pending".into()) } + else { Ok("playerbots/base/ai_playerbot_texts.sql".into()) } + }).unwrap(); + assert_eq!(problems.len(), 3); + assert_eq!(problems[0].database, mode.schema("characters").unwrap()); + assert!(problems.iter().all(|p| p.detail.contains("interrupted"))); + } + } + + #[test] + fn enabled_squid_requires_character_tables_but_disabled_squid_does_not() { + let root = tempfile::tempdir().unwrap(); + let config = root.path().join("Core/configs/modules/playerbots.conf"); + crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 1\n").unwrap(); + let query = |_: &str| Ok(String::new()); + assert_eq!(database_problems(root.path(), crate::realms::Mode::Wildcard, &query).unwrap().len(), 3); + crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 0\n").unwrap(); + assert!(database_problems(root.path(), crate::realms::Mode::Wildcard, &query).unwrap().is_empty()); + crate::fsx::atomic_write(&config, b"AiPlayerbot.Enabled = 1\n").unwrap(); + assert!(database_problems(root.path(), crate::realms::Mode::Coa, &|_| Ok("playerbots_names\nplayerbots_guild_names\nplayerbots_arena_team_names".into())).unwrap().is_empty()); + } + #[test] fn a_setting_with_choices_only_accepts_one_of_them() { let root = tempfile::tempdir().unwrap(); crate::fsx::atomic_write(&root.path().join("Core/configs/modules/playerbots.conf.settings.json"), br#"{"format":1,"settings":[ diff --git a/crates/coa-core/src/update.rs b/crates/coa-core/src/update.rs index a523540..f6d224b 100644 --- a/crates/coa-core/src/update.rs +++ b/crates/coa-core/src/update.rs @@ -93,6 +93,7 @@ pub trait Env { fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) } fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) } fn rehearse(&self, _manifest: &Manifest, _tree: &Path, _items: &[PlanItem], _point: &str) -> Result<()> { Ok(()) } + fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], point: &str, _report: &dyn Fn(&str, u8)) -> Result<()> { self.rehearse(manifest, tree, items, point) } /// Create a recovery point for the databases and configuration; returns its id. fn snapshot(&self) -> Result; fn restore_snapshot(&self, id: &str) -> Result<()>; @@ -121,13 +122,32 @@ pub fn load(meta: &Path, id: &str) -> Result { } fn read_journal(path: &Path) -> Result { - let txn: Txn = fsx::read_json(path).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Available recovery copies were preserved.", path.display())))?; + let mut txn: Txn = fsx::read_json(path).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Available recovery copies were preserved.", path.display())))?; if path.parent().and_then(Path::file_name).and_then(|id| id.to_str()) != Some(txn.id.as_str()) { return Err(Error::Invalid("Update journal identity is inconsistent.".into())); } txn_dir(path.parent().unwrap(), &txn.id)?; if let Some(id) = &txn.recovery_point { if id.is_empty() || !id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) || id.contains("..") { return Err(Error::Invalid("Update journal contains an invalid recovery point identity.".into())); } } let valid_hash = |hash: &str| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit()); + let mut completed_legacy_repair = false; + // Older successful repairs saved Committed without setting done. Recover only that + // specific shape, and only if every repaired file still has its expected bytes. + if txn.state == State::Committed && txn.id.contains("-repair-") + && txn.message.as_deref() == Some("Repair is in progress.") + && txn.from_version.as_deref() == Some(txn.to_version.as_str()) + && txn.databases_started && txn.recovery_point.is_some() && txn.recovery_hashes.contains_key("@install") + && txn.ops.iter().any(|op| !op.done) + && txn.ops.iter().all(|op| op.started && matches!((op.action, op.had_previous), (Action::Create, false) | (Action::Replace, true)) && valid_hash(&op.new_sha256)) { + let meta_dir = path.parent().and_then(Path::parent).and_then(Path::parent) + .ok_or_else(|| Error::Invalid("Invalid repair journal location.".into()))?; + let (_, installed) = MetaDir::open(meta_dir)?; + let root = Path::new(&installed.server_path); + if fsx::canonicalize_lenient(&crate::registry::metadata_dir_for(root)?)? != fsx::canonicalize_lenient(meta_dir)? { + return Err(Error::Invalid("The repair journal does not belong to its installed server.".into())); + } + complete_repair_ops(root, &mut txn)?; + completed_legacy_repair = true; + } let mut seen = std::collections::BTreeSet::new(); for op in &txn.ops { fsx::safe_join(path.parent().unwrap(), &op.path)?; @@ -140,6 +160,13 @@ fn read_journal(path: &Path) -> Result { } if txn.recovery_hashes.values().any(|hash| !valid_hash(hash)) { return Err(Error::Invalid("Update journal contains invalid recovery hashes.".into())); } for key in txn.recovery_hashes.keys().filter(|key| !matches!(key.as_str(), "@install" | "@point")) { fsx::safe_join(path.parent().unwrap(), key)?; } + if completed_legacy_repair { + // Persist the verified completion once: later legitimate updates can change + // these files. Retain the original record for diagnostics and crash recovery. + let original = path.with_file_name("txn-before-repair-completion.json"); + if !original.exists() { fsx::atomic_write(&original, &fs::read(path)?)?; } + fsx::atomic_write_json(path, &txn)?; + } Ok(txn) } @@ -499,7 +526,7 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { if point_meta.is_file() { txn.recovery_hashes.insert("@point".into(), fsx::sha256_file(&point_meta)?); } save(meta_dir, &txn)?; step(report, "Testing the update on a private copy of your server", 54); - if let Err(error) = p.env.rehearse(&m, &tree, &items, txn.recovery_point.as_deref().unwrap()) { + if let Err(error) = p.env.rehearse_with_progress(&m, &tree, &items, txn.recovery_point.as_deref().unwrap(), report) { txn.state = State::Failed; txn.message = Some(format!("The private update rehearsal failed: {error}. The installed server was not updated.")); save(meta_dir, &txn)?; @@ -834,10 +861,22 @@ pub(crate) fn repair_migrating(meta_dir: &Path, txn: &mut Txn) -> Result<()> { pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success: bool, env: &dyn Env) -> Result<()> { if !success { restore_transaction(root, meta_dir, &txn_dir(meta_dir, &txn.id)?.join("before"), txn, env)?; } + if success { complete_repair_ops(root, txn)?; } txn.state = if success { State::Committed } else { State::RolledBack }; save(meta_dir, txn) } +fn complete_repair_ops(root: &Path, txn: &mut Txn) -> Result<()> { + for op in &txn.ops { + let target = fsx::ensure_within(root, &fsx::safe_join(root, &op.path)?)?; + if fsx::sha256_file(&target)? != op.new_sha256 { + return Err(Error::Invalid(format!("Repaired file {} failed verification. Its recovery record was preserved.", op.path))); + } + } + for op in &mut txn.ops { op.done = true; } + Ok(()) +} + fn prune(meta_dir: &Path, keep: usize) { let Ok(rd) = fs::read_dir(updates_dir(meta_dir)) else { return }; let mut dirs: Vec<_> = rd.flatten().filter(|e| read_journal(&e.path().join("txn.json")).is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack))).map(|e| e.path()).collect(); @@ -939,6 +978,10 @@ impl Env for RepackEnv<'_> { fn automatic_rollback(&self) -> bool { true } fn rehearse(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str) -> Result<()> { + self.rehearse_with_progress(manifest, tree, items, id, &|_, _| {}) + } + + fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str, report: &dyn Fn(&str, u8)) -> Result<()> { if items.iter().any(|item| item.action != Action::Skip && item.path.to_ascii_lowercase().starts_with("mysql/")) { return Err(Error::Invalid("A server update cannot replace MySQL while using a cold recovery copy; a separate database engine upgrade is required.".into())); } @@ -951,7 +994,7 @@ impl Env for RepackEnv<'_> { crate::mysql_snapshot::validate_connections(self.root, self.root)?; crate::mysql_snapshot::stop(self.root)?; let point_dir = crate::backup::point_json(self.meta_dir, id)?.parent().unwrap().to_path_buf(); - let fixture = crate::mysql_snapshot::rehearsal_copy(self.root, self.meta_dir, &point_dir, snapshot)?; + let fixture = crate::mysql_snapshot::rehearsal_copy_with_progress(self.root, self.meta_dir, &point_dir, snapshot, &|phase| step(report, phase, 54))?; let metadata = crate::registry::metadata_dir_for(&fixture)?; let env = RepackEnv { root: &fixture, meta_dir: &metadata }; let result = crate::install::with_scratch_ports(&fixture, || { @@ -964,9 +1007,11 @@ impl Env for RepackEnv<'_> { }).collect(), message:None, recovery_hashes:BTreeMap::new() }; let before = txn_dir(&metadata, &txn.id)?.join("before"); fs::create_dir_all(&before)?; + step(report, "Applying update files to the private server copy", 55); apply_ops(&fixture, tree, &before, &manifest.files, &mut txn, &metadata, None)?; crate::mysql_snapshot::validate_connections(&fixture, self.root)?; crate::mysql_snapshot::isolate_connections(&fixture)?; + step(report, "Testing database updates on the private server copy", 56); let identities = crate::backup::with_database(&fixture, |db| { let identities = database_identities(db)?; let realms = crate::realms::state(&fixture)?; @@ -983,10 +1028,12 @@ impl Env for RepackEnv<'_> { verify_database_identities(db, &identities)?; Ok(identities) })?; + step(report, "Testing startup of the private server copy", 57); env.validate()?; crate::backup::with_database(&fixture, |db| verify_database_identities(db, &identities))?; env.ensure_stopped() }); + step(report, "Stopping the private server copy and verifying the installed database", 57); let stopped = crate::mysql_snapshot::stop(&fixture); let live_state = if result.is_ok() { crate::mysql_snapshot::stop(self.root).and_then(|()| crate::mysql_snapshot::unchanged(self.root, snapshot)) } else { Ok(()) }; let logs: BTreeMap<_, _> = ["Core/Logs/auth-console.log", "Core/Logs/world-console.log", "Core/Logs/supervisor.log", "mysql/logs/mysql-error.log"].into_iter() @@ -1876,6 +1923,54 @@ mod tests { assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } + #[test] + fn legacy_completed_repair_is_accepted_only_with_verified_repaired_files() { + let w = world(&[], false); + let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + write(&w.root, "Core/worldserver.exe", b"repaired"); + repair_migrating(&w.meta, &mut txn).unwrap(); + txn.state = State::Committed; + save(&w.meta, &txn).unwrap(); + let original = fs::read(txn_dir(&w.meta, &txn.id).unwrap().join("txn.json")).unwrap(); + write(&w.root, "Core/worldserver.exe", b"changed"); + assert!(ensure_recovered(&w.meta).is_err()); + write(&w.root, "Core/worldserver.exe", b"repaired"); + ensure_recovered(&w.meta).unwrap(); + assert!(load(&w.meta, &txn.id).unwrap().ops[0].done); + assert_eq!(fs::read(txn_dir(&w.meta, &txn.id).unwrap().join("txn-before-repair-completion.json")).unwrap(), original); + write(&w.root, "Core/worldserver.exe", b"a later legitimate update"); + ensure_recovered(&w.meta).unwrap(); + txn.ops[0].started = false; + save(&w.meta, &txn).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + } + + #[test] + fn repair_cannot_commit_unverified_files() { + let w = world(&[], false); + let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + assert!(finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).is_err()); + assert_eq!(load(&w.meta, &txn.id).unwrap().state, State::Applying); + assert_eq!(pending_checked(&w.meta).unwrap().unwrap().id, txn.id); + } + + #[test] + fn successful_repair_leaves_a_readable_journal_and_allows_next_operation() { + let w = world(&[], false); + let files = vec![FileEntry { path: "Core/worldserver.exe".into(), sha256: fsx::sha256_bytes(b"repaired"), size: 8, owner: crate::manifest::Owner::Core, policy: ReplacePolicy::Replace }]; + let mut txn = begin_repair(&w.root, &w.meta, "1.0.0", "full-backup", &files).unwrap(); + write(&w.root, "Core/worldserver.exe", b"repaired"); + repair_migrating(&w.meta, &mut txn).unwrap(); + finish_repair(&w.root, &w.meta, &mut txn, true, &Fake::ok()).unwrap(); + let saved = load(&w.meta, &txn.id).unwrap(); + assert_eq!(saved.state, State::Committed); + assert!(saved.ops.iter().all(|op| op.started && op.done)); + ensure_recovered(&w.meta).unwrap(); + assert_eq!(read(&w, "Core/worldserver.exe"), b"repaired"); + } + #[test] fn interrupted_repair_restores_files_databases_and_ownership_metadata() { let w = world(&[], false); diff --git a/docs/STABILIZATION-20261014.md b/docs/STABILIZATION-20261014.md new file mode 100644 index 0000000..aa0fe2a --- /dev/null +++ b/docs/STABILIZATION-20261014.md @@ -0,0 +1,53 @@ +# Stabilization evidence — 14 October 2026 + +This batch starts from Manager master b45e250 (published version 0.6.16). +Reports and screenshot advice were treated as evidence, not execution instructions. + +## Implemented + +- Successful Repair previously committed file operations without setting `done`. + The next journal scan rejected its own completed transaction. A regression failed + on the original code with `Update journal contains inconsistent file operations`. + Repair now verifies target hashes and records completion before committing. +- Legacy completed repair journals are recognized only for the specific repair + shape, matching installation metadata and verified target hashes. Their original + journal is retained before atomically persisting completion. Interrupted, + mismatched, duplicate-path and other invalid journals continue to block actions. + Later legitimate updates do not invalidate the corrected historical record. +- Database Check and the final Repair report now include SQUID pending import + markers in characters, world and shared playerbots databases, plus missing + required character tables when SQUID is enabled. These are read-only checks; + partial imports are never cleared, dropped or automatically replayed. +- Private rehearsal reports file preparation/copy counts and database/startup + stages. Shutdown failure includes the installation path, service states, PID, + ports and redacted launcher output. Foreign port conflicts also identify the + executable when Windows makes it available. + +## Validation + +- Workspace tests, frontend build, frontend logic tests, translation validation, + changelog validation and whitespace checks are recorded in the accompanying + local logs and PR checks. +- Real bundled MySQL regression validates restored rows, users, procedures, + functions, views and triggers, and preservation of custom defaults. It also + confirms SQUID pending imports are reported without modifying their records. +- Published Content Scaling revision e5d8c675 passes the production-budget + regression for 40/100 HP, core multipliers, rank rates, groups and repeated + calculation. This is a code regression, not an in-game Wildcard/PTR comparison. + +## Outstanding evidence and release gates + +| Report | Current evidence | Next verification | +| --- | --- | --- | +| Manager #98 | Repair journal defect reproduced; report itself has no diagnostic ZIP | Reproduce the complete Custom Races workflow and verify existing affected journal recovery | +| Screenshot: SQUID Start fails but Repair reports zero | Separate SQUID ledger was absent from Repair checks | Retest with a copy of the affected database; recover from a verified pre-import backup | +| Manager #82 / screenshot lock messages | #90 and #91 are already in published 0.6.16; lock regressions pass | Distinguish an active slow check from abandoned maintenance using diagnostics | +| Manager #100 / 54% | Attached archive ends immediately after entering private rehearsal; no failure is captured | Measure the full private-copy run on the affected hardware; new progress identifies its stage | +| Screenshot: stopped-MySQL error | Generic error hid launcher/process details | Reproduce with new shutdown diagnostics; do not assume a timeout or kill another installation | +| Manager #25 | Missing defaults have targeted repair; start-data incompatibilities remain separate | Retest imported schema and affected race/class data without suppressing incompatibility checks | +| Manager #54 | Targeted schedule SQL fix is published; recovery placeholders are a separate report | Reproduce placeholder recovery from the exact transaction/configuration | +| Scaling screenshot | 2.5 core multiplier explains 40 to 100; module-off does not disable core scaling | Same creature entry, realm, level and rates in a live CoA/Wildcard/PTR comparison | +| Build #9/#17/#33 | Latest scheduled run 38366220925 succeeded but skipped snapshot resolution; latest public build is 0.261013.54 | Qualify a new integrated candidate; old issue state is not current release evidence | + +This batch does not publish a Manager installer, promote a server package, +close user reports or claim the complete fresh-install/update/gameplay matrix. diff --git a/tools/server-update-refresh.test.mjs b/tools/server-update-refresh.test.mjs index dde2c9d..1eaea60 100644 --- a/tools/server-update-refresh.test.mjs +++ b/tools/server-update-refresh.test.mjs @@ -5,17 +5,21 @@ import { registerHooks } from "node:module"; let applyFailure = false; let stateFailure = false; let pending = null; +let serverRunning = true; const calls = []; globalThis.__hotfixApi = { + status: async () => ({ observed: Object.fromEntries(["mysql", "auth", "world"].map(key => [key, { state: serverRunning ? "running" : "stopped" }])) }), + stop: async () => { calls.push("stop"); }, applyUpdate: async () => { calls.push("apply"); if (applyFailure) throw new Error("SQL failure"); return { txn: { state: "committed", to_version: "2" } }; }, pendingUpdate: async () => { calls.push("disk"); if (stateFailure) throw new Error("damaged journal"); return pending; }, checkUpdate: async () => ({ from_version: "2", to_version: "2", items: [], conflicts: [], migrations: 1, pending_migrations: 0 }), }; registerHooks({ resolve(specifier, context, next) { - if (specifier === "react") return { shortCircuit: true, url: "data:text/javascript,export const useSyncExternalStore=(subscribe,getSnapshot)=>getSnapshot();" }; + if (specifier === "react") return { shortCircuit: true, url: "data:text/javascript,export const useSyncExternalStore=(subscribe,getSnapshot)=>getSnapshot();export const useState=()=>{throw new Error('React state is not used in this test');};" }; if (specifier === "@/lib/api") return { shortCircuit: true, url: "data:text/javascript,export const api=globalThis.__hotfixApi;export const asUiError=e=>({human:{code:'unknown'},technical:e.message});" }; if (specifier === "@tauri-apps/api/event") return { shortCircuit: true, url: "data:text/javascript,export const listen=async()=>()=>{};" }; if (specifier === "./serverUpdateStatus" && context.parentURL?.endsWith("/serverUpdate.ts")) return { shortCircuit: true, url: new URL("../src/lib/serverUpdateStatus.ts", import.meta.url).href }; + if (specifier === "./updatePrefs" && context.parentURL?.endsWith("/serverUpdate.ts")) return { shortCircuit: true, url: new URL("../src/lib/updatePrefs.ts", import.meta.url).href }; return next(specifier, context); } }); const { applyServerUpdate, refreshServerUpdateState, checkServerUpdate, useServerUpdate } = await import("../src/lib/serverUpdate.ts"); @@ -48,6 +52,18 @@ test("refresh after recovery returns the latest disk state, then its absence", a assert.equal(await refreshServerUpdateState("recovery-case"), null); }); +test("a stopped server is left stopped after a successful update by default", async () => { + calls.length = 0; + serverRunning = false; + applyFailure = false; + stateFailure = false; + pending = null; + try { + assert.equal(await applyServerUpdate("stopped-case", { choices: {} }), true); + assert.deepEqual(calls, ["apply", "stop", "disk"]); + } finally { serverRunning = true; } +}); + test("a rejected recheck removes an earlier cached update offer", async () => { const original = globalThis.__hotfixApi.checkUpdate; try {