diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1809a02..1d81b74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,16 +12,32 @@ permissions: jobs: installer: - runs-on: windows-latest - timeout-minutes: 60 + runs-on: [self-hosted, Windows, X64, coa-update-acceptance] + timeout-minutes: 1440 steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 26.10.0 - run: npm ci - name: Tests - run: cargo test --workspace -- --test-threads=1 + run: cargo test --workspace --locked -- --test-threads=1 + - name: Windows server upgrade and recovery gate + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + cargo build -p coa-release --locked + if ($LASTEXITCODE) { throw 'Verification tools failed to build.' } + gh release download base --repo Corfirean/coa-server-build --dir _acceptance-base + if ($LASTEXITCODE) { throw 'Signed base download failed.' } + & target/debug/coa-release.exe resolve --source https://github.com/Corfirean/coa-server-build/releases/download/edge --out _candidate-manifest.json + if ($LASTEXITCODE) { throw 'Cannot pin the candidate server release.' } + $version = (Get-Content _candidate-manifest.json -Raw | ConvertFrom-Json).version + gh release download "server-$version" --repo Corfirean/coa-server-build --dir _acceptance-candidate + if ($LASTEXITCODE) { throw 'Candidate download failed.' } + & tools/update-chaos-matrix.ps1 -BasePackage _acceptance-base -CandidatePackage _acceptance-candidate + if ($LASTEXITCODE) { throw 'Update/recovery acceptance failed; manager will not be published.' } - name: Build, sign and publish uses: tauri-apps/tauri-action@v0 env: @@ -32,7 +48,29 @@ jobs: tagName: ${{ github.ref_name }} releaseName: CoA Server Manager ${{ github.ref_name }} releaseBody: See the assets to download the installer. Existing installations update themselves. - releaseDraft: false + releaseDraft: true prerelease: false args: --bundles nsis includeUpdaterJson: true + - name: Verify the exact packaged installer + shell: pwsh + run: | + $installer = Get-ChildItem target/release/bundle/nsis/*-setup.exe | Select-Object -First 1 + if (-not $installer) { throw 'Built installer is missing.' } + $destination = Join-Path $env:RUNNER_TEMP ('coa-manager-installer-' + [guid]::NewGuid().ToString('N')) + $process = Start-Process -FilePath $installer.FullName -ArgumentList @('/S', "/D=$destination") -WindowStyle Hidden -Wait -PassThru + if ($process.ExitCode) { throw 'Silent installer failed.' } + $installed = Join-Path $destination 'coa-server-manager.exe' + if (-not (Test-Path $installed)) { throw 'Installed manager binary is missing.' } + if ((Get-FileHash $installed).Hash -ne (Get-FileHash target/release/coa-server-manager.exe).Hash) { throw 'Installer contains a different manager binary.' } + $report = @{ installerSha256 = (Get-FileHash $installer.FullName).Hash.ToLower(); managerSha256 = (Get-FileHash $installed).Hash.ToLower(); source = '${{ github.sha }}'; serverManifestSha256 = (Get-FileHash _acceptance-candidate/manifest.json).Hash.ToLower(); result = 'passed' } + $report | ConvertTo-Json | Set-Content installer-acceptance.json -Encoding utf8NoBOM + - name: Publish validated draft + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload '${{ github.ref_name }}' installer-acceptance.json --clobber + if ($LASTEXITCODE) { throw 'Acceptance report upload failed.' } + gh release edit '${{ github.ref_name }}' --draft=false + if ($LASTEXITCODE) { throw 'Validated draft publication failed.' } diff --git a/Cargo.lock b/Cargo.lock index 97bff9f..1c38530 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "coa-core" -version = "0.6.4" +version = "0.6.5" dependencies = [ "base64 0.22.1", "chrono", @@ -519,16 +519,19 @@ dependencies = [ [[package]] name = "coa-release" -version = "0.6.4" +version = "0.6.5" dependencies = [ + "base64 0.22.1", "chrono", "coa-core", + "ed25519-dalek", "serde_json", + "tempfile", ] [[package]] name = "coa-server-manager" -version = "0.6.4" +version = "0.6.5" dependencies = [ "coa-core", "serde", diff --git a/Cargo.toml b/Cargo.toml index 2f57649..73bccfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["crates/coa-core", "crates/coa-release", "src-tauri"] [workspace.package] -version = "0.6.4" +version = "0.6.5" edition = "2021" license = "AGPL-3.0-only" diff --git a/changelog.d/20261005-safe-server-updates.md b/changelog.d/20261005-safe-server-updates.md new file mode 100644 index 0000000..4c2e023 --- /dev/null +++ b/changelog.d/20261005-safe-server-updates.md @@ -0,0 +1,7 @@ +--- +area: manager +type: changed +audience: admins +title: Server updates are tested on a private copy and automatically restored after failure +--- +Updates keep a fixed verified package, preserve migration history, and recover interrupted changes before starting the server. Recovery problems show a specific reason and keep the available backups. diff --git a/crates/coa-core/Cargo.toml b/crates/coa-core/Cargo.toml index 1bea4ee..d4471dd 100644 --- a/crates/coa-core/Cargo.toml +++ b/crates/coa-core/Cargo.toml @@ -31,7 +31,7 @@ rand_core = { version = "0.6", features = ["getrandom"] } ed25519-dalek = { version = "2", features = ["rand_core"] } [target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.60", features = ["Win32_Foundation", "Win32_System_Threading", "Win32_System_SystemInformation", "Win32_System_Diagnostics_ToolHelp", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock"] } +windows-sys = { version = "0.60", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Threading", "Win32_System_SystemInformation", "Win32_System_Diagnostics_ToolHelp", "Win32_NetworkManagement_IpHelper", "Win32_Networking_WinSock"] } [features] testkit = [] diff --git a/crates/coa-core/examples/migrate_e2e.rs b/crates/coa-core/examples/migrate_e2e.rs index 290bd3e..91b1adc 100644 --- a/crates/coa-core/examples/migrate_e2e.rs +++ b/crates/coa-core/examples/migrate_e2e.rs @@ -29,7 +29,7 @@ fn main() { let dst = staging.join(db).join(format!("{id}.sql")); std::fs::create_dir_all(dst.parent().unwrap()).unwrap(); std::fs::write(&dst, &bytes).unwrap(); - list.push(Migration { id, db: db.into(), sha256: fsx::sha256_bytes(&bytes), destructive: false }); + list.push(Migration { compatible_sha256: vec![], id, db: db.into(), sha256: fsx::sha256_bytes(&bytes), destructive: false }); }; for (kind, dirs) in [("auth", ["db_auth", "pending_db_auth"]), ("characters", ["db_characters", "pending_db_characters"]), ("world", ["db_world", "pending_db_world"])] { for d in dirs { diff --git a/crates/coa-core/examples/realm_e2e.rs b/crates/coa-core/examples/realm_e2e.rs index af08337..b5fab7c 100644 --- a/crates/coa-core/examples/realm_e2e.rs +++ b/crates/coa-core/examples/realm_e2e.rs @@ -58,7 +58,7 @@ fn main() -> Result<()> { let path = staged.join(format!("{kind}/manager_realm_fixture.sql")); std::fs::create_dir_all(path.parent().unwrap())?; std::fs::write(&path, sql)?; - manifest.migrations.push(Migration { id: "manager_realm_fixture".into(), db: kind.into(), sha256: coa_core::fsx::sha256_file(&path)?, destructive: false }); + manifest.migrations.push(Migration { compatible_sha256: vec![], id: "manager_realm_fixture".into(), db: kind.into(), sha256: coa_core::fsx::sha256_file(&path)?, destructive: false }); } let result = env.migrate(&manifest, &staged)?; assert!(result.failed.is_none()); diff --git a/crates/coa-core/examples/repair_e2e.rs b/crates/coa-core/examples/repair_e2e.rs index 300a771..06499e2 100644 --- a/crates/coa-core/examples/repair_e2e.rs +++ b/crates/coa-core/examples/repair_e2e.rs @@ -20,7 +20,7 @@ fn main() -> Result<()> { fsx::atomic_write(&update_tree.join(format!("_migrations/world/{id}.sql")), sql.as_bytes())?; let options = |kind, version: &str, migrations| BuildOptions { kind, version: version.into(), core_commit: None, built_at: "fixture".into(), part_size: 1 << 20, bots_commit: None, migrations }; let base = package::build(&base_tree, &base_pkg, &options(Kind::Base, "1.0.0", vec![]), &|_| {})?; - let update = package::build(&update_tree, &update_pkg, &options(Kind::Update, "1.0.1", vec![Migration { id: id.into(), db: "world".into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }]), &|_| {})?; + let update = package::build(&update_tree, &update_pkg, &options(Kind::Update, "1.0.1", vec![Migration { compatible_sha256: vec![], id: id.into(), db: "world".into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }]), &|_| {})?; // A test-only trust key, unrelated to production signing material. let key = SigningKey::from_bytes(&[23; 32]); let public = STANDARD.encode(key.verifying_key().as_bytes()); diff --git a/crates/coa-core/examples/update_e2e.rs b/crates/coa-core/examples/update_e2e.rs index 547284e..a398d99 100644 --- a/crates/coa-core/examples/update_e2e.rs +++ b/crates/coa-core/examples/update_e2e.rs @@ -33,7 +33,7 @@ fn main() { let sql = "CREATE TABLE IF NOT EXISTS coa_manager_update_test (id INT PRIMARY KEY); INSERT IGNORE INTO coa_manager_update_test VALUES (1);"; put(&src, "_migrations/world/2026_09_30_00_update_e2e.sql", sql); let mut m = build(&src, &pkg, &BuildOptions { kind: Kind::Update, version: "0.2.0".into(), core_commit: None, built_at: chrono::Utc::now().to_rfc3339(), part_size: 1 << 20, bots_commit: None, migrations: vec![] }, &|_| {}).unwrap(); - m.migrations.push(Migration { id: "2026_09_30_00_update_e2e".into(), db: "world".into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); + m.migrations.push(Migration { compatible_sha256: vec![], id: "2026_09_30_00_update_e2e".into(), db: "world".into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); std::fs::write(pkg.join("manifest.json"), serde_json::to_vec_pretty(&m).unwrap()).unwrap(); let key = format!("{}/.coa-manager/signing/manifest-signing.key", std::env::var("USERPROFILE").unwrap()); let seed: [u8; 32] = STANDARD.decode(std::fs::read_to_string(key).unwrap().trim()).unwrap().try_into().unwrap(); diff --git a/crates/coa-core/examples/update_recovery_e2e.rs b/crates/coa-core/examples/update_recovery_e2e.rs index 9cbf310..7e24705 100644 --- a/crates/coa-core/examples/update_recovery_e2e.rs +++ b/crates/coa-core/examples/update_recovery_e2e.rs @@ -37,7 +37,7 @@ fn exercise(root: &Path) -> Result<()> { ("world", "probe_world", "DROP TABLE manager_recovery_probe; THIS IS INTENTIONALLY INVALID SQL;"), ] { fsx::atomic_write(&source.join(format!("_migrations/{db}/{id}.sql")), sql.as_bytes())?; - migrations.push(Migration { db: db.into(), id: id.into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); + migrations.push(Migration { compatible_sha256: vec![], db: db.into(), id: id.into(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); } let pkg = root.join("fixture-package"); let manifest = package::build(&source, &pkg, &BuildOptions { diff --git a/crates/coa-core/examples/wildcard_repair_e2e.rs b/crates/coa-core/examples/wildcard_repair_e2e.rs index d9ebec1..7373c40 100644 --- a/crates/coa-core/examples/wildcard_repair_e2e.rs +++ b/crates/coa-core/examples/wildcard_repair_e2e.rs @@ -22,7 +22,7 @@ fn main() -> Result<()> { let path = old_sql.join(format!("{name}.sql")); let hash = fsx::sha256_file(&path)?; db.put(&LedgerRow { db: "characters".into(), id: name.into(), sha256: hash.clone(), status: Status::Applied, error: None, baseline: false })?; - migrations.push(Migration { db: "characters".into(), id: name.into(), sha256: hash, destructive: false }); + migrations.push(Migration { compatible_sha256: vec![], db: "characters".into(), id: name.into(), sha256: hash, destructive: false }); } for (_, tables) in schema_check::WILDCARD_TABLES.iter().filter(|(kind,_)| *kind == "characters") { for table in *tables { db.query(&format!("DROP TABLE IF EXISTS acore_characters.`{table}`;"))?; } @@ -33,7 +33,7 @@ fn main() -> Result<()> { let files = release_schema::collect(empty_core.path(), None, Some(&repairs))?; assert_eq!(files.len(), 1); let repair = &files[0]; - migrations.push(Migration { db: repair.db.clone(), id: repair.id.clone(), sha256: repair.sha256.clone(), destructive: false }); + migrations.push(Migration { compatible_sha256: vec![], db: repair.db.clone(), id: repair.id.clone(), sha256: repair.sha256.clone(), destructive: false }); let staged = tree.path().join("sql/characters"); fsx::atomic_write(&staged.join(format!("{}.sql", repair.id)), &std::fs::read(&repair.abs)?)?; let report = migrations::apply_pending(&db, &migrations, &tree.path().join("sql"), &|| Ok("disposable fixture".into()))?; diff --git a/crates/coa-core/src/backup.rs b/crates/coa-core/src/backup.rs index 5fcad0d..ec950b5 100644 --- a/crates/coa-core/src/backup.rs +++ b/crates/coa-core/src/backup.rs @@ -51,6 +51,8 @@ pub struct Component { pub sha256: Option, pub tables: Option, pub files: Option>, + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub file_sha256: std::collections::BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -91,7 +93,10 @@ pub fn list(meta: &Path) -> Vec { } pub fn get(meta: &Path, id: &str) -> Result { - fsx::read_json(&point_dir(meta, id)?.join("backup.json")).map_err(|_| Error::Invalid(format!("backup {id} was not found"))) + let path = fsx::ensure_within(meta, &point_dir(meta, id)?.join("backup.json"))?; + let point: RecoveryPoint = fsx::read_json(&path).map_err(|_| Error::Invalid(format!("backup {id} was not found or its metadata is damaged")))?; + if point.id != id || point.schema != 1 { return Err(Error::Invalid(format!("Backup {id} has inconsistent identity or an unsupported schema."))); } + Ok(point) } /// Files that make up "configuration": everything a user or the Manager may have changed, minus secrets. @@ -132,6 +137,7 @@ pub fn config_files(root: &Path) -> Vec { walk(root, &root.join("Settings/realm-profiles"), &mut out); if root.join("Settings/realm-profile.json").is_file() { out.push("Settings/realm-profile.json".into()); } out.sort(); + out.dedup(); out } @@ -139,13 +145,17 @@ fn copy_configs(root: &Path, dir: &Path) -> Result { let files = config_files(root); let target = dir.join("files"); let mut bytes = 0; + let mut file_sha256 = std::collections::BTreeMap::new(); for rel in &files { let src = fsx::safe_join(root, rel)?; let dst = fsx::safe_join(&target, rel)?; fs::create_dir_all(dst.parent().unwrap())?; - bytes += fs::copy(&src, &dst)?; + let contents = fs::read(&src)?; + fsx::atomic_write(&dst, &contents)?; + bytes += contents.len() as u64; + file_sha256.insert(rel.clone(), fsx::sha256_bytes(&contents)); } - Ok(Component { name: "configs".into(), path: "files".into(), bytes, sha256: None, tables: None, files: Some(files) }) + Ok(Component { name: "configs".into(), path: "files".into(), bytes, sha256: None, tables: None, files: Some(files), file_sha256 }) } /// Make sure the database is up for `f`. If we had to start it (and nothing else is running), stop it again. @@ -215,7 +225,7 @@ pub fn create(root: &Path, meta: &Path, kind: Kind, trigger: Trigger, label: Opt let tables = db.tables(schema)?.len(); let file = format!("{name}.sql.zst"); let (bytes, sha) = db.dump_to(schema, &partial.join(&file))?; - Ok(Component { name, path: file, bytes, sha256: Some(sha), tables: Some(tables), files: None }) + Ok(Component { name, path: file, bytes, sha256: Some(sha), tables: Some(tables), files: None, file_sha256: Default::default() }) })?; components.push(component); } @@ -245,7 +255,7 @@ pub fn create(root: &Path, meta: &Path, kind: Kind, trigger: Trigger, label: Opt components, }; fsx::atomic_write_json(&partial.join("backup.json"), &point)?; - fs::rename(&partial, &final_dir)?; + fsx::durable_replace(&partial, &final_dir)?; tracing::info!(%id, ?kind, ?trigger, "recovery point created"); if trigger == Trigger::Automatic { prune_automatic(meta, 10); @@ -255,6 +265,7 @@ pub fn create(root: &Path, meta: &Path, kind: Kind, trigger: Trigger, label: Opt /// Keep the newest `keep` automatic recovery points; manual and safety ones are never pruned. pub fn prune_automatic(meta: &Path, keep: usize) { + if crate::update::ensure_recovered(meta).is_err() { return; } let autos: Vec<_> = list(meta).into_iter().filter(|p| p.trigger == Trigger::Automatic).collect(); for p in autos.into_iter().skip(keep) { if let Ok(dir) = point_dir(meta, &p.id) { @@ -274,7 +285,7 @@ pub fn verify(meta: &Path, id: &str) -> Result { let dir = point_dir(meta, id)?; let mut problems = Vec::new(); for c in &point.components { - let path = dir.join(&c.path); + let path = fsx::ensure_within(&dir, &fsx::safe_join(&dir, &c.path)?)?; match (&c.sha256, &c.files) { (Some(sha), _) => match fsx::sha256_file(&path) { Ok(actual) if actual.eq_ignore_ascii_case(sha) => {} @@ -285,6 +296,8 @@ pub fn verify(meta: &Path, id: &str) -> Result { for f in files { if !fsx::safe_join(&path, f).map(|p| p.is_file()).unwrap_or(false) { problems.push(format!("configuration file {f} is missing")); + } else if let Some(expected) = c.file_sha256.get(f) { + if fsx::sha256_file(&fsx::safe_join(&path, f)?).ok().as_deref() != Some(expected.as_str()) { problems.push(format!("configuration file {f} is damaged")); } } } } @@ -296,6 +309,9 @@ pub fn verify(meta: &Path, id: &str) -> Result { /// Delete one recovery point (its own folder only). pub fn delete(meta: &Path, id: &str) -> Result<()> { + // A damaged journal cannot reliably identify which copies recovery needs. + // Preserve every available point until the installation is consistent. + crate::update::ensure_recovered(meta)?; if crate::update::unfinished(meta).is_some_and(|t| t.recovery_point.as_deref() == Some(id)) { return Err(Error::Invalid("This recovery point is required by an unfinished update and cannot be deleted.".into())); } @@ -324,6 +340,20 @@ pub fn restore_configs(root: &Path, meta: &Path, id: &str) -> Result Result<()> { + let point = get(meta, id)?; + if !verify(meta, id)?.ok { return Err(Error::Invalid("The update recovery point is damaged.".into())); } + let comp = point.components.iter().find(|c| c.name == "configs").ok_or_else(|| Error::Invalid("The update recovery point has no configuration.".into()))?; + if comp.files.as_deref().unwrap_or_default().iter().any(|file| !comp.file_sha256.contains_key(file)) { + return Err(Error::Invalid("This legacy recovery point has no configuration checksums. Automatic recovery cannot verify the original settings; copies were preserved for inspection.".into())); + } + for rel in comp.files.as_deref().unwrap_or_default() { + fsx::atomic_write(&fsx::ensure_within(root, &fsx::safe_join(root, rel)?)?, &fs::read(fsx::safe_join(&point_dir(meta, id)?.join(&comp.path), rel)?)?)?; + } + Ok(()) +} + #[derive(Debug, Serialize)] pub struct DbRestore { /// The database the restore replaced, kept intact under this name. @@ -335,6 +365,14 @@ pub struct DbRestore { /// Restore one database from a recovery point without dropping anything: /// import into a staging schema, sanity-check it, then swap tables atomically and keep the old schema. pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Result { + restore_database_inner(root, meta, id, name, true) +} + +pub(crate) fn restore_update_database(root: &Path, meta: &Path, id: &str, name: &str) -> Result { + restore_database_inner(root, meta, id, name, false) +} + +fn restore_database_inner(root: &Path, meta: &Path, id: &str, name: &str, safety_copy: bool) -> Result { let point = get(meta, id)?; if point.realm != crate::realms::state(root)?.active { return Err(Error::Invalid("Select the realm this backup belongs to before restoring it.".into())); @@ -350,21 +388,48 @@ pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Resul let live = if name.contains('-') || name == "playerbots" { db::schema_of(name)? } else { point.realm.schema(name)? }; let expected_tables = comp.tables.unwrap_or(0); let dump = point_dir(meta, id)?.join(&comp.path); - let stamp = uuid::Uuid::new_v4().simple().to_string()[..16].to_string(); + let reconstructed_hash = point_dir(meta, id)?.join(format!("restore-{name}.sha256")); + let stamp = if safety_copy { uuid::Uuid::new_v4().simple().to_string()[..16].to_string() } + else { fsx::sha256_bytes(format!("update-recovery:{id}:{name}").as_bytes())[..16].to_string() }; // 1. Safety copy of the current state, so even a wrong restore is reversible. - let safety = create(root, meta, Kind::Database, Trigger::BeforeRestore, Some(format!("before restoring {name} from {id}")), &|_| {})?; + let safety = if safety_copy { Some(create(root, meta, Kind::Database, Trigger::BeforeRestore, Some(format!("before restoring {name} from {id}")), &|_| {})?.id) } else { None }; with_database(root, |db| { let db = db.clone().for_realm(crate::realms::Mode::Coa); let staging = format!("{live}_restore_{stamp}"); let old = format!("{live}_before_restore_{stamp}"); - if db.schema_exists(&staging)? || db.schema_exists(&old)? { + if safety_copy && (db.schema_exists(&staging)? || db.schema_exists(&old)?) { return Err(Error::Invalid("A previous restore left its work schemas behind; try again in a moment.".into())); } + if !safety_copy { + if db.schema_exists(&old)? { + // RENAME TABLE is atomic. An empty staging schema proves that the old tables + // were moved out and the complete restored set was moved into live together. + if reconstructed_hash.is_file() && db.schema_exists(&staging)? && db.tables(&staging)?.is_empty() && db.tables(live)?.len() == expected_tables { + db.query(&format!("DROP DATABASE `{old}`; DROP DATABASE `{staging}`;"))?; + return Ok(DbRestore { previous_schema: old, safety_backup: String::new(), tables_restored: expected_tables }); + } + // A server crash can interrupt a rename involving MyISAM tables. + // The original dump was verified above and is authoritative: discard only + // this recovery's deterministic work schemas and import a complete set again. + db.query(&format!("DROP DATABASE `{old}`;"))?; + } + if db.schema_exists(&staging)? { db.query(&format!("DROP DATABASE `{staging}`;"))?; } + } // 2. Import into staging. db.query(&format!("CREATE DATABASE `{staging}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"))?; db.import_from(&staging, &dump)?; + if !safety_copy { + // SHOW CREATE TABLE can make implicit character sets explicit on import. + // Compare live against the independently reconstructed snapshot, keeping all + // schema and row bytes rather than accepting a table-count approximation. + let check = point_dir(meta, id)?.join(format!("restore-{name}.sql.zst")); + let (_, hash) = db.dump_to(&staging, &check)?; + fsx::atomic_write(&reconstructed_hash, hash.as_bytes())?; + fs::remove_file(check)?; + } + crate::update_fault::checkpoint(root, &format!("restore-database:{name}:imported")); // 3. Sanity checks. let staged = db.tables(&staging)?; if comp.tables.is_some_and(|expected| staged.len() != expected) || (comp.tables.is_none() && staged.is_empty()) { @@ -384,7 +449,11 @@ pub fn restore_database(root: &Path, meta: &Path, id: &str, name: &str) -> Resul renames.push(format!("`{staging}`.`{t}` TO `{live}`.`{t}`")); } if !renames.is_empty() { db.query(&format!("RENAME TABLE {};", renames.join(", ")))?; } - Ok(DbRestore { previous_schema: old, safety_backup: safety.id.clone(), tables_restored: staged.len() }) + crate::update_fault::checkpoint(root, &format!("restore-database:{name}:swapped")); + // For emergency recovery, completed temporary schemas are disposable. A crash before + // cleanup leaves the original dump intact, and another recovery can repeat the swap. + if !safety_copy { db.query(&format!("DROP DATABASE `{old}`; DROP DATABASE `{staging}`;"))?; } + Ok(DbRestore { previous_schema: old, safety_backup: safety.clone().unwrap_or_default(), tables_restored: staged.len() }) }) } @@ -405,6 +474,18 @@ mod tests { (dir, root, meta) } + #[test] + fn corrupt_update_journal_preserves_available_backups() { + let (_d, root, meta) = setup(); + let point = create(&root, &meta, Kind::Config, Trigger::Automatic, None, &|_| {}).unwrap(); + let updates = meta.join("updates").join("interrupted"); + fs::create_dir_all(&updates).unwrap(); + fs::write(updates.join("txn.json"), b"{damaged").unwrap(); + assert!(delete(&meta, &point.id).is_err()); + prune_automatic(&meta, 0); + assert!(get(&meta, &point.id).is_ok()); + } + #[test] fn config_backup_excludes_secrets_and_clutter_and_restores_byte_exact() { let (_d, root, meta) = setup(); diff --git a/crates/coa-core/src/cleanbase.rs b/crates/coa-core/src/cleanbase.rs index 9519ff7..ced4b04 100644 --- a/crates/coa-core/src/cleanbase.rs +++ b/crates/coa-core/src/cleanbase.rs @@ -181,7 +181,7 @@ pub fn build(p: &Params, say: &dyn Fn(&str)) -> Result<()> { if let Some(b) = p.bots { sql.extend(collect_bots_sql(b)?); } - let migrations: Vec<_> = sql.iter().map(|f| crate::manifest::Migration { id: f.id.clone(), db: f.db.clone(), sha256: f.sha256.clone(), destructive: false }).collect(); + let migrations: Vec<_> = sql.iter().map(|f| crate::manifest::Migration { compatible_sha256: vec![], id: f.id.clone(), db: f.db.clone(), sha256: f.sha256.clone(), destructive: false }).collect(); let staging = std::env::temp_dir().join("coa-cleanbase-sql"); let _ = fs::remove_dir_all(&staging); for f in &sql { diff --git a/crates/coa-core/src/db.rs b/crates/coa-core/src/db.rs index 0024b7a..b9a7a9b 100644 --- a/crates/coa-core/src/db.rs +++ b/crates/coa-core/src/db.rs @@ -89,6 +89,9 @@ pub(crate) fn credentials(root: &Path) -> Result<(String, String)> { } impl Db { + pub(crate) fn native_root(&self) -> Option { + if self.container.is_some() { None } else { self.bin.parent().and_then(|p| p.parent()).map(Path::to_path_buf) } + } pub fn from_repack(root: &Path, account: Account) -> Result { let creds: Credentials = fsx::read_json(&root.join("Settings/database.json")) .map_err(|_| Error::Invalid("The database settings of this server could not be read.".into()))?; @@ -265,7 +268,7 @@ impl Db { let file = File::create(out)?; let mut enc = zstd::Encoder::new(BufWriter::new(file), 3)?; let copy = std::io::copy(&mut stdout, &mut enc); - let finish = enc.finish().and_then(|mut w| w.flush()); + let finish = enc.finish().and_then(|mut w| { w.flush()?; w.get_ref().sync_all() }); let status = child.wait()?; let stderr_bytes = err_thread.join().unwrap_or_default(); if !status.success() { @@ -280,6 +283,8 @@ impl Db { /// Run a plain SQL file against `schema`; the first error aborts and is returned. pub fn run_sql_file(&self, schema: &str, file: &Path) -> Result<()> { + let injected = if let Some(root) = self.native_root() { crate::update_fault::failing_sql(&root, file)? } else { None }; + let file = injected.as_ref().map(|f| f.path()).unwrap_or(file); let schema = self.realm_schema(schema); if !ident_ok(schema) { return Err(Error::Invalid("bad schema name".into())); diff --git a/crates/coa-core/src/diag.rs b/crates/coa-core/src/diag.rs index 431f097..4c0787c 100644 --- a/crates/coa-core/src/diag.rs +++ b/crates/coa-core/src/diag.rs @@ -249,6 +249,14 @@ pub fn export_package(root: &Path, meta_dir: &Path, manager_log: &Path, meta: &I if let Ok(b) = fs::read(meta_dir.join("logs/database-checks.json")) { add("database-checks.json", redact(&String::from_utf8_lossy(&b)).as_bytes())?; } + if let Ok(b) = fs::read(meta_dir.join("logs/update-result.json")) { + add("update-result.json", redact(&String::from_utf8_lossy(&b)).as_bytes())?; + } + if let Ok(Some(txn)) = crate::update::latest(meta_dir) { + if let Ok(b) = fs::read(meta_dir.join("updates").join(&txn.id).join("diagnostic.json")) { + add("update-diagnostic.json", redact(&String::from_utf8_lossy(&b)).as_bytes())?; + } + } zip.finish().map_err(|e| Error::Invalid(e.to_string()))?; Ok(n) } diff --git a/crates/coa-core/src/driver.rs b/crates/coa-core/src/driver.rs index 573f699..fc9119f 100644 --- a/crates/coa-core/src/driver.rs +++ b/crates/coa-core/src/driver.rs @@ -81,20 +81,41 @@ fn launcher(root: &Path) -> Result<(PathBuf, PathBuf)> { /// Run a launcher verb to completion (blocking; call from a worker thread). pub fn run(root: &Path, verb: Verb) -> Result { - run_inner(root, verb, false) + run_inner(root, verb, false, false) } pub(crate) fn validate_update(root: &Path) -> Result { - run_inner(root, Verb::StartAll, true) + run_inner(root, Verb::StartAll, true, true) } -fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result { +pub(crate) fn start_after_update(root: &Path) -> Result { + run_inner(root, Verb::StartAll, true, false) +} + +pub(crate) fn validate_secondary(root: &Path) -> Result { + run_inner(root, Verb::StartWorld, true, true) +} + +/// Validate without accepting game connections or submitting reports. Child supervisors inherit the flag. +pub(crate) fn offline_launcher(text: &str) -> Result { + let normalized = text.replace("\r\n", "\n"); + let text = normalized.as_str(); + if text.contains("# coa-offline-validation") { return Ok(text.into()); } + if !text.contains("def spawn(name, command, cwd, log):") || !text.contains("def relay_worker():") { + return Err(Error::Invalid("This launcher cannot isolate an update validation. Update was not applied.".into())); + } + let spawn = "def spawn(name, command, cwd, log):\n # coa-offline-validation\n if os.environ.get('COA_UPDATE_VALIDATION') == '1':\n for filename in ('worldserver.conf', 'authserver.conf'):\n path = ROOT / 'Core/configs' / filename\n if path.exists():\n text = path.read_text(encoding='utf-8-sig')\n for key, value in (('BindIP', '\"127.0.0.1\"'), ('Ra.IP', '\"127.0.0.1\"'), ('SOAP.Enabled', '0')):\n text = re.sub(r'(?m)^\\s*' + re.escape(key) + r'\\s*=.*$', '', text)\n text += '\\n' + key + ' = ' + value + '\\n'\n path.write_text(text, encoding='utf-8')\n"; + let relay = "def relay_worker():\n if os.environ.get('COA_UPDATE_VALIDATION') == '1':\n save_process('relay-ready', os.getpid())\n while not (STATE / 'stop-relay').exists():\n time.sleep(0.5)\n return\n"; + Ok(text.replacen("def spawn(name, command, cwd, log):\n", spawn, 1).replacen("def relay_worker():\n", relay, 1)) +} + +fn run_inner(root: &Path, verb: Verb, validating_update: bool, offline: bool) -> Result { let root = fsx::canonicalize_lenient(root)?; let _update_lock = if !validating_update && matches!(verb, Verb::StartAll | Verb::StartWorld) { Some(crate::update::operation_lock(&crate::registry::metadata_dir_for(&root)?)?) } else { None }; if !validating_update && matches!(verb, Verb::StartAll | Verb::StartWorld) { - crate::update::ensure_recovered(&crate::registry::metadata_dir_for(&root)?)?; + crate::update::recover_before_start(&root, &crate::registry::metadata_dir_for(&root)?)?; } if matches!(verb, Verb::StartAll | Verb::StartWorld) { crate::modules::ensure_bot_exclusivity(&root)?; } if crate::docker::is_docker(&root) { @@ -115,7 +136,8 @@ fn run_inner(root: &Path, verb: Verb, validating_update: bool) -> Result Result Result Result { pub(crate) fn launcher_matches(signed: &str, actual: &[u8]) -> bool { if signed.as_bytes() == actual { return true; } if patch_launcher_imports(signed).is_ok_and(|s| s.as_bytes() == actual) { return true; } + if patch_launcher_imports(signed).and_then(|s| offline_launcher(&s)).is_ok_and(|s| s.as_bytes() == actual) { return true; } crate::realms::patch_launcher(signed).is_ok_and(|s| s.as_bytes() == actual || patch_launcher_imports(&s).is_ok_and(|p| p.as_bytes() == actual)) } diff --git a/crates/coa-core/src/fsx.rs b/crates/coa-core/src/fsx.rs index 983fa0c..47925a1 100644 --- a/crates/coa-core/src/fsx.rs +++ b/crates/coa-core/src/fsx.rs @@ -123,7 +123,7 @@ pub fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> { let tmp = parent.join(format!( ".{}.{}.tmp", path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(), - std::process::id() + uuid::Uuid::new_v4().simple() )); let write = || -> std::io::Result<()> { let mut f = OpenOptions::new().write(true).create_new(true).open(&tmp)?; @@ -135,13 +135,41 @@ pub fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> { let _ = fs::remove_file(&tmp); return Err(e.into()); } - if let Err(e) = fs::rename(&tmp, path) { + if let Err(e) = durable_replace(&tmp, path) { let _ = fs::remove_file(&tmp); return Err(e.into()); } Ok(()) } +/// The temporary file must already be flushed, on the same volume as the destination. +pub(crate) fn durable_replace(from: &Path, to: &Path) -> std::io::Result<()> { + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; + fn wide(path: &Path) -> std::io::Result> { + let absolute = std::path::absolute(path)?; + let text = absolute.as_os_str().to_string_lossy().replace('/', "\\"); + let extended = if text.starts_with("\\\\?\\") { text } + else if let Some(unc) = text.strip_prefix("\\\\") { format!("\\\\?\\UNC\\{unc}") } + else { format!("\\\\?\\{text}") }; + Ok(std::ffi::OsStr::new(&extended).encode_wide().chain(Some(0)).collect()) + } + let src = wide(from)?; + let dst = wide(to)?; + if unsafe { MoveFileExW(src.as_ptr(), dst.as_ptr(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH) } == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } + #[cfg(not(windows))] + { + fs::rename(from, to)?; + File::open(to.parent().unwrap())?.sync_all() + } +} + pub fn atomic_write_json(path: &Path, value: &T) -> Result<()> { let mut bytes = serde_json::to_vec_pretty(value)?; bytes.push(b'\n'); @@ -199,6 +227,16 @@ pub fn require_space(path: &Path, needed: u64) -> Result<()> { mod tests { use super::*; + #[test] + fn durable_replacement_handles_long_transaction_backup_paths() { + let dir = tempfile::tempdir().unwrap(); + let parent = dir.path().join("a".repeat(90)).join("b".repeat(90)); + let path = parent.join("configuration-with-a-long-name.template"); + atomic_write(&path, b"old").unwrap(); + atomic_write(&path, b"new").unwrap(); + assert_eq!(fs::read(&path).unwrap(), b"new"); + } + #[test] fn safe_join_accepts_normal_and_rejects_traversal() { let root = Path::new("C:/srv"); diff --git a/crates/coa-core/src/install.rs b/crates/coa-core/src/install.rs index bea801d..abc5bd3 100644 --- a/crates/coa-core/src/install.rs +++ b/crates/coa-core/src/install.rs @@ -148,10 +148,6 @@ pub(crate) fn bootstrap_database(root: &Path) -> Result<()> { with_scratch_ports(root, || bootstrap_database_inner(root)) } -fn free_port() -> Result { - Ok(std::net::TcpListener::bind("127.0.0.1:0")?.local_addr()?.port()) -} - /// Run `f` with the staging copy listening on unused ports, so an installation never collides with a server (or any /// other program) already using the shipped ports. The shipped ports are put back afterwards. pub(crate) fn with_scratch_ports(root: &Path, f: impl FnOnce() -> Result) -> Result { @@ -159,10 +155,14 @@ pub(crate) fn with_scratch_ports(root: &Path, f: impl FnOnce() -> Result) let path = root.join("Settings/repack.json"); let original: serde_json::Value = fsx::read_json(&path)?; let mut scratch = original.clone(); + let mut listeners = Vec::new(); for k in KEYS { - scratch[k] = free_port()?.into(); + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + scratch[k] = listener.local_addr()?.port().into(); + listeners.push(listener); } fsx::atomic_write_json(&path, &scratch)?; + drop(listeners); let result = f(); // keep whatever else changed meanwhile (console credentials), restore only the ports let mut now: serde_json::Value = fsx::read_json(&path).unwrap_or(scratch); diff --git a/crates/coa-core/src/lib.rs b/crates/coa-core/src/lib.rs index 38502dc..65f2549 100644 --- a/crates/coa-core/src/lib.rs +++ b/crates/coa-core/src/lib.rs @@ -25,6 +25,7 @@ pub mod population; pub mod process; pub mod manifest; pub mod migrations; +pub mod migration_registry; pub mod multiworld; pub mod repair; pub mod schema_check; @@ -39,6 +40,8 @@ pub mod registry; pub mod release; pub mod release_schema; pub mod update; +mod update_trial; +mod update_fault; pub mod upnp; pub mod signing; pub mod srp6; diff --git a/crates/coa-core/src/manifest.rs b/crates/coa-core/src/manifest.rs index 9f5fac4..b87dbe9 100644 --- a/crates/coa-core/src/manifest.rs +++ b/crates/coa-core/src/manifest.rs @@ -48,6 +48,8 @@ pub struct FileEntry { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Migration { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub compatible_sha256: Vec, pub id: String, pub db: String, pub sha256: String, @@ -134,6 +136,11 @@ impl Manifest { if parse_version(&self.min_manager_version).is_none() { return bad(format!("bad minManagerVersion {:?}", self.min_manager_version)); } + for migration in &self.migrations { + if !is_sha256(&migration.sha256) || migration.compatible_sha256.iter().any(|h| !is_sha256(h) || h == &"0".repeat(64)) { + return bad(format!("{}: invalid migration checksum compatibility", migration.id)); + } + } for (label, rev) in [("core", Some(&self.core)), ("bots", self.bots.as_ref())] { if let Some(Revision { commit: Some(c) }) = rev { if !is_commit(c) { diff --git a/crates/coa-core/src/migrations.rs b/crates/coa-core/src/migrations.rs index d7d8d9b..a77c1ed 100644 --- a/crates/coa-core/src/migrations.rs +++ b/crates/coa-core/src/migrations.rs @@ -37,6 +37,7 @@ pub trait Store { fn load(&self) -> Result>; fn put(&self, row: &LedgerRow) -> Result<()>; fn run_file(&self, schema: &str, file: &Path) -> Result<()>; + fn checkpoint(&self, _point: &str) {} } fn hex_of(s: &str) -> String { @@ -48,6 +49,9 @@ fn ident_ok(s: &str) -> bool { } impl Store for Db { + fn checkpoint(&self, point: &str) { + if let Some(root) = self.native_root() { crate::update_fault::checkpoint(&root, point); } + } fn load(&self) -> Result> { self.query(&format!( "CREATE TABLE IF NOT EXISTS `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db` VARCHAR(32) NOT NULL, `id` VARCHAR(190) NOT NULL, `sha256` CHAR(64) NOT NULL, `status` VARCHAR(16) NOT NULL, `applied_at` DATETIME NOT NULL, `error` TEXT NULL, `baseline` TINYINT NOT NULL DEFAULT 0, PRIMARY KEY (`db`,`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" @@ -148,7 +152,7 @@ fn recorded<'a>(rows: &'a [LedgerRow], m: &Migration) -> Option<&'a LedgerRow> { } fn hash_changed(row: &LedgerRow, m: &Migration) -> bool { - row.status == Status::Applied && row.sha256 != "0".repeat(64) && !row.sha256.eq_ignore_ascii_case(&m.sha256) + row.status == Status::Applied && row.sha256 != "0".repeat(64) && !row.sha256.eq_ignore_ascii_case(&m.sha256) && !m.compatible_sha256.iter().any(|h| h.eq_ignore_ascii_case(&row.sha256)) } /// Record migrations as already present without running them (adopting an existing server whose database @@ -217,10 +221,13 @@ pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot for (m, schema, path) in files { tracing::info!(migration = %m.id, database = %m.db, "database migration starting"); store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Running, error: Some("Interrupted SQL must be recovered before replay.".into()), baseline: false })?; + store.checkpoint(&format!("before-sql:{}", m.id)); match store.run_file(schema, &path) { Ok(()) => { + store.checkpoint(&format!("after-sql:{}", m.id)); store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Applied, error: None, baseline: false })?; report.applied.push(m.id.clone()); + store.checkpoint(&format!("recorded-sql:{}", m.id)); tracing::info!(migration = %m.id, database = %m.db, "database migration applied"); } Err(e) => { @@ -273,7 +280,7 @@ mod tests { for (db, id, sql) in files { std::fs::create_dir_all(d.path().join(db)).unwrap(); std::fs::write(d.path().join(db).join(format!("{id}.sql")), sql).unwrap(); - list.push(Migration { id: id.to_string(), db: db.to_string(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); + list.push(Migration { compatible_sha256: vec![], id: id.to_string(), db: db.to_string(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); } (d, list) } @@ -304,6 +311,19 @@ mod tests { assert!(store.ran.borrow().is_empty()); } + #[test] + fn only_explicit_published_compatibility_skips_changed_hash_without_rewriting_history() { + let (d, mut list) = setup(&[("characters", "repair", "SELECT 1;\n")]); + let store = Mem::default(); + let previous = fsx::sha256_bytes(b"SELECT 1;\r\n"); + store.rows.borrow_mut().push(LedgerRow { db: "characters".into(), id: "repair".into(), sha256: previous.clone(), status: Status::Applied, error: None, baseline: false }); + assert!(apply_pending(&store, &list, d.path(), &no_snapshot).is_err()); + list[0].compatible_sha256.push(previous.clone()); + assert!(apply_pending(&store, &list, d.path(), &no_snapshot).unwrap().applied.is_empty()); + assert!(store.ran.borrow().is_empty()); + assert_eq!(store.rows.borrow()[0].sha256, previous); + } + #[test] fn applies_in_order_once_and_records_the_ledger() { let (d, list) = setup(&[("world", "2026_09_01_00_a", "CREATE TABLE t (id INT);"), ("world", "2026_09_02_00_b", "ALTER TABLE t ADD c INT;")]); diff --git a/crates/coa-core/src/multiworld.rs b/crates/coa-core/src/multiworld.rs index 15fcfca..ae3dd15 100644 --- a/crates/coa-core/src/multiworld.rs +++ b/crates/coa-core/src/multiworld.rs @@ -125,6 +125,14 @@ fn secondary_launcher(source: &str) -> Result { } pub fn start(main: &Path) -> Result<()> { + start_inner(main, false) +} + +pub(crate) fn start_validation(main: &Path) -> Result<()> { + start_inner(main, true) +} + +fn start_inner(main: &Path, offline: bool) -> Result<()> { if !realms::state(main)?.simultaneous { return Ok(()); } if is_running(main) { let second = root(main); @@ -132,7 +140,7 @@ pub fn start(main: &Path) -> Result<()> { return Err(Error::Invalid("The second realm is still starting or stopping.".into())); } let second = prepare(main)?; - let out = driver::run(&second, Verb::StartWorld)?; + let out = if offline { driver::validate_secondary(&second)? } else { driver::run(&second, Verb::StartWorld)? }; if !out.ok { return Err(Error::Invalid(format!("The second realm did not start: {}", out.output))); } Ok(()) } diff --git a/crates/coa-core/src/pkgsource.rs b/crates/coa-core/src/pkgsource.rs index 85fac7a..c4bfce4 100644 --- a/crates/coa-core/src/pkgsource.rs +++ b/crates/coa-core/src/pkgsource.rs @@ -4,6 +4,8 @@ use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; +use base64::{engine::general_purpose::STANDARD, Engine}; +use serde::{Deserialize, Serialize}; use crate::download::{self, Cancel, Job, Progress}; use crate::error::{Error, Result}; @@ -20,6 +22,77 @@ pub enum Source { Dir(PathBuf), } +#[derive(Debug, Serialize, Deserialize)] +pub struct Channel { + pub schema: u32, + pub version: String, + pub release: String, + pub manifest_sha256: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub acceptance_sha256: Option, +} + +#[derive(Serialize, Deserialize)] +pub struct SignedChannel { + pub payload: String, + pub signature: String, +} + +pub fn verify_channel(bytes: &[u8], trusted_key: &str) -> Result { + let signed: SignedChannel = serde_json::from_slice(bytes).map_err(|e| Error::InvalidManifest(e.to_string()))?; + let payload = STANDARD.decode(&signed.payload).map_err(|e| Error::InvalidManifest(e.to_string()))?; + signing::verify(&payload, &signed.signature, trusted_key)?; + let pointer: Channel = serde_json::from_slice(&payload).map_err(|e| Error::InvalidManifest(e.to_string()))?; + let required = format!("https://github.com/Corfirean/coa-server-build/releases/download/server-{}/", pointer.version); + if pointer.schema != 1 || crate::manifest::parse_version(&pointer.version).is_none() || pointer.release != required { + return Err(Error::InvalidManifest("Channel does not point to an immutable server release.".into())); + } + Ok(pointer) +} + +/// A channel is one atomically replaced signed envelope. Its target is immutable. +pub struct PinnedPackage { + pub source: Source, + pub manifest: Manifest, + pub bytes: Vec, + pub signature: Vec, +} + +pub fn fetch_package(source: &Source, trusted_key: &str) -> Result { + let channel = match source { + Source::Dir(d) if !d.join("channel.json").exists() => None, + _ => match fetch_small(source, "channel.json") { + Ok(bytes) => Some(bytes), + Err(Error::PackageNotPublished(_)) => None, + Err(e) => return Err(e), + }, + }; + let (mut resolved, expected) = if let Some(bytes) = channel { + let pointer = verify_channel(&bytes, trusted_key)?; + (Source::Url(pointer.release.clone()), Some(pointer)) + } else { (source.clone(), None) }; + let mut bytes = fetch_small(&resolved, "manifest.json")?; + let mut signature = fetch_small(&resolved, "manifest.json.sig")?; + signing::verify(&bytes, &String::from_utf8_lossy(&signature), trusted_key)?; + let manifest = Manifest::parse(&bytes)?; + // Legacy moving channels are pinned by matching their signed manifest to the immutable tag. + if matches!(&resolved, Source::Url(url) if matches!(url.trim_end_matches('/'), "https://github.com/Corfirean/coa-server-build/releases/download/stable" | "https://github.com/Corfirean/coa-server-build/releases/download/edge")) { + resolved = Source::Url(format!("https://github.com/Corfirean/coa-server-build/releases/download/server-{}/", manifest.version)); + let immutable = fetch_small(&resolved, "manifest.json")?; + let sig = fetch_small(&resolved, "manifest.json.sig")?; + signing::verify(&immutable, &String::from_utf8_lossy(&sig), trusted_key)?; + if immutable != bytes { return Err(Error::InvalidManifest("Moving channel differs from its immutable release.".into())); } + bytes = immutable; + signature = sig; + } + if let Some(pointer) = expected { + if pointer.version != manifest.version || pointer.manifest_sha256 != fsx::sha256_bytes(&bytes) { + return Err(Error::InvalidManifest("Immutable release does not match the signed channel.".into())); + } + } + Ok(PinnedPackage { source: resolved, manifest, bytes, signature }) +} + /// Read a small file (manifest, signature) from the source. pub fn fetch_small(source: &Source, name: &str) -> Result> { match source { @@ -44,11 +117,8 @@ pub fn fetch_small(source: &Source, name: &str) -> Result> { /// Download and verify the manifest. Returns the parsed manifest and its exact bytes. pub fn fetch_manifest(source: &Source, trusted_key: &str) -> Result<(Manifest, Vec)> { - let bytes = fetch_small(source, "manifest.json")?; - let sig = String::from_utf8_lossy(&fetch_small(source, "manifest.json.sig")?).into_owned(); - signing::verify(&bytes, &sig, trusted_key)?; - let manifest = Manifest::parse(&bytes)?; - Ok((manifest, bytes)) + let pinned = fetch_package(source, trusted_key)?; + Ok((pinned.manifest, pinned.bytes)) } /// Make every archive part available in a verified folder: a local source is used in place, a remote one is diff --git a/crates/coa-core/src/release.rs b/crates/coa-core/src/release.rs index 427b03c..06f517e 100644 --- a/crates/coa-core/src/release.rs +++ b/crates/coa-core/src/release.rs @@ -13,6 +13,7 @@ use crate::package::{self, BuildOptions}; #[derive(Debug, Clone)] pub struct SqlFile { + pub compatible_sha256: Vec, pub db: String, pub id: String, /// Path relative to the checkout it came from (used only for ordering and display). @@ -49,7 +50,7 @@ pub fn collect_core_sql(core: &Path) -> Result> { let mut out = Vec::new(); let mut push = |db: &str, id: String, abs: PathBuf, origin: String| -> Result<()> { let sha256 = fsx::sha256_file(&abs)?; - out.push(SqlFile { db: db.into(), id, origin, abs, sha256 }); + out.push(SqlFile { compatible_sha256: vec![], db: db.into(), id, origin, abs, sha256 }); Ok(()) }; for (kind, dirs) in [("auth", ["db_auth", "pending_db_auth"]), ("characters", ["db_characters", "pending_db_characters"]), ("world", ["db_world", "pending_db_world"])] { @@ -92,13 +93,13 @@ pub fn collect_bots_sql(bots: &Path) -> Result> { let mut out = Vec::new(); for f in sql_in(&bots.join("dist/sql")) { let stem = f.file_stem().unwrap().to_string_lossy().into_owned(); - out.push(SqlFile { db: "characters".into(), id: format!("mod_coa_playerbots__{stem}"), origin: format!("dist/sql/{stem}.sql"), sha256: fsx::sha256_file(&f)?, abs: f }); + out.push(SqlFile { compatible_sha256: vec![], db: "characters".into(), id: format!("mod_coa_playerbots__{stem}"), origin: format!("dist/sql/{stem}.sql"), sha256: fsx::sha256_file(&f)?, abs: f }); } Ok(out) } fn to_migrations(files: &[SqlFile]) -> Vec { - files.iter().map(|f| Migration { id: f.id.clone(), db: f.db.clone(), sha256: f.sha256.clone(), destructive: false }).collect() + files.iter().map(|f| Migration { compatible_sha256: f.compatible_sha256.clone(), id: f.id.clone(), db: f.db.clone(), sha256: f.sha256.clone(), destructive: false }).collect() } pub struct UpdateParams<'a> { @@ -226,7 +227,7 @@ mod tests { write(&tree, "Core/newlib.dll", "n"); write(&tree, crate::schema_check::CONTRACT, "same contract"); write(&sqldir, "world.sql", "SELECT 1;"); - let sql = vec![SqlFile { db: "world".into(), id: "m1".into(), origin: "x".into(), abs: sqldir.join("world.sql"), sha256: fsx::sha256_file(&sqldir.join("world.sql")).unwrap() }]; + let sql = vec![SqlFile { compatible_sha256: vec![], db: "world".into(), id: "m1".into(), origin: "x".into(), abs: sqldir.join("world.sql"), sha256: fsx::sha256_file(&sqldir.join("world.sql")).unwrap() }]; let m = pack_update(&UpdateParams { tree: &tree, base_manifest: &base, sql: &sql, out: &upd_out, version: "1.1.0".into(), core_commit: Some("a".repeat(40)), bots_commit: Some("b".repeat(40)), part_size: 1 << 20 }, &|_| {}).unwrap(); let paths: Vec<&str> = m.files.iter().map(|f| f.path.as_str()).collect(); assert!(paths.contains(&"Core/worldserver.exe") && paths.contains(&"Core/newlib.dll")); diff --git a/crates/coa-core/src/release_schema.rs b/crates/coa-core/src/release_schema.rs index 3b9c429..3e6881a 100644 --- a/crates/coa-core/src/release_schema.rs +++ b/crates/coa-core/src/release_schema.rs @@ -16,7 +16,7 @@ pub fn collect(core: &Path, bots: Option<&Path>, repairs: Option<&Path>) -> Resu paths.sort(); for path in paths.into_iter().filter(|p| p.extension().is_some_and(|e| e == "sql")) { let stem = path.file_stem().unwrap().to_string_lossy(); - files.push(SqlFile { db: kind.into(), id: format!("manager_repair__{stem}"), origin: format!("{kind}/{stem}.sql"), sha256: fsx::sha256_file(&path)?, abs: path }); + files.push(SqlFile { compatible_sha256: vec![], db: kind.into(), id: format!("manager_repair__{stem}"), origin: format!("{kind}/{stem}.sql"), sha256: fsx::sha256_file(&path)?, abs: path }); } } } @@ -47,7 +47,7 @@ pub fn capture_release(fixture: &Path, tree: &Path, sql: &[SqlFile]) -> Result<( return Err(Error::Invalid("Refusing to apply release SQL to a non-fixture server.".into())); } let staging = tempfile::tempdir()?; - let migrations: Vec<_> = sql.iter().map(|f| Migration { db: f.db.clone(), id: f.id.clone(), sha256: f.sha256.clone(), destructive: false }).collect(); + let migrations: Vec<_> = sql.iter().map(|f| Migration { compatible_sha256: f.compatible_sha256.clone(), db: f.db.clone(), id: f.id.clone(), sha256: f.sha256.clone(), destructive: false }).collect(); for file in sql { let path = fsx::safe_join(staging.path(), &format!("{}/{}.sql", file.db, file.id))?; fsx::atomic_write(&path, &std::fs::read(&file.abs)?)?; diff --git a/crates/coa-core/src/update.rs b/crates/coa-core/src/update.rs index 3ef3b09..6b88dae 100644 --- a/crates/coa-core/src/update.rs +++ b/crates/coa-core/src/update.rs @@ -27,7 +27,14 @@ const STAGED_PREFIX: &str = "_migrations/"; #[serde(rename_all = "kebab-case")] pub enum State { Prepared, + Stopping, + Snapshotting, + Trialing, Applying, + Migrating, + Validating, + Committing, + Restoring, /// Files replaced; waiting for migrations / health check. Applied, /// Files replaced but the new build did not become healthy; the owner decides (rollback is offered). @@ -66,6 +73,10 @@ pub enum Resolution { Replace, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Disposition { Updated, Rejected, Restored, RecoveryRequired } + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Txn { pub id: String, @@ -77,6 +88,24 @@ pub struct Txn { pub databases_started: bool, pub ops: Vec, pub message: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_stage: Option, + #[serde(default)] + pub manifest_sha256: Option, + #[serde(default)] + pub was_running: bool, + #[serde(default)] + pub server_stopped: bool, + #[serde(default)] + pub recovery_verified: bool, + #[serde(default)] + pub trial_completed: bool, + #[serde(default)] + pub protected_files: Vec, + #[serde(default)] + pub recovery_hashes: BTreeMap, + #[serde(default)] + pub outcome: Option, } fn legacy_databases_started() -> bool { true } @@ -84,6 +113,14 @@ fn legacy_databases_started() -> bool { true } /// Everything the transaction needs from the outside world; tests provide a fake. pub trait Env { fn ensure_stopped(&self) -> Result<()>; + fn was_running(&self) -> Result { Ok(false) } + fn stop(&self) -> Result<()> { self.ensure_stopped() } + fn trial(&self, _manifest: &Manifest, _tree: &Path, _before: &Path, _point: &str, _resolutions: &BTreeMap) -> Result<()> { Ok(()) } + fn check_capacity(&self, _manifest: &Manifest) -> Result<()> { Ok(()) } + fn reserve_recovery(&self, _before: &Path) -> Result<()> { Ok(()) } + fn restore_running_state(&self, _running: bool) -> Result<()> { Ok(()) } + fn verify_restored(&self) -> Result<()> { Ok(()) } + fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) } /// Create a recovery point for the databases and configuration; returns its id. fn snapshot(&self) -> Result; fn restore_snapshot(&self, id: &str) -> Result<()>; @@ -100,38 +137,165 @@ fn txn_dir(meta: &Path, id: &str) -> Result { if id.is_empty() || !id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) { return Err(Error::PathRejected(format!("bad update id {id:?}"))); } - Ok(updates_dir(meta).join(id)) + fsx::ensure_within(meta, &updates_dir(meta).join(id)) } fn save(meta: &Path, t: &Txn) -> Result<()> { - fsx::atomic_write_json(&txn_dir(meta, &t.id)?.join("txn.json"), t) + let mut saved = t.clone(); + saved.outcome = match saved.state { + State::Committed if saved.server_stopped => Some(Disposition::RecoveryRequired), + State::Committed => Some(Disposition::Updated), + State::RolledBack => Some(Disposition::Restored), + State::Restoring => Some(Disposition::RecoveryRequired), + State::Failed if saved.server_stopped || saved.databases_started || saved.ops.iter().any(|op| op.started) => Some(Disposition::RecoveryRequired), + State::Failed => Some(Disposition::Rejected), + _ => None, + }; + fsx::atomic_write_json(&txn_dir(meta, &t.id)?.join("txn.json"), &saved) } pub fn load(meta: &Path, id: &str) -> Result { - fsx::read_json(&txn_dir(meta, id)?.join("txn.json")) + read_journal(&txn_dir(meta, id)?.join("txn.json")) +} + +fn read_journal(path: &Path) -> Result { + let txn: Txn = fsx::read_json(path)?; + let directory_id = path.parent().and_then(Path::file_name).and_then(|s| s.to_str()); + if directory_id != Some(txn.id.as_str()) || txn.id.is_empty() + || !txn.id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) + { + return Err(Error::Invalid("Update journal identity is inconsistent. Recovery requires intervention.".into())); + } + let parent = path.parent().ok_or_else(|| Error::Invalid("Update journal has no directory.".into()))?; + if let Some(point) = &txn.recovery_point { + if point.is_empty() || !point.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) { + return Err(Error::Invalid("Update journal contains an invalid recovery point identity.".into())); + } + } + let valid_hash = |hash: &str| hash.len() == 64 && hash.bytes().all(|b| b.is_ascii_hexdigit()); + if txn.manifest_sha256.as_deref().is_some_and(|hash| !valid_hash(hash)) + || txn.recovery_hashes.values().any(|hash| !valid_hash(hash)) { + return Err(Error::Invalid("Update journal contains an invalid content hash.".into())); + } + let mut paths = std::collections::BTreeSet::new(); + for op in &txn.ops { + fsx::safe_join(parent, &op.path)?; + if !paths.insert(op.path.replace('\\', "/").to_ascii_lowercase()) || !valid_hash(&op.new_sha256) { + return Err(Error::Invalid("Update journal contains duplicate paths or an invalid file hash.".into())); + } + if op.done && !op.started && !matches!(op.action, Action::Skip | Action::Conflict) { + return Err(Error::Invalid("Update journal contains an inconsistent file operation.".into())); + } + } + for protected in &txn.protected_files { fsx::safe_join(parent, protected)?; } + if txn.state == State::RolledBack && txn.manifest_sha256.is_some() && !txn.recovery_verified { + return Err(Error::Invalid("Update journal claims an unverified recovery.".into())); + } + if txn.state == State::Prepared && (txn.server_stopped || txn.ops.iter().any(|op| op.started)) { + return Err(Error::Invalid("Update journal marks a changed installation as unstarted.".into())); + } + if txn.state == State::Committed && txn.ops.iter().any(|op| !op.done && !matches!(op.action, Action::Skip | Action::Conflict)) { + return Err(Error::Invalid("Update journal marks incomplete file operations as committed.".into())); + } + Ok(txn) +} + +pub fn latest(meta: &Path) -> Result> { + let mut transactions = read_journals(meta)?; + transactions.sort_by(|a, b| b.id.cmp(&a.id)); + Ok(transactions.into_iter().next()) +} + +#[derive(Serialize, Deserialize)] +struct CleanupMarker { id: String, state: State, server_stopped: bool } + +fn cleanup_marker(meta: &Path, id: &str) -> Result { + txn_dir(meta, id)?; + fsx::ensure_within(meta, &updates_dir(meta).join(format!("{id}.cleanup.json"))) +} + +fn read_journals(meta: &Path) -> Result> { + let entries = match fs::read_dir(updates_dir(meta)) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(e.into()), + }; + let mut transactions = Vec::new(); + for entry in entries { + let entry = entry?; + let kind = entry.file_type()?; + if kind.is_symlink() { return Err(Error::Invalid("Linked update journal storage is not supported.".into())); } + if !kind.is_dir() { continue; } + let directory = fsx::ensure_within(meta, &entry.path())?; + let path = directory.join("txn.json"); + if !path.is_file() { + let id = directory.file_name().and_then(|s| s.to_str()).ok_or_else(|| Error::Invalid("Invalid update directory identity.".into()))?; + let completed_cleanup = fsx::read_json::(&cleanup_marker(meta, id)?).is_ok_and(|m| + m.id == id && !m.server_stopped && matches!(m.state, State::Committed | State::RolledBack)); + if completed_cleanup { continue; } + return Err(Error::Invalid(format!("Update journal {} is missing. Recovery requires intervention; available copies were preserved.", path.display()))); + } + transactions.push(read_journal(&path).map_err(|_| Error::Invalid("An update journal cannot be read or is inconsistent. Recovery is required.".into()))?); + } + Ok(transactions) +} + +/// Called while holding the installation lock. Never replays interrupted SQL. +pub(crate) fn recover_before_start(root: &Path, meta: &Path) -> Result<()> { + if let Some(mut txn) = latest(meta)? { + // Commit has already happened: resume admission, never replay SQL or undo it. + if txn.state == State::Committed && txn.server_stopped { + release_reserve(root, &txn_dir(meta, &txn.id)?.join("before"))?; + if let Err(e) = (RepackEnv { root, meta_dir: meta }).restore_running_state(txn.was_running) { + txn.failure_stage = Some(State::Committed); + txn.message = Some(format!("The update is committed, but restoring the previous running state failed: {e}")); + save(meta, &txn)?; + write_diagnostic(root, meta, &txn)?; + return Err(e); + } + txn.server_stopped = false; + txn.message = Some("Committed update startup was resumed.".into()); + save(meta, &txn)?; + write_diagnostic(root, meta, &txn)?; + } + } + if let Some(mut txn) = unfinished(meta) { + if txn.failure_stage.is_none() && txn.state != State::Restoring { txn.failure_stage = Some(txn.state); } + txn.state = State::Restoring; + save(meta, &txn)?; + let env = RepackEnv { root, meta_dir: meta }; + if let Err(e) = restore_transaction(root, meta, &txn_dir(meta, &txn.id)?.join("before"), &mut txn, &env) { + txn.message = Some(format!("Automatic recovery requires intervention: {e}")); + save(meta, &txn)?; + write_diagnostic(root, meta, &txn)?; + return Err(e); + } + txn.state = State::RolledBack; + txn.message = Some("Interrupted update was automatically restored.".into()); + save(meta, &txn)?; + write_diagnostic(root, meta, &txn)?; + } + ensure_recovered(meta) +} + +pub fn recover(root: &Path, meta: &Path) -> Result> { + let _lock = operation_lock(meta)?; + recover_before_start(root, meta)?; + latest(meta) } /// A transaction that started but never reached a final state (crash, power loss). The UI offers rollback. pub fn unfinished(meta: &Path) -> Option { - let rd = fs::read_dir(updates_dir(meta)).ok()?; - let mut all: Vec = rd.flatten().filter_map(|e| fsx::read_json::(&e.path().join("txn.json")).ok()).collect(); + let mut all = read_journals(meta).ok()?; all.sort_by(|a, b| b.id.cmp(&a.id)); - all.into_iter().find(|t| !matches!(t.state, State::Committed | State::RolledBack | State::Prepared) && (t.state != State::Failed || t.databases_started || t.ops.iter().any(|o| o.started))) + all.into_iter().find(|t| !matches!(t.state, State::Committed | State::RolledBack | State::Prepared) && (t.state != State::Failed || t.server_stopped || t.databases_started || t.ops.iter().any(|o| o.started))) } pub fn ensure_recovered(meta: &Path) -> Result<()> { - match fs::read_dir(updates_dir(meta)) { - Ok(entries) => { - for entry in entries { - let entry = entry?; - let journal = entry.path().join("txn.json"); - if journal.exists() { - fsx::read_json::(&journal).map_err(|e| Error::Invalid(format!("Update journal {} cannot be read: {e}. Recover it before changing or starting the server.", journal.display())))?; - } - } + for txn in read_journals(meta)? { + if txn.state == State::Committed && txn.server_stopped { + return Err(Error::Invalid(format!("Update {} is committed but has not restored the previous running state. Resume recovery before changing the server.", txn.id))); } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e.into()), } if let Some(t) = unfinished(meta) { return Err(Error::Invalid(format!("Resolve unfinished update {} before changing or starting the server.", t.id))); @@ -250,13 +414,57 @@ pub struct Outcome { pub migrations: Option, } +#[derive(Debug, Serialize, Deserialize)] +pub struct ExecutionResult { + pub disposition: Disposition, + pub txn: Option, + pub message: Option, +} + +pub fn last_execution_result(meta: &Path) -> Result> { + let path = meta.join("logs/update-result.json"); + if !path.exists() { return Ok(None); } + fsx::read_json(&path).map(Some) +} + +/// UI contract: application errors retain their actual recovery outcome instead of +/// forcing the caller to infer it from a translated error string. +pub fn execute(p: &Params, report: &dyn Fn(&str, u8)) -> Result { + let previous = latest(p.meta_dir); + let previous_id = previous.as_ref().ok().and_then(|t| t.as_ref()).map(|t| t.id.clone()); + match apply(p, report) { + Ok(out) => { + let result = ExecutionResult { disposition: Disposition::Updated, txn: Some(out.txn), message: None }; + fsx::atomic_write_json(&p.meta_dir.join("logs/update-result.json"), &result)?; + Ok(result) + }, + Err(error) => { + let journal = latest(p.meta_dir); + let corrupted = previous.is_err() || journal.is_err(); + let txn = journal.ok().flatten().filter(|t| Some(&t.id) != previous_id.as_ref() || t.server_stopped || unfinished(p.meta_dir).is_some()); + let disposition = if corrupted { Disposition::RecoveryRequired } else { + txn.as_ref().and_then(|t| t.outcome).unwrap_or(Disposition::Rejected) + }; + let mut message = error.to_string(); + if let Ok((a,b)) = crate::db::credentials(p.root) { + for secret in [a,b] { if !secret.is_empty() { message = message.replace(&secret, "[redacted]"); } } + } + let mut result = ExecutionResult { disposition, txn, message: Some(crate::diag::redact(&message)) }; + if let Some(txn) = &mut result.txn { txn.message = result.message.clone(); } + // Even signature/network rejection, before any transaction exists, gets a local report. + fsx::atomic_write_json(&p.meta_dir.join("logs").join(format!("update-result-{}.json", uuid::Uuid::new_v4().simple())), &result)?; + fsx::atomic_write_json(&p.meta_dir.join("logs/update-result.json"), &result)?; + Ok(result) + } + } +} + fn step(report: &dyn Fn(&str, u8), name: &str, pct: u8) { tracing::info!(step = name, percent = pct, "server update progress"); report(name, pct); } -/// Run the whole update. On failure before the new build is validated the files are rolled back automatically; -/// a build that is applied but unhealthy is left in `NeedsDecision` so the owner chooses. +/// Run the private trial and apply the same package. Failures before commit restore automatically. pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let (root, meta_dir) = (p.root, p.meta_dir); let _lock = operation_lock(meta_dir)?; @@ -265,7 +473,8 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let _ = md; step(report, "Checking the update", 2); - let (m, manifest_bytes) = fetch_manifest(&p.source, p.trusted_key)?; + let pinned = crate::pkgsource::fetch_package(&p.source, p.trusted_key)?; + let (m, manifest_bytes, signature, source) = (pinned.manifest, pinned.bytes, pinned.signature, pinned.source); tracing::info!(root = %root.display(), from_version = ?meta.core.version, to_version = %m.version, migrations = m.migrations.len(), "server update selected"); check_manifest(&m)?; let archive = m.archive.clone().ok_or_else(|| Error::InvalidManifest("no archive".into()))?; @@ -273,19 +482,23 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let tdir = txn_dir(meta_dir, &id)?; let (tree, before) = (tdir.join("tree"), tdir.join("before")); fs::create_dir_all(&before)?; - let signature = crate::pkgsource::fetch_small(&p.source, "manifest.json.sig")?; crate::signing::verify(&manifest_bytes, &String::from_utf8_lossy(&signature), p.trusted_key)?; fsx::atomic_write(&tdir.join("manifest.json"), &manifest_bytes)?; fsx::atomic_write(&tdir.join("manifest.json.sig"), &signature)?; fsx::atomic_write(&before.join("manager-install.json"), &fs::read(meta_dir.join("install.json"))?)?; + let previous_manifest = meta_dir.join("manifests").join(format!("update-{}.json", m.version)); + fsx::atomic_write(&before.join("manager-manifest-existed"), if previous_manifest.exists() { b"yes" } else { b"no" })?; + if previous_manifest.exists() { fsx::atomic_write(&before.join("manager-manifest.json"), &fs::read(previous_manifest)?)?; } - let mut txn = Txn { id: id.clone(), state: State::Prepared, from_version: meta.core.version.clone(), to_version: m.version.clone(), recovery_point: None, databases_started: false, ops: Vec::new(), message: None }; + let mut txn = Txn { id: id.clone(), state: State::Prepared, from_version: meta.core.version.clone(), to_version: m.version.clone(), recovery_point: None, databases_started: false, ops: Vec::new(), message: None, failure_stage: None, manifest_sha256: Some(fsx::sha256_bytes(&manifest_bytes)), was_running: false, server_stopped: false, recovery_verified: false, trial_completed: false, protected_files: Vec::new(), recovery_hashes: BTreeMap::new(), outcome: None }; + txn.recovery_hashes.insert("@install".into(), fsx::sha256_file(&before.join("manager-install.json"))?); + if before.join("manager-manifest.json").exists() { txn.recovery_hashes.insert("@manifest".into(), fsx::sha256_file(&before.join("manager-manifest.json"))?); } save(meta_dir, &txn)?; // Nothing has touched the installation yet; failures up to here just discard the staging area. let staged = (|| -> Result { step(report, "Downloading the update", 5); - let parts = fetch_parts(&p.source, &m, &meta_dir.join("staging").join("download"), &p.cancel, &|f, _| step(report, "Downloading the update", 5 + (f * 35.0) as u8))?; + let parts = fetch_parts(&source, &m, &meta_dir.join("staging").join("download"), &p.cancel, &|f, _| step(report, "Downloading the update", 5 + (f * 35.0) as u8))?; step(report, "Verifying the update", 42); package::extract(&parts, &m, &tree, &|_, _| {})?; Ok(tree.clone()) @@ -301,6 +514,7 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { } }; + crate::update_fault::checkpoint(root, "verified"); let items = plan(root, &meta, &m, &p.resolutions, Some(&tree))?; if let Some(c) = items.iter().find(|i| i.action == Action::Conflict) { txn.state = State::Failed; @@ -313,10 +527,56 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { fsx::require_space(root, need)?; fsx::require_space(meta_dir, need)?; - step(report, "Making sure the server is stopped", 48); - p.env.ensure_stopped()?; - step(report, "Saving a recovery point", 52); - txn.recovery_point = Some(p.env.snapshot().map_err(|e| Error::Invalid(format!("The update was not applied because the safety backup failed: {e}")))?); + p.env.check_capacity(&m)?; + txn.was_running = p.env.was_running()?; + let preparation = (|| -> Result<()> { + // Validation renders files not necessarily listed in the incoming package. + txn.protected_files = crate::backup::config_files(root); + txn.protected_files.extend(crate::update_trial::secondary_files(root)?); + for path in ["Scripts/manage.py", "Settings/realm-profile.json", "Settings/realm-switch.json", "mysql/my.ini", "mysql/admin-client.ini"] { + if root.join(path).is_file() && !txn.protected_files.iter().any(|p| p == path) { txn.protected_files.push(path.into()); } + } + for rel in &txn.protected_files { + let bytes = fs::read(fsx::safe_join(root, rel)?)?; + fsx::atomic_write(&fsx::safe_join(&before.join("validation-originals"), rel)?, &bytes)?; + txn.recovery_hashes.insert(format!("protected:{rel}"), fsx::sha256_bytes(&bytes)); + } + // Save originals completely before persisting the intent to stop or render files. + txn.server_stopped = true; + txn.state = State::Stopping; + save(meta_dir, &txn)?; + step(report, "Making sure the server is stopped", 48); + p.env.stop()?; + crate::update_fault::checkpoint(root, "stopped"); + txn.state = State::Snapshotting; + save(meta_dir, &txn)?; + step(report, "Saving a recovery point", 52); + txn.recovery_point = Some(p.env.snapshot().map_err(|e| Error::Invalid(format!("The update was not applied because the safety backup failed: {e}")))?); + let point_metadata = meta_dir.join("backups").join(txn.recovery_point.as_deref().unwrap()).join("backup.json"); + if point_metadata.is_file() { txn.recovery_hashes.insert("@recovery-point".into(), fsx::sha256_file(&point_metadata)?); } + crate::update_fault::checkpoint(root, "snapshot"); + p.env.reserve_recovery(&before)?; + save(meta_dir, &txn)?; + crate::update_fault::checkpoint(root, "snapshot-recorded"); + step(report, "Testing the update on an isolated copy", 54); + txn.state = State::Trialing; + save(meta_dir, &txn)?; + p.env.trial(&m, &tree, &before, txn.recovery_point.as_deref().unwrap(), &p.resolutions)?; + crate::update_fault::checkpoint(root, "trial"); + txn.trial_completed = true; + save(meta_dir, &txn)?; + Ok(()) + })(); + if let Err(e) = preparation { + if !txn.server_stopped { + txn.state = State::Failed; + txn.message = Some(e.to_string()); + save(meta_dir, &txn)?; + write_diagnostic(root, meta_dir, &txn)?; + return Err(e); + } + return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), None); + } txn.ops = items .iter() @@ -335,6 +595,7 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { step(report, "Undoing the update", 70); return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), None); } + crate::update_fault::checkpoint(root, "applied"); txn.state = State::Applied; save(meta_dir, &txn)?; @@ -342,8 +603,10 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { let mut migrated = None; if !m.migrations.is_empty() { step(report, "Updating the database", 75); + txn.state = State::Migrating; txn.databases_started = true; save(meta_dir, &txn)?; + crate::update_fault::checkpoint(root, "before-migrations"); match p.env.migrate(&m, &tree.join("_migrations")) { Ok(r) if r.failed.is_none() => migrated = Some(r), Ok(r) => { @@ -354,15 +617,27 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result { } } + crate::update_fault::checkpoint(root, "migrated"); step(report, "Starting the updated server", 85); + txn.state = State::Validating; + // Startup can change database state even when there are no SQL files. + txn.databases_started = true; + save(meta_dir, &txn)?; if let Err(e) = p.env.validate() { - txn.state = State::NeedsDecision; - txn.message = Some(format!("The updated server did not start correctly: {e}")); - save(meta_dir, &txn)?; - return Ok(Outcome { txn, migrations: migrated }); + return fail_after_apply(p, &mut txn, &before, &tree, format!("The updated server did not start correctly: {e}"), migrated); } - - finish(root, meta_dir, &mut meta, &m, &tree, &mut txn)?; + crate::update_fault::checkpoint(root, "validated"); + txn.state = State::Committing; + save(meta_dir, &txn)?; + if let Err(e) = finish(root, meta_dir, &mut meta, &m, &tree, &mut txn) { + return fail_after_apply(p, &mut txn, &before, &tree, e.to_string(), migrated); + } + // Commit is durable before game connections are allowed again. + release_reserve(root, &before)?; + p.env.restore_running_state(txn.was_running)?; + txn.server_stopped = false; + save(meta_dir, &txn)?; + txn = load(meta_dir, &txn.id)?; step(report, "Done", 100); let _ = archive; Ok(Outcome { txn, migrations: migrated }) @@ -389,7 +664,9 @@ fn finish(root: &Path, meta_dir: &Path, meta: &mut InstallMeta, m: &Manifest, tr } } } + crate::update_fault::checkpoint(root, "before-commit"); fsx::atomic_write_json(&meta_dir.join("install.json"), meta)?; + crate::update_fault::checkpoint(root, "commit-metadata"); fsx::atomic_write(&meta_dir.join("manifests").join(format!("update-{}.json", m.version)), &serde_json::to_vec_pretty(m)?)?; txn.state = State::Committed; txn.message = None; @@ -440,11 +717,16 @@ pub fn retry_validation(root: &Path, meta_dir: &Path, id: &str, fallback: &Sourc fn fail_after_apply(p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: String, report: Option) -> Result { tracing::error!(transaction = %txn.id, databases_started = txn.databases_started, "server update failed; restoring recovery point"); + txn.failure_stage = Some(txn.state); + txn.state = State::Restoring; + txn.message = Some(why.clone()); + save(p.meta_dir, txn)?; let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env); tracing::info!(transaction = %txn.id, recovered = restored.is_ok(), "server update recovery finished"); txn.state = if restored.is_ok() { State::RolledBack } else { State::Failed }; txn.message = Some(match &restored { Ok(()) => why.clone(), Err(e) => format!("{why} Recovery failed: {e}") }); save(p.meta_dir, txn)?; + write_diagnostic(p.root, p.meta_dir, txn)?; if restored.is_ok() { let _ = fs::remove_dir_all(tree); } let _ = report; Err(Error::Invalid(match restored { @@ -455,19 +737,131 @@ fn fail_after_apply(p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: } fn restore_transaction(root: &Path, meta: &Path, before: &Path, txn: &mut Txn, env: &dyn Env) -> Result<()> { - env.ensure_stopped()?; + for (key, expected) in &txn.recovery_hashes { + let saved = match key.as_str() { + "@recovery-point" => { + let id = txn.recovery_point.as_deref().ok_or_else(|| Error::Invalid("The recovery point identity is missing.".into()))?; + fsx::safe_join(&meta.join("backups"), id)?.join("backup.json") + } + "@install" => before.join("manager-install.json"), + "@manifest" => before.join("manager-manifest.json"), + _ if key.starts_with("protected:") => fsx::safe_join(&before.join("validation-originals"), &key[10..])?, + _ => fsx::safe_join(before, key)?, + }; + if fsx::sha256_file(&saved)? != *expected { return Err(Error::Invalid(format!("Recovery copy {key} is damaged. Available copies were preserved."))); } + } + if let Some(id) = &txn.recovery_point { env.verify_snapshot(id)?; } + release_reserve(root, before)?; + let trial = before.parent().unwrap().join("trial"); + if trial.exists() && trial.join("Scripts/manage.py").is_file() { + let out = crate::driver::run(&trial, crate::driver::Verb::StopAll)?; + if !out.ok { return Err(Error::Invalid("The interrupted isolated trial could not be stopped.".into())); } + } + env.stop()?; + crate::update_fault::checkpoint(root, "restore-stopped"); if txn.databases_started { let id = txn.recovery_point.as_deref().ok_or_else(|| Error::Invalid("The database recovery point is missing.".into()))?; env.restore_snapshot(id)?; + crate::update_fault::checkpoint(root, "restore-databases"); } restore_files(root, before, txn)?; + crate::update_fault::checkpoint(root, "restore-files"); + if !txn.protected_files.is_empty() { + let mut generated = crate::backup::config_files(root); + generated.extend(crate::update_trial::secondary_files(root)?); + if txn.manifest_sha256.is_some() { + for rel in ["Settings/realm-profile.json", "Settings/realm-switch.json", "mysql/my.ini", "mysql/admin-client.ini"] { if root.join(rel).is_file() { generated.push(rel.into()); } } + } + generated.sort(); + generated.dedup(); + for rel in generated { + if !txn.protected_files.contains(&rel) && !txn.ops.iter().any(|o| o.path == rel && o.had_previous) { + fs::remove_file(fsx::ensure_within(root, &fsx::safe_join(root, &rel)?)?)?; + } + } + for rel in &txn.protected_files { + fsx::atomic_write(&fsx::ensure_within(root, &fsx::safe_join(root, rel)?)?, &fs::read(fsx::safe_join(&before.join("validation-originals"), rel)?)?)?; + } + } let saved_meta = before.join("manager-install.json"); if saved_meta.exists() { - fsx::atomic_write(&meta.join("install.json"), &fs::read(saved_meta)?)?; + fsx::atomic_write(&meta.join("install.json"), &fs::read(&saved_meta)?)?; + } + let manifest = meta.join("manifests").join(format!("update-{}.json", txn.to_version)); + match fs::read(before.join("manager-manifest-existed")).ok().as_deref() { + Some(b"yes") => fsx::atomic_write(&manifest, &fs::read(before.join("manager-manifest.json"))?)?, + Some(b"no") if manifest.exists() => fs::remove_file(manifest)?, + _ => {}, // Legacy journals did not record this optional metadata file. + } + env.verify_restored()?; + env.restore_running_state(txn.was_running)?; + // Database verification and restarting either world regenerate configuration + // and realm snapshots. Restore those bytes after the last process command. + // Avoid replacing unchanged runtime binaries that the restarted server holds. + let finish_protection = (|| -> Result<()> { + let mut generated = crate::backup::config_files(root); + generated.extend(crate::update_trial::secondary_files(root)?); + if txn.manifest_sha256.is_some() { + for rel in ["Settings/realm-profile.json", "Settings/realm-switch.json", "mysql/my.ini", "mysql/admin-client.ini"] { if root.join(rel).is_file() { generated.push(rel.into()); } } + } + generated.sort(); + generated.dedup(); + for rel in generated { + if !txn.protected_files.contains(&rel) && !txn.ops.iter().any(|o| o.path == rel && o.had_previous) { + fs::remove_file(fsx::ensure_within(root, &fsx::safe_join(root, &rel)?)?)?; + } + } + for rel in &txn.protected_files { + let original = fs::read(fsx::safe_join(&before.join("validation-originals"), rel)?)?; + let target = fsx::ensure_within(root, &fsx::safe_join(root, rel)?)?; + if !target.is_file() || fsx::sha256_file(&target)? != fsx::sha256_bytes(&original) { + fsx::atomic_write(&target, &original)?; + } + } + if saved_meta.exists() { + let original = fs::read(&saved_meta)?; + if fsx::sha256_file(&meta.join("install.json"))? != fsx::sha256_bytes(&original) { + fsx::atomic_write(&meta.join("install.json"), &original)?; + } + } + Ok(()) + })(); + if let Err(error) = finish_protection { + let _ = env.stop(); + return Err(error); + } + txn.recovery_verified = true; + txn.server_stopped = false; + Ok(()) +} + +fn release_reserve(root: &Path, before: &Path) -> Result<()> { + for file in [before.join("recovery.reserve"), root.join(".state/coa-update.reserve")] { + match fs::remove_file(file) { + Ok(()) => {}, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}, + Err(e) => return Err(e.into()), + } } Ok(()) } +fn write_diagnostic(root: &Path, meta: &Path, txn: &Txn) -> Result<()> { + let recorded = load(meta, &txn.id)?; + let mut report = serde_json::to_value(&recorded).map_err(|e| Error::Invalid(e.to_string()))?; + // Do not put SQL output, configs, or credentials into the automatically collected report. + if let Some(message) = report.get_mut("message") { + let mut text = message.as_str().unwrap_or_default().to_string(); + if let Ok((a, b)) = crate::db::credentials(root) { + for secret in [a, b] { if !secret.is_empty() { text = text.replace(&secret, "[redacted]"); } } + } + *message = crate::diag::redact(&text).into(); + } + fsx::atomic_write_json(&txn_dir(meta, &txn.id)?.join("diagnostic.json"), &report)?; + let sanitized: Txn = serde_json::from_value(report)?; + fsx::atomic_write_json(&meta.join("logs/update-result.json"), &ExecutionResult { disposition: sanitized.outcome.unwrap_or(Disposition::RecoveryRequired), message: sanitized.message.clone(), txn: Some(sanitized) }) +} + fn apply_ops(root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: &mut Txn, meta_dir: &Path, fail_after: Option) -> Result<()> { let mut completed = 0usize; for i in 0..txn.ops.len() { @@ -481,19 +875,24 @@ fn apply_ops(root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: if target.is_file() { let keep = fsx::safe_join(before, &path)?; fs::create_dir_all(keep.parent().unwrap())?; - fsx::atomic_write(&keep, &fs::read(&target)?)?; + let bytes = fs::read(&target)?; + fsx::atomic_write(&keep, &bytes)?; + txn.recovery_hashes.insert(path.clone(), fsx::sha256_bytes(&bytes)); } // Journal only after the old bytes are safely saved; a crash during backup must not // turn an untouched file into a rollback operation with a missing saved copy. + crate::update_fault::checkpoint(root, &format!("before-file:{i}")); txn.ops[i].started = true; save(meta_dir, txn)?; + crate::update_fault::checkpoint(root, &format!("journal-file:{i}")); fs::create_dir_all(target.parent().unwrap())?; match action { Action::Create | Action::Replace => { // Same-volume temp file + rename: the target is either the old or the new file, never half of one. let tmp = target.with_file_name(format!("{}.coa-new", target.file_name().unwrap().to_string_lossy())); fs::copy(&staged, &tmp)?; - if let Err(e) = fs::rename(&tmp, &target) { + fs::OpenOptions::new().write(true).open(&tmp)?.sync_all()?; + if let Err(e) = fsx::durable_replace(&tmp, &target) { let _ = fs::remove_file(&tmp); return Err(e.into()); } @@ -510,13 +909,33 @@ fn apply_ops(root: &Path, tree: &Path, before: &Path, files: &[FileEntry], txn: } Action::Skip | Action::Conflict => {} } + crate::update_fault::checkpoint(root, &format!("written-file:{i}")); txn.ops[i].done = true; save(meta_dir, txn)?; + crate::update_fault::checkpoint(root, &format!("completed-file:{i}")); completed += 1; } Ok(()) } +pub(crate) fn apply_trial_files(root: &Path, meta: &InstallMeta, m: &Manifest, tree: &Path, resolutions: &BTreeMap) -> Result<()> { + for item in plan(root, meta, m, resolutions, Some(tree))? { + let target = fsx::ensure_within(root, &fsx::safe_join(root, &item.path)?)?; + let source = fsx::safe_join(tree, &item.path)?; + match item.action { + Action::Create | Action::Replace => fsx::atomic_write(&target, &fs::read(source)?)?, + Action::MergeConfig => { + let mut conf = ConfFile::parse_bytes(&fs::read(&target)?)?; + merge::apply(&mut conf, &ConfFile::parse_bytes(&fs::read(source)?)?); + fsx::atomic_write(&target, conf.to_text().as_bytes())?; + } + Action::Conflict => return Err(Error::Invalid(format!("{} needs a decision in the isolated copy.", item.path))), + Action::Skip => {} + } + } + Ok(()) +} + /// Put every touched file back the way it was. Files created by the update (no previous version) are removed only if /// they still hold exactly what the update wrote. fn restore_files(root: &Path, before: &Path, txn: &mut Txn) -> Result<()> { @@ -543,12 +962,20 @@ pub fn rollback(root: &Path, meta_dir: &Path, id: &str, env: &dyn Env) -> Result if matches!(txn.state, State::Committed | State::RolledBack) { return Err(Error::Invalid("This update cannot be rolled back.".into())); } - restore_transaction(root, meta_dir, &txn_dir(meta_dir, id)?.join("before"), &mut txn, env)?; + txn.state = State::Restoring; + save(meta_dir, &txn)?; + if let Err(e) = restore_transaction(root, meta_dir, &txn_dir(meta_dir, id)?.join("before"), &mut txn, env) { + txn.message = Some(format!("Recovery requires intervention: {e}")); + save(meta_dir, &txn)?; + write_diagnostic(root, meta_dir, &txn)?; + return Err(e); + } txn.state = State::RolledBack; txn.message = Some("rolled back".into()); save(meta_dir, &txn)?; let _ = fs::remove_dir_all(txn_dir(meta_dir, id)?.join("tree")); - Ok(txn) + write_diagnostic(root, meta_dir, &txn)?; + load(meta_dir, id) } pub(crate) fn operation_lock(meta_dir: &Path) -> Result { @@ -572,7 +999,7 @@ pub(crate) fn begin_repair(root: &Path, meta_dir: &Path, version: &str, backup: if had_previous { fsx::atomic_write(&fsx::safe_join(&before, &file.path)?, &fs::read(&target)?)?; } ops.push(Op { path: file.path.clone(), action: if had_previous { Action::Replace } else { Action::Create }, reason: None, new_sha256: file.sha256.clone(), had_previous, started: true, done: false }); } - let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()) }; + let txn = Txn { id, state: State::Applying, from_version: Some(version.into()), to_version: version.into(), recovery_point: Some(backup.into()), databases_started: false, ops, message: Some("Repair is in progress.".into()), failure_stage: None, manifest_sha256: None, was_running: false, server_stopped: false, recovery_verified: false, trial_completed: false, protected_files: Vec::new(), recovery_hashes: BTreeMap::new(), outcome: None }; save(meta_dir, &txn)?; Ok(txn) } @@ -591,10 +1018,15 @@ pub(crate) fn finish_repair(root: &Path, meta_dir: &Path, txn: &mut Txn, success fn prune(meta_dir: &Path, keep: usize) { let Ok(rd) = fs::read_dir(updates_dir(meta_dir)) else { return }; - let mut dirs: Vec<_> = rd.flatten().filter(|e| fsx::read_json::(&e.path().join("txn.json")).is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack))).map(|e| e.path()).collect(); + let mut dirs: Vec<_> = rd.flatten().filter(|e| read_journal(&e.path().join("txn.json")).is_ok_and(|t| matches!(t.state, State::Committed | State::RolledBack) && !t.server_stopped)).map(|e| e.path()).collect(); dirs.sort(); while dirs.len() > keep { - let _ = fs::remove_dir_all(dirs.remove(0)); + if let Ok(path) = fsx::ensure_within(meta_dir, &dirs.remove(0)) { + let Ok(txn) = read_journal(&path.join("txn.json")) else { continue }; + let Ok(marker) = cleanup_marker(meta_dir, &txn.id) else { continue }; + if fsx::atomic_write_json(&marker, &CleanupMarker { id: txn.id, state: txn.state, server_stopped: txn.server_stopped }).is_err() { continue; } + if fs::remove_dir_all(path).is_ok() { let _ = fs::remove_file(marker); } + } } } @@ -605,11 +1037,32 @@ pub struct RepackEnv<'a> { } impl RepackEnv<'_> { - pub(crate) fn verify_recovery_point(&self, point: &crate::backup::RecoveryPoint) -> Result<()> { + fn verify_snapshot_contents(&self, point: &crate::backup::RecoveryPoint) -> Result<()> { if point.kind != crate::backup::Kind::Full || !["characters", "auth", "world", "configs"].iter().all(|name| point.components.iter().any(|c| c.name == *name)) { return Err(Error::Invalid("The update recovery point does not contain every required database and configuration.".into())); } - if !crate::backup::verify(self.meta_dir, &point.id)?.ok { return Err(Error::Invalid("The recovery point failed verification.".into())); } + let mut names = std::collections::BTreeSet::new(); + for component in &point.components { + if !names.insert(&component.name) { return Err(Error::Invalid("The recovery point contains duplicate components.".into())); } + if component.name == "configs" { + let files = component.files.as_ref().ok_or_else(|| Error::Invalid("The recovery point has no configuration inventory.".into()))?; + if files.iter().any(|file| !component.file_sha256.contains_key(file)) { + return Err(Error::Invalid("The recovery point lacks configuration checksums; automatic restoration requires inspection.".into())); + } + } else { + if component.sha256.as_ref().is_none_or(|hash| hash.len() != 64 || !hash.bytes().all(|b| b.is_ascii_hexdigit())) || component.tables.is_none() { + return Err(Error::Invalid(format!("Recovery database {} has incomplete integrity metadata.", component.name))); + } + if component.name.contains('-') || component.name == "playerbots" { crate::db::schema_of(&component.name)?; } + else { point.realm.schema(&component.name)?; } + } + } + if !crate::backup::verify(self.meta_dir, &point.id)?.ok { return Err(Error::Invalid("The recovery point failed verification. Available copies were preserved.".into())); } + Ok(()) + } + + pub(crate) fn verify_recovery_point(&self, point: &crate::backup::RecoveryPoint) -> Result<()> { + self.verify_snapshot_contents(point)?; crate::backup::with_database(self.root, |db| { let db = db.clone().for_realm(crate::realms::Mode::Coa); for component in point.components.iter().filter(|c| c.sha256.is_some()) { @@ -633,7 +1086,130 @@ impl RepackEnv<'_> { } } +impl RepackEnv<'_> { + fn wait_for_running(&self) -> bool { + use crate::process::{observe, ServiceState}; + let ports = crate::layout::read_ports(self.root); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); + let mut ready_since = None; + loop { + let o = observe(self.root, &ports); + if o.mysql.state == ServiceState::Running && o.auth.state == ServiceState::Running && o.world.state == ServiceState::Running && o.secondary_world.as_ref().is_none_or(|s| s.state == ServiceState::Running) { + let since = ready_since.get_or_insert_with(std::time::Instant::now); + if since.elapsed() >= std::time::Duration::from_secs(10) { return true; } + } else { ready_since = None; } + if std::time::Instant::now() > deadline { return false; } + std::thread::sleep(std::time::Duration::from_secs(2)); + } + } + + fn validate_isolated(&self) -> Result<()> { + crate::backup::with_database(self.root, |db| { + let mut modes = vec![crate::realms::Mode::Coa]; + if crate::realms::state(self.root)?.wildcard_created { modes.push(crate::realms::Mode::Wildcard); } + for mode in modes { + if let Some(p) = crate::schema_check::check(&db.clone().for_realm(mode), self.root)?.first() { + return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail))); + } + } + Ok(()) + })?; + // The check start is the first start of the new build: give the module configs the settings the update added + // (as a normal start does), or the server logs a "missing property" line for every one of them. + match crate::registry::MetaDir::open(self.meta_dir) { + Ok((_, meta)) if meta.kind == crate::registry::InstallKind::New => { + crate::config::materialize_module_configs(self.root)?; + } + // An imported server gets the files it lacks (nothing it has is changed). + _ => { + crate::config::create_missing_module_configs(self.root)?; + } + } + let started = match crate::driver::validate_update(self.root) { + Ok(out) => out, + Err(e) => { + let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); + return Err(e); + } + }; + let healthy = started.ok && self.wait_for_running(); + if healthy { + let out = crate::driver::run(self.root, crate::driver::Verb::StopAll)?; + if !out.ok { return Err(Error::Invalid("The validated server could not be stopped before commit.".into())); } + return Ok(()); + } + let cause = crate::driver::startup_failure(self.root, &started); + // Leave the files unlocked so that a rollback can replace them. + let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); + Err(Error::Invalid(cause)) + } +} + impl Env for RepackEnv<'_> { + fn was_running(&self) -> Result { + let o = crate::process::observe(self.root, &crate::layout::read_ports(self.root)); + Ok(o.world.state == crate::process::ServiceState::Running || crate::multiworld::is_running(self.root)) + } + + fn stop(&self) -> Result<()> { + let out = crate::driver::run(self.root, crate::driver::Verb::StopAll)?; + if !out.ok { return Err(Error::Invalid("The server could not be stopped for update or recovery.".into())); } + self.ensure_stopped() + } + + fn check_capacity(&self, manifest: &Manifest) -> Result<()> { + crate::update_trial::capacity(self.root, self.meta_dir, manifest) + } + + fn reserve_recovery(&self, before: &Path) -> Result<()> { + crate::update_trial::reserve(self.root, before) + } + + fn trial(&self, manifest: &Manifest, tree: &Path, before: &Path, point: &str, resolutions: &BTreeMap) -> Result<()> { + crate::update_trial::run(self.root, self.meta_dir, manifest, tree, before, point, resolutions) + } + + fn restore_running_state(&self, running: bool) -> Result<()> { + if running { + let out = crate::driver::start_after_update(self.root)?; + if !out.ok { return Err(Error::Invalid(format!("The restored server could not start: {}", crate::driver::startup_failure(self.root, &out)))); } + if !self.wait_for_running() { + let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); + return Err(Error::Invalid("The original running state could not be restored: the server did not stay healthy. Recovery copies were preserved.".into())); + } + } else { self.stop()?; } + Ok(()) + } + + fn verify_restored(&self) -> Result<()> { + let transaction = unfinished(self.meta_dir); + if let Some(id) = transaction.as_ref().and_then(|t| t.recovery_point.as_ref()) { + let point = crate::backup::get(self.meta_dir, &id)?; + crate::backup::with_database(self.root, |db| { + let db = db.clone().for_realm(crate::realms::Mode::Coa); + for component in point.components.iter().filter(|c| c.sha256.is_some()) { + let schema = if component.name.contains('-') || component.name == "playerbots" { crate::db::schema_of(&component.name)? } else { point.realm.schema(&component.name)? }; + if component.tables.is_some_and(|expected| db.tables(schema).map(|t| t.len() != expected).unwrap_or(true)) { + return Err(Error::Invalid(format!("Restored database {schema} does not match the original snapshot."))); + } + // The same deterministic dump includes schema, rows and migration history. + // Table counts alone cannot prove that accounts and inventory survived. + let check = self.meta_dir.join("backups").join(&id).join(format!("verify-{}.sql.zst", component.name)); + let (_, actual) = db.dump_to(schema, &check)?; + let _ = fs::remove_file(&check); + let expected = if transaction.as_ref().is_some_and(|t| t.databases_started) { + fs::read_to_string(self.meta_dir.join("backups").join(id).join(format!("restore-{}.sha256", component.name)))? + } else { component.sha256.clone().unwrap_or_default() }; + if expected.trim() != actual { + return Err(Error::Invalid(format!("Restored database {schema} content differs from the recovery point. Copies were preserved."))); + } + } + Ok(()) + })?; + } + Ok(()) + } + fn ensure_stopped(&self) -> Result<()> { use crate::process::{observe, ServiceState}; let o = observe(self.root, &crate::layout::read_ports(self.root)); @@ -652,17 +1228,18 @@ impl Env for RepackEnv<'_> { fn restore_snapshot(&self, id: &str) -> Result<()> { let point = crate::backup::get(self.meta_dir, id)?; - if point.kind != crate::backup::Kind::Full { return Err(Error::Invalid("Database rollback requires a full recovery point.".into())); } - if !["characters", "auth", "world", "configs"].iter().all(|name| point.components.iter().any(|c| c.name == *name)) || !crate::backup::verify(self.meta_dir, id)?.ok { - return Err(Error::Invalid("The full database recovery point is incomplete or damaged.".into())); - } + self.verify_snapshot_contents(&point)?; for component in point.components.iter().filter(|c| c.sha256.is_some()) { - crate::backup::restore_database(self.root, self.meta_dir, id, &component.name)?; + crate::backup::restore_update_database(self.root, self.meta_dir, id, &component.name)?; } - crate::backup::restore_configs(self.root, self.meta_dir, id)?; + crate::backup::restore_update_configs(self.root, self.meta_dir, id)?; Ok(()) } + fn verify_snapshot(&self, id: &str) -> Result<()> { + self.verify_snapshot_contents(&crate::backup::get(self.meta_dir, id)?) + } + fn migrate(&self, manifest: &Manifest, staged: &Path) -> Result { let root = self.root; crate::backup::with_database(root, |db| { @@ -699,61 +1276,9 @@ impl Env for RepackEnv<'_> { } fn validate(&self) -> Result<()> { - use crate::process::{observe, ServiceState}; - crate::backup::with_database(self.root, |db| { - let mut modes = vec![crate::realms::Mode::Coa]; - if crate::realms::state(self.root)?.wildcard_created { modes.push(crate::realms::Mode::Wildcard); } - for mode in modes { - if let Some(p) = crate::schema_check::check(&db.clone().for_realm(mode), self.root)?.first() { - return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail))); - } - } - Ok(()) - })?; - // The check start is the first start of the new build: give the module configs the settings the update added - // (as a normal start does), or the server logs a "missing property" line for every one of them. - match crate::registry::MetaDir::open(self.meta_dir) { - Ok((_, meta)) if meta.kind == crate::registry::InstallKind::New => { - let _ = crate::config::materialize_module_configs(self.root); - } - // An imported server gets the files it lacks (nothing it has is changed). - _ => { - let _ = crate::config::create_missing_module_configs(self.root); - } - } - let started = match crate::driver::validate_update(self.root) { - Ok(out) => out, - Err(e) => { - let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); - return Err(e); - } - }; - let ports = crate::layout::read_ports(self.root); - let healthy = started.ok && { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); - let mut ready_since = None; - loop { - let o = observe(self.root, &ports); - if o.mysql.state == ServiceState::Running && o.auth.state == ServiceState::Running && o.world.state == ServiceState::Running && o.secondary_world.as_ref().is_none_or(|s| s.state == ServiceState::Running) { - let since = ready_since.get_or_insert_with(std::time::Instant::now); - if since.elapsed() >= std::time::Duration::from_secs(10) { break true; } - } else { - ready_since = None; - } - if std::time::Instant::now() > deadline { - break false; - } - std::thread::sleep(std::time::Duration::from_secs(2)); - } - }; - if healthy { - return Ok(()); - } - let cause = crate::driver::startup_failure(self.root, &started); - // Leave the files unlocked so that a rollback can replace them. - let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll); - Err(Error::Invalid(cause)) + crate::install::with_scratch_ports(self.root, || self.validate_isolated()) } + } #[cfg(test)] @@ -851,7 +1376,7 @@ mod tests { let mut m = build(&src, &pkg, &BuildOptions { kind: Kind::Update, version: "2.0.0".into(), core_commit: None, built_at: "x".into(), part_size: 1 << 20, bots_commit: None, migrations: vec![] }, &|_| {}).unwrap(); // Settings/* are merge-config by the packager; make the migration entry visible to the runner if migrations { - m.migrations.push(crate::manifest::Migration { id: "m1".into(), db: "world".into(), sha256: fsx::sha256_bytes(b"SELECT 1;"), destructive: false }); + m.migrations.push(crate::manifest::Migration { compatible_sha256: vec![], id: "m1".into(), db: "world".into(), sha256: fsx::sha256_bytes(b"SELECT 1;"), destructive: false }); fs::write(pkg.join("manifest.json"), serde_json::to_vec_pretty(&m).unwrap()).unwrap(); } let sk = SigningKey::generate(&mut rand_core::OsRng); @@ -887,6 +1412,17 @@ mod tests { assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } + fn legacy_pending(w: &World, env: &Fake) -> Outcome { + let was_healthy = env.healthy.replace(true); + let mut out = run(w, env, BTreeMap::new(), None).unwrap(); + env.healthy.set(was_healthy); + out.txn.state = State::NeedsDecision; + let (m, _) = fetch_manifest(&Source::Dir(w.pkg.clone()), &w.key).unwrap(); + package::extract(&w.pkg, &m, &txn_dir(&w.meta, &out.txn.id).unwrap().join("tree"), &|_, _| {}).unwrap(); + save(&w.meta, &out.txn).unwrap(); + out + } + #[test] fn happy_path_replaces_creates_merges_and_records_ownership() { let w = world(&[], false); @@ -952,22 +1488,18 @@ mod tests { } #[test] - fn unhealthy_new_build_is_kept_until_the_owner_rolls_back() { + fn unhealthy_new_build_is_automatically_restored() { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); - assert_eq!(out.txn.state, State::NeedsDecision); - assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2", "not undone automatically"); - assert_eq!(unfinished(&w.meta).unwrap().id, out.txn.id); - let (_, meta) = MetaDir::open(&w.meta).unwrap(); - assert_eq!(meta.core.version.as_deref(), Some("1.0.0"), "not committed"); - - rollback(&w.root, &w.meta, &out.txn.id, &env).unwrap(); + let error = run(&w, &env, BTreeMap::new(), None).unwrap_err(); + assert!(error.to_string().contains("did not start correctly")); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); - assert!(!w.root.join("Core/newfile.dll").exists()); assert!(unfinished(&w.meta).is_none()); - assert!(rollback(&w.root, &w.meta, &out.txn.id, &env).is_err(), "already rolled back"); + let t = latest(&w.meta).unwrap().unwrap(); + assert_eq!(t.state, State::RolledBack); + assert!(t.recovery_verified); + assert!(txn_dir(&w.meta, &t.id).unwrap().join("diagnostic.json").is_file()); } #[test] @@ -975,7 +1507,7 @@ mod tests { let w = world(&[], true); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let out = legacy_pending(&w, &env); let calls = env.calls.borrow().len(); env.healthy.set(true); let done = retry_validation(&w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env).unwrap(); @@ -991,7 +1523,7 @@ mod tests { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let out = legacy_pending(&w, &env); let dir = txn_dir(&w.meta, &out.txn.id).unwrap(); fs::remove_file(dir.join("manifest.json")).unwrap(); fs::remove_file(dir.join("manifest.json.sig")).unwrap(); @@ -1008,7 +1540,7 @@ mod tests { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let out = legacy_pending(&w, &env); write(&w.root, "Core/worldserver.exe", b"unexpected binary"); assert!(retry_validation(&w.root, &w.meta, &out.txn.id, &Source::Dir(w.pkg.clone()), &w.key, &env).is_err()); write(&w.root, "Core/worldserver.exe", b"world-v2"); @@ -1029,6 +1561,7 @@ mod tests { t.state = State::Applying; for op in t.ops.iter_mut() { op.started = op.path == "Core/worldserver.exe"; + op.done = op.started; } save(&w.meta, &t).unwrap(); assert_eq!(unfinished(&w.meta).unwrap().id, id); @@ -1036,6 +1569,115 @@ mod tests { assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); } + #[test] + fn syntactically_valid_corrupt_journal_blocks_startup() { + let w = world(&[], false); + let out = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap(); + let path = txn_dir(&w.meta, &out.txn.id).unwrap().join("txn.json"); + let mut txn = load(&w.meta, &out.txn.id).unwrap(); + txn.id = "another-transaction".into(); + fsx::atomic_write_json(&path, &txn).unwrap(); + assert!(latest(&w.meta).is_err()); + assert!(ensure_recovered(&w.meta).is_err()); + txn.id = out.txn.id.clone(); + txn.ops[0].path = "../outside.exe".into(); + fsx::atomic_write_json(&path, &txn).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + txn = out.txn.clone(); + txn.state = State::Prepared; + fsx::atomic_write_json(&path, &txn).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + txn = out.txn.clone(); + txn.ops[0].done = false; + fsx::atomic_write_json(&path, &txn).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + txn = out.txn.clone(); + txn.recovery_point = Some("../another-backup".into()); + fsx::atomic_write_json(&path, &txn).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + } + + #[test] + fn damaged_snapshot_blocks_recovery_before_stopping_or_changing_files() { + struct Damaged; + impl Env for Damaged { + fn ensure_stopped(&self) -> Result<()> { panic!("recovery must verify first") } + fn snapshot(&self) -> Result { unreachable!() } + fn restore_snapshot(&self, _: &str) -> Result<()> { panic!("damaged snapshot must not be imported") } + fn migrate(&self, _: &Manifest, _: &Path) -> Result { unreachable!() } + fn validate(&self) -> Result<()> { unreachable!() } + fn verify_snapshot(&self, _: &str) -> Result<()> { Err(Error::Invalid("damaged database archive".into())) } + } + for databases_started in [false, true] { + let w = world(&[], false); + let mut txn = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap().txn; + txn.databases_started = databases_started; + txn.recovery_point = Some("rp-1".into()); + let before = txn_dir(&w.meta, &txn.id).unwrap().join("before"); + let current = read(&w, "Core/worldserver.exe"); + assert!(restore_transaction(&w.root, &w.meta, &before, &mut txn, &Damaged).is_err()); + assert_eq!(read(&w, "Core/worldserver.exe"), current); + } + } + + #[test] + fn missing_journal_is_not_confused_with_completed_cleanup() { + let w = world(&[], false); + let out = run(&w, &Fake::ok(), BTreeMap::new(), None).unwrap(); + fs::remove_file(txn_dir(&w.meta, &out.txn.id).unwrap().join("txn.json")).unwrap(); + assert!(latest(&w.meta).is_err()); + assert!(ensure_recovered(&w.meta).is_err()); + let marker = cleanup_marker(&w.meta, &out.txn.id).unwrap(); + fsx::atomic_write_json(&marker, &CleanupMarker { id: out.txn.id.clone(), state: State::Committed, server_stopped: false }).unwrap(); + assert!(latest(&w.meta).unwrap().is_none()); + assert!(ensure_recovered(&w.meta).is_ok()); + fsx::atomic_write_json(&marker, &CleanupMarker { id: out.txn.id, state: State::Applying, server_stopped: false }).unwrap(); + assert!(ensure_recovered(&w.meta).is_err()); + } + + #[test] + fn recovery_restores_configs_after_verification_and_stop_regenerate_them() { + struct Regenerates<'a>(&'a Path, bool); + impl Env for Regenerates<'_> { + fn ensure_stopped(&self) -> Result<()> { Ok(()) } + fn was_running(&self) -> Result { Ok(self.1) } + fn snapshot(&self) -> Result { Ok("rp-1".into()) } + fn restore_snapshot(&self, _: &str) -> Result<()> { Ok(()) } + fn migrate(&self, _: &Manifest, _: &Path) -> Result { unreachable!() } + fn validate(&self) -> Result<()> { Err(Error::Invalid("startup failed".into())) } + fn verify_restored(&self) -> Result<()> { + fsx::atomic_write(&self.0.join("Core/configs/modules/generated.conf"), b"new verification config")?; + fsx::atomic_write(&self.0.join("Settings/realm-profiles/coa.json"), b"new realm profile")?; + fsx::atomic_write(&self.0.join("Settings/realm-profiles/wildcard.json"), b"newly generated profile")?; + fsx::atomic_write(&self.0.join("mysql/my.ini"), b"generated mysql configuration")?; + fsx::atomic_write(&self.0.join("Core/configs/worldserver.conf"), b"generated by verification") + } + fn restore_running_state(&self, running: bool) -> Result<()> { + assert_eq!(running, self.1); + fsx::atomic_write(&self.0.join("Settings/realm-profiles/coa.json"), b"profile regenerated on restart")?; + fsx::atomic_write(&self.0.join("Core/configs/worldserver.conf"), b"generated by stop") + } + } + for was_running in [false, true] { + let w = world(&[], false); + write(&w.root, "Core/configs/worldserver.conf", b"original user config\r\n"); + write(&w.root, "Settings/realm-profiles/coa.json", b"original realm profile"); + write(&w.root, "mysql/my.ini", b"original mysql configuration"); + let env = Regenerates(&w.root, was_running); + assert!(apply(&Params { root: &w.root, meta_dir: &w.meta, source: Source::Dir(w.pkg.clone()), trusted_key: &w.key, + cancel: Cancel::default(), resolutions: BTreeMap::new(), env: &env, fail_after_ops: None }, &|_, _| {}).is_err()); + assert_eq!(read(&w, "Core/configs/worldserver.conf"), b"original user config\r\n"); + assert!(!w.root.join("Core/configs/modules/generated.conf").exists()); + assert!(!w.root.join("Settings/realm-profiles/wildcard.json").exists()); + assert_eq!(read(&w, "Settings/realm-profiles/coa.json"), b"original realm profile"); + assert_eq!(read(&w, "mysql/my.ini"), b"original mysql configuration"); + let recovered = latest(&w.meta).unwrap().unwrap(); + assert!(recovered.recovery_verified); + assert_eq!(recovered.failure_stage, Some(State::Validating)); + assert_eq!(recovered.was_running, was_running); + } + } + #[test] fn a_running_server_or_failed_backup_blocks_the_update_before_any_change() { let w = world(&[], false); @@ -1043,6 +1685,7 @@ mod tests { env.stopped.set(false); assert!(run(&w, &env, BTreeMap::new(), None).is_err()); assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + let w = world(&[], false); let env = Fake::ok(); env.snapshot_ok.set(false); let e = run(&w, &env, BTreeMap::new(), None).unwrap_err(); @@ -1090,7 +1733,7 @@ mod tests { let w = world(&[], true); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let out = legacy_pending(&w, &env); let recovery = Fake::ok(); rollback(&w.root, &w.meta, &out.txn.id, &recovery).unwrap(); assert_eq!(*recovery.calls.borrow(), ["stop", "restore-databases"]); @@ -1142,7 +1785,7 @@ mod tests { let w = world(&[], false); let env = Fake::ok(); env.healthy.set(false); - let out = run(&w, &env, BTreeMap::new(), None).unwrap(); + let out = legacy_pending(&w, &env); let (_, mut meta) = MetaDir::open(&w.meta).unwrap(); meta.core.version = Some("2.0.0".into()); meta.core.commit = Some("a".repeat(40)); @@ -1153,6 +1796,20 @@ mod tests { assert_eq!(restored.core.commit, None); } + #[test] + fn damaged_original_file_blocks_recovery_without_announcing_success() { + let w = world(&[], false); + let out = legacy_pending(&w, &Fake::ok()); + write(&txn_dir(&w.meta, &out.txn.id).unwrap().join("before"), "Core/worldserver.exe", b"damaged backup"); + let error = rollback(&w.root, &w.meta, &out.txn.id, &Fake::ok()).unwrap_err(); + assert!(error.to_string().contains("damaged")); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v2"); + let journal = load(&w.meta, &out.txn.id).unwrap(); + assert_eq!(journal.state, State::Restoring); + assert!(!journal.recovery_verified); + assert!(ensure_recovered(&w.meta).is_err()); + } + #[test] fn bad_signature_and_wrong_package_kind_change_nothing() { let w = world(&[], false); @@ -1168,4 +1825,29 @@ mod tests { assert_eq!(p.to_version, "2.0.0"); assert!(p.items.iter().any(|i| i.path == "Core/newfile.dll" && i.action == Action::Create)); } + + #[test] + fn execution_reports_rejection_restoration_and_required_intervention() { + let attempt = |w: &World, env: &Fake| execute(&Params { root: &w.root, meta_dir: &w.meta, source: Source::Dir(w.pkg.clone()), trusted_key: &w.key, cancel: Cancel::default(), resolutions: BTreeMap::new(), env, fail_after_ops: None }, &|_, _| {}).unwrap(); + let w = world(&[], false); + fs::write(w.pkg.join("manifest.json.sig"), "invalid signature").unwrap(); + let rejected = attempt(&w, &Fake::ok()); + assert!(matches!(rejected.disposition, Disposition::Rejected)); + assert!(rejected.txn.is_none()); + assert!(w.meta.join("logs/update-result.json").is_file()); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + let w = world(&[], true); + let env = Fake::ok(); + env.migrate_fail.set(true); + let restored = attempt(&w, &env); + assert!(matches!(restored.disposition, Disposition::Restored)); + assert!(restored.txn.unwrap().recovery_verified); + assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1"); + let w = world(&[], true); + env.restore_fail.set(true); + let blocked = attempt(&w, &env); + assert!(matches!(blocked.disposition, Disposition::RecoveryRequired)); + assert!(!blocked.txn.unwrap().recovery_verified); + assert!(ensure_recovered(&w.meta).is_err()); + } } diff --git a/crates/coa-release/Cargo.toml b/crates/coa-release/Cargo.toml index a338081..51d0e50 100644 --- a/crates/coa-release/Cargo.toml +++ b/crates/coa-release/Cargo.toml @@ -7,4 +7,7 @@ license.workspace = true [dependencies] coa-core = { path = "../coa-core" } serde_json = "1" +base64 = "0.22" +ed25519-dalek = "2" +tempfile = "3" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } diff --git a/crates/coa-release/src/main.rs b/crates/coa-release/src/main.rs index 2956080..4fcea12 100644 --- a/crates/coa-release/src/main.rs +++ b/crates/coa-release/src/main.rs @@ -46,6 +46,65 @@ fn run() -> Result<(), String> { let e = |x: coa_core::Error| x.to_string(); let part = a.get("part-size").and_then(|v| v.parse().ok()).unwrap_or(DEFAULT_PART_SIZE); match cmd.as_str() { + "verify-registry" => { + let sql = coa_core::migration_registry::reconcile(&PathBuf::from(need(&a, "registry")?), Vec::new()).map_err(e)?; + println!("Verified {} historical migrations, order and signed byte variants", sql.len()); + } + "resolve" => { + let package = coa_core::pkgsource::fetch_package(&coa_core::pkgsource::Source::Url(need(&a, "source")?.clone()), coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + coa_core::fsx::atomic_write(&PathBuf::from(need(&a, "out")?), &package.bytes).map_err(e)?; + } + "extract-game-data" => { + let source = PathBuf::from(need(&a, "package")?); + let package = coa_core::pkgsource::fetch_package(&coa_core::pkgsource::Source::Dir(source.clone()), coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + if !package.manifest.files.iter().any(|f| f.path.starts_with("Data/")) { return Err("Signed base has no game data.".into()); } + coa_core::package::extract_selected(&source, &package.manifest, &PathBuf::from(need(&a, "fixture")?), &|p| p.starts_with("Data/"), &|_, _| {}).map_err(e)?; + } + "repackage" => { + let source = PathBuf::from(need(&a, "package")?); + let pinned = coa_core::pkgsource::fetch_package(&coa_core::pkgsource::Source::Dir(source.clone()), coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + let work = tempfile::tempdir().map_err(|e| e.to_string())?; + coa_core::package::extract(&source, &pinned.manifest, work.path(), &|_, _| {}).map_err(e)?; + let mut sql: Vec<_> = pinned.manifest.migrations.iter().map(|m| coa_core::release::SqlFile { + compatible_sha256: m.compatible_sha256.clone(), db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), origin: "signed package".into(), abs: work.path().join("_migrations").join(&m.db).join(format!("{}.sql", m.id)), + }).collect(); + let repairs = PathBuf::from(need(&a, "repairs")?); + for repair in coa_core::release_schema::collect(&work.path().join("no-core"), None, Some(&repairs)).map_err(e)? { + if !sql.iter().any(|m| m.db == repair.db && m.id == repair.id) { sql.push(repair); } + } + let sql = coa_core::migration_registry::reconcile(&PathBuf::from(need(&a, "registry")?), sql).map_err(e)?; + let migrations = sql.iter().map(|m| coa_core::manifest::Migration { compatible_sha256: m.compatible_sha256.clone(), db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), destructive: false }).collect(); + for file in &sql { coa_core::fsx::atomic_write(&work.path().join("_migrations").join(&file.db).join(format!("{}.sql", file.id)), &std::fs::read(&file.abs).map_err(|e| e.to_string())?).map_err(e)?; } + let out = PathBuf::from(need(&a, "out")?); + let opts = BuildOptions { kind: Kind::Update, version: need(&a, "version")?.clone(), core_commit: pinned.manifest.core.commit, bots_commit: pinned.manifest.bots.and_then(|b| b.commit), built_at: chrono_now(), part_size: part, migrations }; + let mut manifest = build(work.path(), &out, &opts, &|s| eprintln!("{s}")).map_err(e)?; + manifest.min_manager_version = coa_core::MANAGER_VERSION.into(); + coa_core::fsx::atomic_write_json(&out.join("manifest.json"), &manifest).map_err(e)?; + } + "archive-migrations" => { + coa_core::migration_registry::import(&PathBuf::from(need(&a, "package")?), &PathBuf::from(need(&a, "registry")?), coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + println!("Published migration bytes and compatibility archived"); + } + "verify-channel" => { + let pointer = coa_core::pkgsource::verify_channel(&std::fs::read(need(&a, "file")?).map_err(|e| e.to_string())?, coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + let dir = PathBuf::from(need(&a, "dir")?); + let manifest: Manifest = coa_core::fsx::read_json(&dir.join("manifest.json")).map_err(e)?; + if manifest.version != pointer.version || coa_core::fsx::sha256_file(&dir.join("manifest.json")).map_err(e)? != pointer.manifest_sha256 || pointer.acceptance_sha256.as_deref() != Some(&coa_core::fsx::sha256_file(&dir.join("acceptance.json")).map_err(e)?) { + return Err("Signed channel differs from the tested package/report".into()); + } + } + "channel" => { + use base64::{engine::general_purpose::STANDARD, Engine}; + use ed25519_dalek::{Signer, SigningKey}; + let dir = PathBuf::from(need(&a, "dir")?); + let pinned = coa_core::pkgsource::fetch_package(&coa_core::pkgsource::Source::Dir(dir.clone()), coa_core::signing::EMBEDDED_PUBLIC_KEY).map_err(e)?; + let pointer = coa_core::pkgsource::Channel { schema: 1, version: pinned.manifest.version.clone(), release: format!("https://github.com/Corfirean/coa-server-build/releases/download/server-{}/", pinned.manifest.version), manifest_sha256: coa_core::fsx::sha256_bytes(&pinned.bytes), acceptance_sha256: Some(coa_core::fsx::sha256_file(&dir.join("acceptance.json")).map_err(e)?) }; + let payload = serde_json::to_vec(&pointer).map_err(|e| e.to_string())?; + let seed: [u8; 32] = STANDARD.decode(signing_key()?.trim()).map_err(|e| e.to_string())?.try_into().map_err(|_| "Wrong signing key length")?; + let signed = coa_core::pkgsource::SignedChannel { payload: STANDARD.encode(&payload), signature: STANDARD.encode(SigningKey::from_bytes(&seed).sign(&payload).to_bytes()) }; + coa_core::signing::verify_embedded(&payload, &signed.signature).map_err(e)?; + coa_core::fsx::atomic_write_json(&PathBuf::from(need(&a, "out")?), &signed).map_err(e)?; + } "extract-schema-base" => { coa_core::release_schema::extract_base( &PathBuf::from(need(&a, "package")?), &PathBuf::from(need(&a, "fixture")?), @@ -59,7 +118,8 @@ fn run() -> Result<(), String> { if let Some(core) = a.get("core") { let bots = a.get("bots").map(PathBuf::from); let repairs = a.get("repairs").map(PathBuf::from); - let sql = coa_core::release_schema::collect(&PathBuf::from(core), bots.as_deref(), repairs.as_deref()).map_err(e)?; + let mut sql = coa_core::release_schema::collect(&PathBuf::from(core), bots.as_deref(), repairs.as_deref()).map_err(e)?; + if let Some(registry) = a.get("registry") { sql = coa_core::migration_registry::reconcile(&PathBuf::from(registry), sql).map_err(e)?; } coa_core::release_schema::capture_release(&repack, &tree, &sql).map_err(e)?; } else { coa_core::backup::with_database(&repack, |db| coa_core::schema_check::capture(db, &tree)).map_err(e)?; @@ -77,7 +137,8 @@ fn run() -> Result<(), String> { coa_core::schema_check::require_release_contract(&PathBuf::from(need(&a, "tree")?)).map_err(e)?; let bots = a.get("bots").map(PathBuf::from); let repairs = a.get("repairs").map(PathBuf::from); - let sql = coa_core::release_schema::collect(&PathBuf::from(need(&a, "core")?), bots.as_deref(), repairs.as_deref()).map_err(e)?; + let mut sql = coa_core::release_schema::collect(&PathBuf::from(need(&a, "core")?), bots.as_deref(), repairs.as_deref()).map_err(e)?; + if let Some(registry) = a.get("registry") { sql = coa_core::migration_registry::reconcile(&PathBuf::from(registry), sql).map_err(e)?; } let out = PathBuf::from(need(&a, "out")?); let m = pack_update( &UpdateParams { tree: &PathBuf::from(need(&a, "tree")?), base_manifest: &base, sql: &sql, out: &out, version: need(&a, "version")?.clone(), core_commit: a.get("core-commit").cloned(), bots_commit: a.get("bots-commit").cloned(), part_size: part }, diff --git a/docs/RELIABLE_UPDATES.md b/docs/RELIABLE_UPDATES.md new file mode 100644 index 0000000..32a5baa --- /dev/null +++ b/docs/RELIABLE_UPDATES.md @@ -0,0 +1,94 @@ +# Reliable updates and Windows acceptance + +An operation resolves one signed manifest and immutable release before downloading its +archive. The same verified bytes are used for the private trial and the real application. +The installation lock covers stop, snapshot, trial, application, validation and recovery. +Native Windows installations with the bundled Python and MySQL runtime are supported; +Docker and unrecognised launchers are rejected before applying files. + +The manager measures the installation and database directories, checks free space and +allocates recovery reserves. The private copy contains independent runtime, configuration, +game-data and database files. It imports the verified original recovery point, runs the +candidate SQL and uses loopback ports for startup. The known launcher reporting relay is +disabled during validation. No junctions or hardlinks are created. + +Transactions keep the exact manifest hash, original running state, protected generated +files, database recovery point and individual file operations. SQL has its own migration +ledger. Stop, snapshot and trial have separate durable states. The original failure stage +is retained when the current state changes to recovery or recovered. +Interrupted SQL is restored, never automatically repeated. Windows file replacement +uses a synced temporary file and MoveFileExW with WRITE_THROUGH, including long paths. +Recovery is repeatable and does not require backing up the damaged state. Corrupt journals +or snapshots block startup and preserve available copies. + +Recovery verifies the original dump before importing it into a private recovery schema, +then compares the live database with a deterministic dump of that schema. This accounts +for MySQL's normalisation of character-set declarations while checking schema and rows. +Configuration backups carry individual content hashes. Legacy update snapshots without +those hashes require manual inspection rather than an unverified automatic restore. +Database dumps and configuration copies are flushed to disk before the snapshot is usable. +A committed operation interrupted before admission +resumes startup without undoing the committed database. Diagnostic JSON is saved locally; +database passwords are removed and reports leave the machine only through explicit export. + +## Migration publication + +`coa-release archive-migrations --package DIR --registry DIR` verifies the released +signature and preserves original SQL bytes, fixed order, hashes and signed provenance. +Only byte variants differing in UTF-8 BOM or line endings can share an ID. Other changes +fail the build and need a new migration ID. Compatibility hashes are included in the signed +manifest, leaving existing ledger checksums unchanged. + +Published packages use `server-VERSION` releases. Channels contain one signed envelope; +it binds the immutable manifest and acceptance report hashes. Stable is switched only +after package verification and acceptance. Never replace assets of an existing immutable +release or rewrite historical registry SQL. + +## Disposable acceptance fixture + +Run `tools/update-chaos-gate.ps1 -BasePackage DIR -CandidatePackage DIR -Target NEW_PATH` +on Windows. NEW_PATH must start with `coa-schema-fixture-chaos-`; the harness creates +both disposable markers itself. Base and candidate signatures are required. The harness +extracts game data independently and seeds an account, character and inventory item. +Optional `-DataSource DIR` reads an existing game-data folder without modifying it. + +Crash injection is compiled only with `coa-core/testkit` and additionally requires the +disposable marker. `COA_UPDATE_CRASH_AT` terminates the test process with exit 91 at a +named checkpoint. Production builds do not enable this mechanism. The harness exercises +file journaling, SQL phases, commit and repeated recovery interruption. Backups and failed +trial copies remain available for diagnosis. + +The release jobs must run on a Windows worker with enough free space for the actual +signed base, private copy, full snapshots and recovery reserves. Insufficient space is a +test failure, not permission to bypass acceptance. +The workflows require a dedicated runner labelled `coa-update-acceptance` (Windows, +X64). Provision at least 300 GB of free workspace and 16 GB of RAM for the retained +ten-scenario matrix, plus the normal Rust/Tauri build prerequisites. The timeout is +24 hours: full snapshot, import and recovery checks take minutes per injected fault. +A default hosted runner's disk and four-hour budget are insufficient for this gate. + +`tools/update-chaos-matrix.ps1` runs separate fixtures for CoA, Wildcard, both worlds, +SQUID and imported installations. With `-PreviousPackage`, it also runs each mode from +that signed previous release. Its acceptance report binds every scenario to the same +candidate manifest hash and manager source commit. Publication requires all ten scenarios. +It also requires every mandatory fault class in each scenario's coverage record. Missing +coverage produces an `incomplete` report and fails publication. The current harness still +lacks real disk exhaustion, denied writes, database-child and new-server crash coverage, +OS network containment and coverage of every file-operation phase. +The gate includes real occupied ports and locked files, a Python startup failure, and SQL +that creates and populates a table before failing. These faults must preserve seeded +accounts, characters and inventory in every applicable database. + +Validation currently isolates listeners on loopback and disables the recognised reporting +relay. It does not enforce an operating-system firewall against arbitrary outbound actions +in custom binaries; additional network containment is required for that release criterion. + +## Release acceptance still required + +Unit tests and compilation do not establish the full release criterion. Record results +for exact package and installer hashes, previous supported releases and skipped versions, +CoA/Wildcard/both/SQUID, imported repacks and modified configurations. Exercise occupied +ports/files, partial DDL, disk exhaustion, write errors, corrupt packages/journals/backups, +Python failure and server crashes. Verify schema, all player data, migration history, +metadata and startup after every injected failure. A passing subset must not be described +as completion of this matrix or as a guarantee against all possible failures. diff --git a/docs/UPDATE_VALIDATION_20261005.md b/docs/UPDATE_VALIDATION_20261005.md new file mode 100644 index 0000000..e061d45 --- /dev/null +++ b/docs/UPDATE_VALIDATION_20261005.md @@ -0,0 +1,61 @@ +# Local update validation, 2026-10-05 + +These results are a partial validation record, not a release acceptance report. +Tests use the independently copied, doubly marked disposable fixture +`C:\games\coa-schema-fixture-chaos-20261005-v1`. User installations are not modified. + +Candidate: `0.261005.24`, signed manifest SHA-256 +`1284b1273815209c9621b6d173a2211bae316f33462ccbe8b578e53b8d26fb5b`. +Published migration provenance was verified for `0.261005.0` and `0.261005.22`. +Their disputed repair differs only by line endings; both published hashes are retained. + +Completed real-runtime checks: + +- Base-to-candidate update, isolated trial, SQL, startup and commit. +- Repeat candidate application without replaying applied migrations. +- CoA and Wildcard together, including both world processes and player databases. +- Hard process termination after stop and after corrective SQL. +- Repeated recovery interruption after database restoration, followed by successful recovery. +- Occupied world port and exclusively locked configuration-default file, rejected before stop. +- Partial DDL and inserted data followed by invalid SQL in the live corrective migration. + Automatic rollback completed and removed the injected table. +- Bundled Python failure during live startup after a successful private two-world trial. + Automatic recovery passed exact pre-operation file and installation-metadata hashes, + restored all six databases, and preserved seeded accounts, characters and inventory. +- Modified user template values survived recovery. The fixture changes the template, + not the launcher-generated active file; this corrected an initial harness assertion. +- A modified manifest with its original signature was rejected before application. +- Hard termination immediately after recording a complete snapshot, followed by separately + corrupted transaction JSON and compressed character database archive. Both recovery + attempts refused without starting services or changing the world executable/installation + metadata. After removing the injected corruption, rollback completed and preserved the + seeded accounts, characters and inventory in both worlds. +- A live Python failure with both worlds originally running restored their ready state, + all six databases, player data, configuration and installation metadata. An initial + strict comparison exposed realm-profile regeneration after restart; the final protection + pass was corrected and the real-runtime case passed on repetition. Test processes were + explicitly stopped after verification. + +Checks compare restored database content and preserve a seeded account, character and +inventory item in each applicable realm. Recovery also checks original file hashes and +installation metadata. The core unit suite passed 260 tests; frontend build and translation +validation passed. Changes after these runs require the relevant checks again. +The full 260-test run used `--test-threads=1`. A parallel run during native database +recovery encountered connection resets in two existing RA socket tests; those failures +are recorded rather than treated as update failures or hidden by the passing rerun. + +Not yet established: the complete five-mode, base/previous-release Windows matrix; +published installer execution against fixtures; actual disk exhaustion and denied writes; +database-child termination during table swaps; every individual file mutation checkpoint; +operating-system containment of arbitrary external actions. Publication must stay blocked +until the required matrix and remaining release invariants have been demonstrated. + +The matrix report now explicitly records fault coverage. Missing mandatory fault classes +produce an `incomplete` report and fail the gate, even when every scenario that was executed +passed. Disk exhaustion, denied writes, database-child and new-server crashes, +network containment and all file-operation phases still require actual evidence. Snapshot +corruption guards now have real-runtime coverage. The later backup-metadata hash and +backup-retention guards require their own runtime injection checks before release. + +Infrastructure inspection found zero registered repository runners for both projects. +The dedicated Windows acceptance runner must be provisioned before these workflows can run. diff --git a/package-lock.json b/package-lock.json index a98828c..7ef1c3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "coa-server-manager", - "version": "0.6.4", + "version": "0.6.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "coa-server-manager", - "version": "0.6.4", + "version": "0.6.5", "dependencies": { "@tauri-apps/api": "^2.12.0", "@tauri-apps/plugin-dialog": "^2.8.0", diff --git a/package.json b/package.json index 650bb96..d5ed230 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "coa-server-manager", "private": true, - "version": "0.6.4", + "version": "0.6.5", "license": "AGPL-3.0-only", "type": "module", "scripts": { diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..f71b0da --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.98.1" +profile = "minimal" diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 874169e..8e96619 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -697,7 +697,28 @@ async fn check_update(state: State<'_, AppState>, id: String, source: Option, id: String) -> std::result::Result, UiError> { let root = path_of(&state, &id)?; - Ok(update::unfinished(&meta_dir(&root)?)) + let dir = meta_dir(&root)?; + let latest = update::latest(&dir)?; + Ok(update::unfinished(&dir).or_else(|| latest.filter(|t| t.state == update::State::Committed && t.server_stopped))) +} + +#[tauri::command] +async fn recover_update(state: State<'_, AppState>, id: String) -> std::result::Result, UiError> { + let root = path_of(&state, &id)?; + let _guard = BusyGuard::acquire(&state, &id)?; + blocking(move || update::recover(&root, &meta_dir(&root)?)).await.map_err(UiError::from) +} + +#[tauri::command] +fn last_update(state: State<'_, AppState>, id: String) -> std::result::Result, UiError> { + let root = path_of(&state, &id)?; + Ok(update::latest(&meta_dir(&root)?)?) +} + +#[tauri::command] +fn last_update_result(state: State<'_, AppState>, id: String) -> std::result::Result, UiError> { + let root = path_of(&state, &id)?; + Ok(update::last_execution_result(&meta_dir(&root)?)?) } #[tauri::command] @@ -707,22 +728,14 @@ async fn apply_update( id: String, source: Option, resolutions: BTreeMap, -) -> std::result::Result { +) -> std::result::Result { let root = path_of(&state, &id)?; let src = update_source(source); let _guard = BusyGuard::acquire(&state, &id)?; blocking(move || { let (dir, _) = install_meta(&root)?; - // Files cannot be replaced while the server runs: stop it first (gracefully), like the Stop button. - let observed = coa_core::process::observe(&root, &layout::read_ports(&root)); - if observed.world.state != coa_core::process::ServiceState::Stopped || observed.auth.state != coa_core::process::ServiceState::Stopped || coa_core::multiworld::is_running(&root) { - let out = driver::run(&root, Verb::StopAll)?; - if !out.ok { - return Err(Error::Invalid("The server could not be stopped, so the update was not started.".into())); - } - } let env = update::RepackEnv { root: &root, meta_dir: &dir }; - update::apply( + update::execute( &update::Params { root: &root, meta_dir: &dir, source: src, trusted_key: coa_core::signing::EMBEDDED_PUBLIC_KEY, cancel: Cancel::default(), resolutions, env: &env, fail_after_ops: None }, &|step, percent| { let _ = app.emit("update-progress", serde_json::json!({ "step": step, "percent": percent })); @@ -1569,6 +1582,9 @@ pub fn run() { create_account, check_update, pending_update, + recover_update, + last_update, + last_update_result, apply_update, rollback_update, retry_update_validation, diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 6340112..0f3103d 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "CoA Server Manager", - "version": "0.6.4", + "version": "0.6.5", "identifier": "com.coa.servermanager", "build": { "beforeDevCommand": "npm run dev", diff --git a/src/dev/mock.ts b/src/dev/mock.ts index 4996c1c..f2b46aa 100644 --- a/src/dev/mock.ts +++ b/src/dev/mock.ts @@ -47,7 +47,8 @@ export {}; if(cmd==="companion_sizes") return {hardware:{cores:16,ram_gb:31,free_ram_gb:12},sizes:[{id:"small",title:"Small",bots:50,warning:null},{id:"medium",title:"Medium",bots:250,warning:null},{id:"large",title:"Large",bots:500,warning:"500 companions may need more CPU and memory than this computer has (8 cores, 16 GB)."}]}; if(cmd==="add_companions"){const base=(window as any).__bots??85;(window as any).__target=base+args.count;return {spawned:"queued",created:null,baseline:base};} if(cmd==="check_update") return {from_version:(window as any).__updated?"0.261001.8":"0.260930.5",to_version:"0.261001.8",items:[],conflicts:[],migrations:5,download_bytes:6000000}; - if(cmd==="apply_update"){const w=window as any;for(let i=1;i<=8;i++){await new Promise((r: any)=>setTimeout(r,700));(w.__handlers?.["update-progress"]??[]).forEach((h: any)=>h({event:"update-progress",id:1,payload:{step:i<4?"Downloading the update":i<7?"Applying the update":"Finishing",percent:i*12}}));}(window as any).__updated=true;return {txn:{state:"committed",to_version:"0.261001.8"},migrations:null};} + if(cmd==="apply_update"){const w=window as any;for(let i=1;i<=8;i++){await new Promise((r: any)=>setTimeout(r,700));(w.__handlers?.["update-progress"]??[]).forEach((h: any)=>h({event:"update-progress",id:1,payload:{step:i<4?"Downloading the update":i<7?"Applying the update":"Finishing",percent:i*12}}));}(window as any).__updated=true;return {disposition:"updated",txn:{state:"committed",to_version:"0.261001.8"},message:null};} + if(cmd==="pending_update" || cmd==="last_update" || cmd==="last_update_result" || cmd==="recover_update") return null; if(cmd==="get_population"){if(!running) return null;let b=(window as any).__bots??85;const t=(window as any).__target??0;if(t>b){b=Math.min(t,b+25);(window as any).__bots=b;}return {online_total:b,bots_online:Math.max(0,b-7),players_online:0,bots_total:b};} if(cmd==="get_performance") return running?{mean_ms:14,median_ms:15,p95_ms:24,p99_ms:30,max_ms:61,ticks_per_sec:71.4}:null; if(cmd==="run_diagnostics") return {problems:2,checks:[{id:"files",title:"Server files",level:"ok",detail:"All expected server parts were found."},{id:"world",title:"World server",level:"warn",detail:"Not running."},{id:"disk",title:"Free disk space",level:"warn",detail:"Only 3 GB free; backups and updates need room."},{id:"exposure",title:"Private services",level:"ok",detail:"Database and server console are not reachable from the network."}]}; diff --git a/src/i18n/locales/de.ts b/src/i18n/locales/de.ts index e825ce1..0c6619e 100644 --- a/src/i18n/locales/de.ts +++ b/src/i18n/locales/de.ts @@ -371,6 +371,7 @@ const de: Partial> = { "upd.backNote": "Die Rückkehr stellt Dateien und Einstellungen wieder her. Wurden Datenbankupdates begonnen, werden auch alle Datenbanken aus der Sicherung vor dem Update wiederhergestellt. Späterer Fortschritt geht verloren.", "upd.goBackTo": "Zu Version {v} zurückkehren", "upd.goBackBefore": "Zur Version vor dem Update zurückkehren", + "upd.autoRestored": "Die Aktualisierung ist fehlgeschlagen. Der vorherige Server wurde automatisch wiederhergestellt.", "upd.title": "Server-Updates", "upd.text": "Updates werden auf Echtheit geprüft, vorher gesichert und können rückgängig gemacht werden.", "upd.check": "Nach Updates suchen", diff --git a/src/i18n/locales/en.ts b/src/i18n/locales/en.ts index d63d4b4..15ccfe7 100644 --- a/src/i18n/locales/en.ts +++ b/src/i18n/locales/en.ts @@ -404,6 +404,7 @@ const en = { "upd.backNote": "Going back restores files and settings. If database updates started, it also restores all databases to the backup taken before the update. Later progress will be lost.", "upd.goBackTo": "Go back to version {v}", "upd.goBackBefore": "Go back to the version before the update", + "upd.autoRestored": "The update failed validation. Your previous server was restored automatically.", "upd.title": "Server updates", "upd.text": "Updates are checked for authenticity, backed up first, and can be undone.", "upd.check": "Check for updates", diff --git a/src/i18n/locales/es.ts b/src/i18n/locales/es.ts index c68bcfa..9735bcd 100644 --- a/src/i18n/locales/es.ts +++ b/src/i18n/locales/es.ts @@ -371,6 +371,7 @@ const es: Partial> = { "upd.backNote": "Volver restaura los archivos y los ajustes. Si comenzaron las actualizaciones de las bases de datos, también se restauran todas las bases desde la copia anterior a la actualización. Se perderá el progreso posterior.", "upd.goBackTo": "Volver a la versión {v}", "upd.goBackBefore": "Volver a la versión anterior a la actualización", + "upd.autoRestored": "La actualización falló. El servidor anterior se restauró automáticamente.", "upd.title": "Actualizaciones del servidor", "upd.text": "Las actualizaciones se comprueban, se respaldan antes y se pueden deshacer.", "upd.check": "Buscar actualizaciones", diff --git a/src/i18n/locales/fr.ts b/src/i18n/locales/fr.ts index 6f0ab00..3a5eee1 100644 --- a/src/i18n/locales/fr.ts +++ b/src/i18n/locales/fr.ts @@ -371,6 +371,7 @@ const fr: Partial> = { "upd.backNote": "Le retour restaure les fichiers et les paramètres. Si la mise à jour des bases a commencé, toutes les bases sont aussi restaurées depuis la sauvegarde précédant la mise à jour. La progression ultérieure sera perdue.", "upd.goBackTo": "Revenir à la version {v}", "upd.goBackBefore": "Revenir à la version d’avant la mise à jour", + "upd.autoRestored": "La mise à jour a échoué. Le serveur précédent a été restauré automatiquement.", "upd.title": "Mises à jour du serveur", "upd.text": "Les mises à jour sont vérifiées, sauvegardées au préalable et peuvent être annulées.", "upd.check": "Rechercher des mises à jour", diff --git a/src/i18n/locales/ru.ts b/src/i18n/locales/ru.ts index bef5677..b3d2cff 100644 --- a/src/i18n/locales/ru.ts +++ b/src/i18n/locales/ru.ts @@ -402,6 +402,7 @@ const ru: Partial> = { "upd.backNote": "Возврат восстанавливает файлы и настройки. Если обновление баз уже началось, все базы также восстанавливаются из копии перед обновлением. Более поздний прогресс будет потерян.", "upd.goBackTo": "Вернуться к версии {v}", "upd.goBackBefore": "Вернуться к версии до обновления", + "upd.autoRestored": "Обновление не прошло проверку. Прежний сервер автоматически восстановлен.", "upd.title": "Обновления сервера", "upd.text": "Подлинность обновлений проверяется, перед ними делается резервная копия, и их можно отменить.", "upd.check": "Проверить обновления", diff --git a/src/i18n/locales/zh.ts b/src/i18n/locales/zh.ts index 45c397a..78726bc 100644 --- a/src/i18n/locales/zh.ts +++ b/src/i18n/locales/zh.ts @@ -383,6 +383,7 @@ const zh = { "upd.backNote": "回滚将恢复文件和设置。如果数据库更新已开始,所有数据库也将从更新前的备份恢复。之后的进度将丢失。", "upd.goBackTo": "回滚至版本 {v}", "upd.goBackBefore": "回滚至更新前的版本", + "upd.autoRestored": "更新验证失败。之前的服务器已自动恢复。", "upd.title": "服务器更新", "upd.text": "更新均经过真实性签名校验,并在执行前自动备份,随时可安全撤销。", "upd.check": "检查更新", diff --git a/src/lib/api.ts b/src/lib/api.ts index 554608e..74bc603 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -95,6 +95,12 @@ export interface UiError { fields?: FieldError[]; } +export interface UpdateExecutionResult { + disposition: "updated" | "rejected" | "restored" | "recovery-required"; + txn: UpdateTxn | null; + message: string | null; +} + export type Scope = "bots" | "server"; export type Restart = "runtime" | "world" | "full"; export type JsonValue = string | number | boolean; @@ -275,12 +281,18 @@ export interface UpdatePreview { export interface UpdateTxn { id: string; - state: "prepared" | "applying" | "applied" | "needs-decision" | "committed" | "rolled-back" | "failed"; + state: "prepared" | "stopping" | "snapshotting" | "trialing" | "applying" | "applied" | "migrating" | "validating" | "committing" | "restoring" | "needs-decision" | "committed" | "rolled-back" | "failed"; + failure_stage?: UpdateTxn["state"]; from_version: string | null; to_version: string; recovery_point: string | null; databases_started: boolean; message: string | null; + outcome?: "updated" | "rejected" | "restored" | "recovery-required" | null; + manifest_sha256?: string | null; + was_running?: boolean; + recovery_verified?: boolean; + trial_completed?: boolean; } export interface Population { @@ -429,8 +441,11 @@ export const api = { invoke("create_account", { id, username, password, administrator }), checkUpdate: (id: string, source?: string) => invoke("check_update", { id, source: source ?? null }), pendingUpdate: (id: string) => invoke("pending_update", { id }), + lastUpdate: (id: string) => invoke("last_update", { id }), + recoverUpdate: (id: string) => invoke("recover_update", { id }), applyUpdate: (id: string, resolutions: Record, source?: string) => - invoke<{ txn: UpdateTxn }>("apply_update", { id, source: source ?? null, resolutions }), + invoke("apply_update", { id, source: source ?? null, resolutions }), + lastUpdateResult: (id: string) => invoke("last_update_result", { id }), rollbackUpdate: (id: string, txn: string) => invoke("rollback_update", { id, txn }), retryUpdateValidation: (id: string, txn: string) => invoke("retry_update_validation", { id, txn }), population: (id: string) => invoke("get_population", { id }), diff --git a/src/lib/serverUpdate.ts b/src/lib/serverUpdate.ts index 49727ff..ca2ce9b 100644 --- a/src/lib/serverUpdate.ts +++ b/src/lib/serverUpdate.ts @@ -17,7 +17,7 @@ export interface ServerUpdateState { /** The full error of the last update attempt, for the Settings card. */ uiError: UiError | null; /** How the last update ended; kept here so the Settings card can show it after the user left and came back. */ - result: { committed: string } | { pending: UpdateTxn } | null; + result: { committed: string } | { pending: UpdateTxn } | { recovered: UpdateTxn } | null; } const EMPTY: ServerUpdateState = { preview: null, available: false, checking: false, applying: false, step: null, percent: 0, error: null, uiError: null, result: null }; @@ -48,7 +48,21 @@ export async function checkServerUpdate(id: string): Promise { /** Check now and then every five minutes. Returns the function that stops it. */ export function startServerUpdatePolling(id: string): () => void { - void checkServerUpdate(id); + void (async () => { + try { + const pending = await api.pendingUpdate(id); + if (pending) { + set(id, { applying: true, step: "Recovering the interrupted update", result: null }); + const recovered = await api.recoverUpdate(id); + if (recovered) set(id, { result: recovered.state === "committed" ? { committed: recovered.to_version } : { recovered } }); + } + } catch (e) { + const ui = asUiError(e); + set(id, { error: ui.human.code, uiError: ui }); + return; + } finally { set(id, { applying: false, step: null }); } + await checkServerUpdate(id); + })(); const t = setInterval(() => void checkServerUpdate(id), CHECK_EVERY_MS); return () => clearInterval(t); } @@ -76,8 +90,14 @@ export async function applyServerUpdate(id: string, opts?: { choices: Record { - void api.pendingUpdate(serverId).then(setPending).catch(() => {}); + void api.pendingUpdate(serverId).then(setPending).catch((e) => setError(asUiError(e))); + void api.lastUpdateResult(serverId).then(async (result) => { + const txn = result?.txn ?? await api.lastUpdate(serverId); + if (result?.disposition === "rejected" || result?.disposition === "recovery-required") { + setDone(null); + if (result.message) setError(asUiError(result.message)); + } else if (txn?.outcome === "restored" && txn.recovery_verified) setDone(t("upd.autoRestored")); + else if (result?.disposition === "updated" && txn) setDone(t("upd.updatedTo", { v: txn.to_version })); + }).catch((e) => setError(asUiError(e))); }, [serverId]); // the update finished (possibly while another tab was open): drop the stale preview, show what is left to decide @@ -44,7 +52,10 @@ export function SettingsHome({ serverId }: { serverId: string }) { if (!upd.result) return; setPreview(null); if ("pending" in upd.result) setPending(upd.result.pending); - else void api.pendingUpdate(serverId).then(setPending).catch(() => {}); + else { + if ("recovered" in upd.result && upd.result.recovered.recovery_verified) setDone(t("upd.autoRestored")); + void api.pendingUpdate(serverId).then(setPending).catch((e) => setError(asUiError(e))); + } }, [upd.result, serverId]); async function check() { @@ -68,6 +79,9 @@ export function SettingsHome({ serverId }: { serverId: string }) { setError(null); setDone(null); await applyServerUpdate(serverId, { choices, source: source.trim() || undefined }); + const txn = await api.lastUpdate(serverId).catch(() => null); + if (txn?.outcome === "restored" && txn.recovery_verified) setDone(t("upd.autoRestored")); + setPending(await api.pendingUpdate(serverId).catch(() => null)); } async function rollback(txn: UpdateTxn) { diff --git a/tools/update-chaos-gate.ps1 b/tools/update-chaos-gate.ps1 new file mode 100644 index 0000000..a8e3a16 --- /dev/null +++ b/tools/update-chaos-gate.ps1 @@ -0,0 +1,133 @@ +param( + [Parameter(Mandatory)][string]$BasePackage, + [Parameter(Mandatory)][string]$CandidatePackage, + [string]$DataSource = '', + [string]$PreviousPackage = '', + [ValidateSet('coa','wildcard','dual','squid','imported')][string]$Mode = 'coa', + [switch]$SkipBuild, + [string]$ReportFile = '', + [string]$Target = (Join-Path $env:RUNNER_TEMP ('coa-schema-fixture-chaos-' + [guid]::NewGuid().ToString('N'))) +) +$ErrorActionPreference = 'Stop' +$project = Split-Path $PSScriptRoot -Parent +$tool = Join-Path $project 'target/debug/coa-release.exe' +$test = Join-Path $project 'target/debug/examples/update_acceptance_e2e.exe' +if (Test-Path -LiteralPath $Target) { throw 'Use a new disposable fixture directory.' } +if ((Split-Path $Target -Leaf) -notlike 'coa-schema-fixture-chaos-*') { throw 'Invalid disposable fixture name.' } +if (-not $SkipBuild) { + & cargo build --manifest-path "$project/Cargo.toml" -p coa-release -p coa-core --example update_acceptance_e2e --features coa-core/testkit --locked + if ($LASTEXITCODE) { throw 'Could not build acceptance tools.' } + & cargo build --manifest-path "$project/Cargo.toml" -p coa-release --locked + if ($LASTEXITCODE) { throw 'Could not build release tools.' } +} +if (-not (Test-Path -LiteralPath $tool) -or -not (Test-Path -LiteralPath $test)) { throw 'Acceptance tools are missing.' } +& $tool extract-schema-base --package $BasePackage --fixture $Target +if ($LASTEXITCODE) { throw 'Could not prepare the signed base.' } +[IO.File]::WriteAllText((Join-Path $Target '.coa-chaos-fixture'), 'disposable update chaos fixture') +# A real independent copy: a junction would allow validation to modify source data. +if ($DataSource) { + & robocopy $DataSource (Join-Path $Target 'Data') /E /R:1 /W:1 /NFL /NDL /NJH /NJS /NP | Out-Null + if ($LASTEXITCODE -ge 8) { throw 'Could not copy isolated game data.' } +} else { + & $tool extract-game-data --package $BasePackage --fixture $Target + if ($LASTEXITCODE) { throw 'Could not extract signed game data.' } +} +& $test init $Target +if ($LASTEXITCODE) { throw 'Could not initialize sentinels.' } +& $test seed-players $Target +if ($LASTEXITCODE) { throw 'Could not seed real account, character and inventory records.' } +if ($PreviousPackage) { + & $test baseline-release $Target $PreviousPackage + if ($LASTEXITCODE) { throw 'Previous published release could not be established.' } +} +& $test apply $Target $CandidatePackage +if ($LASTEXITCODE) { throw 'Candidate failed acceptance.' } +& $test prepare-mode $Target $Mode +if ($LASTEXITCODE) { throw "Could not prepare $Mode installation." } +if ($PreviousPackage) { + # Re-establish the old signed release after mode provisioning, so the actual + # upgrade below runs in this mode rather than only in the initial CoA setup. + & $test baseline-release $Target $PreviousPackage + if ($LASTEXITCODE) { throw "Previous release could not be established in $Mode mode." } +} +& $test seed-config $Target +if ($LASTEXITCODE) { throw 'Could not seed a modified user configuration.' } +& $test break-legacy-baseline $Target +if ($LASTEXITCODE) { throw 'Could not inject missing-table baseline.' } +& $test apply $Target $CandidatePackage +if ($LASTEXITCODE) { throw 'Legacy missing-table baseline was not repaired.' } +& $test verify $Target +if ($LASTEXITCODE) { throw 'Sentinels were lost.' } +foreach ($rejection in @('expect-corrupt-package','expect-locked-file','expect-occupied-port')) { + & $test $rejection $Target $CandidatePackage + if ($LASTEXITCODE) { throw "$rejection did not reject safely." } +} +$oldPythonFailure = $env:COA_UPDATE_PYTHON_FAILURE +try { + $env:COA_UPDATE_PYTHON_FAILURE = '1' + & $test expect-rejection $Target $CandidatePackage + if ($LASTEXITCODE) { throw 'Python failure was not restored.' } +} finally { $env:COA_UPDATE_PYTHON_FAILURE = $oldPythonFailure } +$oldValidation = $env:COA_UPDATE_VALIDATION +try { + $env:COA_UPDATE_VALIDATION = '1' + & $test start-fixture $Target + if ($LASTEXITCODE) { throw 'Original offline fixture could not start.' } + $env:COA_UPDATE_PYTHON_FAILURE = '1' + & $test expect-rejection $Target $CandidatePackage + if ($LASTEXITCODE) { throw 'The originally running server was not restored.' } +} finally { + $env:COA_UPDATE_PYTHON_FAILURE = $oldPythonFailure + & $test stop-fixture $Target + $stopResult = $LASTEXITCODE + $env:COA_UPDATE_VALIDATION = $oldValidation + if ($stopResult) { throw 'The offline running-state fixture could not be stopped.' } +} +$oldSqlFailure = $env:COA_UPDATE_SQL_FAILURE +try { + & $test break-legacy-baseline $Target + if ($LASTEXITCODE) { throw 'Could not prepare partial-DDL failure.' } + $env:COA_UPDATE_SQL_FAILURE = 'manager_repair__20261005_reconcile_wildcard_tables' + & $test expect-rejection $Target $CandidatePackage + if ($LASTEXITCODE) { throw 'Partial DDL/SQL failure was not restored.' } +} finally { $env:COA_UPDATE_SQL_FAILURE = $oldSqlFailure } +$oldCrash = $env:COA_UPDATE_CRASH_AT +try { + $env:COA_UPDATE_CRASH_AT = 'snapshot-recorded' + & $test apply $Target $CandidatePackage + if ($LASTEXITCODE -ne 91) { throw 'The recorded-snapshot interruption was not exercised.' } + $env:COA_UPDATE_CRASH_AT = '' + foreach ($corruption in @('expect-corrupt-journal','expect-corrupt-recovery-point')) { + & $test $corruption $Target + if ($LASTEXITCODE) { throw "Corruption guard $corruption failed." } + } + & $test rollback $Target + if ($LASTEXITCODE) { throw 'Recovery after removing injected corruption failed.' } + foreach ($point in @('stopped','snapshot','trial','before-file:0','journal-file:0','written-file:0','completed-file:0','applied','before-migrations','before-sql:manager_repair__20261005_reconcile_wildcard_tables','after-sql:manager_repair__20261005_reconcile_wildcard_tables','recorded-sql:manager_repair__20261005_reconcile_wildcard_tables','migrated','validated','before-commit','commit-metadata')) { + $env:COA_UPDATE_CRASH_AT = '' + & $test break-legacy-baseline $Target + if ($LASTEXITCODE) { throw 'Could not reset isolated baseline.' } + & $test dirty-fixture $Target $CandidatePackage + if ($LASTEXITCODE) { throw 'Could not arrange a real file replacement.' } + $env:COA_UPDATE_CRASH_AT = $point + & $test apply $Target $CandidatePackage + if ($LASTEXITCODE -ne 91) { throw "Fault point $point was not exercised (exit $LASTEXITCODE)." } + foreach ($recoveryPoint in @('restore-stopped','restore-databases','restore-files')) { + $env:COA_UPDATE_CRASH_AT = $recoveryPoint + & $test rollback $Target + # restore-databases may not be reached before any live database changes. + if ($LASTEXITCODE -notin @(0,91)) { throw "Recovery fault $recoveryPoint failed." } + if ($LASTEXITCODE -eq 0) { break } + } + $env:COA_UPDATE_CRASH_AT = '' + & $test rollback $Target + if ($LASTEXITCODE) { throw "Recovery from $point failed." } + } +} finally { $env:COA_UPDATE_CRASH_AT = $oldCrash } +$manifest = Get-Content (Join-Path $CandidatePackage 'manifest.json') -Raw | ConvertFrom-Json +$managerSource = (& git -C $project rev-parse HEAD).Trim() +if ($LASTEXITCODE -or $managerSource -notmatch '^[0-9a-f]{40}$') { throw 'Cannot identify the tested manager source.' } +$report = @{ schema = 1; version = $manifest.version; manifestSha256 = (Get-FileHash (Join-Path $CandidatePackage 'manifest.json') -Algorithm SHA256).Hash.ToLower(); manager = (Get-FileHash $test -Algorithm SHA256).Hash.ToLower(); managerSource = $managerSource; mode = $Mode; baseline = $(if ($PreviousPackage) { (Get-Content (Join-Path $PreviousPackage 'manifest.json') -Raw | ConvertFrom-Json).version } else { 'base' }); result = 'passed' } +$report.coverage = @('signed-package-corruption','locked-file','occupied-port','python-startup-failure','partial-ddl-invalid-sql','update-process-crash','recovery-process-crash','modified-config-preservation','running-state-recovery','player-data-preservation','corrupt-journal','corrupt-recovery-point') +if (-not $ReportFile) { $ReportFile = Join-Path $CandidatePackage 'acceptance.json' } +$report | ConvertTo-Json | Set-Content $ReportFile -Encoding utf8NoBOM diff --git a/tools/update-chaos-matrix.ps1 b/tools/update-chaos-matrix.ps1 new file mode 100644 index 0000000..3102ba6 --- /dev/null +++ b/tools/update-chaos-matrix.ps1 @@ -0,0 +1,81 @@ +param( + [Parameter(Mandatory)][string]$BasePackage, + [Parameter(Mandatory)][string]$CandidatePackage, + [string]$PreviousPackage = '', + [string]$DataSource = '', + [ValidateRange(1,5)][int]$MaxParallel = 2, + [string]$WorkDirectory = (Join-Path $env:RUNNER_TEMP ('coa-acceptance-' + [guid]::NewGuid().ToString('N'))) +) +$ErrorActionPreference = 'Stop' +$project = Split-Path $PSScriptRoot -Parent +& git -C $project diff --quiet HEAD -- +if ($LASTEXITCODE) { throw 'Release acceptance requires committed manager source; tracked changes are present.' } +$untrackedSource = & git -C $project ls-files --others --exclude-standard -- crates src src-tauri tools .github +if ($LASTEXITCODE -or $untrackedSource) { throw 'Release acceptance requires committed manager source; untracked source files are present.' } +$base = (Resolve-Path -LiteralPath $BasePackage).Path +$candidate = (Resolve-Path -LiteralPath $CandidatePackage).Path +$previous = if ($PreviousPackage) { (Resolve-Path -LiteralPath $PreviousPackage).Path } else { '' } +$data = if ($DataSource) { (Resolve-Path -LiteralPath $DataSource).Path } else { '' } +if (Test-Path -LiteralPath $WorkDirectory) { throw 'Use a fresh acceptance work directory.' } +New-Item -ItemType Directory -Path $WorkDirectory | Out-Null +$work = (Resolve-Path -LiteralPath $WorkDirectory).Path +& cargo build --manifest-path "$project/Cargo.toml" -p coa-core --example update_acceptance_e2e --features testkit --locked +if ($LASTEXITCODE) { throw 'Could not build the marked-fixture acceptance tool.' } +& cargo build --manifest-path "$project/Cargo.toml" -p coa-release --locked +if ($LASTEXITCODE) { throw 'Could not build signature verification tools.' } + +$queue = [Collections.Generic.Queue[object]]::new() +foreach ($mode in @('coa','wildcard','dual','squid','imported')) { + $queue.Enqueue(@{ mode=$mode; baseline='base'; previous='' }) + if ($previous) { $queue.Enqueue(@{ mode=$mode; baseline='previous'; previous=$previous }) } +} +$active = [Collections.Generic.List[object]]::new() +$completed = [Collections.Generic.List[object]]::new() +try { + while ($queue.Count -or $active.Count) { + while ($queue.Count -and $active.Count -lt $MaxParallel) { + $job = $queue.Dequeue() + $name = "$($job.mode)-$($job.baseline)" + $report = Join-Path $work "$name.json" + $target = Join-Path $work "coa-schema-fixture-chaos-$name" + $arguments = @('-NoProfile','-File', ('"' + (Join-Path $PSScriptRoot 'update-chaos-gate.ps1') + '"'), '-SkipBuild', '-BasePackage', ('"' + $base + '"'), '-CandidatePackage', ('"' + $candidate + '"'), '-Target', ('"' + $target + '"'), '-Mode', $job.mode, '-ReportFile', ('"' + $report + '"')) + if ($job.previous) { $arguments += @('-PreviousPackage', ('"' + $job.previous + '"')) } + if ($data) { $arguments += @('-DataSource', ('"' + $data + '"')) } + $process = Start-Process -FilePath (Get-Process -Id $PID).Path -ArgumentList $arguments -WindowStyle Hidden -PassThru -RedirectStandardOutput (Join-Path $work "$name.log") -RedirectStandardError (Join-Path $work "$name.errors.log") + $active.Add(@{ process=$process; name=$name; report=$report }) + Write-Output "Started isolated $name acceptance" + } + foreach ($job in @($active.ToArray())) { + $job.process.Refresh() + if (-not $job.process.HasExited) { continue } + $job.process.WaitForExit() + if ($job.process.ExitCode -ne 0 -or -not (Test-Path -LiteralPath $job.report)) { + throw "Acceptance $($job.name) failed. Logs and marked copies are preserved at $work." + } + $completed.Add((Get-Content -LiteralPath $job.report -Raw | ConvertFrom-Json)) + $active.Remove($job) | Out-Null + Write-Output "Passed isolated $($job.name) acceptance" + } + if ($active.Count) { Start-Sleep -Seconds 1 } + } +} catch { + # Do not kill recovery processes or remove their copies. Each active fixture completes + # independently; the parent fails publication immediately. + throw +} +$manifestHash = (Get-FileHash (Join-Path $candidate 'manifest.json')).Hash.ToLower() +$manifest = Get-Content (Join-Path $candidate 'manifest.json') -Raw | ConvertFrom-Json +foreach ($result in $completed) { + if ($result.result -ne 'passed' -or $result.manifestSha256 -ne $manifestHash -or $result.version -ne $manifest.version -or $result.managerSource -ne $completed[0].managerSource) { throw 'Acceptance results refer to different source or package bytes.' } +} +$expected = if ($previous) { 10 } else { 5 } +if ($completed.Count -ne $expected) { throw 'Incomplete update acceptance matrix.' } +$requiredCoverage = @('signed-package-corruption','locked-file','occupied-port','python-startup-failure','partial-ddl-invalid-sql','update-process-crash','recovery-process-crash','modified-config-preservation','running-state-recovery','player-data-preservation','disk-exhaustion','write-denial','corrupt-journal','corrupt-recovery-point','database-process-crash','new-server-crash','network-action-containment','all-file-operation-phases') +$missingCoverage = @($requiredCoverage | Where-Object { + $required = $_ + @($completed | Where-Object { $required -notin $_.coverage }).Count -gt 0 +}) +$combined = @{ schema=1; result=$(if ($missingCoverage.Count) { 'incomplete' } else { 'passed' }); version=$manifest.version; manifestSha256=$manifestHash; manager=$completed[0].manager; managerSource=$completed[0].managerSource; scenarios=$completed.ToArray(); missingCoverage=$missingCoverage } +$combined | ConvertTo-Json -Depth 10 | Set-Content (Join-Path $candidate 'acceptance.json') -Encoding utf8NoBOM +if ($missingCoverage.Count) { throw "Publication is blocked: release fault coverage is incomplete ($($missingCoverage -join ', ')). Scenario results were preserved." } +Write-Output "Passed all $expected update acceptance scenarios"