mirror of
https://github.com/Corfirean/coa-server-manager.git
synced 2026-11-21 03:25:56 +00:00
Check release compatibility and recover unhealthy updates automatically
This commit is contained in:
1 parent
a9ab7ced7e
commit
02843bb6d1
11 files changed
+150
-11
No files matched your search
@@ -78,9 +78,22 @@ pub struct Revision {
|
||||
pub commit: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct Compatibility {
|
||||
pub schema: u32,
|
||||
pub platform: String,
|
||||
pub core_commit: String,
|
||||
pub module_commits: std::collections::BTreeMap<String, String>,
|
||||
pub client_patch_version: String,
|
||||
pub source_versions: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Manifest {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub compatibility: Option<Compatibility>,
|
||||
pub schema: u32,
|
||||
pub kind: Kind,
|
||||
pub version: String,
|
||||
@@ -136,6 +149,18 @@ impl Manifest {
|
||||
if parse_version(&self.min_manager_version).is_none() {
|
||||
return bad(format!("bad minManagerVersion {:?}", self.min_manager_version));
|
||||
}
|
||||
if let Some(c) = &self.compatibility {
|
||||
if parse_version(&self.min_manager_version).is_none_or(|v| v < (0, 6, 13)) {
|
||||
return bad("Release compatibility matrices require Manager 0.6.13 or newer".into());
|
||||
}
|
||||
if c.schema != 1 || !matches!(c.platform.as_str(), "windows-x86_64" | "linux-x86_64")
|
||||
|| !is_commit(&c.core_commit) || self.core.commit.as_deref() != Some(&c.core_commit)
|
||||
|| c.module_commits.is_empty() || c.module_commits.values().any(|sha| !is_commit(sha))
|
||||
|| parse_version(&c.client_patch_version).is_none()
|
||||
|| c.source_versions.is_empty() || c.source_versions.iter().any(|v| parse_version(v).is_none()) {
|
||||
return bad("Invalid release compatibility matrix".into());
|
||||
}
|
||||
}
|
||||
for migration in &self.migrations {
|
||||
if !is_sha256(&migration.sha256) || migration.compatible_sha256.iter().any(|h| !is_sha256(h) || h == &"0".repeat(64)) {
|
||||
return bad(format!("{}: invalid migration checksum compatibility", migration.id));
|
||||
@@ -232,6 +257,21 @@ mod tests {
|
||||
assert!(!m.compatible_with_manager("0.0.9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compatibility_matrix_binds_core_and_requires_tested_sources() {
|
||||
let mut m = Manifest::parse(sample("").as_bytes()).unwrap();
|
||||
m.min_manager_version = "0.6.13".into();
|
||||
m.compatibility = Some(Compatibility { schema: 1, platform: "windows-x86_64".into(),
|
||||
core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(),
|
||||
client_patch_version: "1.5.1".into(), source_versions: vec!["0.261010.32".into()] });
|
||||
m.validate().unwrap();
|
||||
m.compatibility.as_mut().unwrap().core_commit = "c".repeat(40);
|
||||
assert!(m.validate().is_err());
|
||||
m.compatibility.as_mut().unwrap().core_commit = "a".repeat(40);
|
||||
m.compatibility.as_mut().unwrap().source_versions.clear();
|
||||
assert!(m.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_traversal_bad_hash_duplicates_and_short_commit() {
|
||||
let ok = "b".repeat(64);
|
||||
|
||||
@@ -217,6 +217,7 @@ pub fn build(src: &Path, out: &Path, opts: &BuildOptions, progress: &dyn Fn(&str
|
||||
split.close_current()?;
|
||||
|
||||
let manifest = Manifest {
|
||||
compatibility: None,
|
||||
schema: crate::manifest::SCHEMA,
|
||||
kind: opts.kind,
|
||||
version: opts.version.clone(),
|
||||
@@ -503,6 +504,7 @@ mod tests {
|
||||
fs::write(d.path().join(name), &bytes).unwrap();
|
||||
let ok_sha = fsx::sha256_bytes(b"data");
|
||||
let m = Manifest {
|
||||
compatibility: None,
|
||||
schema: 1,
|
||||
kind: Kind::Base,
|
||||
version: "0.1.0".into(),
|
||||
|
||||
@@ -85,6 +85,8 @@ fn legacy_databases_started() -> bool { true }
|
||||
|
||||
/// Everything the transaction needs from the outside world; tests provide a fake.
|
||||
pub trait Env {
|
||||
fn automatic_rollback(&self) -> bool { false }
|
||||
fn validate_recovery(&self) -> Result<()> { self.validate() }
|
||||
fn ensure_stopped(&self) -> Result<()>;
|
||||
fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) }
|
||||
fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) }
|
||||
@@ -360,6 +362,15 @@ fn launcher_already_integrated(root: &Path, meta_dir: &Path, meta: &InstallMeta,
|
||||
}
|
||||
|
||||
fn reject_downgrade(meta: &InstallMeta, manifest: &Manifest) -> Result<()> {
|
||||
if let Some(c) = &manifest.compatibility {
|
||||
let platform = if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" };
|
||||
if c.platform != platform {
|
||||
return Err(Error::Invalid(format!("This package targets {} rather than {platform}.", c.platform)));
|
||||
}
|
||||
if meta.core.version.as_ref().is_none_or(|version| !c.source_versions.contains(version)) {
|
||||
return Err(Error::Invalid("This server version is outside the release's tested compatibility matrix.".into()));
|
||||
}
|
||||
}
|
||||
if let Some(installed) = meta.core.version.as_deref() {
|
||||
if let (Some(have), Some(candidate)) = (crate::manifest::parse_version(installed), crate::manifest::parse_version(&manifest.version)) {
|
||||
if candidate < have {
|
||||
@@ -380,6 +391,11 @@ fn check_manifest(m: &Manifest) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_candidate(meta: &InstallMeta, manifest: &Manifest) -> Result<()> {
|
||||
check_manifest(manifest)?;
|
||||
reject_downgrade(meta, manifest)
|
||||
}
|
||||
|
||||
pub struct Params<'a> {
|
||||
pub root: &'a Path,
|
||||
pub meta_dir: &'a Path,
|
||||
@@ -515,6 +531,10 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result<Outcome> {
|
||||
|
||||
step(report, "Starting the updated server", 85);
|
||||
if let Err(e) = p.env.validate() {
|
||||
if p.env.automatic_rollback() {
|
||||
return fail_after_apply(p, &mut txn, &before, &tree,
|
||||
format!("The updated server did not start correctly: {e}"), migrated);
|
||||
}
|
||||
txn.state = State::NeedsDecision;
|
||||
txn.message = Some(format!("The updated server did not start correctly: {e}"));
|
||||
save(meta_dir, &txn)?;
|
||||
@@ -599,7 +619,10 @@ pub fn retry_validation(root: &Path, meta_dir: &Path, id: &str, fallback: &Sourc
|
||||
|
||||
fn fail_after_apply(p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: String, report: Option<ApplyReport>) -> Result<Outcome> {
|
||||
tracing::error!(transaction = %txn.id, databases_started = txn.databases_started, "server update failed; restoring recovery point");
|
||||
let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env);
|
||||
let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env).and_then(|()| {
|
||||
if p.env.automatic_rollback() { p.env.validate_recovery() } else { Ok(()) }
|
||||
});
|
||||
if restored.is_err() { let _ = p.env.ensure_stopped(); }
|
||||
tracing::info!(transaction = %txn.id, recovered = restored.is_ok(), "server update recovery finished");
|
||||
txn.state = if restored.is_ok() { State::RolledBack } else { State::Failed };
|
||||
txn.message = Some(match &restored { Ok(()) => why.clone(), Err(e) => format!("{why} Recovery failed: {e}") });
|
||||
@@ -839,6 +862,7 @@ impl RepackEnv<'_> {
|
||||
}
|
||||
|
||||
impl Env for RepackEnv<'_> {
|
||||
fn automatic_rollback(&self) -> bool { true }
|
||||
fn preflight(&self, manifest: &Manifest) -> Result<()> {
|
||||
self.pending_migrations(manifest).map(|_| ())
|
||||
}
|
||||
@@ -983,6 +1007,8 @@ mod tests {
|
||||
use std::cell::{Cell, RefCell};
|
||||
|
||||
struct Fake {
|
||||
automatic_rollback: Cell<bool>,
|
||||
recovery_healthy: Cell<bool>,
|
||||
stopped: Cell<bool>,
|
||||
healthy: Cell<bool>,
|
||||
snapshot_ok: Cell<bool>,
|
||||
@@ -994,11 +1020,16 @@ mod tests {
|
||||
|
||||
impl Fake {
|
||||
fn ok() -> Fake {
|
||||
Fake { stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false) }
|
||||
Fake { automatic_rollback: Cell::new(false), recovery_healthy: Cell::new(true), stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Env for Fake {
|
||||
fn automatic_rollback(&self) -> bool { self.automatic_rollback.get() }
|
||||
fn validate_recovery(&self) -> Result<()> {
|
||||
self.calls.borrow_mut().push("validate-recovery");
|
||||
if self.recovery_healthy.get() { Ok(()) } else { Err(Error::Invalid("Restored server is unhealthy".into())) }
|
||||
}
|
||||
fn preflight(&self, _: &Manifest) -> Result<()> {
|
||||
if self.preflight_fail.get() { Err(Error::Invalid("Conflicting migration history".into())) } else { Ok(()) }
|
||||
}
|
||||
@@ -1304,6 +1335,56 @@ mod tests {
|
||||
assert!(rollback(&w.root, &w.meta, &out.txn.id, &env).is_err(), "already rolled back");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn automatic_health_failure_restores_databases_files_and_checks_recovery() {
|
||||
let w = world(&[], true);
|
||||
let env = Fake::ok();
|
||||
env.automatic_rollback.set(true);
|
||||
env.healthy.set(false);
|
||||
assert!(run(&w, &env, BTreeMap::new(), None).unwrap_err().to_string().contains("restored"));
|
||||
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
|
||||
assert!(!w.root.join("Core/newfile.dll").exists());
|
||||
assert!(env.calls.borrow().contains(&"restore-databases"));
|
||||
assert!(env.calls.borrow().contains(&"validate-recovery"));
|
||||
assert!(unfinished(&w.meta).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incompatible_source_or_platform_is_rejected_without_touching_installation() {
|
||||
let w = world(&[], false);
|
||||
let (_, meta) = MetaDir::open(&w.meta).unwrap();
|
||||
let mut manifest = Manifest::parse(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap();
|
||||
manifest.min_manager_version = "0.6.13".into();
|
||||
manifest.core.commit = Some("a".repeat(40));
|
||||
manifest.compatibility = Some(crate::manifest::Compatibility { schema: 1,
|
||||
platform: if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" }.into(),
|
||||
core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(),
|
||||
client_patch_version: "1.5.1".into(), source_versions: vec!["1.0.0".into()] });
|
||||
manifest.validate().unwrap();
|
||||
validate_candidate(&meta, &manifest).unwrap();
|
||||
manifest.compatibility.as_mut().unwrap().source_versions = vec!["0.9.0".into()];
|
||||
assert!(validate_candidate(&meta, &manifest).unwrap_err().to_string().contains("compatibility matrix"));
|
||||
manifest.compatibility.as_mut().unwrap().source_versions = vec!["1.0.0".into()];
|
||||
manifest.compatibility.as_mut().unwrap().platform = if cfg!(windows) { "linux-x86_64" } else { "windows-x86_64" }.into();
|
||||
assert!(validate_candidate(&meta, &manifest).unwrap_err().to_string().contains("targets"));
|
||||
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
|
||||
assert!(unfinished(&w.meta).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unhealthy_restoration_preserves_recovery_and_blocks_startup() {
|
||||
let w = world(&[], true);
|
||||
let env = Fake::ok();
|
||||
env.automatic_rollback.set(true);
|
||||
env.healthy.set(false);
|
||||
env.recovery_healthy.set(false);
|
||||
assert!(run(&w, &env, BTreeMap::new(), None).unwrap_err().to_string().contains("Recovery failed"));
|
||||
let pending = unfinished(&w.meta).unwrap();
|
||||
assert_eq!(pending.state, State::Failed);
|
||||
assert!(txn_dir(&w.meta, &pending.id).unwrap().join("before").exists());
|
||||
assert!(ensure_recovered(&w.meta).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retrying_startup_commits_without_reapplying_sql_or_restoring_databases() {
|
||||
let w = world(&[], true);
|
||||
|
||||
Reference in new issue
Block a user