Check release compatibility and recover unhealthy updates automatically

This commit is contained in:
Дмитрий committed 2026-10-10 12:22:14 +05:00
1 parent a9ab7ced7e
commit 02843bb6d1
11 files changed
+150 -11

No files matched your search

+40
View File
@@ -78,9 +78,22 @@ pub struct Revision {
pub commit: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Compatibility {
pub schema: u32,
pub platform: String,
pub core_commit: String,
pub module_commits: std::collections::BTreeMap<String, String>,
pub client_patch_version: String,
pub source_versions: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Manifest {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub compatibility: Option<Compatibility>,
pub schema: u32,
pub kind: Kind,
pub version: String,
@@ -136,6 +149,18 @@ impl Manifest {
if parse_version(&self.min_manager_version).is_none() {
return bad(format!("bad minManagerVersion {:?}", self.min_manager_version));
}
if let Some(c) = &self.compatibility {
if parse_version(&self.min_manager_version).is_none_or(|v| v < (0, 6, 13)) {
return bad("Release compatibility matrices require Manager 0.6.13 or newer".into());
}
if c.schema != 1 || !matches!(c.platform.as_str(), "windows-x86_64" | "linux-x86_64")
|| !is_commit(&c.core_commit) || self.core.commit.as_deref() != Some(&c.core_commit)
|| c.module_commits.is_empty() || c.module_commits.values().any(|sha| !is_commit(sha))
|| parse_version(&c.client_patch_version).is_none()
|| c.source_versions.is_empty() || c.source_versions.iter().any(|v| parse_version(v).is_none()) {
return bad("Invalid release compatibility matrix".into());
}
}
for migration in &self.migrations {
if !is_sha256(&migration.sha256) || migration.compatible_sha256.iter().any(|h| !is_sha256(h) || h == &"0".repeat(64)) {
return bad(format!("{}: invalid migration checksum compatibility", migration.id));
@@ -232,6 +257,21 @@ mod tests {
assert!(!m.compatible_with_manager("0.0.9"));
}
#[test]
fn compatibility_matrix_binds_core_and_requires_tested_sources() {
let mut m = Manifest::parse(sample("").as_bytes()).unwrap();
m.min_manager_version = "0.6.13".into();
m.compatibility = Some(Compatibility { schema: 1, platform: "windows-x86_64".into(),
core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(),
client_patch_version: "1.5.1".into(), source_versions: vec!["0.261010.32".into()] });
m.validate().unwrap();
m.compatibility.as_mut().unwrap().core_commit = "c".repeat(40);
assert!(m.validate().is_err());
m.compatibility.as_mut().unwrap().core_commit = "a".repeat(40);
m.compatibility.as_mut().unwrap().source_versions.clear();
assert!(m.validate().is_err());
}
#[test]
fn rejects_traversal_bad_hash_duplicates_and_short_commit() {
let ok = "b".repeat(64);
+2
View File
@@ -217,6 +217,7 @@ pub fn build(src: &Path, out: &Path, opts: &BuildOptions, progress: &dyn Fn(&str
split.close_current()?;
let manifest = Manifest {
compatibility: None,
schema: crate::manifest::SCHEMA,
kind: opts.kind,
version: opts.version.clone(),
@@ -503,6 +504,7 @@ mod tests {
fs::write(d.path().join(name), &bytes).unwrap();
let ok_sha = fsx::sha256_bytes(b"data");
let m = Manifest {
compatibility: None,
schema: 1,
kind: Kind::Base,
version: "0.1.0".into(),
+83 -2
View File
@@ -85,6 +85,8 @@ fn legacy_databases_started() -> bool { true }
/// Everything the transaction needs from the outside world; tests provide a fake.
pub trait Env {
fn automatic_rollback(&self) -> bool { false }
fn validate_recovery(&self) -> Result<()> { self.validate() }
fn ensure_stopped(&self) -> Result<()>;
fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) }
fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) }
@@ -360,6 +362,15 @@ fn launcher_already_integrated(root: &Path, meta_dir: &Path, meta: &InstallMeta,
}
fn reject_downgrade(meta: &InstallMeta, manifest: &Manifest) -> Result<()> {
if let Some(c) = &manifest.compatibility {
let platform = if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" };
if c.platform != platform {
return Err(Error::Invalid(format!("This package targets {} rather than {platform}.", c.platform)));
}
if meta.core.version.as_ref().is_none_or(|version| !c.source_versions.contains(version)) {
return Err(Error::Invalid("This server version is outside the release's tested compatibility matrix.".into()));
}
}
if let Some(installed) = meta.core.version.as_deref() {
if let (Some(have), Some(candidate)) = (crate::manifest::parse_version(installed), crate::manifest::parse_version(&manifest.version)) {
if candidate < have {
@@ -380,6 +391,11 @@ fn check_manifest(m: &Manifest) -> Result<()> {
Ok(())
}
pub fn validate_candidate(meta: &InstallMeta, manifest: &Manifest) -> Result<()> {
check_manifest(manifest)?;
reject_downgrade(meta, manifest)
}
pub struct Params<'a> {
pub root: &'a Path,
pub meta_dir: &'a Path,
@@ -515,6 +531,10 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result<Outcome> {
step(report, "Starting the updated server", 85);
if let Err(e) = p.env.validate() {
if p.env.automatic_rollback() {
return fail_after_apply(p, &mut txn, &before, &tree,
format!("The updated server did not start correctly: {e}"), migrated);
}
txn.state = State::NeedsDecision;
txn.message = Some(format!("The updated server did not start correctly: {e}"));
save(meta_dir, &txn)?;
@@ -599,7 +619,10 @@ pub fn retry_validation(root: &Path, meta_dir: &Path, id: &str, fallback: &Sourc
fn fail_after_apply(p: &Params, txn: &mut Txn, before: &Path, tree: &Path, why: String, report: Option<ApplyReport>) -> Result<Outcome> {
tracing::error!(transaction = %txn.id, databases_started = txn.databases_started, "server update failed; restoring recovery point");
let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env);
let restored = restore_transaction(p.root, p.meta_dir, before, txn, p.env).and_then(|()| {
if p.env.automatic_rollback() { p.env.validate_recovery() } else { Ok(()) }
});
if restored.is_err() { let _ = p.env.ensure_stopped(); }
tracing::info!(transaction = %txn.id, recovered = restored.is_ok(), "server update recovery finished");
txn.state = if restored.is_ok() { State::RolledBack } else { State::Failed };
txn.message = Some(match &restored { Ok(()) => why.clone(), Err(e) => format!("{why} Recovery failed: {e}") });
@@ -839,6 +862,7 @@ impl RepackEnv<'_> {
}
impl Env for RepackEnv<'_> {
fn automatic_rollback(&self) -> bool { true }
fn preflight(&self, manifest: &Manifest) -> Result<()> {
self.pending_migrations(manifest).map(|_| ())
}
@@ -983,6 +1007,8 @@ mod tests {
use std::cell::{Cell, RefCell};
struct Fake {
automatic_rollback: Cell<bool>,
recovery_healthy: Cell<bool>,
stopped: Cell<bool>,
healthy: Cell<bool>,
snapshot_ok: Cell<bool>,
@@ -994,11 +1020,16 @@ mod tests {
impl Fake {
fn ok() -> Fake {
Fake { stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false) }
Fake { automatic_rollback: Cell::new(false), recovery_healthy: Cell::new(true), stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false) }
}
}
impl Env for Fake {
fn automatic_rollback(&self) -> bool { self.automatic_rollback.get() }
fn validate_recovery(&self) -> Result<()> {
self.calls.borrow_mut().push("validate-recovery");
if self.recovery_healthy.get() { Ok(()) } else { Err(Error::Invalid("Restored server is unhealthy".into())) }
}
fn preflight(&self, _: &Manifest) -> Result<()> {
if self.preflight_fail.get() { Err(Error::Invalid("Conflicting migration history".into())) } else { Ok(()) }
}
@@ -1304,6 +1335,56 @@ mod tests {
assert!(rollback(&w.root, &w.meta, &out.txn.id, &env).is_err(), "already rolled back");
}
#[test]
fn automatic_health_failure_restores_databases_files_and_checks_recovery() {
let w = world(&[], true);
let env = Fake::ok();
env.automatic_rollback.set(true);
env.healthy.set(false);
assert!(run(&w, &env, BTreeMap::new(), None).unwrap_err().to_string().contains("restored"));
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
assert!(!w.root.join("Core/newfile.dll").exists());
assert!(env.calls.borrow().contains(&"restore-databases"));
assert!(env.calls.borrow().contains(&"validate-recovery"));
assert!(unfinished(&w.meta).is_none());
}
#[test]
fn incompatible_source_or_platform_is_rejected_without_touching_installation() {
let w = world(&[], false);
let (_, meta) = MetaDir::open(&w.meta).unwrap();
let mut manifest = Manifest::parse(&fs::read(w.pkg.join("manifest.json")).unwrap()).unwrap();
manifest.min_manager_version = "0.6.13".into();
manifest.core.commit = Some("a".repeat(40));
manifest.compatibility = Some(crate::manifest::Compatibility { schema: 1,
platform: if cfg!(windows) { "windows-x86_64" } else { "linux-x86_64" }.into(),
core_commit: "a".repeat(40), module_commits: [("squid".into(), "b".repeat(40))].into(),
client_patch_version: "1.5.1".into(), source_versions: vec!["1.0.0".into()] });
manifest.validate().unwrap();
validate_candidate(&meta, &manifest).unwrap();
manifest.compatibility.as_mut().unwrap().source_versions = vec!["0.9.0".into()];
assert!(validate_candidate(&meta, &manifest).unwrap_err().to_string().contains("compatibility matrix"));
manifest.compatibility.as_mut().unwrap().source_versions = vec!["1.0.0".into()];
manifest.compatibility.as_mut().unwrap().platform = if cfg!(windows) { "linux-x86_64" } else { "windows-x86_64" }.into();
assert!(validate_candidate(&meta, &manifest).unwrap_err().to_string().contains("targets"));
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
assert!(unfinished(&w.meta).is_none());
}
#[test]
fn unhealthy_restoration_preserves_recovery_and_blocks_startup() {
let w = world(&[], true);
let env = Fake::ok();
env.automatic_rollback.set(true);
env.healthy.set(false);
env.recovery_healthy.set(false);
assert!(run(&w, &env, BTreeMap::new(), None).unwrap_err().to_string().contains("Recovery failed"));
let pending = unfinished(&w.meta).unwrap();
assert_eq!(pending.state, State::Failed);
assert!(txn_dir(&w.meta, &pending.id).unwrap().join("before").exists());
assert!(ensure_recovered(&w.meta).is_err());
}
#[test]
fn retrying_startup_commits_without_reapplying_sql_or_restoring_databases() {
let w = world(&[], true);