Simplify updates to one recovery-backed migration pass (#111)

* Share migration planning and execution across update phases

* Apply updates in one recovery-backed pass without nested server rehearsals

* Preserve applied core SQL and share account migration history across realms
This commit is contained in:
Corfirean authored and GitHub committed 2026-10-16 13:39:43 +05:00
1 parent cda7875a9a
commit a7b56d6aed
20 files changed
+409 -382

No files matched your search

+148 -52
View File
@@ -11,6 +11,7 @@ use crate::fsx;
use crate::manifest::Migration;
const LEDGER_SCHEMA: &str = "acore_world";
const AUTH_LEDGER_SCHEMA: &str = "acore_auth";
const LEDGER_TABLE: &str = "coa_manager_migrations";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
@@ -37,6 +38,10 @@ pub trait Store {
fn load(&self) -> Result<Vec<LedgerRow>>;
fn load_readonly(&self) -> Result<Vec<LedgerRow>> { self.load() }
fn put(&self, row: &LedgerRow) -> Result<()>;
fn put_batch(&self, rows: &[LedgerRow]) -> Result<()> {
for row in rows { self.put(row)?; }
Ok(())
}
fn run_file(&self, schema: &str, file: &Path) -> Result<()>;
}
@@ -48,59 +53,101 @@ fn ident_ok(s: &str) -> bool {
!s.is_empty() && s.len() <= 150 && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-'))
}
pub(crate) fn shared_auth(kind: &str) -> bool {
matches!(db::schema_of(kind), Ok("acore_auth"))
}
fn create_ledger(db: &Db, schema: &str) -> Result<()> {
db.query(&format!("CREATE TABLE IF NOT EXISTS `{schema}`.`{LEDGER_TABLE}` (`db` VARCHAR(32) NOT NULL, `id` VARCHAR(190) NOT NULL, `sha256` CHAR(64) NOT NULL, `status` VARCHAR(16) NOT NULL, `applied_at` DATETIME NOT NULL, `error` TEXT NULL, `baseline` TINYINT NOT NULL DEFAULT 0, PRIMARY KEY (`db`,`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"))?;
Ok(())
}
impl Store for Db {
fn load(&self) -> Result<Vec<LedgerRow>> {
self.query(&format!(
"CREATE TABLE IF NOT EXISTS `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db` VARCHAR(32) NOT NULL, `id` VARCHAR(190) NOT NULL, `sha256` CHAR(64) NOT NULL, `status` VARCHAR(16) NOT NULL, `applied_at` DATETIME NOT NULL, `error` TEXT NULL, `baseline` TINYINT NOT NULL DEFAULT 0, PRIMARY KEY (`db`,`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;"
))?;
self.load_readonly()
create_ledger(self, LEDGER_SCHEMA)?;
create_ledger(self, AUTH_LEDGER_SCHEMA)?;
let rows = self.load_readonly()?;
// Preserve already-recorded core history even if a module later replaces its updates table.
// A zero checksum remains unknown; importing it does not invent a checksum for old SQL.
let selected: std::collections::BTreeMap<_, _> = rows.iter().map(|row| ((row.db.clone(), row.id.to_ascii_lowercase()), row)).collect();
let core: Vec<_> = selected.values().filter(|r| r.status == Status::Applied && r.baseline && r.sha256 == "0".repeat(64)
&& matches!(r.db.as_str(), "world" | "characters") && ident_ok(&r.id)).map(|r| (*r).clone()).collect();
self.put_batch(&core)?;
Ok(rows)
}
fn load_readonly(&self) -> Result<Vec<LedgerRow>> {
let out = if self.tables(LEDGER_SCHEMA)?.iter().any(|name| name == LEDGER_TABLE) {
self.query(&format!("SELECT `db`,`id`,`sha256`,`status`,`baseline`,COALESCE(HEX(`error`),'') FROM `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` ORDER BY `db`,`id`;"))?
} else { String::new() };
let mut rows = Vec::new();
// Files the core's own updater (or whoever prepared this database) already recorded count as applied.
for (kind, schema) in db::SCHEMAS {
if self.tables(schema)?.iter().any(|name| name == "updates") {
let names = self.query(&format!("SELECT name FROM `{schema}`.`updates`;"))?;
for name in names.lines().filter(|n| n.ends_with(".sql")) {
rows.push(LedgerRow { db: kind.into(), id: name.trim_end_matches(".sql").into(), sha256: "0".repeat(64), status: Status::Applied, error: None, baseline: true });
}
// Resolve physical schemas once, then read all history in two client calls.
// Using a neutral connection prevents a Wildcard query from rerouting the other realm.
let connection = self.clone().for_realm(crate::realms::Mode::Coa);
let world = self.realm_schema("acore_world");
let characters = self.realm_schema("acore_characters");
let other = if self.realm() == crate::realms::Mode::Coa { "acore_world_wildcard" } else { "acore_world" };
let schemas: std::collections::BTreeSet<_> = [world, characters, other, AUTH_LEDGER_SCHEMA].into_iter().collect();
let catalog = connection.query(&format!("SELECT table_schema,table_name FROM information_schema.tables WHERE table_schema IN ('{}') AND table_name IN ('updates','{LEDGER_TABLE}') AND table_type='BASE TABLE';", schemas.into_iter().collect::<Vec<_>>().join("','")))?;
let available: std::collections::BTreeSet<_> = catalog.lines().filter_map(|line| line.split_once('\t')).collect();
let text = |expression: &str| format!("CONVERT({expression} USING utf8mb4) COLLATE utf8mb4_bin");
let mut queries = Vec::new();
for (kind, schema) in [("characters", characters), ("auth", AUTH_LEDGER_SCHEMA), ("world", world)] {
if available.contains(&(schema, "updates")) {
queries.push(format!("SELECT 'core','{kind}',{},REPEAT('0',64),'applied',1,'' FROM `{schema}`.`updates` WHERE BINARY RIGHT(name,4)='.sql'",text("LEFT(name,CHAR_LENGTH(name)-4)")));
}
}
for line in out.lines().filter(|l| !l.is_empty()) {
let f: Vec<&str> = line.split('\t').collect();
if f.len() < 5 {
return Err(Error::Invalid("The database migration history is malformed; no SQL was replayed.".into()));
for (origin, schema, filter) in [("local",world,""), ("other",other," WHERE `db` IN ('auth','coa-auth','wildcard-auth')"), ("shared",AUTH_LEDGER_SCHEMA,"")] {
if available.contains(&(schema, LEDGER_TABLE)) {
queries.push(format!("SELECT '{origin}',{},{},{},{},`baseline`,{} FROM `{schema}`.`{LEDGER_TABLE}`{filter}",text("`db`"),text("`id`"),text("`sha256`"),text("`status`"),text("COALESCE(HEX(`error`),'')")));
}
if !matches!(f[3], "applied" | "failed" | "running") || !ident_ok(f[1]) || f[2].len() != 64 || !f[2].chars().all(|c| c.is_ascii_hexdigit()) {
}
if queries.is_empty() { return Ok(vec![]); }
let out = connection.query(&queries.join(" UNION ALL "))?;
let (mut rows, mut local, mut auth, mut shared) = (Vec::new(), Vec::new(), Vec::new(), Vec::new());
for line in out.lines().filter(|l| !l.is_empty()) {
let f: Vec<_> = line.split('\t').collect();
if f.len() < 6 { return Err(Error::Invalid("The database migration history is malformed; no SQL was replayed.".into())); }
// Unrelated legacy filenames cannot match a valid manifest migration ID.
if f[0] == "core" && !ident_ok(f[2]) { continue; }
if !matches!(f[4], "applied" | "failed" | "running") || !ident_ok(f[2]) || f[3].len() != 64 || !f[3].chars().all(|c| c.is_ascii_hexdigit()) {
return Err(Error::Invalid("The database migration history is invalid; no SQL was replayed.".into()));
}
let err = f.get(5).filter(|h| !h.is_empty()).and_then(|h| hex::decode(h).ok()).map(|b| String::from_utf8_lossy(&b).into_owned());
rows.push(LedgerRow {
db: f[0].into(),
id: f[1].into(),
sha256: f[2].into(),
status: match f[3] { "applied" => Status::Applied, "running" => Status::Running, _ => Status::Failed },
baseline: f[4] == "1",
error: err,
});
let error = f.get(6).filter(|h| !h.is_empty()).and_then(|h| hex::decode(h).ok()).map(|b| String::from_utf8_lossy(&b).into_owned());
let row = LedgerRow { db:f[1].into(), id:f[2].into(), sha256:f[3].into(), status:match f[4] { "applied"=>Status::Applied,"running"=>Status::Running,_=>Status::Failed }, baseline:f[5]=="1", error };
match f[0] {
"core" => rows.push(row),
"local" if !shared_auth(&row.db) => local.push(row),
"local" | "other" => auth.push(row),
"shared" => shared.push(row),
_ => return Err(Error::Invalid("Unknown migration history source.".into())),
}
}
rows.extend(local);
// An interrupted legacy auth execution must not be hidden by another realm's applied row.
auth.sort_by_key(|r| (match r.status { Status::Applied=>0,Status::Running=>1,Status::Failed=>2,Status::Pending=>3 },r.sha256!="0".repeat(64)));
for mut row in auth { row.db="auth".into(); rows.push(row); }
// New auth records live with the shared account database and are authoritative.
for mut row in shared {
if !shared_auth(&row.db) { return Err(Error::Invalid("The shared authentication migration history is invalid.".into())); }
rows.retain(|r| !(shared_auth(&r.db) && r.id.eq_ignore_ascii_case(&row.id)));
row.db="auth".into(); rows.push(row);
}
Ok(rows)
}
fn put(&self, r: &LedgerRow) -> Result<()> {
if !ident_ok(&r.id) || !ident_ok(&r.db) || r.sha256.len() != 64 || !r.sha256.chars().all(|c| c.is_ascii_hexdigit()) {
return Err(Error::Invalid("migration record has an invalid identifier".into()));
fn put(&self, row: &LedgerRow) -> Result<()> { self.put_batch(std::slice::from_ref(row)) }
fn put_batch(&self, rows: &[LedgerRow]) -> Result<()> {
let mut groups = std::collections::BTreeMap::<&str, Vec<String>>::new();
for r in rows {
if !ident_ok(&r.id) || !ident_ok(&r.db) || r.sha256.len() != 64 || !r.sha256.chars().all(|c| c.is_ascii_hexdigit()) {
return Err(Error::Invalid("migration record has an invalid identifier".into()));
}
let status = match r.status { Status::Applied => "applied", Status::Running => "running", _ => "failed" };
let error = r.error.as_deref().map(|e| format!("UNHEX('{}')", hex_of(e))).unwrap_or_else(|| "NULL".into());
let (schema, kind) = if shared_auth(&r.db) { (AUTH_LEDGER_SCHEMA, "auth") } else { (LEDGER_SCHEMA, r.db.as_str()) };
groups.entry(schema).or_default().push(format!("('{}','{}','{}','{status}',NOW(),{error},{})",kind,r.id,r.sha256,r.baseline as u8));
}
for (schema, values) in groups {
self.query(&format!("REPLACE INTO `{schema}`.`{LEDGER_TABLE}` (`db`,`id`,`sha256`,`status`,`applied_at`,`error`,`baseline`) VALUES {};",values.join(",")))?;
}
let status = match r.status { Status::Applied => "applied", Status::Running => "running", _ => "failed" };
let err = r.error.as_deref().map(|e| format!("UNHEX('{}')", hex_of(e))).unwrap_or_else(|| "NULL".into());
self.query(&format!(
"REPLACE INTO `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db`,`id`,`sha256`,`status`,`applied_at`,`error`,`baseline`) VALUES ('{}','{}','{}','{status}',NOW(),{err},{});",
r.db, r.id, r.sha256, r.baseline as u8
))?;
Ok(())
}
@@ -137,21 +184,34 @@ pub fn status(store: &dyn Store, list: &[Migration]) -> Result<Vec<Item>> {
.map(|m| {
// Manager records are appended after the core's updates table and are authoritative.
let row = recorded(&rows, m);
let changed = row.is_some_and(|r| hash_changed(r, m));
let history = effective_history(&rows, m);
let changed = history.iter().any(|r| hash_changed(r,m));
let interrupted = history.iter().find(|r| matches!(r.status, Status::Running | Status::Failed));
Item {
id: m.id.clone(),
db: m.db.clone(),
status: if changed || row.is_some_and(|r| r.status == Status::Running) { Status::Failed } else { row.map(|r| r.status).unwrap_or(Status::Pending) },
status: if changed || interrupted.is_some() { Status::Failed } else { row.map(|r| r.status).unwrap_or(Status::Pending) },
destructive: m.destructive,
baseline: row.map(|r| r.baseline).unwrap_or(false),
error: if changed { Some("An already applied SQL file changed. Publish a new migration instead of replaying it.".into()) } else { row.and_then(|r| r.error.clone()) },
error: if changed { Some("An already applied SQL file changed. Publish a new migration instead of replaying it.".into()) } else { interrupted.and_then(|r| r.error.clone()).or_else(|| row.and_then(|r| r.error.clone())) },
}
})
.collect())
}
fn recorded<'a>(rows: &'a [LedgerRow], m: &Migration) -> Option<&'a LedgerRow> {
rows.iter().rev().find(|r| r.db == m.db && r.id == m.id)
rows.iter().rev().find(|r| (r.db == m.db || shared_auth(&r.db) && shared_auth(&m.db)) && r.id.eq_ignore_ascii_case(&m.id))
}
fn effective_history<'a>(rows: &'a [LedgerRow], m: &Migration) -> Vec<&'a LedgerRow> {
if shared_auth(&m.db) { rows.iter().filter(|r| shared_auth(&r.db) && r.id.eq_ignore_ascii_case(&m.id)).collect() }
else { recorded(rows, m).into_iter().collect() }
}
fn preserve_auth_history(store: &dyn Store, rows: &[LedgerRow], list: &[Migration]) -> Result<()> {
let applied: Vec<_> = list.iter().filter(|m| shared_auth(&m.db)).filter_map(|m| recorded(rows,m))
.filter(|r| r.status == Status::Applied).cloned().collect();
store.put_batch(&applied)
}
/// Migrations that were published with different bytes but have the same effect, so a server that applied either
@@ -166,7 +226,7 @@ const SAME_EFFECT: &[(&str, &[&str])] = &[(
)];
fn same_effect(id: &str, a: &str, b: &str) -> bool {
SAME_EFFECT.iter().any(|(known, hashes)| *known == id
SAME_EFFECT.iter().any(|(known, hashes)| known.eq_ignore_ascii_case(id)
&& hashes.iter().any(|h| h.eq_ignore_ascii_case(a)) && hashes.iter().any(|h| h.eq_ignore_ascii_case(b)))
}
@@ -183,8 +243,7 @@ pub fn preflight(store: &dyn Store, list: &[Migration]) -> Result<()> {
pub fn pending_count(store: &dyn Store, list: &[Migration]) -> Result<usize> {
let rows = store.load_readonly()?;
check_history(&rows, list)?;
Ok(list.iter().filter(|m| recorded(&rows, m).is_none_or(|r| r.status != Status::Applied)).count())
Ok(pending_migrations(&rows, list)?.len())
}
fn verified_file(m: &Migration, dir: &Path) -> Result<(&'static str, std::path::PathBuf, bool)> {
@@ -199,22 +258,39 @@ fn verified_file(m: &Migration, dir: &Path) -> Result<(&'static str, std::path::
/// Check every migration artifact, including applied history, before replacing server files.
pub fn verify_files(list: &[Migration], dir: &Path) -> Result<()> {
validate_list(list)?;
for m in list { verified_file(m, dir)?; }
Ok(())
}
fn validate_list(list: &[Migration]) -> Result<()> {
let mut seen = std::collections::BTreeSet::new();
for m in list {
if !seen.insert((&m.db, &m.id)) { return Err(Error::InvalidManifest(format!("duplicate migration {} ({})", m.id, m.db))); }
verified_file(m, dir)?;
if !ident_ok(&m.id) { return Err(Error::InvalidManifest(format!("migration id {:?} is not allowed", m.id))); }
let schema = db::schema_of(&m.db)?;
if !seen.insert((schema, m.id.to_ascii_lowercase())) {
return Err(Error::InvalidManifest(format!("duplicate migration {} ({})", m.id, m.db)));
}
}
Ok(())
}
fn pending_migrations<'a>(rows: &[LedgerRow], list: &'a [Migration]) -> Result<Vec<&'a Migration>> {
check_history(rows, list)?;
Ok(list.iter().filter(|m| recorded(rows, m).is_none_or(|r| r.status != Status::Applied)).collect())
}
fn check_history(rows: &[LedgerRow], list: &[Migration]) -> Result<()> {
validate_list(list)?;
for m in list {
if recorded(rows, m).is_some_and(|r| matches!(r.status, Status::Running | Status::Failed)) {
for row in effective_history(rows,m) {
if matches!(row.status, Status::Running | Status::Failed) {
return Err(Error::Invalid(format!("Database update {} ({}) failed or was interrupted and may be partially applied. Restore its recovery point before retrying.", m.id, m.db)));
}
if recorded(rows, m).is_some_and(|r| hash_changed(r, m)) {
if hash_changed(row,m) {
return Err(Error::Invalid(format!("Applied database update {} ({}) has a different checksum. A new migration is required; it was not replayed.", m.id, m.db)));
}
}
}
Ok(())
}
@@ -223,9 +299,11 @@ fn check_history(rows: &[LedgerRow], list: &[Migration]) -> Result<()> {
/// was brought up to date by hand).
pub fn baseline(store: &dyn Store, list: &[Migration]) -> Result<usize> {
let rows = store.load()?;
check_history(&rows,list)?;
preserve_auth_history(store,&rows,list)?;
let mut n = 0;
for m in list {
if rows.iter().any(|r| r.db == m.db && r.id == m.id && r.status == Status::Applied) {
if recorded(&rows,m).is_some_and(|r| r.status == Status::Applied) {
continue;
}
store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Applied, error: None, baseline: true })?;
@@ -246,8 +324,7 @@ pub struct ApplyReport {
pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot: &dyn Fn() -> Result<String>) -> Result<ApplyReport> {
let rows = store.load()?;
// The list order is the release's order of application (released updates, then pending, then modules).
check_history(&rows, list)?;
let todo: Vec<&Migration> = list.iter().filter(|m| recorded(&rows, m).is_none_or(|r| r.status != Status::Applied)).collect();
let todo = pending_migrations(&rows, list)?;
tracing::info!(total = list.len(), pending = todo.len(), skipped = list.len() - todo.len(), "database migration plan");
// Resolve and verify every file before running anything.
@@ -261,11 +338,13 @@ pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot
let mut report = ApplyReport { applied: Vec::new(), failed: None, snapshot: None };
if files.is_empty() {
preserve_auth_history(store,&rows,list)?;
return Ok(report);
}
if any_destructive {
report.snapshot = Some(snapshot().map_err(|e| Error::Invalid(format!("No database update was applied because the safety backup failed: {e}")))?);
}
preserve_auth_history(store,&rows,list)?;
for (m, schema, path) in files {
tracing::info!(migration = %m.id, database = %m.db, "database migration starting");
store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Running, error: Some("Interrupted SQL must be recovered before replay.".into()), baseline: false })?;
@@ -368,6 +447,23 @@ mod tests {
assert!(store.ran.borrow().is_empty());
}
#[test]
fn every_entry_point_rejects_duplicate_sql_before_execution() {
let (dir, original) = setup(&[("world", "once", "SELECT 1;\n")]);
for alias in ["world", "coa-world"] {
let mut list = original.clone();
let mut duplicate = original[0].clone();
duplicate.db = alias.into();
list.push(duplicate);
let store = Mem::default();
assert!(preflight(&store, &list).is_err());
assert!(pending_count(&store, &list).is_err());
assert!(apply_pending(&store, &list, dir.path(), &no_snapshot).is_err());
assert!(store.ran.borrow().is_empty());
assert!(store.rows.borrow().is_empty());
}
}
#[test]
fn staged_files_reject_missing_corrupt_duplicate_and_invalid_migrations() {
let (dir, list) = setup(&[("characters", "repair", "SELECT 1;\n")]);
+45 -155
View File
@@ -129,10 +129,6 @@ pub(crate) fn inventory(root: &Path) -> Result<BTreeMap<String, String>> {
}
pub(crate) fn copy_verified(source: &Path, target: &Path, files: &BTreeMap<String, String>) -> Result<()> {
copy_verified_with_progress(source, target, files, &|_| {})
}
fn copy_verified_with_progress(source: &Path, target: &Path, files: &BTreeMap<String, String>, report: &dyn Fn(&str)) -> Result<()> {
if target.exists() { return Err(Error::Invalid("Database snapshot destination already exists.".into())); }
let bytes = files.keys().try_fold(0u64, |total, name| -> Result<u64> {
let path = fsx::ensure_within(source, &fsx::safe_join(source, name)?)?;
@@ -142,15 +138,13 @@ fn copy_verified_with_progress(source: &Path, target: &Path, files: &BTreeMap<St
fs::create_dir_all(parent)?;
fsx::require_space(parent, bytes)?;
fs::create_dir(target)?;
for (index, (relative, expected)) in files.iter().enumerate() {
if index % 250 == 0 { report(&format!("Copying private server files: {} of {}", index, files.len())); }
for (relative, expected) in files {
let source_file = fsx::ensure_within(source, &fsx::safe_join(source, relative)?)?;
let target_file = fsx::ensure_within(target, &fsx::safe_join(target, relative)?)?;
fs::create_dir_all(target_file.parent().unwrap())?;
fs::copy(&source_file, &target_file)?;
if fsx::sha256_file(&target_file)? != *expected { return Err(Error::Invalid(format!("Database snapshot file {relative} failed verification."))); }
}
report("Verifying the complete private file copy");
if inventory(target)? != *files { return Err(Error::Invalid("Database snapshot inventory differs.".into())); }
Ok(())
}
@@ -250,103 +244,6 @@ pub(crate) fn verify(point: &Path, snapshot: &Snapshot) -> Result<()> {
Ok(())
}
pub(crate) fn rehearsal_copy_with_progress(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot, report: &dyn Fn(&str)) -> Result<PathBuf> {
report("Verifying the database recovery copy before rehearsal");
verify(point, snapshot)?;
let canonical = fsx::canonicalize_lenient(root)?;
let root = canonical.as_path();
let destination = meta.join("rehearsals").join(uuid::Uuid::new_v4().simple().to_string()).join("server");
let mut files = BTreeMap::new();
let mut file_bytes = 0u64;
let mut directories = vec![root.to_path_buf()];
while let Some(directory) = directories.pop() {
for entry in fs::read_dir(directory)? {
let entry = entry?;
let raw = entry.path();
let relative = raw.strip_prefix(root).map_err(|e| Error::Invalid(e.to_string()))?.to_string_lossy().replace('\\', "/");
let lower = relative.to_ascii_lowercase();
let first = lower.split('/').next().unwrap();
if entry.file_type()?.is_dir() && !relative.contains('/')
&& !["core", "data", "scripts", "settings", "runtime", "mysql", ".realms", "bugreport", "extras", "licenses"].contains(&first) { continue; }
if lower == "mysql/data" || lower.starts_with("mysql/data-")
|| lower.split('/').any(|part| part == ".state")
|| lower == "bugreport/reports" || lower == "core/logs" || lower == "mysql/logs" { continue; }
let metadata = fs::symlink_metadata(&raw)?;
reject_link(&metadata)?;
let path = fsx::ensure_within(root, &raw)?;
if metadata.is_dir() { directories.push(path); }
else if metadata.is_file() {
file_bytes = file_bytes.checked_add(metadata.len()).ok_or_else(|| Error::Invalid("Private copy size overflow.".into()))?;
files.insert(relative, fsx::sha256_file(&path)?);
if files.len() % 250 == 0 { report(&format!("Preparing the private copy: {} files checked", files.len())); }
}
else { return Err(Error::Invalid("Unsupported rehearsal file.".into())); }
}
}
let needed = file_bytes.checked_add(snapshot.expanded_bytes.max(snapshot.bytes)).ok_or_else(|| Error::Invalid("Private copy size overflow.".into()))?;
fsx::require_space(meta, needed)?;
copy_verified_with_progress(root, &destination, &files, report)?;
report("Restoring MySQL into the private server copy");
copy_snapshot(point, snapshot, &destination.join("mysql/data"))?;
let (_, mut installed) = crate::registry::MetaDir::open(meta)?;
installed.id = uuid::Uuid::new_v4().to_string();
installed.server_path = destination.to_string_lossy().into_owned();
installed.client_path = None;
crate::registry::MetaDir::create(&destination, &installed)?;
Ok(destination)
}
pub(crate) fn isolate_connections(root: &Path) -> Result<()> {
for relative in ["Settings/worldserver.conf.template", "Settings/authserver.conf.template", "Core/configs/worldserver.conf", "Core/configs/authserver.conf"] {
let path = fsx::ensure_within(root, &root.join(relative))?;
if !path.is_file() {
if relative.ends_with(".template") { return Err(Error::Invalid("The private update copy lacks connection templates.".into())); }
continue;
}
let mut config = crate::config::parser::ConfFile::parse_bytes(&fs::read(&path)?)?;
for key in ["LoginDatabaseInfo", "WorldDatabaseInfo", "CharacterDatabaseInfo"] {
if relative.contains("worldserver") || key == "LoginDatabaseInfo" {
config.set(key, &format!("\"@{key}@\""), &[]);
}
}
config.set("BindIP", "\"127.0.0.1\"", &[]);
if relative.contains("worldserver") {
config.set("Ra.IP", "\"127.0.0.1\"", &[]);
config.set("DataDir", "\"@DATA@\"", &[]);
}
fsx::atomic_write(&path, config.to_text().as_bytes())?;
}
Ok(())
}
pub(crate) fn unchanged(root: &Path, snapshot: &Snapshot) -> Result<()> {
let data = fsx::ensure_within(root, &root.join("mysql/data"))?;
if inventory(&data)? != snapshot.files || directories(&data)? != snapshot.directories {
return Err(Error::Invalid("The installed database changed during the private rehearsal. Retry after stopping other database access.".into()));
}
Ok(())
}
pub(crate) fn validate_connections(templates: &Path, installation: &Path) -> Result<()> {
let port = crate::layout::read_ports(installation).mysql.to_string();
let (_, password) = crate::db::credentials(installation)?;
let realm = crate::realms::state(installation)?.active;
for relative in ["Settings/worldserver.conf.template", "Settings/authserver.conf.template"] {
let path = fsx::ensure_within(templates, &templates.join(relative))?;
let config = crate::config::parser::ConfFile::parse_bytes(&fs::read(path)?)?;
for (key, kind) in [("LoginDatabaseInfo", "auth"), ("WorldDatabaseInfo", "world"), ("CharacterDatabaseInfo", "characters")] {
if relative.contains("authserver") && kind != "auth" { continue; }
let value = config.get(key).ok_or_else(|| Error::Invalid(format!("The update cannot isolate the missing {key} connection.")))?.trim().trim_matches('"');
if value == format!("@{key}@") { continue; }
let fields: Vec<_> = value.split(';').collect();
if fields.len() != 5 || !matches!(fields[0], "127.0.0.1" | "localhost") || fields[1] != port || fields[2] != "acore" || fields[3] != password || fields[4] != realm.schema(kind)? {
return Err(Error::Invalid(format!("The {key} connection does not use this repack's bundled database. A private update cannot safely qualify an external database.")));
}
}
}
Ok(())
}
// Keep the replaced data directory. If interrupted between renames, a retry can install a new verified copy.
pub(crate) fn swap(root: &Path, meta: &Path, point: &Path, snapshot: &Snapshot, fail_after_move: bool) -> Result<PathBuf> {
verify(point, snapshot)?;
@@ -574,6 +471,46 @@ mod tests {
};
let mut server = start();
db.query("CREATE DATABASE acore_characters; CREATE DATABASE acore_world; CREATE DATABASE acore_auth; CREATE TABLE acore_characters.probe(id INT PRIMARY KEY, note VARCHAR(40)); CREATE TABLE acore_world.probe(id INT NOT NULL DEFAULT 0); INSERT INTO acore_world.probe VALUES(43); CREATE TABLE acore_auth.probe(id INT); INSERT INTO acore_auth.probe VALUES(44); CREATE USER 'fixture_user'@'localhost' IDENTIFIED BY 'fixture'; CREATE TRIGGER acore_characters.probe_insert BEFORE INSERT ON acore_characters.probe FOR EACH ROW SET NEW.note='triggered'; INSERT INTO acore_characters.probe VALUES(42,'original'); CREATE VIEW acore_characters.probe_view AS SELECT c.id,w.id AS world_id FROM acore_characters.probe c CROSS JOIN acore_world.probe w; CREATE PROCEDURE acore_characters.probe_proc() SELECT id FROM acore_characters.probe; CREATE FUNCTION acore_characters.probe_func() RETURNS INT DETERMINISTIC RETURN 73;").unwrap();
// Shared auth must not run again when the active realm changes.
use crate::migrations::Store;
db.query("CREATE DATABASE acore_world_wildcard;").unwrap();
let migration_dir = root.join("migration-fixture");
fs::create_dir_all(migration_dir.join("auth")).unwrap();
fs::create_dir_all(migration_dir.join("world")).unwrap();
let auth_sql = b"INSERT INTO acore_auth.probe VALUES(45);";
fs::write(migration_dir.join("auth/shared_once.sql"), auth_sql).unwrap();
let auth_list = vec![crate::manifest::Migration { id:"shared_once".into(), db:"auth".into(), sha256:fsx::sha256_bytes(auth_sql), destructive:false, compatible_sha256:vec![] }];
assert_eq!(crate::migrations::apply_pending(&db, &auth_list, &migration_dir, &|| Ok("fixture".into())).unwrap().applied.len(), 1);
let wildcard = db.clone().for_realm(crate::realms::Mode::Wildcard);
assert_eq!(crate::migrations::pending_count(&wildcard, &auth_list).unwrap(), 0, "auth is shared by both realms");
assert!(crate::migrations::apply_pending(&wildcard, &auth_list, &migration_dir, &|| Ok("fixture".into())).unwrap().applied.is_empty());
assert_eq!(db.query("SELECT COUNT(*) FROM acore_auth.probe WHERE id=45;").unwrap(), "1");
let mut failed = crate::migrations::LedgerRow { db:"auth".into(), id:"shared_once".into(), sha256:auth_list[0].sha256.clone(), status:crate::migrations::Status::Failed, error:Some("interrupted fixture".into()), baseline:false };
db.put(&failed).unwrap();
assert!(crate::migrations::pending_count(&wildcard, &auth_list).is_err());
failed.status = crate::migrations::Status::Applied; failed.error = None; db.put(&failed).unwrap();
// Legacy per-world auth records are adopted without replaying their SQL.
let legacy_sql = b"INSERT INTO acore_auth.probe VALUES(46);";
fs::write(migration_dir.join("auth/legacy_auth.sql"), legacy_sql).unwrap();
db.query(&format!("INSERT INTO acore_auth.probe VALUES(46); REPLACE INTO acore_world.coa_manager_migrations VALUES('auth','legacy_auth','{}','applied',NOW(),NULL,0);", fsx::sha256_bytes(legacy_sql))).unwrap();
let legacy_auth = vec![crate::manifest::Migration { id:"legacy_auth".into(), db:"auth".into(), sha256:fsx::sha256_bytes(legacy_sql), destructive:false, compatible_sha256:vec![] }];
assert_eq!(crate::migrations::pending_count(&wildcard, &legacy_auth).unwrap(), 0);
assert!(crate::migrations::apply_pending(&wildcard, &legacy_auth, &migration_dir, &|| Ok("fixture".into())).unwrap().applied.is_empty());
assert_eq!(db.query("SELECT COUNT(*) FROM acore_auth.probe WHERE id=46;").unwrap(), "1");
// A core history cleanup cannot make previously recorded SQL pending again.
db.query("CREATE TABLE acore_world.updates(name VARCHAR(150) PRIMARY KEY) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; INSERT INTO acore_world.updates VALUES('LEGACY_WORLD.sql'); INSERT INTO acore_world.probe VALUES(99);").unwrap();
db.load().unwrap();
db.query("DROP TABLE acore_world.updates;").unwrap();
let legacy_world_sql = b"INSERT INTO acore_world.probe VALUES(99);";
fs::write(migration_dir.join("world/legacy_world.sql"), legacy_world_sql).unwrap();
let legacy_world = vec![crate::manifest::Migration { id:"legacy_world".into(), db:"world".into(), sha256:fsx::sha256_bytes(legacy_world_sql), destructive:false, compatible_sha256:vec![] }];
assert_eq!(crate::migrations::pending_count(&db, &legacy_world).unwrap(), 0);
assert!(crate::migrations::apply_pending(&db, &legacy_world, &migration_dir, &|| Ok("fixture".into())).unwrap().applied.is_empty());
assert_eq!(db.query("SELECT COUNT(*) FROM acore_world.probe WHERE id=99;").unwrap(), "1");
db.query("DELETE FROM acore_world.probe WHERE id=99;").unwrap();
assert_eq!(db.recovery_objects("acore_characters").unwrap().len(), 4);
db.query("CREATE TABLE acore_characters.coa_squid_pending(path VARCHAR(255), sha256 CHAR(64)); INSERT INTO acore_characters.coa_squid_pending VALUES('playerbots/base/ai_playerbot_texts.sql', REPEAT('a',64));").unwrap();
let squid_problems = crate::squid::database_problems(&root, crate::realms::Mode::Coa, &|sql| {
@@ -664,57 +601,10 @@ mod tests {
fsx::atomic_write(&archive_path, b"damaged").unwrap();
assert!(verify(&point, &snapshot).is_err());
}
#[test]
fn private_rehearsals_reject_external_database_connections() {
let (_temp, root, _meta, _point, _snapshot) = fixture();
fsx::atomic_write_json(&root.join("Settings/repack.json"), &serde_json::json!({"mysqlPort":3307})).unwrap();
fsx::atomic_write_json(&root.join("Settings/database.json"), &serde_json::json!({"rootPassword":"test-root","appPassword":"test-app"})).unwrap();
fsx::atomic_write(&root.join("Settings/worldserver.conf.template"), b"LoginDatabaseInfo = \"@LoginDatabaseInfo@\"\nWorldDatabaseInfo = \"@WorldDatabaseInfo@\"\nCharacterDatabaseInfo = \"@CharacterDatabaseInfo@\"\n").unwrap();
fsx::atomic_write(&root.join("Settings/authserver.conf.template"), b"LoginDatabaseInfo = \"127.0.0.1;3307;acore;test-app;acore_auth\"\n").unwrap();
validate_connections(&root, &root).unwrap();
fsx::atomic_write(&root.join("Settings/authserver.conf.template"), b"LoginDatabaseInfo = \"192.0.2.1;3307;acore;test-app;acore_auth\"\n").unwrap();
let error = validate_connections(&root, &root).unwrap_err().to_string();
assert!(error.contains("external database"));
assert!(!error.contains("test-app"));
}
#[test]
fn a_changed_live_database_invalidates_the_rehearsal() {
let (_temp, root, _meta, _point, snapshot) = fixture();
assert!(unchanged(&root, &snapshot).is_err());
fsx::atomic_write(&root.join("mysql/data/characters.ibd"), b"saved").unwrap();
fs::create_dir_all(root.join("mysql/data/empty_schema")).unwrap();
unchanged(&root, &snapshot).unwrap();
fsx::atomic_write(&root.join("mysql/data/extra.ibd"), b"new data").unwrap();
assert!(unchanged(&root, &snapshot).is_err());
}
#[test]
fn rehearsal_copy_succeeds_with_nested_directories_and_unnormalized_root() {
let (_temp, root, meta, point, snapshot) = fixture();
let meta_obj = crate::registry::InstallMeta::new(crate::registry::InstallKind::New, &root);
fsx::atomic_write_json(&meta.join("install.json"), &meta_obj).unwrap();
fsx::atomic_write(&root.join("Core/worldserver.exe"), b"exe").unwrap();
fsx::atomic_write(&root.join("Core/configs/worldserver.conf"), b"conf").unwrap();
fsx::atomic_write(&root.join("Settings/repack.json"), b"{}").unwrap();
let phases = std::cell::RefCell::new(Vec::new());
let dest = rehearsal_copy_with_progress(&root, &meta, &point, &snapshot, &|phase| phases.borrow_mut().push(phase.to_string())).unwrap();
assert_eq!(fs::read(dest.join("Core/configs/worldserver.conf")).unwrap(), b"conf");
assert_eq!(fs::read(dest.join("mysql/data/characters.ibd")).unwrap(), b"saved");
let phases = phases.into_inner();
assert!(phases[0].contains("Verifying the database"));
assert!(phases.iter().any(|phase| phase.contains("Copying private server files")));
assert!(phases.last().unwrap().contains("Restoring MySQL"));
}
#[test]
fn rehearsal_checks_combined_program_and_database_space_before_copying() {
let (_temp, root, meta, point, mut snapshot) = fixture();
fs::create_dir_all(&meta).unwrap();
fsx::atomic_write(&root.join("Core/worldserver.exe"), b"exe").unwrap();
snapshot.expanded_bytes = fsx::free_space(&meta).unwrap().saturating_add(1024 * 1024 * 1024);
let error = rehearsal_copy_with_progress(&root, &meta, &point, &snapshot, &|_| {}).unwrap_err();
assert!(matches!(error, Error::InsufficientSpace { .. }));
assert!(!meta.join("rehearsals").exists());
assert_eq!(fs::read(root.join("mysql/data/characters.ibd")).unwrap(), b"live");
}
}
+2 -1
View File
@@ -348,7 +348,8 @@ AC>";
let (port, _h) = fake_ra("Account not found.");
let mut ra = Ra::connect_to(port, "u", "p").unwrap();
assert!(ra.set_account_password("x", "newpass9").is_err(), "the name is validated before anything is sent");
assert!(ra.set_account_password("Nobody", "newpass9").unwrap_err().to_string().contains("not found"));
let error = ra.set_account_password("Nobody", "newpass9").unwrap_err().to_string();
assert!(error.contains("not found"), "Unexpected console error: {error}");
}
#[test]
+1 -3
View File
@@ -14,8 +14,6 @@ impl crate::update::Env for QualificationEnv<'_> {
fn preflight(&self, manifest: &crate::manifest::Manifest) -> Result<()> { self.0.preflight(manifest) }
fn snapshot(&self) -> Result<String> { self.0.snapshot() }
fn verify_snapshot(&self, id: &str) -> Result<()> { self.0.verify_snapshot(id) }
fn rehearse(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str) -> Result<()> { self.0.rehearse(manifest, tree, items, point) }
fn rehearse_with_progress(&self, manifest: &crate::manifest::Manifest, tree: &Path, items: &[crate::update::PlanItem], point: &str, report: &dyn Fn(&str, u8)) -> Result<()> { self.0.rehearse_with_progress(manifest, tree, items, point, report) }
fn restore_snapshot(&self, id: &str) -> Result<()> { self.0.restore_snapshot(id) }
fn migrate(&self, manifest: &crate::manifest::Manifest, staged: &Path) -> Result<crate::migrations::ApplyReport> { self.0.migrate(manifest, staged) }
fn validate(&self) -> Result<()> { self.0.validate() }
@@ -87,7 +85,7 @@ pub fn qualify_upgrade(base: &Path, source: &Path, candidate: &Path, fixture: &P
"schema": 1, "result": "passed", "fromVersion": source_manifest.version,
"toVersion": candidate_manifest.version, "fromManifestSha256": fsx::sha256_bytes(&source_bytes),
"fromSchemaSha256": source_manifest.files.iter().find(|file| file.path == schema_check::CONTRACT).map(|file| &file.sha256),
"checks": ["manager-transaction", "private-user-database-rehearsal", "database-schema", "private-auth-world-startup", "existing-identities", "user-file-preservation"],
"checks": ["manager-transaction", "recovery-backed-database-update", "database-schema", "private-auth-world-startup", "existing-identities", "user-file-preservation"],
}))
}
+64 -161
View File
@@ -92,9 +92,6 @@ pub trait Env {
fn ensure_stopped(&self) -> Result<()>;
fn preflight(&self, _manifest: &Manifest) -> Result<()> { Ok(()) }
fn verify_snapshot(&self, _id: &str) -> Result<()> { Ok(()) }
fn rehearse(&self, _manifest: &Manifest, _tree: &Path, _items: &[PlanItem], _point: &str) -> Result<()> { Ok(()) }
fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], point: &str, _report: &dyn Fn(&str, u8)) -> Result<()> { self.rehearse(manifest, tree, items, point) }
/// Create a recovery point for the databases and configuration; returns its id.
fn snapshot(&self) -> Result<String>;
fn restore_snapshot(&self, id: &str) -> Result<()>;
fn migrate(&self, manifest: &Manifest, staged_migrations: &Path) -> Result<ApplyReport>;
@@ -512,14 +509,6 @@ pub fn apply(p: &Params, report: &dyn Fn(&str, u8)) -> Result<Outcome> {
let point_meta = crate::backup::point_json(meta_dir, txn.recovery_point.as_deref().unwrap())?;
if point_meta.is_file() { txn.recovery_hashes.insert("@point".into(), fsx::sha256_file(&point_meta)?); }
save(meta_dir, &txn)?;
step(report, "Testing the update on a private copy of your server", 54);
if let Err(error) = p.env.rehearse_with_progress(&m, &tree, &items, txn.recovery_point.as_deref().unwrap(), report) {
txn.state = State::Failed;
txn.message = Some(format!("The private update rehearsal failed: {error}. The installed server was not updated."));
save(meta_dir, &txn)?;
return Err(Error::Invalid(txn.message.unwrap()));
}
txn.ops = items
.iter()
.filter(|i| i.action != Action::Skip)
@@ -907,7 +896,7 @@ impl RepackEnv<'_> {
if realms.wildcard_created { modes.push(if realms.active == crate::realms::Mode::Coa { crate::realms::Mode::Wildcard } else { crate::realms::Mode::Coa }); }
let mut count = 0;
for (index, mode) in modes.into_iter().enumerate() {
let list: Vec<_> = manifest.migrations.iter().filter(|m| index == 0 || m.db != "auth").cloned().collect();
let list: Vec<_> = manifest.migrations.iter().filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)).cloned().collect();
count += crate::migrations::pending_count(&db.clone().for_realm(mode), &list)?;
}
Ok(count)
@@ -973,85 +962,6 @@ impl Env for RepackEnv<'_> {
}
fn automatic_rollback(&self) -> bool { true }
fn rehearse(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str) -> Result<()> {
self.rehearse_with_progress(manifest, tree, items, id, &|_, _| {})
}
fn rehearse_with_progress(&self, manifest: &Manifest, tree: &Path, items: &[PlanItem], id: &str, report: &dyn Fn(&str, u8)) -> Result<()> {
if items.iter().any(|item| item.action != Action::Skip && item.path.to_ascii_lowercase().starts_with("mysql/")) {
return Err(Error::Invalid("A server update cannot replace MySQL while using a cold recovery copy; a separate database engine upgrade is required.".into()));
}
let point = crate::backup::get(self.meta_dir, id)?;
let Some(snapshot) = &point.mysql_snapshot else {
// Docker uses its separate experimental release channel until its clone runner is qualified.
if crate::docker::is_docker(self.root) { return Ok(()); }
return Err(Error::Invalid("This installation has no complete MySQL recovery copy for an update rehearsal.".into()));
};
crate::mysql_snapshot::validate_connections(self.root, self.root)?;
crate::mysql_snapshot::stop(self.root)?;
let point_dir = crate::backup::point_json(self.meta_dir, id)?.parent().unwrap().to_path_buf();
let fixture = crate::mysql_snapshot::rehearsal_copy_with_progress(self.root, self.meta_dir, &point_dir, snapshot, &|phase| step(report, phase, 54))?;
let metadata = crate::registry::metadata_dir_for(&fixture)?;
let env = RepackEnv { root: &fixture, meta_dir: &metadata };
let result = crate::install::with_scratch_ports(&fixture, || {
let mut txn = Txn { id: "rehearsal".into(), state: State::Applying, from_version: None,
to_version: manifest.version.clone(), recovery_point: None, databases_started: false,
ops: items.iter().filter(|item| item.action != Action::Skip).map(|item| Op {
path: item.path.clone(), action: item.action, reason: item.reason.clone(),
new_sha256: manifest.files.iter().find(|file| file.path == item.path).unwrap().sha256.clone(),
had_previous: fsx::safe_join(&fixture, &item.path).is_ok_and(|path| path.is_file()), started:false, done:false,
}).collect(), message:None, recovery_hashes:BTreeMap::new() };
let before = txn_dir(&metadata, &txn.id)?.join("before");
fs::create_dir_all(&before)?;
step(report, "Applying update files to the private server copy", 55);
apply_ops(&fixture, tree, &before, &manifest.files, &mut txn, &metadata, None)?;
crate::mysql_snapshot::validate_connections(&fixture, self.root)?;
crate::mysql_snapshot::isolate_connections(&fixture)?;
step(report, "Testing database updates on the private server copy", 56);
let identities = crate::backup::with_database(&fixture, |db| {
let identities = database_identities(db)?;
let realms = crate::realms::state(&fixture)?;
let mut modes = vec![realms.active];
if realms.wildcard_created { modes.push(if realms.active == crate::realms::Mode::Coa { crate::realms::Mode::Wildcard } else { crate::realms::Mode::Coa }); }
for (index, mode) in modes.into_iter().enumerate() {
let db = db.clone().for_realm(mode);
crate::schema_check::repair_missing_defaults(&db, &fixture, manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str()))?;
let list: Vec<_> = manifest.migrations.iter().filter(|migration| index == 0 || migration.db != "auth").cloned().collect();
let report = crate::migrations::apply_pending(&db, &list, &tree.join("_migrations"), &|| Ok("private rehearsal".into()))?;
if let Some((migration, error)) = report.failed { return Err(Error::Invalid(format!("Rehearsal migration {migration} failed: {error}"))); }
crate::schema_check::repair_missing_defaults(&db, &fixture, manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str()))?;
}
verify_database_identities(db, &identities)?;
Ok(identities)
})?;
step(report, "Testing startup of the private server copy", 57);
env.validate()?;
crate::backup::with_database(&fixture, |db| verify_database_identities(db, &identities))?;
env.ensure_stopped()
});
step(report, "Stopping the private server copy and verifying the installed database", 57);
let stopped = crate::mysql_snapshot::stop(&fixture);
let live_state = if result.is_ok() { crate::mysql_snapshot::stop(self.root).and_then(|()| crate::mysql_snapshot::unchanged(self.root, snapshot)) } else { Ok(()) };
let logs: BTreeMap<_, _> = ["Core/Logs/auth-console.log", "Core/Logs/world-console.log", "Core/Logs/supervisor.log", "mysql/logs/mysql-error.log"].into_iter()
.map(|path| (path, crate::diag::redact(&crate::health::tail(&fixture.join(path), 32768)))).collect();
let mut schema_reports = Vec::<serde_json::Value>::new();
if let Ok(entries) = fs::read_dir(metadata.join("diagnostics")) {
for entry in entries.flatten().filter(|entry| entry.file_name().to_string_lossy().starts_with("schema-validation-")) {
let path = fsx::ensure_within(&metadata, &entry.path())?;
if fs::metadata(&path)?.len() <= 4 * 1024 * 1024 { schema_reports.push(fsx::read_json(&path)?); }
}
}
let report = serde_json::json!({"schema":1,"checkedAt":chrono::Utc::now().to_rfc3339(),"toVersion":manifest.version,"recoveryPoint":id,
"result":if result.is_ok() && stopped.is_ok() && live_state.is_ok() {"passed"} else {"failed"}, "error":result.as_ref().err().map(ToString::to_string), "stopError":stopped.as_ref().err().map(ToString::to_string), "liveStateError":live_state.as_ref().err().map(ToString::to_string), "logs":logs,"schemaReports":schema_reports,"fixture":fixture});
let report_path = self.meta_dir.join("diagnostics").join(format!("database-rehearsal-{}.json", uuid::Uuid::new_v4()));
fsx::atomic_write_json(&report_path, &report)?;
stopped?;
let parent = fixture.parent().unwrap();
let owned = fsx::ensure_within(&self.meta_dir.join("rehearsals"), parent)?;
fs::remove_dir_all(owned)?;
live_state?;
result
}
fn preflight(&self, manifest: &Manifest) -> Result<()> {
crate::mysql_snapshot::stop_previous_rehearsals(self.meta_dir)?;
if let Some(compatibility) = &manifest.compatibility {
@@ -1071,7 +981,7 @@ impl Env for RepackEnv<'_> {
if crate::docker::is_docker(self.root) {
return Err(Error::Invalid(format!("The installed {} database does not match its upgrade source: {}.{}: {}", mode.name(), problem.database, problem.table, problem.detail)));
}
tracing::warn!(realm = mode.name(), table = %problem.table, "Installed schema differs; the private candidate rehearsal must qualify this database");
tracing::warn!(realm = mode.name(), table = %problem.table, "Installed schema differs; pending migrations and target validation will run with a full recovery point");
}
}
Ok(())
@@ -1126,43 +1036,10 @@ impl Env for RepackEnv<'_> {
}
fn migrate(&self, manifest: &Manifest, staged: &Path) -> Result<ApplyReport> {
let root = self.root;
crate::backup::with_database(root, |db| {
let realms = crate::realms::state(root)?;
if let Some(contract) = manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT) {
self.migration_snapshot()?;
crate::schema_check::repair_missing_defaults(db, root, Some(&contract.sha256))?;
}
let mut result = crate::migrations::apply_pending(db, &manifest.migrations, staged, &|| {
self.migration_snapshot()
})?;
if realms.wildcard_created && result.failed.is_none() {
let other = if realms.active == crate::realms::Mode::Coa { crate::realms::Mode::Wildcard } else { crate::realms::Mode::Coa };
let other_db = db.clone().for_realm(other);
crate::schema_check::repair_missing_defaults(&other_db, root, manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str()))?;
let shared_snapshot = result.snapshot.clone();
let migrations: Vec<_> = manifest.migrations.iter().filter(|m| m.db != "auth").cloned().collect();
let extra = crate::migrations::apply_pending(&other_db, &migrations, staged, &|| {
if let Some(id) = &shared_snapshot { return Ok(id.clone()); }
self.migration_snapshot()
})?;
result.applied.extend(extra.applied);
result.failed = extra.failed;
if result.snapshot.is_none() { result.snapshot = extra.snapshot; }
if result.failed.is_none() {
crate::schema_check::repair_missing_defaults(&other_db, root, manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str()))?;
if let Some(p) = crate::schema_check::check_with_report(&other_db, root)?.first() {
return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", other.name(), p.table, p.column, p.detail)));
}
}
}
if result.failed.is_none() {
crate::schema_check::repair_missing_defaults(db, root, manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str()))?;
let problems = crate::schema_check::check_with_report(db, root)?;
if let Some(p) = problems.first() {
return Err(Error::Invalid(format!("Database validation failed: {}.{}.{}: {} ({} problems).", p.database, p.table, p.column, p.detail, problems.len())));
}
}
crate::backup::with_database(self.root, |db| {
let identities = database_identities(db)?;
let result = migrate_databases(db, self.root, manifest, staged, &|| self.migration_snapshot())?;
if result.failed.is_none() { verify_database_identities(db, &identities)?; }
Ok(result)
})
}
@@ -1178,23 +1055,24 @@ impl Env for RepackEnv<'_> {
impl RepackEnv<'_> {
fn validate_server(&self, check_schema: bool) -> Result<()> {
let stopped = crate::driver::run(self.root, crate::driver::Verb::StopAll)?;
if !stopped.ok { return Err(Error::Invalid("Could not stop the server before isolated validation.".into())); }
crate::mysql_snapshot::stop(self.root)?;
self.ensure_stopped()?;
self.recover_validation()?;
let result = (|| -> Result<()> {
crate::update_isolation::begin(self.root, self.meta_dir)?;
use crate::process::{observe, ServiceState};
if check_schema { crate::backup::with_database(self.root, |db| {
let mut modes = vec![crate::realms::Mode::Coa];
if crate::realms::state(self.root)?.wildcard_created { modes.push(crate::realms::Mode::Wildcard); }
for mode in modes {
if let Some(p) = crate::schema_check::check_with_report(&db.clone().for_realm(mode), self.root)?.first() {
return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail)));
let identities = crate::backup::with_database(self.root, |db| {
if check_schema {
let mut modes = vec![crate::realms::Mode::Coa];
if crate::realms::state(self.root)?.wildcard_created { modes.push(crate::realms::Mode::Wildcard); }
for mode in modes {
if let Some(p) = crate::schema_check::check_with_report(&db.clone().for_realm(mode), self.root)?.first() {
return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.database, p.table, p.detail)));
}
}
}
Ok(())
})?; }
database_identities(db)
})?;
// The check start is the first start of the new build: give the module configs the settings the update added
// (as a normal start does), or the server logs a "missing property" line for every one of them.
match crate::registry::MetaDir::open(self.meta_dir) {
@@ -1232,6 +1110,7 @@ impl RepackEnv<'_> {
}
};
if healthy {
crate::backup::with_database(self.root, |db| verify_database_identities(db, &identities))?;
return Ok(());
}
let cause = crate::driver::startup_failure(self.root, &started);
@@ -1239,10 +1118,7 @@ impl RepackEnv<'_> {
let _ = crate::driver::run(self.root, crate::driver::Verb::StopAll);
Err(Error::Invalid(cause))
})();
let stopped = crate::driver::run(self.root, crate::driver::Verb::StopAll)?;
if !stopped.ok {
return Err(Error::Invalid("Isolated validation could not stop the server; recovery configurations were preserved.".into()));
}
crate::mysql_snapshot::stop(self.root)?;
self.ensure_stopped()?;
self.recover_validation()?;
result
@@ -1250,6 +1126,40 @@ impl RepackEnv<'_> {
}
/// One migration pass for both realms, with shared history checks and one recovery point.
fn migrate_databases(db: &crate::db::Db, root: &Path, manifest: &Manifest, staged: &Path, snapshot: &dyn Fn() -> Result<String>) -> Result<ApplyReport> {
let realms = crate::realms::state(root)?;
let mut modes = vec![realms.active];
if realms.wildcard_created {
modes.push(if realms.active == crate::realms::Mode::Coa { crate::realms::Mode::Wildcard } else { crate::realms::Mode::Coa });
}
// Reject a broken history in either realm before changing the first one.
for (index, mode) in modes.iter().enumerate() {
let list: Vec<_> = manifest.migrations.iter().filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)).cloned().collect();
crate::migrations::preflight(&db.clone().for_realm(*mode), &list)?;
}
let contract = manifest.files.iter().find(|file| file.path == crate::schema_check::CONTRACT).map(|file| file.sha256.as_str());
let mut result = ApplyReport { applied: vec![], failed: None, snapshot: None };
if contract.is_some() { result.snapshot = Some(snapshot()?); }
for (index, mode) in modes.into_iter().enumerate() {
let db = db.clone().for_realm(mode);
let list: Vec<_> = manifest.migrations.iter().filter(|m| index == 0 || !crate::migrations::shared_auth(&m.db)).cloned().collect();
crate::schema_check::repair_missing_defaults(&db, root, contract)?;
let extra = crate::migrations::apply_pending(&db, &list, staged, &|| {
if let Some(id) = &result.snapshot { Ok(id.clone()) } else { snapshot() }
})?;
result.applied.extend(extra.applied);
result.failed = extra.failed;
if result.snapshot.is_none() { result.snapshot = extra.snapshot; }
if result.failed.is_some() { return Ok(result); }
crate::schema_check::repair_missing_defaults(&db, root, contract)?;
if let Some(p) = crate::schema_check::check_with_report(&db, root)?.first() {
return Err(Error::Invalid(format!("Database validation failed on {}: {}.{}: {}", mode.name(), p.table, p.column, p.detail)));
}
}
Ok(result)
}
fn database_identities(db: &crate::db::Db) -> Result<BTreeMap<String, std::collections::BTreeSet<String>>> {
let db = db.clone().for_realm(crate::realms::Mode::Coa);
let mut identities = BTreeMap::new();
@@ -1269,7 +1179,7 @@ fn verify_database_identities(db: &crate::db::Db, before: &BTreeMap<String, std:
let after = database_identities(db)?;
for (schema, expected) in before {
if after.get(schema).is_none_or(|actual| !expected.is_subset(actual)) {
return Err(Error::Invalid(format!("The private update changed or removed existing character or account identities in {schema}.")));
return Err(Error::Invalid(format!("The update changed or removed existing character or account identities in {schema}.")));
}
}
Ok(())
@@ -1297,19 +1207,15 @@ mod tests {
migrate_fail: Cell<bool>,
preflight_fail: Cell<bool>,
restore_fail: Cell<bool>,
rehearsal_fail: Cell<bool>,
}
impl Fake {
fn ok() -> Fake {
Fake { activation_expected: RefCell::new(None), activation_failed: Cell::new(false), activated: Cell::new(false), automatic_rollback: Cell::new(false), recovery_healthy: Cell::new(true), stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false), rehearsal_fail: Cell::new(false) }
Fake { activation_expected: RefCell::new(None), activation_failed: Cell::new(false), activated: Cell::new(false), automatic_rollback: Cell::new(false), recovery_healthy: Cell::new(true), stopped: Cell::new(true), healthy: Cell::new(true), snapshot_ok: Cell::new(true), calls: Default::default(), migrate_fail: Cell::new(false), preflight_fail: Cell::new(false), restore_fail: Cell::new(false) }
}
}
impl Env for Fake {
fn rehearse(&self, _: &Manifest, _: &Path, _: &[PlanItem], _: &str) -> Result<()> {
if self.rehearsal_fail.get() { Err(Error::Invalid("SQL failed in the private copy".into())) } else { Ok(()) }
}
fn activate(&self) -> Result<()> {
if let Some((meta, version)) = self.activation_expected.borrow().as_ref() {
ensure_recovered(meta)?;
@@ -2058,20 +1964,17 @@ mod tests {
}
#[test]
fn failed_rehearsal_preserves_installed_files_version_and_recovery_point() {
fn update_saves_recovery_then_migrates_and_validates_once() {
let w = world(&[], true);
let env = Fake::ok();
env.rehearsal_fail.set(true);
let original = fsx::sha256_file(&w.meta.join("install.json")).unwrap();
let error = run(&w, &env, BTreeMap::new(), None).unwrap_err();
assert!(error.to_string().contains("installed server was not updated"));
assert_eq!(read(&w, "Core/worldserver.exe"), b"world-v1");
assert_eq!(fsx::sha256_file(&w.meta.join("install.json")).unwrap(), original);
assert_eq!(&*env.calls.borrow(), &["stop", "snapshot"]);
assert!(pending_checked(&w.meta).unwrap().is_none());
let journal = journals(&w.meta).unwrap().remove(0);
assert_eq!(journal.recovery_point.as_deref(), Some("rp-1"));
assert!(!journal.databases_started && journal.ops.is_empty());
let result = run(&w, &env, BTreeMap::new(), None).unwrap();
assert_eq!(result.txn.state, State::Committed);
let calls = env.calls.borrow();
assert_eq!(calls.iter().filter(|call| **call == "snapshot").count(), 1);
assert_eq!(calls.iter().filter(|call| **call == "migrate").count(), 1);
assert_eq!(calls.iter().filter(|call| **call == "validate").count(), 1);
assert!(calls.iter().position(|call| *call == "snapshot") < calls.iter().position(|call| *call == "migrate"));
assert!(!w.meta.join("rehearsals").exists());
}
#[test]