diff --git a/crates/coa-core/examples/migrate_e2e.rs b/crates/coa-core/examples/migrate_e2e.rs new file mode 100644 index 0000000..290bd3e --- /dev/null +++ b/crates/coa-core/examples/migrate_e2e.rs @@ -0,0 +1,82 @@ +//! Bring a DISPOSABLE fixture database up to the core checkout's SQL, through the tracked migration runner. +//! usage: migrate_e2e +use coa_core::backup; +use coa_core::db::{Account, Db}; +use coa_core::fsx; +use coa_core::manifest::Migration; +use coa_core::migrations::{apply_pending, status, Status}; +use std::path::{Path, PathBuf}; + +fn sql_files(dir: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(dir).map(|rd| rd.flatten().map(|e| e.path()).filter(|p| p.extension().map(|e| e == "sql").unwrap_or(false)).collect()).unwrap_or_default(); + v.sort(); + v +} + +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + assert!(a[0].to_lowercase().contains("fixture"), "refusing: not a fixture folder"); + let (root, core) = (PathBuf::from(&a[0]), PathBuf::from(&a[1])); + let staging = std::env::temp_dir().join("coa-migrate-e2e"); + let _ = std::fs::remove_dir_all(&staging); + + // Release order: released updates, then pending ones, then module updates. + let mut list: Vec = Vec::new(); + let mut origin: std::collections::HashMap<(String, String), String> = Default::default(); + let mut add = |db: &str, id: String, src: &Path| { + origin.insert((db.to_string(), id.clone()), src.strip_prefix(&core).unwrap().to_string_lossy().replace('\\', "/")); + let bytes = std::fs::read(src).unwrap(); + let dst = staging.join(db).join(format!("{id}.sql")); + std::fs::create_dir_all(dst.parent().unwrap()).unwrap(); + std::fs::write(&dst, &bytes).unwrap(); + list.push(Migration { id, db: db.into(), sha256: fsx::sha256_bytes(&bytes), destructive: false }); + }; + for (kind, dirs) in [("auth", ["db_auth", "pending_db_auth"]), ("characters", ["db_characters", "pending_db_characters"]), ("world", ["db_world", "pending_db_world"])] { + for d in dirs { + for f in sql_files(&core.join("data/sql/updates").join(d)) { + add(kind, f.file_stem().unwrap().to_string_lossy().into_owned(), &f); + } + } + } + for m in std::fs::read_dir(core.join("modules")).unwrap().flatten() { + let mname = m.file_name().to_string_lossy().replace('-', "_"); + for kind in ["auth", "characters", "world"] { + let base = m.path().join(format!("data/sql/db-{kind}")); + for f in sql_files(&base.join("base")) { + add(kind, format!("mod_{mname}__base__{}", f.file_stem().unwrap().to_string_lossy()), &f); + } + for f in sql_files(&base) { + add(kind, format!("mod_{mname}__{}", f.file_stem().unwrap().to_string_lossy()), &f); + } + } + } + // Application order = the order the files were introduced in git history (a pending core script may rely on a + // table created by a module script that arrived earlier). The release pipeline computes exactly this. + let out = std::process::Command::new("git") + .current_dir(&core) + .args(["log", "--reverse", "--diff-filter=A", "--name-only", "--pretty=format:", "--", "data/sql/updates", "modules"]) + .output() + .unwrap(); + let mut rank = std::collections::HashMap::new(); + for (i, path) in String::from_utf8_lossy(&out.stdout).lines().filter(|l| l.ends_with(".sql")).enumerate() { + rank.entry(path.to_string()).or_insert(i); + } + let key = |m: &Migration| -> usize { + origin.get(&(m.db.clone(), m.id.clone())).and_then(|p| rank.get(p)).copied().unwrap_or(usize::MAX) + }; + list.sort_by_key(|m| key(m)); + println!("candidate migrations: {} ({} not found in history)", list.len(), list.iter().filter(|m| key(m) == usize::MAX).count()); + + let admin = Db::from_repack(&root, Account::Admin).unwrap(); + let _ = &admin; + let t = std::time::Instant::now(); + let report = backup::with_database(&root, |db| { + let before = status(db, &list)?; + let pending = before.iter().filter(|i| i.status == Status::Pending).count(); + println!("already recorded by the database: {}; pending: {pending}", list.len() - pending); + let pending_list: Vec = list.iter().zip(&before).filter(|(_, i)| i.status == Status::Pending).map(|(m, _)| m.clone()).collect(); + apply_pending(db, &pending_list, &staging, &|| Ok("(fixture: no backup)".into())) + }) + .expect("apply"); + println!("applied {} in {:?}; failed: {:?}", report.applied.len(), t.elapsed(), report.failed); +} diff --git a/crates/coa-core/examples/ra_account.rs b/crates/coa-core/examples/ra_account.rs new file mode 100644 index 0000000..b94e488 --- /dev/null +++ b/crates/coa-core/examples/ra_account.rs @@ -0,0 +1,12 @@ +//! Create an account through the Manager's RA client on a RUNNING disposable fixture. +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + assert!(a[0].to_lowercase().contains("fixture"), "refusing: not a fixture folder"); + let root = std::path::Path::new(&a[0]); + let mut ra = coa_core::ra::Ra::connect(root).expect("connect"); + ra.create_account(&a[1], &a[2]).expect("create"); + ra.make_administrator(&a[1]).expect("gm"); + println!("created {} (administrator)", a[1]); + let out = coa_core::backup::with_database(root, |db| db.query(&format!("SELECT a.username, aa.gmlevel FROM acore_auth.account a LEFT JOIN acore_auth.account_access aa ON aa.id=a.id WHERE a.username='{}';", a[1].to_uppercase()))).unwrap(); + println!("in database: {out}"); +} diff --git a/crates/coa-core/examples/sqlq.rs b/crates/coa-core/examples/sqlq.rs new file mode 100644 index 0000000..d2291c2 --- /dev/null +++ b/crates/coa-core/examples/sqlq.rs @@ -0,0 +1,8 @@ +//! Read-only helper for a DISPOSABLE fixture: run one query. usage: sqlq +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + assert!(a[0].to_lowercase().contains("fixture"), "refusing: not a fixture folder"); + let root = std::path::Path::new(&a[0]); + let out = coa_core::backup::with_database(root, |db| db.query(&a[1])).expect("query"); + println!("{out}"); +} diff --git a/crates/coa-core/src/db.rs b/crates/coa-core/src/db.rs index 37a842b..e9da00f 100644 --- a/crates/coa-core/src/db.rs +++ b/crates/coa-core/src/db.rs @@ -201,13 +201,29 @@ impl Db { Ok((size, fsx::sha256_file(out)?)) } + /// Run a plain SQL file against `schema`; the first error aborts and is returned. + pub fn run_sql_file(&self, schema: &str, file: &Path) -> Result<()> { + if !ident_ok(schema) { + return Err(Error::Invalid("bad schema name".into())); + } + let mut c = self.command("mysql.exe"); + c.args(["--default-character-set=utf8mb4", "--init-command=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci", "--max-allowed-packet=128M"]); + c.arg(schema); + c.stdin(Stdio::from(File::open(file)?)); + let out = c.output()?; + if !out.status.success() { + return Err(self.fail(&out.stderr)); + } + Ok(()) + } + /// Import a dump produced by `dump_to` into `schema` (which must already exist and be empty). pub fn import_from(&self, schema: &str, dump: &Path) -> Result<()> { if !ident_ok(schema) { return Err(Error::Invalid("bad schema name".into())); } let mut c = self.command("mysql.exe"); - c.args(["--default-character-set=utf8mb4", "--max-allowed-packet=128M"]); + c.args(["--default-character-set=utf8mb4", "--init-command=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci", "--max-allowed-packet=128M"]); c.arg(schema); c.stdin(Stdio::piped()); let mut child = c.spawn()?; diff --git a/crates/coa-core/src/lib.rs b/crates/coa-core/src/lib.rs index b1b01a2..55b0feb 100644 --- a/crates/coa-core/src/lib.rs +++ b/crates/coa-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod layout; pub mod logging; pub mod process; pub mod manifest; +pub mod migrations; pub mod package; pub mod ra; pub mod registry; diff --git a/crates/coa-core/src/migrations.rs b/crates/coa-core/src/migrations.rs new file mode 100644 index 0000000..9ac8561 --- /dev/null +++ b/crates/coa-core/src/migrations.rs @@ -0,0 +1,327 @@ +//! Tracked SQL migrations. The core's own database updater is disabled in this project, so schema changes +//! shipped with a release are applied here, one file at a time, recorded in a ledger table, and never twice. + +use std::path::Path; + +use serde::Serialize; + +use crate::db::{self, Db}; +use crate::error::{Error, Result}; +use crate::fsx; +use crate::manifest::Migration; + +const LEDGER_SCHEMA: &str = "acore_world"; +const LEDGER_TABLE: &str = "coa_manager_migrations"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Status { + Applied, + Pending, + Failed, +} + +#[derive(Debug, Clone)] +pub struct LedgerRow { + pub db: String, + pub id: String, + pub sha256: String, + pub status: Status, + pub error: Option, + pub baseline: bool, +} + +/// Storage for the ledger and the ability to run a SQL file; implemented by the real database and by tests. +pub trait Store { + fn load(&self) -> Result>; + fn put(&self, row: &LedgerRow) -> Result<()>; + fn run_file(&self, schema: &str, file: &Path) -> Result<()>; +} + +fn hex_of(s: &str) -> String { + hex::encode(s.as_bytes()) +} + +fn ident_ok(s: &str) -> bool { + !s.is_empty() && s.len() <= 150 && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '-')) +} + +impl Store for Db { + fn load(&self) -> Result> { + self.query(&format!( + "CREATE TABLE IF NOT EXISTS `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db` VARCHAR(32) NOT NULL, `id` VARCHAR(190) NOT NULL, `sha256` CHAR(64) NOT NULL, `status` VARCHAR(16) NOT NULL, `applied_at` DATETIME NOT NULL, `error` TEXT NULL, `baseline` TINYINT NOT NULL DEFAULT 0, PRIMARY KEY (`db`,`id`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;" + ))?; + let out = self.query(&format!("SELECT `db`,`id`,`sha256`,`status`,`baseline`,COALESCE(HEX(`error`),'') FROM `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` ORDER BY `db`,`id`;"))?; + let mut rows = Vec::new(); + // Files the core's own updater (or whoever prepared this database) already recorded count as applied. + for (kind, schema) in db::SCHEMAS { + if let Ok(names) = self.query(&format!("SELECT name FROM `{schema}`.`updates`;")) { + for name in names.lines().filter(|n| n.ends_with(".sql")) { + rows.push(LedgerRow { db: kind.into(), id: name.trim_end_matches(".sql").into(), sha256: "0".repeat(64), status: Status::Applied, error: None, baseline: true }); + } + } + } + for line in out.lines().filter(|l| !l.is_empty()) { + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 5 { + continue; + } + let err = f.get(5).filter(|h| !h.is_empty()).and_then(|h| hex::decode(h).ok()).map(|b| String::from_utf8_lossy(&b).into_owned()); + rows.push(LedgerRow { + db: f[0].into(), + id: f[1].into(), + sha256: f[2].into(), + status: if f[3] == "applied" { Status::Applied } else { Status::Failed }, + baseline: f[4] == "1", + error: err, + }); + } + Ok(rows) + } + + fn put(&self, r: &LedgerRow) -> Result<()> { + if !ident_ok(&r.id) || !ident_ok(&r.db) || r.sha256.len() != 64 || !r.sha256.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(Error::Invalid("migration record has an invalid identifier".into())); + } + let status = if r.status == Status::Applied { "applied" } else { "failed" }; + let err = r.error.as_deref().map(|e| format!("UNHEX('{}')", hex_of(e))).unwrap_or_else(|| "NULL".into()); + self.query(&format!( + "REPLACE INTO `{LEDGER_SCHEMA}`.`{LEDGER_TABLE}` (`db`,`id`,`sha256`,`status`,`applied_at`,`error`,`baseline`) VALUES ('{}','{}','{}','{status}',NOW(),{err},{});", + r.db, r.id, r.sha256, r.baseline as u8 + ))?; + Ok(()) + } + + fn run_file(&self, schema: &str, file: &Path) -> Result<()> { + self.run_sql_file(schema, file) + } +} + +/// Statements that can lose data. Detection is deliberately conservative: a false positive only costs a backup. +pub fn looks_destructive(sql: &str) -> bool { + let upper: String = sql + .lines() + .filter(|l| !l.trim_start().starts_with("--") && !l.trim_start().starts_with('#')) + .collect::>() + .join(" ") + .to_uppercase(); + ["DROP TABLE", "DROP DATABASE", "DROP SCHEMA", "TRUNCATE", "DELETE FROM", "DROP COLUMN", "DROP INDEX", "DROP PRIMARY", "DROP FOREIGN"].iter().any(|k| upper.contains(k)) +} + +#[derive(Debug, Clone, Serialize)] +pub struct Item { + pub id: String, + pub db: String, + pub status: Status, + pub destructive: bool, + pub baseline: bool, + pub error: Option, +} + +pub fn status(store: &dyn Store, list: &[Migration]) -> Result> { + let rows = store.load()?; + Ok(list + .iter() + .map(|m| { + let row = rows.iter().find(|r| r.db == m.db && r.id == m.id); + Item { + id: m.id.clone(), + db: m.db.clone(), + status: row.map(|r| r.status).unwrap_or(Status::Pending), + destructive: m.destructive, + baseline: row.map(|r| r.baseline).unwrap_or(false), + error: row.and_then(|r| r.error.clone()), + } + }) + .collect()) +} + +/// Record migrations as already present without running them (adopting an existing server whose database +/// was brought up to date by hand). +pub fn baseline(store: &dyn Store, list: &[Migration]) -> Result { + let rows = store.load()?; + let mut n = 0; + for m in list { + if rows.iter().any(|r| r.db == m.db && r.id == m.id && r.status == Status::Applied) { + continue; + } + store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Applied, error: None, baseline: true })?; + n += 1; + } + Ok(n) +} + +#[derive(Debug, Serialize)] +pub struct ApplyReport { + pub applied: Vec, + pub failed: Option<(String, String)>, + pub snapshot: Option, +} + +/// Apply every pending migration in list order, stopping at the first failure. `snapshot` runs once, before the +/// first pending migration, when any of them is (or looks) destructive. +pub fn apply_pending(store: &dyn Store, list: &[Migration], dir: &Path, snapshot: &dyn Fn() -> Result) -> Result { + let rows = store.load()?; + // The list order is the release's order of application (released updates, then pending, then modules). + let todo: Vec<&Migration> = list.iter().filter(|m| !rows.iter().any(|r| r.db == m.db && r.id == m.id && r.status == Status::Applied)).collect(); + + // Resolve and verify every file before running anything. + let mut files = Vec::new(); + let mut any_destructive = false; + for m in &todo { + if !ident_ok(&m.id) { + return Err(Error::InvalidManifest(format!("migration id {:?} is not allowed", m.id))); + } + let schema = db::schema_of(&m.db)?; + let path = fsx::safe_join(dir, &format!("{}/{}.sql", m.db, m.id))?; + let bytes = std::fs::read(&path).map_err(|_| Error::Invalid(format!("migration file {} is missing", m.id)))?; + let actual = fsx::sha256_bytes(&bytes); + if !actual.eq_ignore_ascii_case(&m.sha256) { + return Err(Error::HashMismatch { path: path.display().to_string(), expected: m.sha256.clone(), actual }); + } + any_destructive |= m.destructive || looks_destructive(&String::from_utf8_lossy(&bytes)); + files.push((*m, schema, path)); + } + + let mut report = ApplyReport { applied: Vec::new(), failed: None, snapshot: None }; + if files.is_empty() { + return Ok(report); + } + if any_destructive { + report.snapshot = Some(snapshot().map_err(|e| Error::Invalid(format!("No database update was applied because the safety backup failed: {e}")))?); + } + for (m, schema, path) in files { + match store.run_file(schema, &path) { + Ok(()) => { + store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Applied, error: None, baseline: false })?; + report.applied.push(m.id.clone()); + } + Err(e) => { + let msg = e.to_string(); + let _ = store.put(&LedgerRow { db: m.db.clone(), id: m.id.clone(), sha256: m.sha256.clone(), status: Status::Failed, error: Some(msg.clone()), baseline: false }); + report.failed = Some((m.id.clone(), msg)); + break; + } + } + } + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[derive(Default)] + struct Mem { + rows: RefCell>, + ran: RefCell>, + fail_on: Option<&'static str>, + } + + impl Store for Mem { + fn load(&self) -> Result> { + Ok(self.rows.borrow().clone()) + } + fn put(&self, r: &LedgerRow) -> Result<()> { + let mut rows = self.rows.borrow_mut(); + rows.retain(|x| !(x.db == r.db && x.id == r.id)); + rows.push(r.clone()); + Ok(()) + } + fn run_file(&self, _schema: &str, file: &Path) -> Result<()> { + let name = file.file_stem().unwrap().to_string_lossy().into_owned(); + if self.fail_on == Some(Box::leak(name.clone().into_boxed_str())) { + return Err(Error::Invalid("syntax error near 'FOO'".into())); + } + self.ran.borrow_mut().push(name); + Ok(()) + } + } + + fn setup(files: &[(&str, &str, &str)]) -> (tempfile::TempDir, Vec) { + let d = tempfile::tempdir().unwrap(); + let mut list = Vec::new(); + for (db, id, sql) in files { + std::fs::create_dir_all(d.path().join(db)).unwrap(); + std::fs::write(d.path().join(db).join(format!("{id}.sql")), sql).unwrap(); + list.push(Migration { id: id.to_string(), db: db.to_string(), sha256: fsx::sha256_bytes(sql.as_bytes()), destructive: false }); + } + (d, list) + } + + fn no_snapshot() -> Result { + panic!("no snapshot expected") + } + + #[test] + fn applies_in_order_once_and_records_the_ledger() { + let (d, list) = setup(&[("world", "2026_09_01_00_a", "CREATE TABLE t (id INT);"), ("world", "2026_09_02_00_b", "ALTER TABLE t ADD c INT;")]); + let store = Mem::default(); + let r = apply_pending(&store, &list, d.path(), &no_snapshot).unwrap(); + assert_eq!(r.applied, ["2026_09_01_00_a", "2026_09_02_00_b"], "manifest order is application order"); + assert_eq!(*store.ran.borrow(), ["2026_09_01_00_a", "2026_09_02_00_b"]); + let again = apply_pending(&store, &list, d.path(), &no_snapshot).unwrap(); + assert!(again.applied.is_empty(), "never applied twice"); + assert!(status(&store, &list).unwrap().iter().all(|i| i.status == Status::Applied)); + } + + #[test] + fn a_failure_stops_the_batch_and_is_remembered() { + let (d, list) = setup(&[("world", "m1", "SELECT 1;"), ("world", "m2", "BROKEN;"), ("world", "m3", "SELECT 3;")]); + let store = Mem { fail_on: Some("m2"), ..Default::default() }; + let r = apply_pending(&store, &list, d.path(), &no_snapshot).unwrap(); + assert_eq!(r.applied, ["m1"]); + assert_eq!(r.failed.as_ref().unwrap().0, "m2"); + assert_eq!(*store.ran.borrow(), ["m1"], "m3 must not run after m2 failed"); + let st = status(&store, &list).unwrap(); + assert_eq!(st.iter().map(|i| i.status).collect::>(), [Status::Applied, Status::Failed, Status::Pending]); + assert!(st[1].error.as_deref().unwrap().contains("syntax error")); + } + + #[test] + fn destructive_migrations_force_a_backup_first_and_a_failed_backup_blocks_everything() { + let (d, list) = setup(&[("characters", "d1", "-- cleanup\nDELETE FROM old_stuff WHERE 1;")]); + let store = Mem::default(); + let r = apply_pending(&store, &list, d.path(), &|| Ok("backup-1".into())).unwrap(); + assert_eq!(r.snapshot.as_deref(), Some("backup-1")); + assert_eq!(r.applied, ["d1"]); + + let store = Mem::default(); + let e = apply_pending(&store, &list, d.path(), &|| Err(Error::Invalid("disk full".into()))).unwrap_err(); + assert!(e.to_string().contains("safety backup failed")); + assert!(store.ran.borrow().is_empty(), "nothing ran without a backup"); + } + + #[test] + fn tampered_or_missing_files_are_refused_before_anything_runs() { + let (d, mut list) = setup(&[("world", "t1", "SELECT 1;"), ("world", "t2", "SELECT 2;")]); + list[1].sha256 = "0".repeat(64); + let store = Mem::default(); + assert!(matches!(apply_pending(&store, &list, d.path(), &no_snapshot), Err(Error::HashMismatch { .. }))); + assert!(store.ran.borrow().is_empty(), "t1 must not run when t2 is bad"); + std::fs::remove_file(d.path().join("world/t1.sql")).unwrap(); + assert!(apply_pending(&store, &list[..1], d.path(), &no_snapshot).is_err()); + } + + #[test] + fn baseline_marks_without_running_and_hostile_ids_are_rejected() { + let (d, list) = setup(&[("world", "b1", "SELECT 1;")]); + let store = Mem::default(); + assert_eq!(baseline(&store, &list).unwrap(), 1); + assert_eq!(baseline(&store, &list).unwrap(), 0); + assert!(store.ran.borrow().is_empty()); + assert!(status(&store, &list).unwrap()[0].baseline); + let mut evil = list.clone(); + evil[0].id = "../../x".into(); + assert!(apply_pending(&Mem::default(), &evil, d.path(), &no_snapshot).is_err()); + } + + #[test] + fn destructive_detection_ignores_comments() { + assert!(looks_destructive("ALTER TABLE t DROP COLUMN c;")); + assert!(looks_destructive("truncate table t;")); + assert!(!looks_destructive("-- DROP TABLE nothing\nINSERT INTO t VALUES (1);")); + assert!(!looks_destructive("CREATE TABLE IF NOT EXISTS t (id INT);")); + } +}