mirror of
https://github.com/Corfirean/coa-server-manager.git
synced 2026-11-19 11:25:56 +00:00
575 lines
20 KiB
Rust
575 lines
20 KiB
Rust
//! Release engineering: ordered SQL collection, cumulative update packages and a clean base tree.
|
|
//! Used by the `coa-release` command line tool (CI and the maintainer's machine); never by the app at runtime.
|
|
|
|
use std::collections::BTreeMap;
|
|
use std::fs;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
use crate::error::{Error, Result};
|
|
use crate::fsx;
|
|
use crate::manifest::{Kind, Manifest, Migration, Revision};
|
|
use crate::package::{self, BuildOptions};
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct SqlFile {
|
|
pub db: String,
|
|
pub id: String,
|
|
/// Path relative to the checkout it came from (used only for ordering and display).
|
|
pub origin: String,
|
|
pub abs: PathBuf,
|
|
pub sha256: String,
|
|
}
|
|
|
|
fn entries(dir: &Path) -> Result<Vec<fs::DirEntry>> {
|
|
match fs::read_dir(dir) {
|
|
Ok(rd) => Ok(rd.collect::<std::io::Result<Vec<_>>>()?),
|
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(vec![]),
|
|
Err(error) => Err(error.into()),
|
|
}
|
|
}
|
|
|
|
fn sql_in(dir: &Path) -> Result<Vec<PathBuf>> {
|
|
let mut paths: Vec<_> = entries(dir)?
|
|
.into_iter()
|
|
.map(|entry| entry.path())
|
|
.filter(|path| path.extension().is_some_and(|ext| ext == "sql"))
|
|
.collect();
|
|
paths.sort();
|
|
Ok(paths)
|
|
}
|
|
|
|
/// The order in which files were first added to the repository (a pending script may rely on a table a module
|
|
/// created earlier). Files git does not know come last, in name order.
|
|
fn introduction_rank(
|
|
repo: &Path,
|
|
pathspecs: &[&str],
|
|
) -> Result<std::collections::HashMap<String, usize>> {
|
|
if !repo.join(".git").exists() {
|
|
return Ok(Default::default());
|
|
}
|
|
let mut args = vec![
|
|
"log",
|
|
"--full-history",
|
|
"--diff-merges=first-parent",
|
|
"--reverse",
|
|
"--diff-filter=A",
|
|
"--name-only",
|
|
"--pretty=format:",
|
|
"--",
|
|
];
|
|
args.extend_from_slice(pathspecs);
|
|
let out = Command::new("git").current_dir(repo).args(&args).output()?;
|
|
if !out.status.success() {
|
|
return Err(Error::Invalid(format!(
|
|
"Cannot establish SQL migration order: {}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
)));
|
|
}
|
|
let mut rank = std::collections::HashMap::new();
|
|
for (i, line) in String::from_utf8_lossy(&out.stdout)
|
|
.lines()
|
|
.filter(|line| line.ends_with(".sql"))
|
|
.enumerate()
|
|
{
|
|
rank.entry(line.to_string()).or_insert(i);
|
|
}
|
|
Ok(rank)
|
|
}
|
|
|
|
/// Incremental SQL of a core checkout. Installation-only module bases must never be replayed on a live database.
|
|
pub fn collect_core_sql(core: &Path) -> Result<Vec<SqlFile>> {
|
|
collect_core_sql_mode(core, false)
|
|
}
|
|
|
|
/// A clean installation also needs the module tables and initial seed data.
|
|
pub fn collect_core_install_sql(core: &Path) -> Result<Vec<SqlFile>> {
|
|
collect_core_sql_mode(core, true)
|
|
}
|
|
|
|
fn collect_core_sql_mode(core: &Path, include_base: bool) -> Result<Vec<SqlFile>> {
|
|
let mut out = Vec::new();
|
|
let mut push = |db: &str, id: String, abs: PathBuf, origin: String| -> Result<()> {
|
|
let sha256 = fsx::sha256_file(&abs)?;
|
|
out.push(SqlFile {
|
|
db: db.into(),
|
|
id,
|
|
origin,
|
|
abs,
|
|
sha256,
|
|
});
|
|
Ok(())
|
|
};
|
|
for (kind, dirs) in [
|
|
("auth", ["db_auth", "pending_db_auth"]),
|
|
("characters", ["db_characters", "pending_db_characters"]),
|
|
("world", ["db_world", "pending_db_world"]),
|
|
] {
|
|
for d in dirs {
|
|
for f in sql_in(&core.join("data/sql/updates").join(d))? {
|
|
let stem = f.file_stem().unwrap().to_string_lossy().into_owned();
|
|
let rel = format!(
|
|
"data/sql/updates/{d}/{}",
|
|
f.file_name().unwrap().to_string_lossy()
|
|
);
|
|
push(kind, stem, f, rel)?;
|
|
}
|
|
}
|
|
}
|
|
{
|
|
let mut mods = entries(&core.join("modules"))?;
|
|
mods.sort_by_key(|e| e.file_name());
|
|
for m in mods {
|
|
let name = m.file_name().to_string_lossy().into_owned();
|
|
let mname = name.replace('-', "_");
|
|
for kind in ["auth", "characters", "world"] {
|
|
let base = m.path().join(format!("data/sql/db-{kind}"));
|
|
for f in if include_base {
|
|
sql_in(&base.join("base"))?
|
|
} else {
|
|
vec![]
|
|
} {
|
|
let stem = f.file_stem().unwrap().to_string_lossy().into_owned();
|
|
let rel = format!(
|
|
"modules/{name}/data/sql/db-{kind}/base/{}",
|
|
f.file_name().unwrap().to_string_lossy()
|
|
);
|
|
push(kind, format!("mod_{mname}__base__{stem}"), f, rel)?;
|
|
}
|
|
for f in sql_in(&base)? {
|
|
let stem = f.file_stem().unwrap().to_string_lossy().into_owned();
|
|
let rel = format!(
|
|
"modules/{name}/data/sql/db-{kind}/{}",
|
|
f.file_name().unwrap().to_string_lossy()
|
|
);
|
|
push(kind, format!("mod_{mname}__{stem}"), f, rel)?;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let rank = introduction_rank(core, &["data/sql/updates", "modules"])?;
|
|
out.sort_by_key(|f| rank.get(&f.origin).copied().unwrap_or(usize::MAX));
|
|
Ok(out)
|
|
}
|
|
|
|
/// Dump the three game databases of `server` (a repack or a Docker server, started if needed) into `out` as the starting
|
|
/// databases of a Linux package: `<kind>.sql.zst`. The databases must be in the state in which every migration of the
|
|
/// package is applied. Returns (kind, compressed bytes) per database.
|
|
pub fn export_baseline(server: &Path, out: &Path) -> Result<Vec<(String, u64)>> {
|
|
fs::create_dir_all(out)?;
|
|
crate::backup::with_database(server, |db| {
|
|
let mut done = Vec::new();
|
|
for (kind, schema) in crate::db::SCHEMAS {
|
|
let (bytes, _) = db.dump_to(schema, &out.join(format!("{kind}.sql.zst")))?;
|
|
done.push((kind.to_string(), bytes));
|
|
}
|
|
Ok(done)
|
|
})
|
|
}
|
|
|
|
/// SQL a bots module checkout ships for the characters database (its `dist/sql`).
|
|
pub fn collect_bots_sql(bots: &Path) -> Result<Vec<SqlFile>> {
|
|
let mut out = Vec::new();
|
|
for f in sql_in(&bots.join("dist/sql"))? {
|
|
let stem = f.file_stem().unwrap().to_string_lossy().into_owned();
|
|
out.push(SqlFile {
|
|
db: "characters".into(),
|
|
id: format!("mod_coa_playerbots__{stem}"),
|
|
origin: format!("dist/sql/{stem}.sql"),
|
|
sha256: fsx::sha256_file(&f)?,
|
|
abs: f,
|
|
});
|
|
}
|
|
Ok(out)
|
|
}
|
|
|
|
fn to_migrations(files: &[SqlFile]) -> Vec<Migration> {
|
|
files
|
|
.iter()
|
|
.map(|f| Migration {
|
|
compatible_sha256: vec![],
|
|
id: f.id.clone(),
|
|
db: f.db.clone(),
|
|
sha256: f.sha256.clone(),
|
|
destructive: false,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
pub struct UpdateParams<'a> {
|
|
/// Directory laid out like the server folder that holds the release's files (binaries, dist configs, addon...).
|
|
pub tree: &'a Path,
|
|
/// Manifest of the base package this update is cumulative against.
|
|
pub base_manifest: &'a Manifest,
|
|
pub sql: &'a [SqlFile],
|
|
pub out: &'a Path,
|
|
pub version: String,
|
|
pub core_commit: Option<String>,
|
|
pub bots_commit: Option<String>,
|
|
pub part_size: u64,
|
|
}
|
|
|
|
fn walk(root: &Path, dir: &Path, out: &mut Vec<String>) -> Result<()> {
|
|
for e in fs::read_dir(dir)? {
|
|
let e = e?;
|
|
let p = e.path();
|
|
if e.file_type()?.is_dir() {
|
|
walk(root, &p, out)?;
|
|
} else {
|
|
out.push(
|
|
p.strip_prefix(root)
|
|
.unwrap()
|
|
.to_string_lossy()
|
|
.replace('\\', "/"),
|
|
);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Build a cumulative update: every file of `tree` that differs from the base, plus every SQL script. Unsigned.
|
|
pub fn pack_update(p: &UpdateParams, progress: &dyn Fn(&str)) -> Result<Manifest> {
|
|
if let Some(file) = p
|
|
.sql
|
|
.iter()
|
|
.find(|file| file.id.starts_with("mod_") && file.id.contains("__base__"))
|
|
{
|
|
return Err(Error::Invalid(format!(
|
|
"Module installation SQL {} cannot be included in an update package.",
|
|
file.id
|
|
)));
|
|
}
|
|
let base: BTreeMap<String, &str> = p
|
|
.base_manifest
|
|
.files
|
|
.iter()
|
|
.map(|f| (f.path.to_lowercase(), f.sha256.as_str()))
|
|
.collect();
|
|
let work = p.out.with_file_name(format!(
|
|
"{}-staging",
|
|
p.out.file_name().unwrap().to_string_lossy()
|
|
));
|
|
let _ = fs::remove_dir_all(&work);
|
|
fs::create_dir_all(&work)?;
|
|
|
|
let mut rels = Vec::new();
|
|
walk(p.tree, p.tree, &mut rels)?;
|
|
let mut changed = 0;
|
|
for rel in &rels {
|
|
let src = fsx::safe_join(p.tree, rel)?;
|
|
if rel != crate::schema_check::CONTRACT
|
|
&& base
|
|
.get(&rel.to_lowercase())
|
|
.map(|h| h.eq_ignore_ascii_case(&fsx::sha256_file(&src).unwrap_or_default()))
|
|
.unwrap_or(false)
|
|
{
|
|
continue;
|
|
}
|
|
let dst = fsx::safe_join(&work, rel)?;
|
|
fs::create_dir_all(dst.parent().unwrap())?;
|
|
fs::copy(&src, &dst)?;
|
|
changed += 1;
|
|
}
|
|
progress(&format!(
|
|
"{changed} of {} files differ from the base",
|
|
rels.len()
|
|
));
|
|
for f in p.sql {
|
|
let dst = fsx::safe_join(&work, &format!("_migrations/{}/{}.sql", f.db, f.id))?;
|
|
fs::create_dir_all(dst.parent().unwrap())?;
|
|
fs::copy(&f.abs, dst)?;
|
|
}
|
|
let opts = BuildOptions {
|
|
kind: Kind::Update,
|
|
version: p.version.clone(),
|
|
core_commit: p.core_commit.clone(),
|
|
built_at: chrono::Utc::now().to_rfc3339(),
|
|
part_size: p.part_size,
|
|
bots_commit: p.bots_commit.clone(),
|
|
migrations: to_migrations(p.sql),
|
|
};
|
|
let m = package::build(&work, p.out, &opts, &|s| progress(s))?;
|
|
let _ = fs::remove_dir_all(&work);
|
|
Ok(m)
|
|
}
|
|
|
|
/// Sign `manifest.json` in `dir` with the Ed25519 seed (base64) and write `manifest.json.sig`.
|
|
pub fn sign_manifest(dir: &Path, seed_b64: &str) -> Result<()> {
|
|
use base64::{engine::general_purpose::STANDARD, Engine};
|
|
use ed25519_dalek::{Signer, SigningKey};
|
|
let seed: [u8; 32] = STANDARD
|
|
.decode(seed_b64.trim())
|
|
.map_err(|_| Error::Invalid("signing key is not base64".into()))?
|
|
.try_into()
|
|
.map_err(|_| Error::Invalid("signing key has the wrong length".into()))?;
|
|
let bytes = fs::read(dir.join("manifest.json"))?;
|
|
let sig = SigningKey::from_bytes(&seed).sign(&bytes);
|
|
fsx::atomic_write(
|
|
&dir.join("manifest.json.sig"),
|
|
format!("{}\n", STANDARD.encode(sig.to_bytes())).as_bytes(),
|
|
)?;
|
|
// never publish something the app would reject
|
|
crate::signing::verify_embedded(&bytes, &fs::read_to_string(dir.join("manifest.json.sig"))?)
|
|
.map_err(|_| Error::Invalid("The signature does not match the public key built into the app; wrong signing key.".into()))
|
|
}
|
|
|
|
pub fn revision(commit: &str) -> Revision {
|
|
Revision {
|
|
commit: Some(commit.to_string()),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn migrations_added_by_a_merge_precede_later_dependent_updates() {
|
|
let directory = tempfile::tempdir().unwrap();
|
|
let root = directory.path();
|
|
let git = |args: &[&str]| {
|
|
let output = Command::new("git")
|
|
.current_dir(root)
|
|
.args(args)
|
|
.output()
|
|
.unwrap();
|
|
assert!(
|
|
output.status.success(),
|
|
"git {args:?}: {}",
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
};
|
|
git(&["init", "-b", "main"]);
|
|
git(&["config", "user.name", "Migration Test"]);
|
|
git(&["config", "user.email", "migration-test@example.invalid"]);
|
|
write(
|
|
root,
|
|
"data/sql/updates/pending_db_world/00_schema.sql",
|
|
"SELECT 1;",
|
|
);
|
|
git(&["add", "."]);
|
|
git(&["commit", "-m", "schema"]);
|
|
git(&["switch", "-c", "side"]);
|
|
git(&["commit", "--allow-empty", "-m", "side"]);
|
|
git(&["switch", "main"]);
|
|
git(&["commit", "--allow-empty", "-m", "main"]);
|
|
git(&["merge", "--no-ff", "--no-commit", "side"]);
|
|
write(
|
|
root,
|
|
"data/sql/updates/pending_db_world/10_snapshot.sql",
|
|
"SELECT 2;",
|
|
);
|
|
git(&["add", "."]);
|
|
git(&["commit", "-m", "merge with snapshot"]);
|
|
write(
|
|
root,
|
|
"data/sql/updates/pending_db_world/20_progression.sql",
|
|
"SELECT 3;",
|
|
);
|
|
git(&["add", "."]);
|
|
git(&["commit", "-m", "new progression"]);
|
|
let sql = collect_core_sql(root).unwrap();
|
|
let ids: Vec<_> = sql.iter().map(|file| file.id.as_str()).collect();
|
|
assert_eq!(ids, ["00_schema", "10_snapshot", "20_progression"]);
|
|
}
|
|
|
|
fn write(root: &Path, rel: &str, c: &str) {
|
|
let p = root.join(rel);
|
|
fs::create_dir_all(p.parent().unwrap()).unwrap();
|
|
fs::write(p, c).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn sql_collection_rejects_broken_history_in_a_git_checkout() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
write(dir.path(), ".git", "gitdir: missing-history");
|
|
write(dir.path(), "data/sql/updates/db_world/one.sql", "SELECT 1;");
|
|
assert!(collect_core_sql(dir.path())
|
|
.unwrap_err()
|
|
.to_string()
|
|
.contains("migration order"));
|
|
}
|
|
|
|
#[test]
|
|
fn sql_collection_rejects_unreadable_directories_instead_of_omitting_migrations() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
write(dir.path(), "data/sql/updates/db_world", "not a directory");
|
|
assert!(collect_core_sql(dir.path()).is_err());
|
|
let bots = tempfile::tempdir().unwrap();
|
|
write(bots.path(), "dist/sql", "not a directory");
|
|
assert!(collect_bots_sql(bots.path()).is_err());
|
|
let modules = tempfile::tempdir().unwrap();
|
|
write(modules.path(), "modules", "not a directory");
|
|
assert!(collect_core_sql(modules.path()).is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn collects_core_and_module_sql_with_module_base_first_and_stable_ids() {
|
|
let d = tempfile::tempdir().unwrap();
|
|
let c = d.path();
|
|
write(
|
|
c,
|
|
"data/sql/updates/db_world/2026_01_01_00.sql",
|
|
"SELECT 1;",
|
|
);
|
|
write(
|
|
c,
|
|
"data/sql/updates/pending_db_world/rev_1.sql",
|
|
"SELECT 2;",
|
|
);
|
|
write(
|
|
c,
|
|
"data/sql/updates/db_characters/2026_01_02_00.sql",
|
|
"SELECT 3;",
|
|
);
|
|
write(
|
|
c,
|
|
"modules/mod-x/data/sql/db-world/base/01_a.sql",
|
|
"SELECT 4;",
|
|
);
|
|
write(
|
|
c,
|
|
"modules/mod-x/data/sql/db-world/2026_02_02_00_b.sql",
|
|
"SELECT 5;",
|
|
);
|
|
let v = collect_core_install_sql(c).unwrap();
|
|
let ids: Vec<(&str, &str)> = v.iter().map(|f| (f.db.as_str(), f.id.as_str())).collect();
|
|
assert!(
|
|
ids.contains(&("world", "2026_01_01_00"))
|
|
&& ids.contains(&("world", "rev_1"))
|
|
&& ids.contains(&("characters", "2026_01_02_00"))
|
|
);
|
|
assert!(
|
|
ids.contains(&("world", "mod_mod_x__base__01_a"))
|
|
&& ids.contains(&("world", "mod_mod_x__2026_02_02_00_b"))
|
|
);
|
|
// without git history, order falls back to a stable one (all "unknown" keep insertion order)
|
|
let base_pos = ids
|
|
.iter()
|
|
.position(|x| x.1 == "mod_mod_x__base__01_a")
|
|
.unwrap();
|
|
let upd_pos = ids
|
|
.iter()
|
|
.position(|x| x.1 == "mod_mod_x__2026_02_02_00_b")
|
|
.unwrap();
|
|
assert!(
|
|
base_pos < upd_pos,
|
|
"a module's base script precedes its updates"
|
|
);
|
|
let updates = collect_core_sql(c).unwrap();
|
|
assert!(!updates.iter().any(|f| f.id.contains("__base__")));
|
|
assert!(updates.iter().any(|f| f.id == "mod_mod_x__2026_02_02_00_b"));
|
|
}
|
|
|
|
#[test]
|
|
fn update_packaging_rejects_module_seed_sql_before_creating_output() {
|
|
let d = tempfile::tempdir().unwrap();
|
|
let base: Manifest = serde_json::from_value(serde_json::json!({
|
|
"schema": 1, "kind": "base", "version": "1.0.0", "core": {"commit": null},
|
|
"builtAt": "test", "minManagerVersion": "0.1.0", "files": []
|
|
}))
|
|
.unwrap();
|
|
let out = d.path().join("out");
|
|
let sql = vec![SqlFile {
|
|
db: "world".into(),
|
|
id: "mod_mod_raid__base__03_boss_schedule".into(),
|
|
origin: "modules/mod-raid/data/sql/db-world/base/03_boss_schedule.sql".into(),
|
|
abs: d.path().join("seed.sql"),
|
|
sha256: "a".repeat(64),
|
|
}];
|
|
let result = pack_update(
|
|
&UpdateParams {
|
|
tree: d.path(),
|
|
base_manifest: &base,
|
|
sql: &sql,
|
|
out: &out,
|
|
version: "1.1.0".into(),
|
|
core_commit: None,
|
|
bots_commit: None,
|
|
part_size: 1024,
|
|
},
|
|
&|_| {},
|
|
);
|
|
assert!(result.unwrap_err().to_string().contains("installation SQL"));
|
|
assert!(!out.exists());
|
|
assert!(!d.path().join("out-staging").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn cumulative_update_contains_only_what_differs_from_the_base_plus_all_sql() {
|
|
let d = tempfile::tempdir().unwrap();
|
|
let (base_src, base_out, tree, upd_out, sqldir) = (
|
|
d.path().join("bs"),
|
|
d.path().join("bo"),
|
|
d.path().join("tree"),
|
|
d.path().join("uo"),
|
|
d.path().join("sql"),
|
|
);
|
|
write(&base_src, "Core/worldserver.exe", "v1");
|
|
write(&base_src, "Core/authserver.exe", "same");
|
|
write(&base_src, crate::schema_check::CONTRACT, "same contract");
|
|
let base = package::build(
|
|
&base_src,
|
|
&base_out,
|
|
&BuildOptions {
|
|
kind: Kind::Base,
|
|
version: "1.0.0".into(),
|
|
core_commit: None,
|
|
built_at: "x".into(),
|
|
part_size: 1 << 20,
|
|
bots_commit: None,
|
|
migrations: vec![],
|
|
},
|
|
&|_| {},
|
|
)
|
|
.unwrap();
|
|
write(&tree, "Core/worldserver.exe", "v2");
|
|
write(&tree, "Core/authserver.exe", "same");
|
|
write(&tree, "Core/newlib.dll", "n");
|
|
write(&tree, crate::schema_check::CONTRACT, "same contract");
|
|
write(&sqldir, "world.sql", "SELECT 1;");
|
|
let sql = vec![SqlFile {
|
|
db: "world".into(),
|
|
id: "m1".into(),
|
|
origin: "x".into(),
|
|
abs: sqldir.join("world.sql"),
|
|
sha256: fsx::sha256_file(&sqldir.join("world.sql")).unwrap(),
|
|
}];
|
|
let m = pack_update(
|
|
&UpdateParams {
|
|
tree: &tree,
|
|
base_manifest: &base,
|
|
sql: &sql,
|
|
out: &upd_out,
|
|
version: "1.1.0".into(),
|
|
core_commit: Some("a".repeat(40)),
|
|
bots_commit: Some("b".repeat(40)),
|
|
part_size: 1 << 20,
|
|
},
|
|
&|_| {},
|
|
)
|
|
.unwrap();
|
|
let paths: Vec<&str> = m.files.iter().map(|f| f.path.as_str()).collect();
|
|
assert!(paths.contains(&"Core/worldserver.exe") && paths.contains(&"Core/newlib.dll"));
|
|
assert!(
|
|
!paths.contains(&"Core/authserver.exe"),
|
|
"identical to the base: not shipped"
|
|
);
|
|
assert!(paths.contains(&"_migrations/world/m1.sql"));
|
|
assert!(
|
|
paths.contains(&crate::schema_check::CONTRACT),
|
|
"the contract ships even when identical to the base"
|
|
);
|
|
assert_eq!(m.kind, Kind::Update);
|
|
assert_eq!(m.migrations.len(), 1);
|
|
assert_eq!(
|
|
m.bots.as_ref().and_then(|b| b.commit.clone()),
|
|
Some("b".repeat(40))
|
|
);
|
|
assert!(
|
|
!upd_out.with_file_name("uo-staging").exists(),
|
|
"staging removed"
|
|
);
|
|
}
|
|
}
|