diff --git a/crates/coa-core/src/portable/error.rs b/crates/coa-core/src/portable/error.rs index fcba2c5..d1b95d3 100644 --- a/crates/coa-core/src/portable/error.rs +++ b/crates/coa-core/src/portable/error.rs @@ -1,5 +1,5 @@ -use super::ids::CharacterId; -use super::realm::Blocker; +use super::ids::{CharacterId, ImportId}; +use super::realm::{Blocker, ImportProblem}; pub type Result = std::result::Result; @@ -44,6 +44,19 @@ pub enum PortableError { #[error("the realm database is a {found} realm but a {expected} character was requested")] WrongRealm { expected: String, found: String }, + #[error("character {character} is already on server {server_id}")] + AlreadyOnRealm { character: CharacterId, server_id: String }, + #[error("import {import_id} of this character into this server has not finished; recover it first")] + ImportInProgress { import_id: ImportId }, + #[error("import {0} does not exist")] + UnknownImport(ImportId), + #[error("import {import_id} is {state}, which does not allow this")] + ImportState { import_id: ImportId, state: String }, + #[error("the import was refused before anything was written: {}", .0.iter().map(|p| p.to_string()).collect::>().join("; "))] + ImportRefused(Vec), + #[error("import {import_id} needs attention: {detail}")] + ImportNeedsAttention { import_id: ImportId, detail: String }, + #[error("invalid data: {0}")] Invalid(String), #[error("limit exceeded: {0}")] diff --git a/crates/coa-core/src/portable/identity.rs b/crates/coa-core/src/portable/identity.rs index 67ee0ab..13f0092 100644 --- a/crates/coa-core/src/portable/identity.rs +++ b/crates/coa-core/src/portable/identity.rs @@ -1,25 +1,24 @@ //! Content identity of an item: what it *was* when it was mapped to a portable item id. //! //! Local item guids are recycled by the realm. A mapping is only trusted while the item behind the guid still has the -//! same identity. The identity covers the traits that cannot change during an item's life (entry, random property or -//! suffix, crafter); count, durability, enchantments and position change all the time and are deliberately left out. -//! Two different items that share all of these are interchangeable for mapping purposes. +//! same identity. The identity covers what cannot change during an item's life *and* survives a trip through a realm: +//! the entry and the random property / suffix. Count, durability, enchantments, position and the crafter's name are +//! deliberately left out (the crafter is a guid of another realm's character and is not stored at the destination). +//! Two different items that share both traits are interchangeable for mapping purposes. use sha2::{Digest, Sha256}; use super::ids::ContentId; -const VERSION: &str = "v1"; +const VERSION: &str = "v2"; -pub fn item_identity(entry: &ContentId, random_property_id: i32, creator_name: Option<&str>) -> String { +pub fn item_identity(entry: &ContentId, random_property_id: i32) -> String { let mut h = Sha256::new(); h.update(VERSION.as_bytes()); h.update([0]); h.update(entry.to_string().as_bytes()); h.update([0]); h.update(random_property_id.to_be_bytes()); - h.update([0]); - h.update(creator_name.unwrap_or("").as_bytes()); format!("{VERSION}:{}", hex::encode(&h.finalize()[..16])) } @@ -33,19 +32,17 @@ mod tests { #[test] fn same_traits_same_identity_and_any_difference_changes_it() { - let base = item_identity(&entry(100), -5, Some("Smith")); - assert_eq!(base, item_identity(&entry(100), -5, Some("Smith"))); - assert_ne!(base, item_identity(&entry(101), -5, Some("Smith")), "another entry"); - assert_ne!(base, item_identity(&entry(100), -6, Some("Smith")), "another suffix"); - assert_ne!(base, item_identity(&entry(100), -5, Some("Other")), "another crafter"); - assert_ne!(base, item_identity(&entry(100), -5, None), "no crafter"); - assert!(base.starts_with("v1:") && base.len() == 3 + 32); + let base = item_identity(&entry(100), -5); + assert_eq!(base, item_identity(&entry(100), -5)); + assert_ne!(base, item_identity(&entry(101), -5), "another entry"); + assert_ne!(base, item_identity(&entry(100), -6), "another suffix"); + assert!(base.starts_with("v2:") && base.len() == 3 + 32); } #[test] fn the_namespace_is_part_of_the_identity() { let a = ContentId::new("coa", "item", 7).unwrap(); let b = ContentId::new("mod:x", "item", 7).unwrap(); - assert_ne!(item_identity(&a, 0, None), item_identity(&b, 0, None)); + assert_ne!(item_identity(&a, 0), item_identity(&b, 0)); } } diff --git a/crates/coa-core/src/portable/ids.rs b/crates/coa-core/src/portable/ids.rs index 7f3e899..cae612b 100644 --- a/crates/coa-core/src/portable/ids.rs +++ b/crates/coa-core/src/portable/ids.rs @@ -92,6 +92,10 @@ uuid7_id!( /// The global identity of one pet of a portable character. PortablePetId ); +uuid7_id!( + /// One import of a portable character into a realm (the key of the import journal). + ImportId +); /// A reference to a piece of game content that cannot be mistaken for another namespace's numbering: /// `core:wotlk:item:19019`, `coa:wardrobe:10581`, `mod::item:7`. diff --git a/crates/coa-core/src/portable/migrations/003_import_journal.sql b/crates/coa-core/src/portable/migrations/003_import_journal.sql new file mode 100644 index 0000000..66111d3 --- /dev/null +++ b/crates/coa-core/src/portable/migrations/003_import_journal.sql @@ -0,0 +1,28 @@ +-- Schema 3: the import journal. An import changes two systems that cannot share one transaction: the realm's MySQL and +-- this local store. The journal is the bridge: a row is written BEFORE the realm transaction ("prepared"), the realm +-- transaction stores a marker that carries the same nonce, and the row is completed in the same local transaction that +-- records the realm mappings. After a crash between the two commits, the marker in the realm decides what happened. + +CREATE TABLE import_journal ( + import_id TEXT PRIMARY KEY, + character_id TEXT NOT NULL REFERENCES character(character_id) ON DELETE CASCADE, + server_id TEXT NOT NULL, + revision INTEGER NOT NULL, + -- the text of the realm's `coa.portable.import` settings row: nonce words and revision + marker TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('prepared', 'committed', 'aborted', 'needs_attention')), + -- JSON, in plan order: item i is realm guid item_base + i, pet i is pet_base + i. Items carry what the mapping needs + -- ({id, entry, identity}) so completing the import never depends on a snapshot that may have been pruned meanwhile. + items TEXT NOT NULL, + pet_ids TEXT NOT NULL, + local_guid INTEGER, + item_base INTEGER, + pet_base INTEGER, + detail TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +) STRICT; + +-- one unfinished import per (character, realm) +CREATE UNIQUE INDEX import_one_open ON import_journal(character_id, server_id) WHERE state = 'prepared'; +CREATE INDEX import_by_server ON import_journal(server_id, state); diff --git a/crates/coa-core/src/portable/mod.rs b/crates/coa-core/src/portable/mod.rs index 160a649..a8cedbf 100644 --- a/crates/coa-core/src/portable/mod.rs +++ b/crates/coa-core/src/portable/mod.rs @@ -21,6 +21,6 @@ pub(crate) mod fixtures; pub use collection::IdSet; pub use error::{PortableError, Result}; -pub use ids::{CharacterId, ContentId, PortableItemId, PortablePetId, ProfileId}; +pub use ids::{CharacterId, ContentId, ImportId, PortableItemId, PortablePetId, ProfileId}; pub use model::{PortableCharacter, Ruleset}; pub use store::Store; diff --git a/crates/coa-core/src/portable/realm/export.rs b/crates/coa-core/src/portable/realm/export.rs index 51edab7..a18debd 100644 --- a/crates/coa-core/src/portable/realm/export.rs +++ b/crates/coa-core/src/portable/realm/export.rs @@ -272,7 +272,7 @@ fn items(raw: &RawExport, req: &ExportRequest<'_>, ns: &str, warnings: &mut Vec< let entry = ContentId::new(ns, "item", r.u64("entry")?)?; let random_property_id = r.i32("random_property")?; let creator_name = if r.u64("creator_guid")? != 0 { r.opt_text("creator_name")?.filter(|n| !n.is_empty()) } else { None }; - let identity = item_identity(&entry, random_property_id, creator_name.as_deref()); + let identity = item_identity(&entry, random_property_id); // keep the portable id when this local guid is still the same item; a recycled guid gets a new one let id = match req.prior_items.get(&p.guid) { diff --git a/crates/coa-core/src/portable/realm/import.rs b/crates/coa-core/src/portable/realm/import.rs new file mode 100644 index 0000000..577f53c --- /dev/null +++ b/crates/coa-core/src/portable/realm/import.rs @@ -0,0 +1,406 @@ +//! Phase 3: importing a portable character into a **stopped, disposable** realm (the offline importer). +//! +//! This is *not* the production importer. Production imports into a running realm are done by an in-core service +//! (`docs/PORTABLE_IMPORT_SERVICE.md`); this path exists to prove the model end to end and for realms that are stopped. +//! +//! ```text +//! preflight (read-only snapshot) -> journal "prepared" -> ONE realm transaction -> mappings + journal "committed" +//! (marker inside it) (ONE local transaction) +//! ``` +//! +//! * Goes through the existing [`Db`] abstraction; no driver is added or exposed. +//! * Everything written is generated by the typed encoder; untrusted text and blobs are hex literals only. +//! * The realm transaction allocates every guid from the realm's own tables and remaps every reference; it asserts its own +//! result before `COMMIT`. +//! * Manager mappings are recorded **only after** the realm commit. If the process dies between the two commits, the +//! marker row written inside the realm transaction tells [`recover_imports`] what happened. + +use std::collections::BTreeSet; +use std::fmt; +use std::time::Duration; + +use crate::db::Db; + +use super::super::error::{PortableError, Result}; +use super::super::ids::{CharacterId, ContentId, ImportId}; +use super::super::model::{PortableCharacter, Ruleset}; +use super::super::store::{ImportAllocation, ImportState, JournalEntry, PlannedItem, Store}; +use super::plan::{build_plan, parse_report, Allocation, ImportPlan, PlanContext, IMPORT_LOCK}; +use super::script::{self, parse_output, Query}; +use super::sqlenc::Val; +use super::{realm_error, ruleset_of}; + +#[derive(Debug, Clone)] +pub struct ImportOptions { + /// A realm refuses new characters beyond this per account. + pub max_characters_per_account: u32, + /// Database users of the game servers. A session of one of them means a realm is running. + pub game_server_users: Vec, + /// Recovery never declares an import "not committed" before it is this old (an orphaned client process may still be + /// finishing it). + pub recovery_grace: Duration, + /// How long recovery waits for an import that is still running in the realm. + pub lock_wait_seconds: u32, +} + +impl Default for ImportOptions { + fn default() -> Self { + Self { max_characters_per_account: 10, game_server_users: vec!["acore".to_string()], recovery_grace: Duration::from_secs(30), lock_wait_seconds: 30 } + } +} + +/// Why an import was refused before anything was written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImportProblem { + /// The character belongs to another ruleset than this realm. + WrongRuleset { character: Ruleset, realm: Ruleset }, + /// A session of the game server's database user exists: the realm is running. Only a stopped realm can be imported into + /// from outside (a running realm hands out guids from memory). + RealmRunning { sessions: u64 }, + OnlineCharacters(u64), + AccountMissing(u32), + /// Bot and Manager accounts are not destinations. + InternalAccount(u32), + AccountFull { count: u64, max: u32 }, + /// The realm's world data has no start position for this race/class. + UnknownRaceClass { race: u64, class: u64 }, + /// Item entries the realm's `item_template` does not know: it would delete such items on load. + MissingItems(Vec), + MissingPetCreatures(Vec), + /// An id of a namespace this importer does not apply (mod content, another ruleset). + UnsupportedContent(String), +} + +impl fmt::Display for ImportProblem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let list = |v: &[u32]| v.iter().take(10).map(u32::to_string).collect::>().join(", ") + if v.len() > 10 { ", ..." } else { "" }; + match self { + ImportProblem::WrongRuleset { character, realm } => write!(f, "the character is a {character} character and this is a {realm} realm"), + ImportProblem::RealmRunning { sessions } => write!(f, "the realm is running ({sessions} session(s) of the game server's database user); stop it first"), + ImportProblem::OnlineCharacters(n) => write!(f, "{n} character(s) are marked online; stop the realm first"), + ImportProblem::AccountMissing(a) => write!(f, "game account {a} does not exist on this realm"), + ImportProblem::InternalAccount(a) => write!(f, "account {a} belongs to a bot or the Manager"), + ImportProblem::AccountFull { count, max } => write!(f, "the account already has {count} characters (limit {max})"), + ImportProblem::UnknownRaceClass { race, class } => write!(f, "this realm has no start position for race {race} / class {class}"), + ImportProblem::MissingItems(v) => write!(f, "the realm does not know these item entries: {}", list(v)), + ImportProblem::MissingPetCreatures(v) => write!(f, "the realm does not know these pet creatures: {}", list(v)), + ImportProblem::UnsupportedContent(s) => write!(f, "{s}"), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct PreflightReport { + pub problems: Vec, + pub warnings: Vec, + /// The name is taken (or reserved) on the realm: the character will arrive under a temporary name and rename at login. + pub will_rename: bool, +} + +/// Namespace problems that can be found without looking at the realm. +fn content_problems(model: &PortableCharacter, ruleset: Ruleset) -> Vec { + let ns = ruleset.as_str(); + let mut bad: BTreeSet = BTreeSet::new(); + let mut check = |what: &str, id: &ContentId, kind: &str| { + if id.namespace() != ns || id.kind() != kind { + bad.insert(format!("{what} {id} cannot be imported into a {ns} realm")); + } + }; + check("race", &model.identity.race, "race"); + check("class", &model.identity.class, "class"); + for item in &model.items { + check("item", &item.entry, "item"); + if let Some(g) = &item.gift { + check("gift", &g.entry, "item"); + } + } + for pet in &model.pets { + check("pet", &pet.entry, "creature"); + } + bad.into_iter().take(10).map(ImportProblem::UnsupportedContent).collect() +} + +fn count_query(name: &str, sql: String) -> Query { + Query { name: name.to_string(), columns: vec!["n"], sql } +} + +fn users_sql(users: &[String]) -> String { + users.iter().map(|u| Val::text(u.clone()).sql()).collect::>().join(", ") +} + +fn id_list(ids: &BTreeSet) -> String { + ids.iter().map(u32::to_string).collect::>().join(", ") +} + +/// Look at the realm (one read-only consistent snapshot) and at the character, and say whether the import can start. +pub fn preflight(db: &Db, model: &PortableCharacter, account: u32, opts: &ImportOptions) -> Result { + let mut report = PreflightReport::default(); + let realm_ruleset = ruleset_of(db); + if model.ruleset != realm_ruleset { + report.problems.push(ImportProblem::WrongRuleset { character: model.ruleset, realm: realm_ruleset }); + } + report.problems.extend(content_problems(model, realm_ruleset)); + if !report.problems.is_empty() { + return Ok(report); + } + let race = model.identity.race.id(); + let class = model.identity.class.id(); + let entries: BTreeSet = model.items.iter().map(|i| i.entry.id()).chain(model.items.iter().filter_map(|i| i.gift.as_ref().map(|g| g.entry.id()))).filter_map(|e| u32::try_from(e).ok()).collect(); + let creatures: BTreeSet = model.pets.iter().filter_map(|p| u32::try_from(p.entry.id()).ok()).collect(); + + let probe = super::probe(db)?; + let mut queries = vec![ + Query { name: "account".into(), columns: vec!["n", "username"], sql: format!("SELECT COUNT(*), IFNULL(MAX(CONCAT('x', HEX(username))), '-') FROM acore_auth.account WHERE id = {account}") }, + count_query("online", "SELECT COUNT(*) FROM acore_characters.characters WHERE online <> 0".into()), + count_query("sessions", format!("SELECT COUNT(*) FROM information_schema.processlist WHERE id <> CONNECTION_ID() AND user IN ({})", users_sql(&opts.game_server_users))), + count_query("account_chars", format!("SELECT COUNT(*) FROM acore_characters.characters WHERE account = {account}")), + count_query("createinfo", format!("SELECT COUNT(*) FROM acore_world.playercreateinfo WHERE race = {race} AND class = {class}")), + ]; + let name = Val::text(model.identity.name.clone()).sql(); + let mut taken = format!("EXISTS(SELECT 1 FROM acore_characters.characters WHERE name COLLATE utf8mb4_unicode_ci = {name} COLLATE utf8mb4_unicode_ci)"); + if probe.has("reserved_name") { + taken.push_str(&format!(" OR EXISTS(SELECT 1 FROM acore_characters.reserved_name WHERE name COLLATE utf8mb4_unicode_ci = {name} COLLATE utf8mb4_unicode_ci)")); + } + queries.push(count_query("name_taken", format!("SELECT IF({taken}, 1, 0)"))); + if !entries.is_empty() { + queries.push(Query { name: "item_entries".into(), columns: vec!["entry"], sql: format!("SELECT entry FROM acore_world.item_template WHERE entry IN ({})", id_list(&entries)) }); + } + if !creatures.is_empty() { + queries.push(Query { name: "creature_entries".into(), columns: vec!["entry"], sql: format!("SELECT entry FROM acore_world.creature_template WHERE entry IN ({})", id_list(&creatures)) }); + } + let output = db.query(&script::snapshot_script(&queries)).map_err(realm_error)?; + let raw = parse_output(&output, &queries)?; + let one = |name: &str| -> Result { raw.section(name)?.iter().next().map(|r| r.u64("n")).transpose().map(|v| v.unwrap_or(0)) }; + + let sessions = one("sessions")?; + if sessions > 0 { + report.problems.push(ImportProblem::RealmRunning { sessions }); + } + let online = one("online")?; + if online > 0 { + report.problems.push(ImportProblem::OnlineCharacters(online)); + } + let account_row = raw.section("account")?.iter().next().ok_or_else(|| PortableError::CorruptSnapshot("no account answer".into()))?; + if account_row.u64("n")? == 0 { + report.problems.push(ImportProblem::AccountMissing(account)); + } else if account_row.opt_text("username")?.is_some_and(|u| super::blockers::is_internal_account(&u)) { + report.problems.push(ImportProblem::InternalAccount(account)); + } + let chars = one("account_chars")?; + if chars >= opts.max_characters_per_account as u64 { + report.problems.push(ImportProblem::AccountFull { count: chars, max: opts.max_characters_per_account }); + } + if one("createinfo")? == 0 { + report.problems.push(ImportProblem::UnknownRaceClass { race, class }); + } + report.will_rename = one("name_taken")? != 0; + if report.will_rename { + report.warnings.push("the name is taken or reserved on this realm: the character arrives under a temporary name and must be renamed at login".into()); + } + let known = |name: &str| -> Result> { + if !raw.has(name) { + return Ok(BTreeSet::new()); + } + raw.section(name)?.iter().map(|r| r.u32("entry")).collect() + }; + let missing_items: Vec = entries.difference(&known("item_entries")?).copied().collect(); + if !missing_items.is_empty() { + report.problems.push(ImportProblem::MissingItems(missing_items)); + } + let missing_pets: Vec = creatures.difference(&known("creature_entries")?).copied().collect(); + if !missing_pets.is_empty() { + report.problems.push(ImportProblem::MissingPetCreatures(missing_pets)); + } + Ok(report) +} + +#[derive(Debug)] +pub struct ImportOutcome { + pub import_id: ImportId, + pub local_guid: u32, + pub final_name: String, + pub renamed: bool, + pub items: usize, + pub pets: usize, + /// Settings sources and extensions that are in the snapshot but were deliberately not written to the realm. + pub not_applied: Vec, + pub warnings: Vec, +} + +pub(crate) fn planned_items(model: &PortableCharacter) -> Vec { + model + .items + .iter() + .map(|i| PlannedItem { id: i.id, entry: i.entry.clone(), identity: super::super::identity::item_identity(&i.entry, i.random_property_id) }) + .collect() +} + +/// Run the generated script in the realm. `Ok` carries the realm's report; any SQL error means the transaction rolled back +/// or never committed (the caller still verifies through the marker, because a lost answer is indistinguishable). +pub fn run_realm_import(db: &Db, plan: &ImportPlan) -> Result<(Allocation, bool)> { + let output = db.query(&plan.script).map_err(realm_error)?; + parse_report(&output) +} + +/// Import the character's current canonical revision into a stopped realm. +pub fn import_character(db: &Db, store: &mut Store, character_id: CharacterId, server_id: &str, account: u32, opts: &ImportOptions) -> Result { + let record = store.character(character_id)?; + let model = store.load_snapshot(character_id, record.revision)?; + if store.server_mappings(character_id)?.iter().any(|m| m.server_id == server_id) { + return Err(PortableError::AlreadyOnRealm { character: character_id, server_id: server_id.to_string() }); + } + + let report = preflight(db, &model, account, opts)?; + if !report.problems.is_empty() { + return Err(PortableError::ImportRefused(report.problems)); + } + let probe = super::probe(db)?; + + let items = planned_items(&model); + let pet_ids: Vec<_> = model.pets.iter().map(|p| p.id).collect(); + let ticket = store.begin_import(character_id, server_id, record.revision, &items, &pet_ids)?; + + let plan = match build_plan(&model, &PlanContext { ruleset: ruleset_of(db), account, revision: record.revision, nonce: ticket.nonce, max_characters_per_account: opts.max_characters_per_account, game_server_users: &opts.game_server_users, probe: &probe }) { + Ok(plan) => plan, + Err(e) => { + store.abort_import(ticket.import_id, &format!("the import plan could not be built: {e}"))?; + return Err(e); + } + }; + debug_assert_eq!(plan.marker_data, ticket.marker, "the plan and the journal must use the same marker"); + + let allocation = match run_realm_import(db, &plan) { + Ok((alloc, true)) => alloc, + // not committed, or the answer was lost: the marker in the realm decides + outcome => { + let original = outcome.err(); + return match resolve_import(db, store, ticket.import_id, opts, false)? { + Resolution::Committed(alloc) => { + // the realm's own report was lost: read the final name back + let (final_name, renamed) = read_name(db, alloc.local_guid)?; + Ok(ImportOutcome { import_id: ticket.import_id, local_guid: alloc.local_guid, final_name, renamed, items: plan.counts.items, pets: plan.counts.pets, not_applied: plan.not_applied_settings, warnings: report.warnings }) + } + Resolution::Aborted => Err(original.unwrap_or_else(|| PortableError::RealmRead("the realm did not commit the import".into()))), + Resolution::Pending(why) => Err(PortableError::ImportNeedsAttention { import_id: ticket.import_id, detail: format!("the outcome in the realm is not known yet ({why}); run recovery") }), + }; + } + }; + + store.finish_import(ticket.import_id, ImportAllocation { local_guid: allocation.local_guid, item_base: allocation.item_base, pet_base: allocation.pet_base })?; + Ok(ImportOutcome { + import_id: ticket.import_id, + local_guid: allocation.local_guid, + final_name: allocation.final_name, + renamed: allocation.renamed, + items: plan.counts.items, + pets: plan.counts.pets, + not_applied: plan.not_applied_settings, + warnings: report.warnings, + }) +} + +/// The name and rename flag of a character the realm holds (typed guid, fixed SQL). +fn read_name(db: &Db, guid: u32) -> Result<(String, bool)> { + let out = db.query(&format!("SELECT CONCAT('x', HEX(name)), (at_login & 1) FROM acore_characters.characters WHERE guid = {guid}")).map_err(realm_error)?; + let line = out.lines().next().ok_or_else(|| PortableError::CorruptSnapshot("the imported character is not in the realm".into()))?; + let (name, flag) = line.split_once('\t').ok_or_else(|| PortableError::CorruptSnapshot("unexpected name answer".into()))?; + let bytes = hex::decode(name.trim_start_matches('x')).map_err(|_| PortableError::CorruptSnapshot("invalid name hex".into()))?; + Ok((String::from_utf8(bytes).map_err(|_| PortableError::CorruptSnapshot("name is not UTF-8".into()))?, flag.trim() == "1")) +} + +// ---- recovery ----------------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Resolution { + /// The realm committed this import; the local records are (now) complete. + Committed(ImportAllocation), + /// The realm has no trace of it: it never committed. + Aborted, + /// Cannot be decided yet (too recent, or the realm is still running it). + Pending(String), +} + +#[derive(Debug)] +pub struct RecoveryReport { + pub import_id: ImportId, + pub resolution: Resolution, +} + +fn recovery_queries() -> Vec { + let q = |name: &str, columns: Vec<&'static str>| Query { name: name.to_string(), columns, sql: String::new() }; + vec![q("lock", vec!["n"]), q("marker", vec!["guid"]), q("items", vec!["n", "min", "max"]), q("pets", vec!["n", "min", "max"])] +} + +/// The realm side of recovery: take the import lock (so an import that is still running finishes first), then find the +/// character that carries this import's marker. +fn recovery_script(marker: &str, wait: u32) -> String { + let source = Val::text(super::policy::IMPORT_MARKER_SOURCE).sql(); + let data = Val::text(marker.to_string()).sql(); + format!( + "SELECT '#T:lock';\nSELECT GET_LOCK('{IMPORT_LOCK}', {wait});\n\ + SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;\nSTART TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;\n\ + SET @g := (SELECT MIN(guid) FROM acore_characters.character_settings WHERE source = {source} AND data = {data});\n\ + SELECT '#T:marker';\nSELECT IFNULL(@g, 'none');\n\ + SELECT '#T:items';\nSELECT COUNT(*), IFNULL(MIN(guid), 0), IFNULL(MAX(guid), 0) FROM acore_characters.item_instance WHERE owner_guid = @g;\n\ + SELECT '#T:pets';\nSELECT COUNT(*), IFNULL(MIN(id), 0), IFNULL(MAX(id), 0) FROM acore_characters.character_pet WHERE owner = @g;\n\ + COMMIT;\nDO RELEASE_LOCK('{IMPORT_LOCK}');\n" + ) +} + +/// Decide what became of one unfinished import, and bring the local records in line with the realm. +/// `with_grace`: refuse to call an import "never committed" while it is younger than the recovery grace period. +pub fn resolve_import(db: &Db, store: &mut Store, import_id: ImportId, opts: &ImportOptions, with_grace: bool) -> Result { + let entry: JournalEntry = store.import_entry(import_id)?; + match entry.state { + ImportState::Committed => return Ok(Resolution::Committed(entry.allocation.expect("a committed entry has its allocation"))), + ImportState::Aborted => return Ok(Resolution::Aborted), + ImportState::NeedsAttention => return Err(PortableError::ImportNeedsAttention { import_id, detail: entry.detail.unwrap_or_default() }), + ImportState::Prepared => {} + } + let output = db.query(&recovery_script(&entry.marker, opts.lock_wait_seconds)).map_err(realm_error)?; + let queries = recovery_queries(); + let raw = parse_output(&output, &queries)?; + let lock = raw.section("lock")?.iter().next().ok_or_else(|| PortableError::CorruptSnapshot("no lock answer".into()))?.u64("n")?; + if lock != 1 { + return Ok(Resolution::Pending("an import is still running in the realm".into())); + } + let guid_text = raw.section("marker")?.iter().next().ok_or_else(|| PortableError::CorruptSnapshot("no marker answer".into()))?.plain("guid")?; + if guid_text == "none" { + if with_grace && age_seconds(&entry.created_at) < opts.recovery_grace.as_secs() as i64 { + return Ok(Resolution::Pending("the import is too recent to be called lost".into())); + } + store.abort_import(import_id, "the realm has no trace of this import: it never committed")?; + return Ok(Resolution::Aborted); + } + let local_guid: u32 = guid_text.parse().map_err(|_| PortableError::CorruptSnapshot(format!("marker answer {guid_text:?} is not a guid")))?; + + let triple = |name: &str| -> Result<(u64, u64, u64)> { + let r = raw.section(name)?.iter().next().ok_or_else(|| PortableError::CorruptSnapshot(format!("no {name} answer")))?; + Ok((r.u64("n")?, r.u64("min")?, r.u64("max")?)) + }; + let (n_items, min_item, max_item) = triple("items")?; + let (n_pets, min_pet, max_pet) = triple("pets")?; + let contiguous = |n: u64, min: u64, max: u64| n == 0 || max - min + 1 == n; + if n_items as usize != entry.items.len() || n_pets as usize != entry.pet_ids.len() || !contiguous(n_items, min_item, max_item) || !contiguous(n_pets, min_pet, max_pet) { + let detail = format!("the realm has a character with this import's marker (guid {local_guid}) but {n_items} items / {n_pets} pets where the plan had {} / {}", entry.items.len(), entry.pet_ids.len()); + store.flag_import(import_id, &detail)?; + return Err(PortableError::ImportNeedsAttention { import_id, detail }); + } + let alloc = ImportAllocation { local_guid, item_base: min_item as u32, pet_base: min_pet as u32 }; + store.finish_import(import_id, alloc)?; + Ok(Resolution::Committed(alloc)) +} + +fn age_seconds(created_at: &str) -> i64 { + chrono::DateTime::parse_from_rfc3339(created_at).map(|t| (chrono::Utc::now() - t.with_timezone(&chrono::Utc)).num_seconds()).unwrap_or(i64::MAX) +} + +/// Resolve every unfinished import into `server_id`. Call this before starting any new import and when the Manager starts. +pub fn recover_imports(db: &Db, store: &mut Store, server_id: &str, opts: &ImportOptions) -> Result> { + let mut out = Vec::new(); + for entry in store.open_imports(server_id)? { + let resolution = resolve_import(db, store, entry.import_id, opts, true)?; + out.push(RecoveryReport { import_id: entry.import_id, resolution }); + } + Ok(out) +} diff --git a/crates/coa-core/src/portable/realm/import_tests.rs b/crates/coa-core/src/portable/realm/import_tests.rs new file mode 100644 index 0000000..19816e2 --- /dev/null +++ b/crates/coa-core/src/portable/realm/import_tests.rs @@ -0,0 +1,358 @@ +//! Tests of the import plan and its SQL, without a database. The behaviour against a real MySQL is in `live.rs`. + +use std::collections::BTreeSet; + +use super::plan::*; +use super::policy::IMPORT_MARKER_SOURCE; +use super::script::SchemaProbe; +use super::sqlenc::Val; +use crate::portable::fixtures::geared_level_eighty; +use crate::portable::ids::{ContentId, PortableItemId, PortablePetId}; +use crate::portable::model::*; +use crate::portable::PortableError; + +fn probe_with(tables: &[&str]) -> SchemaProbe { + SchemaProbe { tables: tables.iter().map(|t| t.to_string()).collect(), character_columns: Default::default() } +} + +fn ctx<'a>(probe: &'a SchemaProbe, users: &'a [String]) -> PlanContext<'a> { + PlanContext { ruleset: Ruleset::Coa, account: 5001, revision: 3, nonce: [11, 22, 33, 44], max_characters_per_account: 10, game_server_users: users, probe } +} + +fn users() -> Vec { + vec!["acore".to_string()] +} + +fn all_tables() -> SchemaProbe { + let mut names: Vec<&str> = CHARACTER_KEYED.iter().map(|(t, _)| *t).collect(); + names.extend(ITEM_KEYED.iter().map(|(t, _)| *t)); + names.extend(PET_KEYED.iter().map(|(t, _)| *t)); + names.push("reserved_name"); + probe_with(&names) +} + +fn plan_of(model: &PortableCharacter) -> ImportPlan { + let u = users(); + build_plan(model, &ctx(&all_tables(), &u)).unwrap() +} + +/// The script with every hex literal replaced by `H`: what is left is the fixed scaffold. +fn skeleton(script: &str) -> String { + let bytes = script.as_bytes(); + let mut out = String::new(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'0' && bytes.get(i + 1) == Some(&b'x') && bytes.get(i + 2).is_some_and(u8::is_ascii_hexdigit) && (i == 0 || !bytes[i - 1].is_ascii_alphanumeric()) { + i += 2; + while i < bytes.len() && bytes[i].is_ascii_hexdigit() { + i += 1; + } + out.push('H'); + } else { + out.push(bytes[i] as char); + i += 1; + } + } + out +} + +fn hostile_model() -> PortableCharacter { + let mut m = geared_level_eighty(); + m.identity.name = "Bob');DROP TABLE x;--".into(); + m.items[0].text = Some("\"; DELETE FROM item_instance; -- \\ \0 \n\t' OR '1'='1".into()); + m.client_data.get_mut(&5).unwrap().data = Bytes(b"'; DROP DATABASE acore_characters; -- \x00\xff".to_vec()); + m.settings.insert("core.ascension_build.77".into(), vec![1, 2, 3]); + m.pets.push(PortablePet { + id: PortablePetId::new(), + entry: ContentId::new("coa", "creature", 416).unwrap(), + model_id: 1, + created_by_spell: 2, + pet_type: 0, + level: 1, + exp: 0, + react_state: 1, + name: "p');DROP TABLE x;--".into(), + renamed: false, + slot: 100, + health: 1, + mana: 1, + happiness: 1, + action_bar: "'; DROP TABLE pet_spell; --".into(), + spells: vec![PetSpell { spell: 3110, active: 1 }], + declined_names: Some(["a'".into(), "b\"".into(), "c\\".into(), "d;".into(), "e--".into()]), + }); + m.normalized() +} + +// ---- the script --------------------------------------------------------------------------------------------------- + +#[test] +fn the_script_is_exactly_one_realm_transaction() { + let plan = plan_of(&geared_level_eighty()); + let s = &plan.script; + assert_eq!(s.matches("START TRANSACTION;").count(), 1); + assert_eq!(s.matches("COMMIT;").count(), 1); + assert!(s.find("START TRANSACTION;").unwrap() < s.find("INSERT INTO").unwrap()); + assert!(s.find("COMMIT;").unwrap() > s.rfind("INSERT INTO").unwrap(), "every insert is before the commit"); + // nothing is written after the commit, and nothing implicitly commits before it + let after = &s[s.find("COMMIT;").unwrap()..]; + assert!(!after.contains("INSERT") && !after.contains("DELETE") && !after.contains("UPDATE")); + let upper = s.to_ascii_uppercase(); + for forbidden in ["CREATE ", "ALTER ", "DROP ", "TRUNCATE ", "RENAME TABLE", "LOCK TABLES", "UNLOCK TABLES", "SET AUTOCOMMIT", "SET GLOBAL", "LOAD DATA", "OUTFILE", "GRANT ", "ROLLBACK"] { + assert!(!upper.contains(forbidden), "the script contains {forbidden:?}"); + } + // the transaction proves its own result before it commits + let before_commit = &s[..s.find("COMMIT;").unwrap()]; + assert!(before_commit.matches("DO IF(").count() >= 20, "assertions: {}", before_commit.matches("DO IF(").count()); + assert!(before_commit.contains("(SELECT 1 UNION ALL SELECT 2)"), "a failing assertion raises an SQL error"); + assert!(s.starts_with("DO IF(GET_LOCK('coa_portable_import', 10) = 1")); + assert!(s.trim_end().ends_with("DO RELEASE_LOCK('coa_portable_import');")); +} + +#[test] +fn untrusted_text_and_blobs_exist_only_as_hex_literals() { + let benign = plan_of(&geared_level_eighty()); + let hostile = plan_of(&hostile_model()); + // the scaffold differs only by the rows the hostile model adds (one pet): what the hostile data contains cannot change it + let mut quiet = hostile_model(); + quiet.identity.name = "Quiet".into(); + quiet.items[0].text = Some("plain".into()); + quiet.client_data.get_mut(&5).unwrap().data = Bytes(vec![1, 2, 3]); + quiet.pets[1].name = "pet".into(); + quiet.pets[1].action_bar = "bar".into(); + quiet.pets[1].declined_names = Some(["a".into(), "b".into(), "c".into(), "d".into(), "e".into()]); + let quiet = plan_of(&quiet); + let (a, b) = (skeleton(&hostile.script), skeleton(&quiet.script)); + if a != b { + let at = a.bytes().zip(b.bytes()).position(|(x, y)| x != y).unwrap_or(a.len().min(b.len())); + panic!("scaffolds differ at {at}: {:?} vs {:?}", &a[at.saturating_sub(60)..(at + 60).min(a.len())], &b[at.saturating_sub(60)..(at + 60).min(b.len())]); + } + assert_ne!(hostile.script, quiet.script); + for fragment in ["DROP TABLE", "DELETE FROM item_instance;", "DROP DATABASE", "OR '1'='1", "Bob", "\\", "\0"] { + assert!(!hostile.script.contains(fragment), "{fragment:?} leaked into the SQL"); + } + assert!(benign.script.len() < hostile.script.len()); + // the only quotes in the whole script belong to fixed literals of this crate + let quotes = |s: &str| skeleton(s).matches('\'').count(); + assert_eq!(quotes(&hostile.script), quotes(&quiet.script)); +} + +#[test] +fn every_text_of_the_character_is_in_the_script_as_its_exact_utf8_bytes() { + let plan = plan_of(&hostile_model()); + for text in ["Bob');DROP TABLE x;--", "p');DROP TABLE x;--", "core.ascension_build.77", "1 2 3 "] { + assert!(plan.script.contains(&Val::text(text).sql()), "{text:?}"); + } + assert!(plan.script.contains(&Val::Bytes(b"'; DROP DATABASE acore_characters; -- \x00\xff".to_vec()).sql())); +} + +#[test] +fn every_reference_is_remapped_through_the_realms_own_allocation() { + let model = geared_level_eighty(); + let plan = plan_of(&model); + let n = model.items.len(); + assert_eq!(plan.item_ids.len(), n); + assert_eq!(plan.item_ids, model.items.iter().map(|i| i.id).collect::>()); + // item i is @item_base + i, in the item rows and in the inventory rows; nothing outside 0..n + let refs: BTreeSet = plan.script.match_indices("(@item_base + ").map(|(at, m)| plan.script[at + m.len()..].split(')').next().unwrap().parse().unwrap()).collect(); + assert_eq!(refs, (0..n).collect::>()); + // the script allocates from the realm, never from the canonical data + assert!(plan.script.contains("SET @char := (SELECT IFNULL(MAX(guid), 0) + 1 FROM acore_characters.characters);")); + assert!(plan.script.contains("SET @item_base := GREATEST(0, ")); + assert!(plan.script.contains("SET @pet_base := GREATEST(0, ")); + for table in ["item_instance", "mail_items", "auctionhouse", "guild_bank_item", "character_inventory"] { + assert!(plan.script.contains(&format!("FROM acore_characters.`{table}`)")), "{table} must be considered when allocating item guids"); + } + // the character row and every per-character row use @char; no realm guid of any other character is written + let item_insert = plan.script.lines().find(|l| l.starts_with("INSERT INTO acore_characters.`item_instance`")).unwrap(); + assert!(item_insert.contains("VALUES\n") || plan.script.contains("`item_instance` (`guid`")); + assert!(plan.script.contains("(@char, "), "per-character rows are keyed by the new guid"); + // bags: an item in a bag names the bag's plan index + let bag_index = model.items.iter().position(|i| i.container.is_none() && i.slot == 19).unwrap(); + assert!(plan.script.contains(&format!("(@char, (@item_base + {bag_index}), 0,")), "an item in the first bag points at that bag, slot 0"); +} + +#[test] +fn the_plan_counts_what_it_writes_and_asserts_it() { + let model = geared_level_eighty(); + let plan = plan_of(&model); + let c = &plan.counts; + assert_eq!((c.items, c.inventory, c.gifts), (model.items.len(), model.items.len(), 1)); + assert_eq!((c.spells, c.talents, c.skills, c.glyphs), (1284, 0, 23, 1)); + assert_eq!((c.reputation, c.quests, c.rewarded, c.actions, c.pets, c.macros), (105, 2, 4, 12, 1, 1)); + assert_eq!(c.settings, 4, "active spec, starter, build and bar are the carried keys of the fixture character"); + assert!(plan.script.contains(&format!("acore_characters.item_instance WHERE owner_guid = @char) = {},", model.items.len()))); + assert!(plan.script.contains("acore_characters.character_spell WHERE guid = @char) = 1284,")); + assert!(plan.script.contains("acore_characters.character_settings WHERE guid = @char) = 5,"), "carried settings plus the marker"); + // every inventory row must resolve to an item of this character, every bag to an inventory row of this character + assert!(plan.script.contains("ii.guid IS NULL) = 0")); + assert!(plan.script.contains("b.item IS NULL) = 0")); +} + +#[test] +fn only_the_carried_settings_are_written_and_the_rest_is_reported_as_not_applied() { + let mut model = geared_level_eighty(); + model.settings.insert("core.spell_charge.804197".into(), vec![2, 1_790_000_000]); + model.settings.insert("core.destiny_weaver".into(), vec![1]); + model.settings.insert("core.wildcard.cards".into(), vec![1]); + let plan = plan_of(&model.normalized()); + for carried in ["core.ascension_active_spec", "core.ascension_starter", "core.ascension_build.54", "core.ascension_bar.54"] { + assert!(plan.script.contains(&Val::text(carried).sql()), "{carried}"); + } + for skipped in ["core.spell_charge.804197", "core.destiny_weaver", "core.wildcard.cards"] { + assert!(!plan.script.contains(&Val::text(skipped).sql()), "{skipped} must not reach the realm"); + assert!(plan.not_applied_settings.iter().any(|s| s == skipped), "{skipped}"); + } + assert_eq!(plan.counts.settings, 4); +} + +#[test] +fn the_marker_is_written_in_the_same_transaction_and_matches_the_journals() { + let plan = plan_of(&geared_level_eighty()); + assert_eq!(plan.marker_data, "11 22 33 44 3 "); + assert_eq!(plan.marker_data, crate::portable::store::marker_text([11, 22, 33, 44], 3), "plan and journal must agree on the marker"); + let marker_row = format!("(@char, {}, {})", Val::text(IMPORT_MARKER_SOURCE).sql(), Val::text("11 22 33 44 3 ").sql()); + let at = plan.script.find(&marker_row).expect("the marker row is inserted"); + assert!(at < plan.script.find("COMMIT;").unwrap()); +} + +#[test] +fn the_guards_prove_the_realm_is_stopped_and_the_account_is_usable() { + let plan = plan_of(&geared_level_eighty()); + let s = &plan.script; + assert!(s.contains("WHERE online <> 0) = 0")); + assert!(s.contains(&format!("information_schema.processlist WHERE id <> CONNECTION_ID() AND user IN ({})", Val::text("acore").sql()))); + assert!(s.contains("acore_auth.account WHERE id = 5001) = 1")); + assert!(s.contains("UPPER(username) NOT LIKE 'COABOT%'")); + assert!(s.contains("WHERE account = 5001) < 10")); + // the start position comes from the realm's world data, not from the snapshot + assert!(s.contains("INTO @map, @zone, @x, @y, @z, @o FROM acore_world.playercreateinfo WHERE race = 1 AND class = 21;")); + let none: Vec = vec![]; + assert!(build_plan(&geared_level_eighty(), &ctx(&all_tables(), &none)).is_err(), "a plan without a game server user cannot prove the realm is stopped"); +} + +#[test] +fn names_are_checked_inside_the_transaction_and_renamed_when_taken() { + let s = plan_of(&geared_level_eighty()).script; + assert!(s.contains("SET @name := _utf8mb4 0x")); + assert!(s.contains("name COLLATE utf8mb4_unicode_ci = @name COLLATE utf8mb4_unicode_ci")); + assert!(s.contains("acore_characters.reserved_name")); + assert!(s.contains("SET @final_name := IF(@rename = 1, CONCAT(LEFT(@name, 25 - CHAR_LENGTH(HEX(@char))), HEX(@char)), @name);")); + assert!(s.contains("SET @at_login := IF(@rename = 1, 1, 0);")); + let u = users(); + let without = build_plan(&geared_level_eighty(), &ctx(&probe_with(&[]), &u)).unwrap().script; + assert!(!without.contains("reserved_name"), "tables the realm does not have are not touched"); +} + +#[test] +fn leftovers_of_a_deleted_character_with_the_same_guid_are_removed_first() { + let plan = plan_of(&geared_level_eighty()); + let s = &plan.script; + let first_insert = s.find("INSERT INTO").unwrap(); + for (table, column) in CHARACTER_KEYED { + let delete = format!("DELETE FROM acore_characters.`{table}` WHERE `{column}` = @char;"); + let at = s.find(&delete).unwrap_or_else(|| panic!("no cleanup for {table}")); + assert!(at < first_insert, "{table}"); + } + // tables whose `guid` column is not a character guid must never be touched + for never in ["groups", "creature_respawn", "gameobject_respawn", "guild", "mail", "instance"] { + assert!(!s.contains(&format!("DELETE FROM acore_characters.`{never}`")), "{never}"); + } + // only tables the realm has + let u = users(); + let some = build_plan(&geared_level_eighty(), &ctx(&probe_with(&["characters", "item_instance", "character_inventory"]), &u)).unwrap().script; + assert!(some.contains("DELETE FROM acore_characters.`item_instance` WHERE `owner_guid` = @char;")); + assert!(!some.contains("DELETE FROM acore_characters.`coa_character_challenge`")); +} + +// ---- what is refused ------------------------------------------------------------------------------------------------ + +#[test] +fn a_character_of_another_ruleset_or_namespace_cannot_be_planned() { + let u = users(); + let p = all_tables(); + let wildcard = PlanContext { ruleset: Ruleset::Wildcard, ..ctx(&p, &u) }; + assert!(matches!(build_plan(&geared_level_eighty(), &wildcard), Err(PortableError::WrongRealm { .. }))); + let mut m = geared_level_eighty(); + m.items[0].entry = ContentId::new("mod:other", "item", 5).unwrap(); + assert!(matches!(build_plan(&m, &ctx(&p, &u)), Err(PortableError::Invalid(_))), "a mod item cannot be given a number in the realm's namespace"); + let mut m = geared_level_eighty(); + m.items[0].entry = ContentId::new("coa", "item", u32::MAX as u64 + 1).unwrap(); + assert!(build_plan(&m, &ctx(&p, &u)).is_err(), "entries must fit the realm's 32-bit column"); + let mut m = geared_level_eighty(); + m.identity.class = ContentId::new("coa", "race", 12).unwrap(); + assert!(build_plan(&m, &ctx(&p, &u)).is_err(), "a class must be a class"); +} + +#[test] +fn an_invalid_model_is_not_planned() { + let mut m = geared_level_eighty(); + m.progression.money = u32::MAX; + let u = users(); + assert!(build_plan(&m, &ctx(&all_tables(), &u)).is_err()); +} + +#[test] +fn an_empty_character_makes_a_small_valid_plan() { + let m = crate::portable::fixtures::naked_level_one(); + let plan = plan_of(&m); + assert_eq!(plan.counts, PlanCounts { settings: 0, ..PlanCounts::default() }); + assert!(plan.item_ids.is_empty() && plan.pet_ids.is_empty()); + assert!(!plan.script.contains("INSERT INTO acore_characters.`item_instance`")); + assert!(plan.script.contains("INSERT INTO acore_characters.`characters`")); + assert!(plan.script.contains("acore_characters.character_settings WHERE guid = @char) = 1,"), "only the marker"); +} + +#[test] +fn enchantments_charges_and_texts_are_rendered_the_way_the_realm_stores_them() { + let model = geared_level_eighty(); + let chest = model.items.iter().find(|i| i.container.is_none() && i.slot == 3).unwrap(); + let text: String = { + let mut slots = [[0u32; 3]; 12]; + for e in &chest.enchantments { + slots[e.slot as usize] = [e.id, e.duration, e.charges]; + } + slots.iter().flatten().map(|v| format!("{v} ")).collect() + }; + assert!(text.starts_with("3003 0 0 0 0 0 3878 0 0 3879 0 0 0 0 0 ")); + assert_eq!(text.split_whitespace().count(), 36); + let plan = plan_of(&model); + assert!(plan.script.contains(&Val::text(text).sql())); + // NULL charges stay NULL, real ones are five numbers + let mut m = geared_level_eighty(); + m.items[1].charges = vec![-1, 2]; + let plan = plan_of(&m.normalized()); + assert!(plan.script.contains(&Val::text("-1 2 0 0 0 ").sql())); +} + +// ---- the realm's report ------------------------------------------------------------------------------------------------- + +#[test] +fn the_realms_report_is_parsed_strictly() { + let name_hex = hex::encode("Geared"); + let ok = format!("#R:alloc\t1013\t20064\t3004\t0\tx{name_hex}\n#R:committed\n"); + let (alloc, committed) = parse_report(&ok).unwrap(); + assert!(committed); + assert_eq!(alloc, Allocation { local_guid: 1013, item_base: 20064, pet_base: 3004, renamed: false, final_name: "Geared".into() }); + let (_, committed) = parse_report(&format!("#R:alloc\t1\t2\t3\t1\tx{name_hex}\n")).unwrap(); + assert!(!committed, "no commit line: the commit is not proven"); + assert!(parse_report("").is_err()); + assert!(parse_report("#R:committed\n").is_err(), "a commit without an allocation"); + assert!(parse_report("#R:alloc\t1\t2\t3\n").is_err()); + assert!(parse_report("#R:alloc\tx\t2\t3\t0\txab\n").is_err()); + assert!(parse_report("#R:alloc\t4294967296\t2\t3\t0\txab\n").is_err()); + assert!(parse_report("#R:surprise\n").is_err()); + let _ = (PortableItemId::new(), BTreeSet::::new()); +} + +#[test] +fn stock_talents_are_never_written_because_a_bad_row_kills_the_realm() { + let mut m = geared_level_eighty(); + m.build.talents = vec![(900_001, 1), (900_002, 1)]; + let plan = plan_of(&m.normalized()); + assert!(!plan.script.contains("`character_talent` ("), "no INSERT into character_talent"); + assert_eq!(plan.counts.talents, 0); + assert!(plan.not_applied_settings.iter().any(|s| s.starts_with("character_talent (2 stock talent rows")), "{:?}", plan.not_applied_settings); + // and the transaction asserts that the character has none, so a leftover row of an earlier character cannot survive either + assert!(plan.script.contains("acore_characters.character_talent WHERE guid = @char) = 0,")); +} diff --git a/crates/coa-core/src/portable/realm/live_import.rs b/crates/coa-core/src/portable/realm/live_import.rs new file mode 100644 index 0000000..3b26be6 --- /dev/null +++ b/crates/coa-core/src/portable/realm/live_import.rs @@ -0,0 +1,585 @@ +//! Live tests of the offline importer against **two real disposable MySQL servers**: realm A (the source, loaded with +//! `testdata/realm-fixture.sql`) and realm B (the stopped destination, loaded with `testdata/realm-b-fixture.sql`; both +//! are copies of the CoA schema fixture, so B has the real CoA world data). `#[ignore]`d; see `live.rs` for how to run: +//! +//! ```text +//! COA_PORTABLE_LIVE="|13998|" COA_PORTABLE_LIVE_B="|13997|" cargo test -p coa-core portable::realm::live_import -- --ignored --test-threads=1 +//! ``` +//! +//! Realm B must also have a database user `acore` (password `acore-test`, hosts `localhost` and `127.0.0.1`) to stand in for a running game server. + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; +use std::time::Duration; + +use crate::db::Db; +use crate::portable::ids::ContentId; +use crate::portable::model::*; +use crate::portable::store::{ImportState, Store}; +use crate::realms::Mode; + +use super::import::*; +use super::plan::{build_plan, PlanContext}; +use super::*; + +const B: &str = "realm-b"; +const ACCOUNT: u32 = 7001; + +fn tools(var: &str) -> Option<(PathBuf, u16, String)> { + let spec = std::env::var(var).ok()?; + let p: Vec<&str> = spec.split('|').collect(); + assert_eq!(p.len(), 3, "{var} = ||"); + Some((PathBuf::from(p[0]), p[1].parse().expect("port"), p[2].to_string())) +} + +struct Realms { + a: Db, + b: Db, + /// A session of the game server's database user, to stand in for a running realm. + server_user: Db, +} + +fn realms() -> Option { + let (bin_a, port_a, pw_a) = tools("COA_PORTABLE_LIVE")?; + let (bin_b, port_b, pw_b) = tools("COA_PORTABLE_LIVE_B")?; + Some(Realms { + a: Db::with_tools(bin_a, port_a, "root", &pw_a, Mode::Coa), + server_user: Db::with_tools(bin_b.clone(), port_b, "acore", "acore-test", Mode::Coa), + b: Db::with_tools(bin_b, port_b, "root", &pw_b, Mode::Coa), + }) +} + +fn opts() -> ImportOptions { + ImportOptions { recovery_grace: Duration::ZERO, lock_wait_seconds: 2, ..ImportOptions::default() } +} + +/// Put realm B back to its baseline (the characters of `realm-b-fixture.sql` and the template characters): removes every +/// character with a guid above the fixture's and what hangs off it. Every live test starts with this, so the tests do not +/// depend on each other or on their order. +fn reset_b(db: &Db) { + let mut sql = vec!["DROP TRIGGER IF EXISTS acore_characters.coa_test_fail;".to_string()]; + for (table, column) in super::plan::CHARACTER_KEYED { + sql.push(format!("DELETE FROM acore_characters.`{table}` WHERE `{column}` > 3010;")); + } + sql.push("DELETE FROM acore_characters.characters WHERE guid > 3010;".into()); + sql.push("DELETE FROM acore_characters.pet_spell WHERE guid NOT IN (SELECT id FROM acore_characters.character_pet);".into()); + sql.push("DELETE FROM acore_characters.character_pet_declinedname WHERE id NOT IN (SELECT id FROM acore_characters.character_pet);".into()); + sql.push("DELETE FROM acore_characters.mail_items WHERE item_guid >= 99000;".into()); + sql.push("UPDATE acore_characters.characters SET online = 0;".into()); + db.query(&sql.join(" +")).unwrap(); +} + +fn on_b(store: &Store, id: CharacterId) -> Vec { + store.server_mappings(id).unwrap().into_iter().filter(|m| m.server_id == B).collect() +} + +fn first(db: &Db, sql: &str) -> String { + db.query(sql).unwrap().lines().next().unwrap_or("").to_string() +} + +fn number(db: &Db, sql: &str) -> u64 { + first(db, sql).trim().parse().unwrap_or_else(|_| panic!("not a number: {sql}")) +} + +/// Row counts of every table of the characters schema: the "nothing changed" proof. +fn counts(db: &Db) -> BTreeMap { + let tables = db.tables("acore_characters").unwrap(); + let sql = tables.iter().map(|t| format!("SELECT '{t}', COUNT(*) FROM acore_characters.`{t}`")).collect::>().join(" UNION ALL "); + db.query(&sql).unwrap().lines().map(|l| { + let (t, n) = l.trim().split_once('\t').unwrap(); + (t.to_string(), n.trim().parse().unwrap()) + }).collect() +} + +fn fresh_store() -> (Store, crate::portable::ids::ProfileId) { + let mut store = Store::open_in_memory().unwrap(); + let profile = store.default_profile().unwrap(); + (store, profile) +} + +fn make(r: &Realms, store: &mut Store, profile: crate::portable::ids::ProfileId, guid: u32) -> CharacterId { + make_portable(&r.a, store, profile, "realm-a", guid).unwrap().character_id +} + +/// What a trip through a realm legitimately does not preserve. Everything else must come back identical. +fn expected_after_a_trip(canonical: &PortableCharacter, back: &PortableCharacter) -> PortableCharacter { + let mut c = canonical.clone(); + c.identity.name = back.identity.name.clone(); // a taken or reserved name arrives under a temporary one + for item in &mut c.items { + item.creator_name = None; // the crafter is a character of another realm and is not stored at the destination + } + c.extensions.clear(); // quarantined settings are kept in the snapshot, not written to the realm + c.build.talents.clear(); // stock talent rows are never written: a bad row makes the realm assert + for (mine, theirs) in c.pets.iter_mut().zip(&back.pets) { + mine.id = theirs.id; // pets get new portable ids on every read until pet mappings exist + } + c.normalized() +} + +fn read_back(r: &Realms, store: &Store, id: CharacterId) -> PortableCharacter { + let guid = store.server_mappings(id).unwrap().iter().find(|m| m.server_id == B).unwrap().local_guid; + let prior = store.active_item_lookup(id, B).unwrap(); + export_character(&r.b, guid, Some(id), &prior).unwrap().model +} + +#[test] +#[ignore] +fn characters_survive_a_trip_into_a_second_realm() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + for guid in [1001u32, 1002, 1003, 1004, 1005, 1012] { + let id = make(&r, &mut store, profile, guid); + let canonical = store.load_current(id).unwrap(); + let (max_guid, max_item) = (number(&r.b, "SELECT MAX(guid) FROM acore_characters.characters"), number(&r.b, "SELECT GREATEST(IFNULL((SELECT MAX(guid) FROM acore_characters.item_instance),0), IFNULL((SELECT MAX(item_guid) FROM acore_characters.mail_items),0))")); + + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap_or_else(|e| panic!("import of {guid}: {e}")); + + // allocation: a new guid above everything in B, items right above everything that names an item + assert_eq!(outcome.local_guid as u64, max_guid + 1, "{guid}"); + assert_ne!(outcome.local_guid, guid, "the guid of the source realm is never reused"); + let mapping = on_b(&store, id).remove(0); + assert_eq!((mapping.local_guid, mapping.last_revision), (outcome.local_guid, 1)); + let mapped = store.item_mappings(id, B).unwrap(); + assert_eq!(mapped.len(), canonical.items.len()); + if let Some(min) = mapped.iter().map(|m| m.local_item_guid as u64).min() { + assert!(min > max_item, "{guid}: items start at {min}, above {max_item}"); + assert_eq!(mapped.iter().map(|m| m.local_item_guid as u64).max().unwrap() - min + 1, mapped.len() as u64, "contiguous"); + } + + // every reference in the realm points inside the new character + let g = outcome.local_guid; + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.item_instance WHERE owner_guid = {g}")) as usize, canonical.items.len()); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.character_inventory ci LEFT JOIN acore_characters.item_instance ii ON ii.guid = ci.item AND ii.owner_guid = {g} WHERE ci.guid = {g} AND ii.guid IS NULL")), 0); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.character_inventory ci WHERE ci.guid = {g} AND ci.bag <> 0 AND ci.bag NOT IN (SELECT item FROM acore_characters.character_inventory WHERE guid = {g})")), 0); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.item_instance WHERE owner_guid = {g} AND (creatorGuid <> 0 OR giftCreatorGuid <> 0)")), 0, "no guid of another character is carried"); + if !canonical.pets.is_empty() { + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.character_pet WHERE owner = {g}")) as usize, canonical.pets.len()); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.pet_spell ps JOIN acore_characters.character_pet p ON p.id = ps.guid WHERE p.owner = {g}")) as usize, canonical.pets.iter().map(|p| p.spells.len()).sum::()); + } + + // a fresh arrival: not online, no first-login rewards, the intro seen, standing at the realm's own start position + assert_eq!(first(&r.b, &format!("SELECT online, cinematic, at_login & 32 FROM acore_characters.characters WHERE guid = {g}")), "0\t1\t0"); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.characters c JOIN acore_characters.character_homebind h ON h.guid = c.guid JOIN acore_world.playercreateinfo p ON p.race = c.race AND p.class = c.class WHERE c.guid = {g} AND c.map = p.map AND h.mapId = p.map AND h.posX = p.position_x")), 1); + + // the character is what it was: read it back out of the realm and compare + let back = read_back(&r, &store, id); + assert_eq!(back, expected_after_a_trip(&canonical, &back), "character {guid} changed in the realm"); + assert_eq!(back.character_id, id); + + // names: taken (case-insensitively) or reserved names arrive renamed + match guid { + 1001 | 1012 => { + assert!(outcome.renamed, "{guid}"); + assert_eq!(outcome.final_name, format!("{}{:X}", canonical.identity.name, g)); + assert_eq!(number(&r.b, &format!("SELECT at_login & 1 FROM acore_characters.characters WHERE guid = {g}")), 1); + } + _ => { + assert!(!outcome.renamed, "{guid}"); + assert_eq!(outcome.final_name, canonical.identity.name); + } + } + assert!(!outcome.not_applied.is_empty() || guid != 1004, "1004's unapplied settings are reported"); + // the same character cannot be imported into the same realm twice + assert!(matches!(import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()), Err(PortableError::AlreadyOnRealm { .. }))); + let entry = store.open_imports(B).unwrap(); + assert!(entry.is_empty(), "nothing is left unfinished"); + } +} + +#[test] +#[ignore] +fn leftovers_of_a_deleted_character_with_the_same_guid_do_not_attach_to_the_newcomer() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1005); + let next = number(&r.b, "SELECT MAX(guid) + 1 FROM acore_characters.characters"); + // a deleted character with this very guid left rows behind (module tables are not cleaned by the realm's own delete) + let empty_enchants = "0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 "; + r.b.query(&format!( + "INSERT INTO acore_characters.character_spell (guid, spell, specMask) VALUES ({next}, 99999, 1); + INSERT INTO acore_characters.coa_character_condition (guid, flag) VALUES ({next}, 'orphan'); + INSERT INTO acore_characters.character_settings (guid, source, data) VALUES ({next}, 'core.ascension_build.1', '5 '); + INSERT INTO acore_characters.item_instance (guid, itemEntry, owner_guid, enchantments) VALUES (99001, 30000, {next}, '{empty_enchants}'); + INSERT INTO acore_characters.character_inventory (guid, bag, slot, item) VALUES ({next}, 0, 5, 99001); + INSERT INTO acore_characters.mail_items (mail_id, item_guid, receiver) VALUES (1, 99500, 3001); + INSERT INTO acore_characters.character_pet (id, entry, owner) VALUES (3999, 416, {next}); + INSERT INTO acore_characters.pet_spell (guid, spell, active) VALUES (3999, 1, 1);" + )) + .unwrap(); + + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap(); + let g = outcome.local_guid; + assert_eq!(g as u64, next); + + // the leftovers keyed by the character guid are gone + for sql in [ + format!("SELECT COUNT(*) FROM acore_characters.character_spell WHERE guid = {g} AND spell = 99999"), + format!("SELECT COUNT(*) FROM acore_characters.coa_character_condition WHERE guid = {g}"), + format!("SELECT COUNT(*) FROM acore_characters.character_settings WHERE guid = {g} AND source = 'core.ascension_build.1'"), + "SELECT COUNT(*) FROM acore_characters.item_instance WHERE guid = 99001".to_string(), + format!("SELECT COUNT(*) FROM acore_characters.character_inventory WHERE guid = {g} AND item = 99001"), + "SELECT COUNT(*) FROM acore_characters.character_pet WHERE id = 3999".to_string(), + ] { + assert_eq!(number(&r.b, &sql), 0, "{sql}"); + } + // the newcomer's items start above every item guid anything still names (the mail row names 99500) + let items = store.item_mappings(id, B).unwrap(); + assert!(items.iter().all(|m| m.local_item_guid > 99_500)); + // and its pets above the stale pet number: the stale pet_spell row cannot become the newcomer's + let pet_numbers: Vec = r.b.query(&format!("SELECT id FROM acore_characters.character_pet WHERE owner = {g}")).unwrap().lines().map(|l| l.trim().parse().unwrap()).collect(); + assert_eq!(pet_numbers.len(), 3); + assert!(pet_numbers.iter().all(|n| *n > 3999), "{pet_numbers:?}"); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.pet_spell WHERE guid = 3999"), 1, "unrelated rows are not touched"); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.mail_items WHERE item_guid = 99500"), 1); + r.b.query("DELETE FROM acore_characters.pet_spell WHERE guid = 3999; DELETE FROM acore_characters.mail_items WHERE item_guid = 99500").unwrap(); +} + +#[test] +#[ignore] +fn a_failure_in_the_middle_rolls_everything_back_and_leaves_nothing_mapped() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1002); + let before = counts(&r.b); + + // a trigger that makes one spell row fail: the failure happens after the character and its items were already inserted + r.b.query("DROP TRIGGER IF EXISTS acore_characters.coa_test_fail").unwrap(); + r.b.query("DELIMITER //\nCREATE TRIGGER acore_characters.coa_test_fail BEFORE INSERT ON acore_characters.character_spell FOR EACH ROW BEGIN IF NEW.spell = 500090 THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'injected failure'; END IF; END//\nDELIMITER ;\n").unwrap(); + + let result = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()); + r.b.query("DROP TRIGGER IF EXISTS acore_characters.coa_test_fail").unwrap(); + let error = result.expect_err("the injected failure must fail the import").to_string(); + assert!(error.contains("injected failure") || error.contains("database command failed"), "{error}"); + + assert_eq!(counts(&r.b), before, "not one row of one table may have changed"); + assert!(on_b(&store, id).is_empty(), "no Manager mapping after a failed realm transaction"); + assert!(store.item_mappings(id, B).unwrap().is_empty()); + assert!(store.open_imports(B).unwrap().is_empty(), "the journal entry is closed"); + let journal = r.b.query("SELECT 1").unwrap(); + assert_eq!(journal.trim(), "1"); + + // and the realm is fine afterwards: the same import now works (the lock was released, the journal allows a retry) + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap(); + assert_eq!(on_b(&store, id)[0].local_guid, outcome.local_guid); +} + +#[test] +#[ignore] +fn imports_are_refused_before_anything_is_written() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1003); + let model = store.load_current(id).unwrap(); + let before = counts(&r.b); + let refused = |store: &mut Store, id: CharacterId, account: u32| -> Vec { + match import_character(&r.b, store, id, B, account, &opts()) { + Err(PortableError::ImportRefused(p)) => p, + other => panic!("expected a refusal, got {:?}", other.map(|o| o.local_guid)), + } + }; + + // accounts + assert_eq!(refused(&mut store, id, 99_999), vec![ImportProblem::AccountMissing(99_999)]); + assert_eq!(refused(&mut store, id, 7002), vec![ImportProblem::InternalAccount(7002)]); + assert_eq!(refused(&mut store, id, 7003), vec![ImportProblem::AccountFull { count: 10, max: 10 }]); + + // the realm is running: a session of the game server's database user exists + let server = r.server_user.clone(); + let session = std::thread::spawn(move || server.query("SELECT SLEEP(6)").unwrap()); + std::thread::sleep(Duration::from_millis(1500)); + let running = refused(&mut store, id, ACCOUNT); + assert!(matches!(running.as_slice(), [ImportProblem::RealmRunning { sessions }] if *sessions >= 1), "{running:?}"); + // ... and if the server starts between the preflight and the transaction, the transaction's own guard stops it + let probe = super::probe(&r.b).unwrap(); + let users = vec!["acore".to_string()]; + let plan = build_plan(&model, &PlanContext { ruleset: Ruleset::Coa, account: ACCOUNT, revision: 1, nonce: [1, 2, 3, 4], max_characters_per_account: 10, game_server_users: &users, probe: &probe }).unwrap(); + assert!(run_realm_import(&r.b, &plan).is_err(), "the in-transaction guard must refuse while the game server's session exists"); + session.join().unwrap(); + + // characters marked online + r.b.query("UPDATE acore_characters.characters SET online = 1 WHERE guid = 2002").unwrap(); + let online = refused(&mut store, id, ACCOUNT); + r.b.query("UPDATE acore_characters.characters SET online = 0 WHERE guid = 2002").unwrap(); + assert_eq!(online, vec![ImportProblem::OnlineCharacters(1)]); + + assert_eq!(counts(&r.b), before, "refusals write nothing"); + assert!(store.open_imports(B).unwrap().is_empty(), "and leave no journal entry"); + assert!(on_b(&store, id).is_empty()); + + // content the realm cannot hold + let missing_entry = number(&r.b, "SELECT MAX(entry) + 1000 FROM acore_world.item_template") as u32; + let mut with_unknown_item = export_character(&r.a, 1002, None, &HashMap::new()).unwrap().model; + with_unknown_item.items[0].entry = ContentId::new("coa", "item", missing_entry as u64).unwrap(); + let id2 = store.create_character(profile, with_unknown_item, "realm-a").unwrap(); + assert_eq!(refused(&mut store, id2, ACCOUNT), vec![ImportProblem::MissingItems(vec![missing_entry])]); + + let mut unknown_class = export_character(&r.a, 1003, None, &HashMap::new()).unwrap().model; + unknown_class.identity.class = ContentId::new("coa", "class", 99).unwrap(); + let id3 = store.create_character(profile, unknown_class, "realm-a").unwrap(); + assert_eq!(refused(&mut store, id3, ACCOUNT), vec![ImportProblem::UnknownRaceClass { race: 3, class: 99 }]); + + let mut mod_item = export_character(&r.a, 1002, None, &HashMap::new()).unwrap().model; + mod_item.items[0].entry = ContentId::new("mod:other", "item", 5).unwrap(); + let id4 = store.create_character(profile, mod_item, "realm-a").unwrap(); + assert!(matches!(refused(&mut store, id4, ACCOUNT).as_slice(), [ImportProblem::UnsupportedContent(_)])); + + let mut wildcard = export_character(&r.a, 1003, None, &HashMap::new()).unwrap().model; + wildcard.ruleset = Ruleset::Wildcard; + wildcard.content_namespace = "wildcard".into(); + let id5 = store.create_character(profile, wildcard, "realm-a").unwrap(); + assert!(refused(&mut store, id5, ACCOUNT).iter().any(|p| matches!(p, ImportProblem::WrongRuleset { character: Ruleset::Wildcard, realm: Ruleset::Coa })), "CoA and Wildcard never mix"); + + assert_eq!(counts(&r.b), before); + assert!(store.open_imports(B).unwrap().is_empty()); +} + +#[test] +#[ignore] +fn a_crash_between_the_two_commits_is_recovered_from_the_marker_in_the_realm() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1002); + let model = store.load_current(id).unwrap(); + let probe = super::probe(&r.b).unwrap(); + + // the realm commits ... and the Manager "dies" before it records anything + let ticket = store.begin_import(id, B, 1, &import::planned_items(&model), &[]).unwrap(); + let users = vec!["acore".to_string()]; + let plan = build_plan(&model, &PlanContext { ruleset: Ruleset::Coa, account: ACCOUNT, revision: 1, nonce: ticket.nonce, max_characters_per_account: 10, game_server_users: &users, probe: &probe }).unwrap(); + let (alloc, committed) = run_realm_import(&r.b, &plan).unwrap(); + assert!(committed); + assert_eq!(store.import_entry(ticket.import_id).unwrap().state, ImportState::Prepared); + assert!(on_b(&store, id).is_empty(), "the Manager knows nothing yet"); + + // the next start of the Manager: recovery finds the marker and completes the local side + let report = recover_imports(&r.b, &mut store, B, &opts()).unwrap(); + assert_eq!(report.len(), 1); + // the guid and the item base are recovered exactly; this character has no pets, so there is no pet base to recover + let Resolution::Committed(recovered) = &report[0].resolution else { panic!("{:?}", report[0].resolution) }; + assert_eq!((recovered.local_guid, recovered.item_base), (alloc.local_guid, alloc.item_base)); + let mapping = &on_b(&store, id)[0]; + assert_eq!((mapping.local_guid, mapping.last_revision), (alloc.local_guid, 1)); + assert_eq!(store.item_mappings(id, B).unwrap().len(), model.items.len()); + // exactly what a normal import would have recorded: reading the realm back with these mappings gives the character + let back = read_back(&r, &store, id); + assert_eq!(back, expected_after_a_trip(&model, &back)); + assert!(recover_imports(&r.b, &mut store, B, &opts()).unwrap().is_empty(), "nothing is left to recover"); +} + +#[test] +#[ignore] +fn a_crash_before_the_realm_commit_is_recovered_as_never_happened() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1003); + let model = store.load_current(id).unwrap(); + let before = counts(&r.b); + + let ticket = store.begin_import(id, B, 1, &import::planned_items(&model), &[]).unwrap(); + // too recent to be called lost while an orphaned client might still be finishing it + let patient = ImportOptions { recovery_grace: Duration::from_secs(3600), ..opts() }; + let pending = recover_imports(&r.b, &mut store, B, &patient).unwrap(); + assert!(matches!(pending[0].resolution, Resolution::Pending(_)), "{:?}", pending[0].resolution); + assert_eq!(store.import_entry(ticket.import_id).unwrap().state, ImportState::Prepared); + + // an import that is still running in the realm holds the lock: recovery must wait for it, not decide + let holder = r.b.clone(); + let busy = std::thread::spawn(move || holder.query("SELECT GET_LOCK('coa_portable_import', 5), SLEEP(4)").unwrap()); + std::thread::sleep(Duration::from_millis(1000)); + let blocked = recover_imports(&r.b, &mut store, B, &opts()).unwrap(); + assert!(matches!(&blocked[0].resolution, Resolution::Pending(why) if why.contains("still running")), "{:?}", blocked[0].resolution); + busy.join().unwrap(); + + let report = recover_imports(&r.b, &mut store, B, &opts()).unwrap(); + assert_eq!(report[0].resolution, Resolution::Aborted); + assert_eq!(store.import_entry(ticket.import_id).unwrap().state, ImportState::Aborted); + assert!(on_b(&store, id).is_empty()); + assert_eq!(counts(&r.b), before); + // and the import can simply be run again + assert!(import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).is_ok()); +} + +#[test] +#[ignore] +fn a_realm_that_no_longer_matches_the_plan_is_flagged_not_guessed_about() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1002); + let model = store.load_current(id).unwrap(); + let probe = super::probe(&r.b).unwrap(); + let ticket = store.begin_import(id, B, 1, &import::planned_items(&model), &[]).unwrap(); + let users = vec!["acore".to_string()]; + let plan = build_plan(&model, &PlanContext { ruleset: Ruleset::Coa, account: ACCOUNT, revision: 1, nonce: ticket.nonce, max_characters_per_account: 10, game_server_users: &users, probe: &probe }).unwrap(); + let (alloc, _) = run_realm_import(&r.b, &plan).unwrap(); + // someone deletes an item of the new character before the Manager recovers + r.b.query(&format!("DELETE FROM acore_characters.item_instance WHERE guid = {}", alloc.item_base + 3)).unwrap(); + + let result = recover_imports(&r.b, &mut store, B, &opts()); + assert!(matches!(result, Err(PortableError::ImportNeedsAttention { .. })), "{:?}", result.map(|_| ())); + assert_eq!(store.import_entry(ticket.import_id).unwrap().state, ImportState::NeedsAttention); + assert!(on_b(&store, id).is_empty(), "no mapping is recorded on a guess"); +} + +#[test] +#[ignore] +fn hostile_text_in_names_texts_blobs_and_pets_reaches_the_realm_as_data_only() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let tables_before = r.b.tables("acore_characters").unwrap(); + + let mut m = export_character(&r.a, 1005, None, &HashMap::new()).unwrap().model; + m.identity.name = "Bob');DROP TABLE x;--".into(); + m.pets[0].name = "p');DROP TABLE x;--".into(); + m.pets[0].action_bar = "'; DROP TABLE pet_spell; -- \\".into(); + m.pets[1].declined_names = Some(["a'".into(), "b\"".into(), "c\\".into(), "d;".into(), "e--".into()]); + m.client_data.insert(5, ClientBlob { time: 5, data: Bytes(b"'; DROP DATABASE acore_characters; -- \x00\xff\n\t\\".to_vec()) }); + let hostile_item_text = "\"; DELETE FROM item_instance; -- \\ \n\t' OR '1'='1 \u{1F409}"; + m.items.push(PortableItem { + id: crate::portable::ids::PortableItemId::new(), + container: None, + slot: 23, + entry: ContentId::new("coa", "item", 30_000).unwrap(), + count: 1, + duration: 0, + charges: vec![], + flags: 0, + enchantments: vec![], + random_property_id: 0, + durability: 1, + played_time: 0, + text: Some(hostile_item_text.into()), + creator_name: None, + gift: None, + }); + let id = store.create_character(profile, m.normalized(), "realm-a").unwrap(); + let canonical = store.load_current(id).unwrap(); + let characters_before = number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters"); + let items_before = number(&r.b, "SELECT COUNT(*) FROM acore_characters.item_instance"); + + import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap(); + + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters"), characters_before + 1, "exactly one character was added, nothing deleted"); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.item_instance"), items_before + 1); + assert_eq!(r.b.tables("acore_characters").unwrap(), tables_before, "no table was dropped or created"); + assert!(r.b.schema_exists("acore_characters").unwrap()); + + let back = read_back(&r, &store, id); + assert_eq!(back.identity.name, canonical.identity.name, "the name arrives byte for byte (it was free on the realm)"); + assert_eq!(back.items[0].text.as_deref(), Some(hostile_item_text)); + assert_eq!(back.client_data[&5].data, canonical.client_data[&5].data); + assert_eq!(back.pets.iter().map(|p| p.name.clone()).collect::>(), canonical.pets.iter().map(|p| p.name.clone()).collect::>()); + assert!(back.pets.iter().any(|p| p.declined_names == Some(["a'".into(), "b\"".into(), "c\\".into(), "d;".into(), "e--".into()]))); + assert_eq!(back, expected_after_a_trip(&canonical, &back)); +} + +#[test] +#[ignore] +fn the_realms_own_start_data_and_rules_decide_the_arrival() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1004); + let report = preflight(&r.b, &store.load_current(id).unwrap(), ACCOUNT, &opts()).unwrap(); + assert!(report.problems.is_empty(), "{:?}", report.problems); + assert!(!report.will_rename); + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap(); + let g = outcome.local_guid; + // the marker identifies the import in the realm and is not part of what the character is + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.character_settings WHERE guid = {g} AND source = 'coa.portable.import'")), 1); + let back = read_back(&r, &store, id); + assert!(!back.settings.contains_key("coa.portable.import"), "the marker is dropped by the export policy"); + // quarantined settings are in the snapshot and are not in the realm + assert!(outcome.not_applied.iter().any(|s| s == "coa:unlisted-settings"), "{:?}", outcome.not_applied); + assert_eq!(number(&r.b, &format!("SELECT COUNT(*) FROM acore_characters.character_settings WHERE guid = {g} AND source LIKE 'core.spell_charge.%'")), 0); + assert!(store.load_current(id).unwrap().extensions.contains_key("coa:unlisted-settings"), "nothing was destroyed in the canonical snapshot"); +} + +// ---- the real server ------------------------------------------------------------------------------------------------ + +const SERVER_CHECK_GUIDS: [u32; 6] = [1001, 1002, 1003, 1004, 1005, 1012]; + +fn check_dir() -> PathBuf { + PathBuf::from(std::env::var("COA_PORTABLE_SERVER_CHECK_DIR").expect("COA_PORTABLE_SERVER_CHECK_DIR")) +} + +/// Paths at which two JSON values differ (capped). +fn json_diff(path: &str, a: &serde_json::Value, b: &serde_json::Value, out: &mut Vec) { + use serde_json::Value::{Array, Null, Object}; + if out.len() >= 60 { + return; + } + match (a, b) { + (Object(x), Object(y)) => { + for k in x.keys().chain(y.keys().filter(|k| !x.contains_key(*k))) { + json_diff(&format!("{path}.{k}"), x.get(k).unwrap_or(&Null), y.get(k).unwrap_or(&Null), out); + } + } + (Array(x), Array(y)) if x.len() == y.len() => { + for (i, (p, q)) in x.iter().zip(y).enumerate() { + json_diff(&format!("{path}[{i}]"), p, q, out); + } + } + _ if a != b => out.push(format!("{path}: {} -> {}", a.to_string().chars().take(70).collect::(), b.to_string().chars().take(70).collect::())), + _ => {} + } +} + +/// Step 1 of the real-server check (run with the realm STOPPED): import the six exportable characters into B and keep the +/// store on disk so that step 3 can compare after the real worldserver has loaded and saved them. +#[test] +#[ignore] +fn server_check_1_import_into_the_stopped_realm() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let dir = check_dir(); + let _ = std::fs::remove_dir_all(&dir); + let mut store = Store::open(&dir).unwrap(); + let profile = store.default_profile().unwrap(); + let mut lines = Vec::new(); + for guid in SERVER_CHECK_GUIDS { + let id = make(&r, &mut store, profile, guid); + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts()).unwrap(); + lines.push(format!("{guid} -> guid {} ({} items, {} pets, renamed: {})", outcome.local_guid, outcome.items, outcome.pets, outcome.renamed)); + } + std::fs::write(dir.join("imported.txt"), lines.join("\n")).unwrap(); + println!("{}", lines.join("\n")); +} + +/// Step 3: after the real worldserver has logged the characters in and saved them on shutdown, read them back and say +/// what the server did to them. +#[test] +#[ignore] +fn server_check_3_what_the_real_server_did_to_them() { + let Some(r) = realms() else { return }; + let store = Store::open(&check_dir()).unwrap(); + let profile = store.list_characters_all().unwrap(); + let mut report = Vec::new(); + for record in profile { + let id = record.character_id; + let canonical = store.load_current(id).unwrap(); + let guid = on_b(&store, id)[0].local_guid; + let online = number(&r.b, &format!("SELECT online FROM acore_characters.characters WHERE guid = {guid}")); + let prior = store.active_item_lookup(id, B).unwrap(); + let back = export_character(&r.b, guid, Some(id), &prior); + match back { + Err(e) => report.push(format!("{} (guid {guid}, online {online}): export refused: {e}", record.name)), + Ok(e) => { + let expected = expected_after_a_trip(&canonical, &e.model); + let mut diffs = Vec::new(); + json_diff("", &serde_json::to_value(&expected).unwrap(), &serde_json::to_value(&e.model).unwrap(), &mut diffs); + report.push(format!("{} (guid {guid}, online {online}): {} difference(s) after the server loaded and saved it{}", record.name, diffs.len(), if diffs.is_empty() { String::new() } else { format!(":\n {}", diffs.join("\n ")) })); + } + } + } + println!("{}", report.join("\n")); + std::fs::write(check_dir().join("report.txt"), report.join("\n")).unwrap(); +} diff --git a/crates/coa-core/src/portable/realm/mod.rs b/crates/coa-core/src/portable/realm/mod.rs index 1be7717..2ac166a 100644 --- a/crates/coa-core/src/portable/realm/mod.rs +++ b/crates/coa-core/src/portable/realm/mod.rs @@ -11,9 +11,12 @@ pub mod blockers; pub mod export; +pub mod import; +pub mod plan; pub mod policy; pub mod registry; pub mod script; +pub mod sqlenc; use std::collections::HashMap; @@ -27,6 +30,7 @@ use super::store::{RealmRegistration, Store}; pub use blockers::Blocker; pub use export::{build, ExportRequest, Exported}; +pub use import::{import_character, preflight, recover_imports, resolve_import, ImportOptions, ImportOutcome, ImportProblem, PreflightReport, Resolution}; pub use script::SchemaProbe; /// The ruleset a database belongs to is decided by which realm profile it is (`realms::Mode`), never by the caller. @@ -176,7 +180,11 @@ pub fn register(store: &mut Store, profile: ProfileId, server_id: &str, exported Ok(MadePortable { character_id, revision: 1, warnings: exported.warnings }) } +#[cfg(test)] +mod import_tests; #[cfg(test)] mod live; #[cfg(test)] +mod live_import; +#[cfg(test)] mod tests; diff --git a/crates/coa-core/src/portable/realm/plan.rs b/crates/coa-core/src/portable/realm/plan.rs new file mode 100644 index 0000000..4705a15 --- /dev/null +++ b/crates/coa-core/src/portable/realm/plan.rs @@ -0,0 +1,602 @@ +//! A portable character -> the fixed SQL script that imports it into a **stopped** realm, in **one MySQL transaction**. +//! +//! The script is generated, never hand-assembled from strings: +//! * every value goes through the typed encoder ([`super::sqlenc`]): untrusted text and blobs exist in the script only as +//! hex literals; +//! * every id the realm owns is allocated *inside* the transaction from the realm's own tables (`@char`, `@item_base`, +//! `@pet_base`) and every reference is remapped through them (`@item_base + index`), so no guid of the source realm +//! (or of the canonical snapshot) ever reaches the destination; +//! * the transaction asserts its own preconditions and results (an assertion that fails raises an SQL error, which aborts +//! the script before `COMMIT`, which rolls everything back). +//! +//! Layout of the script: lock -> guards -> allocation -> start position -> name -> orphan cleanup -> inserts -> +//! assertions -> marker -> `COMMIT` -> report. Nothing is written outside the transaction. + +use std::collections::HashMap; + +use super::super::error::{PortableError, Result}; +use super::super::ids::{ContentId, PortableItemId, PortablePetId}; +use super::super::model::*; +use super::policy::{classify_setting, Disposition, IMPORT_MARKER_SOURCE}; +use super::script::SchemaProbe; +use super::sqlenc::{Insert, Val}; + +pub const IMPORT_LOCK: &str = "coa_portable_import"; +/// Longest character name the characters table holds. +pub const NAME_COLUMN_CHARS: u32 = 25; +/// Health/power written for a fresh arrival: the realm clamps to the maximum on load, so this means "full". +const FULL: u64 = 2_000_000_000; + +/// Raises an SQL error ("subquery returns more than 1 row"): used as the failing branch of an assertion. +const FAIL: &str = "(SELECT 1 UNION ALL SELECT 2)"; + +/// Per-character tables whose rows are keyed by the **character guid**. Rows left there by a deleted character with +/// the same guid are removed before the import (the realm's own delete leaves module tables behind). Curated by hand: +/// a column named `guid` is not always a character guid (`groups.guid`, `creature_respawn.guid`, ...). +pub const CHARACTER_KEYED: &[(&str, &str)] = &[ + ("ascension_manastorm_bonus", "guid"), + ("ascension_manastorm_cache", "guid"), + ("ascension_manastorm_clear", "guid"), + ("ascension_manastorm_loadout", "guid"), + ("ascension_manastorm_xp", "guid"), + ("character_account_data", "guid"), + ("character_achievement", "guid"), + ("character_achievement_offline_updates", "guid"), + ("character_achievement_progress", "guid"), + ("character_action", "guid"), + ("character_appearance", "guid"), + ("character_appearance_outfit", "guid"), + ("character_appearance_settings", "guid"), + ("character_arena_stats", "guid"), + ("character_ascension_state", "guid"), + ("character_aura", "guid"), + ("character_banned", "guid"), + ("character_battleground_random", "guid"), + ("character_brew_of_the_month", "guid"), + ("character_coa_lfg_settings", "guid"), + ("character_declinedname", "guid"), + ("character_entry_point", "guid"), + ("character_equipmentsets", "guid"), + ("character_gifts", "guid"), + ("character_glyphs", "guid"), + ("character_homebind", "guid"), + ("character_instance", "guid"), + ("character_inventory", "guid"), + ("character_pet", "owner"), + ("character_pet_declinedname", "owner"), + ("character_queststatus", "guid"), + ("character_queststatus_daily", "guid"), + ("character_queststatus_monthly", "guid"), + ("character_queststatus_rewarded", "guid"), + ("character_queststatus_seasonal", "guid"), + ("character_queststatus_weekly", "guid"), + ("character_reputation", "guid"), + ("character_settings", "guid"), + ("character_skills", "guid"), + ("character_spell", "guid"), + ("character_spell_cooldown", "guid"), + ("character_stats", "guid"), + ("character_talent", "guid"), + ("character_worldforged_loot", "guid"), + ("coa_challenge_completion", "guid"), + ("coa_challenge_failure", "guid"), + ("coa_character_challenge", "guid"), + ("coa_character_condition", "guid"), + ("coa_character_fatigue", "guid"), + ("coa_character_gamemode", "guid"), + ("coa_character_gamemode_lives", "guid"), + ("coa_character_looted_item", "guid"), + ("coa_character_objective", "guid"), + ("coa_character_survival", "guid"), + ("coa_custom_trial", "guid"), + ("coa_custom_trial_active", "guid"), + ("coa_custom_trial_completion", "guid"), + ("coa_custom_trial_entry", "guid"), + ("coa_custom_trial_vote", "guid"), + ("corpse", "guid"), + ("item_instance", "owner_guid"), + ("mod_craftsmans_codex", "guid"), +]; + +/// Tables that name **item guids**. The new item guids start above the highest of all of them, so a stale row (a mail, +/// an auction, a bank slot of a long-deleted item) can never attach itself to an imported item. +pub const ITEM_KEYED: &[(&str, &str)] = &[ + ("item_instance", "guid"), + ("character_inventory", "item"), + ("character_gifts", "item_guid"), + ("mail_items", "item_guid"), + ("auctionhouse", "itemguid"), + ("guild_bank_item", "item_guid"), + ("item_refund_instance", "item_guid"), + ("item_soulbound_trade_data", "itemGuid"), + ("item_loot_storage", "containerGUID"), + ("ascension_manastorm_cache", "item"), + ("highrisk_chest_item", "item_guid"), + ("mod_ascension_bank_item", "item_guid"), + ("coa_character_looted_item", "itemGuid"), +]; + +/// Tables that name **pet numbers**. +pub const PET_KEYED: &[(&str, &str)] = &[ + ("character_pet", "id"), + ("pet_spell", "guid"), + ("pet_aura", "guid"), + ("pet_spell_cooldown", "guid"), + ("character_pet_declinedname", "id"), +]; + +pub struct PlanContext<'a> { + pub ruleset: Ruleset, + /// The destination game account (it must exist; checked in the script and by the preflight). + pub account: u32, + /// The canonical revision being imported. + pub revision: u64, + /// Identifies this import in the realm (see [`marker_data`]). + pub nonce: [u32; 4], + pub max_characters_per_account: u32, + /// Database users of a running game server; any other session of one of these users means the realm is running. + pub game_server_users: &'a [String], + pub probe: &'a SchemaProbe, +} + +/// What the plan will write, for assertions and reports. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PlanCounts { + pub items: usize, + pub inventory: usize, + pub gifts: usize, + pub spells: usize, + pub talents: usize, + pub skills: usize, + pub glyphs: usize, + pub reputation: usize, + pub quests: usize, + pub rewarded: usize, + pub actions: usize, + pub pets: usize, + pub pet_spells: usize, + pub declined: usize, + pub settings: usize, + pub macros: usize, +} + +#[derive(Debug)] +pub struct ImportPlan { + pub script: String, + /// Item `i` of the plan gets the realm guid `@item_base + i`. + pub item_ids: Vec, + /// Pet `i` of the plan gets the pet number `@pet_base + i`. + pub pet_ids: Vec, + pub marker_data: String, + pub counts: PlanCounts, + /// Settings sources that are in the snapshot but are deliberately not applied. + pub not_applied_settings: Vec, +} + +/// The text stored in the realm's `character_settings` row `coa.portable.import` of an imported character: four random +/// words and the canonical revision, all numbers. Recovery finds the character by this exact text. +pub fn marker_data(nonce: [u32; 4], revision: u64) -> String { + format!("{} {} {} {} {} ", nonce[0], nonce[1], nonce[2], nonce[3], revision) +} + +fn to_u32(what: &str, v: u64) -> Result { + u32::try_from(v).map_err(|_| PortableError::Invalid(format!("{what} {v} does not fit the realm's 32-bit column"))) +} + +fn content_id(what: &str, id: &ContentId, ns: &str, kind: &str) -> Result { + if id.namespace() != ns || id.kind() != kind { + return Err(PortableError::Invalid(format!("{what} {id} is not a {ns}:{kind} id and cannot be imported into a {ns} realm"))); + } + to_u32(what, id.id()) +} + +fn settings_text(values: &[u32]) -> String { + values.iter().map(|v| format!("{v} ")).collect() +} + +/// `a b c ... ` for the 12 enchantment slots, as the realm stores them. +fn enchantment_text(item: &PortableItem) -> String { + let mut slots = [[0u32; 3]; 12]; + for e in &item.enchantments { + slots[e.slot as usize] = [e.id, e.duration, e.charges]; + } + slots.iter().flatten().map(|v| format!("{v} ")).collect() +} + +fn charges_text(item: &PortableItem) -> Option { + if item.charges.is_empty() { + return None; + } + let mut c = item.charges.clone(); + c.resize(5, 0); + Some(c.iter().map(|v| format!("{v} ")).collect()) +} + +fn assert_count(table_sql: &str, expected: usize) -> String { + format!("DO IF((SELECT COUNT(*) FROM {table_sql}) = {expected}, 1, {FAIL});") +} + +pub fn build_plan(model: &PortableCharacter, ctx: &PlanContext<'_>) -> Result { + model.validate()?; + if model.ruleset != ctx.ruleset { + return Err(PortableError::WrongRealm { expected: model.ruleset.to_string(), found: ctx.ruleset.to_string() }); + } + let ns = ctx.ruleset.as_str(); + if model.content_namespace != ns { + return Err(PortableError::Invalid(format!("the character's content namespace {} is not {ns}", model.content_namespace))); + } + if ctx.game_server_users.is_empty() { + return Err(PortableError::Invalid("internal: no game server user is named, the realm cannot be proven stopped".into())); + } + + let race = content_id("race", &model.identity.race, ns, "race")?; + let class = content_id("class", &model.identity.class, ns, "class")?; + let item_index: HashMap = model.items.iter().enumerate().map(|(i, it)| (it.id, i as u32)).collect(); + + // ---- rows ----------------------------------------------------------------------------------------------------- + let mut counts = PlanCounts::default(); + let mut inserts: Vec = Vec::new(); + + let p = &model.progression; + let mut characters = Insert::new( + "characters", + &[ + "guid", "account", "name", "race", "class", "gender", "level", "xp", "money", "skin", "face", "hairStyle", "hairColor", "facialStyle", "bankSlots", "restState", + "playerFlags", "position_x", "position_y", "position_z", "map", "instance_id", "instance_mode_mask", "orientation", "taximask", "online", "cinematic", "totaltime", + "leveltime", "logout_time", "is_logout_resting", "rest_bonus", "resettalents_cost", "resettalents_time", "extra_flags", "stable_slots", "at_login", "zone", + "death_expire_time", "arenaPoints", "totalHonorPoints", "todayHonorPoints", "yesterdayHonorPoints", "totalKills", "todayKills", "yesterdayKills", "chosenTitle", + "knownCurrencies", "watchedFaction", "drunk", "health", "power1", "power2", "power3", "power4", "power5", "power6", "power7", "latency", "talentGroupsCount", + "activeTalentGroup", "exploredZones", "equipmentCache", "ammoId", "knownTitles", "actionBars", "grantableLevels", "innTriggerId", "extraBonusTalentCount", + ], + ); + let a = &model.identity.appearance; + characters.row(vec![ + Val::Expr("@char"), + Val::u(ctx.account), + Val::Expr("@final_name"), + Val::u(race), + Val::u(class), + Val::u(model.identity.gender), + Val::u(p.level), + Val::u(p.xp), + Val::u(p.money), + Val::u(a.skin), + Val::u(a.face), + Val::u(a.hair_style), + Val::u(a.hair_color), + Val::u(a.facial_style), + Val::u(p.bank_slots), + Val::u(0u8), + Val::u(model.identity.cosmetic_flags), + Val::Expr("@x"), + Val::Expr("@y"), + Val::Expr("@z"), + Val::Expr("@map"), + Val::u(0u8), + Val::u(0u8), + Val::Expr("@o"), + Val::text(p.taxi_mask.clone()), + Val::u(0u8), + Val::u(1u8), // the intro cinematic has been seen + Val::u(0u8), + Val::u(0u8), + Val::u(0u8), + Val::u(0u8), + Val::u(0u8), + Val::u(model.build.reset_talents_cost), + Val::u(0u8), + Val::u(0u8), + Val::u(p.stable_slots), + Val::Expr("@at_login"), + Val::Expr("@zone"), + Val::u(0u8), + Val::u(p.honor.arena_points), + Val::u(p.honor.total_honor), + Val::u(p.honor.today_honor), + Val::u(p.honor.yesterday_honor), + Val::u(p.honor.total_kills), + Val::u(p.honor.today_kills), + Val::u(p.honor.yesterday_kills), + Val::u(p.chosen_title), + Val::u(p.known_currencies), + Val::u(p.watched_faction), + Val::u(0u8), + Val::u(FULL), // health + Val::u(FULL), // power1 (mana) + Val::u(0u8), // power2 (rage) + Val::u(0u8), // power3 (focus) + Val::u(FULL), // power4 (energy) + Val::u(0u8), + Val::u(0u8), + Val::u(0u8), + Val::u(0u8), + Val::u(model.build.talent_groups_count), + Val::u(model.build.active_talent_group), + Val::text(p.explored_zones.clone()), + Val::text(""), + Val::u(0u8), + Val::text(p.known_titles.clone()), + Val::u(p.action_bars_mask), + Val::u(0u8), + Val::u(0u8), + Val::i(model.build.extra_bonus_talent_count), + ])?; + inserts.push(characters); + + let mut homebind = Insert::new("character_homebind", &["guid", "mapId", "zoneId", "posX", "posY", "posZ"]); + homebind.row(vec![Val::Expr("@char"), Val::Expr("@map"), Val::Expr("@zone"), Val::Expr("@x"), Val::Expr("@y"), Val::Expr("@z")])?; + inserts.push(homebind); + + let mut item_rows = Insert::new( + "item_instance", + &["guid", "itemEntry", "owner_guid", "creatorGuid", "giftCreatorGuid", "count", "duration", "charges", "flags", "enchantments", "randomPropertyId", "durability", "playedTime", "text"], + ); + let mut inventory = Insert::new("character_inventory", &["guid", "bag", "slot", "item"]); + let mut gifts = Insert::new("character_gifts", &["guid", "item_guid", "entry", "flags"]); + for (i, item) in model.items.iter().enumerate() { + let entry = content_id("item", &item.entry, ns, "item")?; + item_rows.row(vec![ + Val::ItemRef(i as u32), + Val::u(entry), + Val::Expr("@char"), + Val::u(0u8), // the crafter named a character of another realm + Val::u(0u8), + Val::u(item.count), + Val::i(item.duration), + Val::opt_text(charges_text(item).as_deref()), + Val::u(item.flags), + Val::text(enchantment_text(item)), + Val::i(item.random_property_id), + Val::u(item.durability), + Val::u(item.played_time), + Val::opt_text(item.text.as_deref()), + ])?; + let bag = match item.container { + None => Val::u(0u8), + Some(c) => Val::ItemRef(*item_index.get(&c).ok_or_else(|| PortableError::Invalid("an item names a container that is not part of the plan".into()))?), + }; + inventory.row(vec![Val::Expr("@char"), bag, Val::u(item.slot), Val::ItemRef(i as u32)])?; + if let Some(g) = &item.gift { + gifts.row(vec![Val::Expr("@char"), Val::ItemRef(i as u32), Val::u(content_id("gift", &g.entry, ns, "item")?), Val::u(g.flags)])?; + } + } + counts.items = item_rows.len(); + counts.inventory = inventory.len(); + counts.gifts = gifts.len(); + inserts.extend([item_rows, inventory, gifts]); + + let mut spells = Insert::new("character_spell", &["guid", "spell", "specMask"]); + for (spell, mask) in &model.build.spells { + spells.row(vec![Val::Expr("@char"), Val::u(*spell), Val::u(*mask)])?; + } + // Stock talents are NOT written. The realm asserts (and the whole worldserver dies) when `character_talent` names a spell + // that is not a talent of its DBC, and nothing in the realm's database can tell whether a spell id is one. CoA keeps its + // talents as spells and its builds in `character_settings`; real CoA characters have no `character_talent` rows at all. + // The rows stay in the canonical snapshot. + let mut not_applied_talents = Vec::new(); + if !model.build.talents.is_empty() { + not_applied_talents.push(format!("character_talent ({} stock talent rows: not applied, see PORTABLE_IMPORT.md)", model.build.talents.len())); + } + let mut skills = Insert::new("character_skills", &["guid", "skill", "value", "max"]); + for s in &model.build.skills { + skills.row(vec![Val::Expr("@char"), Val::u(s.skill), Val::u(s.value), Val::u(s.max)])?; + } + let mut glyphs = Insert::new("character_glyphs", &["guid", "talentGroup", "glyph1", "glyph2", "glyph3", "glyph4", "glyph5", "glyph6"]); + for g in &model.build.glyphs { + let mut row = vec![Val::Expr("@char"), Val::u(g.talent_group)]; + row.extend(g.glyphs.iter().map(|v| Val::u(*v))); + glyphs.row(row)?; + } + (counts.spells, counts.talents, counts.skills, counts.glyphs) = (spells.len(), 0, skills.len(), glyphs.len()); + inserts.extend([spells, skills, glyphs]); + + let mut reputation = Insert::new("character_reputation", &["guid", "faction", "standing", "flags"]); + for r in &model.reputation { + reputation.row(vec![Val::Expr("@char"), Val::u(r.faction), Val::i(r.standing), Val::u(r.flags)])?; + } + let mut quests = Insert::new( + "character_queststatus", + &["guid", "quest", "status", "explored", "timer", "mobcount1", "mobcount2", "mobcount3", "mobcount4", "itemcount1", "itemcount2", "itemcount3", "itemcount4", "itemcount5", "itemcount6", "playercount"], + ); + for q in &model.quests.active { + let mut row = vec![Val::Expr("@char"), Val::u(q.quest), Val::u(q.status), Val::u(q.explored as u8), Val::u(q.timer)]; + row.extend(q.mob_counts.iter().map(|v| Val::u(*v))); + row.extend(q.item_counts.iter().map(|v| Val::u(*v))); + row.push(Val::u(q.player_count)); + quests.row(row)?; + } + let mut rewarded = Insert::new("character_queststatus_rewarded", &["guid", "quest", "active"]); + for q in &model.quests.rewarded { + rewarded.row(vec![Val::Expr("@char"), Val::u(*q), Val::u(1u8)])?; + } + let mut actions = Insert::new("character_action", &["guid", "spec", "button", "action", "type"]); + for a in &model.actions { + actions.row(vec![Val::Expr("@char"), Val::u(a.spec), Val::u(a.button), Val::u(a.action), Val::u(a.kind)])?; + } + counts.reputation = reputation.len(); + counts.quests = quests.len(); + counts.rewarded = rewarded.len(); + counts.actions = actions.len(); + inserts.extend([reputation, quests, rewarded, actions]); + + let mut pets = Insert::new( + "character_pet", + &["id", "entry", "owner", "modelid", "CreatedBySpell", "PetType", "level", "exp", "Reactstate", "name", "renamed", "slot", "curhealth", "curmana", "curhappiness", "savetime", "abdata"], + ); + let mut pet_spells = Insert::new("pet_spell", &["guid", "spell", "active"]); + let mut declined = Insert::new("character_pet_declinedname", &["id", "owner", "genitive", "dative", "accusative", "instrumental", "prepositional"]); + for (i, pet) in model.pets.iter().enumerate() { + let i = i as u32; + pets.row(vec![ + Val::PetRef(i), + Val::u(content_id("pet", &pet.entry, ns, "creature")?), + Val::Expr("@char"), + Val::u(pet.model_id), + Val::u(pet.created_by_spell), + Val::u(pet.pet_type), + Val::u(pet.level), + Val::u(pet.exp), + Val::u(pet.react_state), + Val::text(pet.name.clone()), + Val::u(pet.renamed as u8), + Val::u(pet.slot), + Val::u(pet.health), + Val::u(pet.mana), + Val::u(pet.happiness), + Val::Expr("UNIX_TIMESTAMP()"), + if pet.action_bar.is_empty() { Val::Null } else { Val::text(pet.action_bar.clone()) }, + ])?; + for s in &pet.spells { + pet_spells.row(vec![Val::PetRef(i), Val::u(s.spell), Val::u(s.active)])?; + } + if let Some(names) = &pet.declined_names { + let mut row = vec![Val::PetRef(i), Val::Expr("@char")]; + row.extend(names.iter().map(|n| Val::text(n.clone()))); + declined.row(row)?; + } + } + (counts.pets, counts.pet_spells, counts.declined) = (pets.len(), pet_spells.len(), declined.len()); + inserts.extend([pets, pet_spells, declined]); + + // settings: only what the policy carries is applied; the importer's own marker goes in the same table + let mut settings = Insert::new("character_settings", &["guid", "source", "data"]); + let mut not_applied = not_applied_talents; + for (source, values) in &model.settings { + match classify_setting(source, model.ruleset) { + Disposition::Carry => settings.row(vec![Val::Expr("@char"), Val::text(source.clone()), Val::text(settings_text(values))])?, + _ => not_applied.push(source.clone()), + } + } + // extensions (module data this importer does not apply) stay in the snapshot only + not_applied.extend(model.extensions.keys().cloned()); + counts.settings = settings.len(); + let marker = marker_data(ctx.nonce, ctx.revision); + settings.row(vec![Val::Expr("@char"), Val::text(IMPORT_MARKER_SOURCE), Val::text(marker.clone())])?; + inserts.push(settings); + + let mut macros = Insert::new("character_account_data", &["guid", "type", "time", "data"]); + if let Some(blob) = model.client_data.get(&5) { + macros.row(vec![Val::Expr("@char"), Val::u(5u8), Val::u(blob.time), Val::Bytes(blob.data.0.clone())])?; + } + counts.macros = macros.len(); + inserts.push(macros); + + // ---- the script ----------------------------------------------------------------------------------------------- + let mut s = String::new(); + let mut line = |text: String| { + s.push_str(&text); + s.push('\n'); + }; + let has = |t: &str| ctx.probe.has(t); + + line(format!("DO IF(GET_LOCK('{IMPORT_LOCK}', 10) = 1, 1, {FAIL});")); + line("SET SESSION innodb_lock_wait_timeout = 10;".into()); + line("START TRANSACTION;".into()); + + // guards: the realm is stopped (no online character, no session of the game server's database user), the account exists + line(format!("DO IF((SELECT COUNT(*) FROM acore_characters.characters WHERE online <> 0) = 0, 1, {FAIL});")); + let users = ctx.game_server_users.iter().map(|u| Val::text(u.clone()).sql()).collect::>().join(", "); + line(format!("DO IF((SELECT COUNT(*) FROM information_schema.processlist WHERE id <> CONNECTION_ID() AND user IN ({users})) = 0, 1, {FAIL});")); + line(format!("DO IF((SELECT COUNT(*) FROM acore_auth.account WHERE id = {}) = 1, 1, {FAIL});", ctx.account)); + line(format!("DO IF((SELECT COUNT(*) FROM acore_auth.account WHERE id = {} AND UPPER(username) NOT LIKE 'COABOT%' AND UPPER(username) <> 'COAMANAGER') = 1, 1, {FAIL});", ctx.account)); + line(format!("DO IF((SELECT COUNT(*) FROM acore_characters.characters WHERE account = {}) < {}, 1, {FAIL});", ctx.account, ctx.max_characters_per_account)); + + // allocation: above everything that exists (and everything stale rows still name) + line("SET @char := (SELECT IFNULL(MAX(guid), 0) + 1 FROM acore_characters.characters);".into()); + let max_of = |list: &[(&str, &str)]| -> String { + let parts: Vec = list.iter().filter(|(t, _)| has(t)).map(|(t, c)| format!("IFNULL((SELECT MAX(`{c}`) FROM acore_characters.`{t}`), 0)")).collect(); + format!("GREATEST(0, {})", parts.join(", ")) + }; + line(format!("SET @item_base := {} + 1;", max_of(ITEM_KEYED))); + line(format!("SET @pet_base := {} + 1;", max_of(PET_KEYED))); + line(format!("DO IF(@char < 4294967000 AND @item_base + {} < 4294967000 AND @pet_base + {} < 4294967000, 1, {FAIL});", model.items.len(), model.pets.len())); + + // start position from the destination's own world data + line(format!("SELECT map, zone, position_x, position_y, position_z, orientation INTO @map, @zone, @x, @y, @z, @o FROM acore_world.playercreateinfo WHERE race = {race} AND class = {class};")); + line(format!("DO IF(@map IS NOT NULL, 1, {FAIL});")); + + // name: kept unless it collides (case/accent-insensitively) or is reserved; then a temporary name + rename at login + line(format!("SET @name := {};", Val::text(model.identity.name.clone()).sql())); + let collides = "EXISTS(SELECT 1 FROM acore_characters.characters WHERE name COLLATE utf8mb4_unicode_ci = @name COLLATE utf8mb4_unicode_ci)"; + let reserved = if has("reserved_name") { " OR EXISTS(SELECT 1 FROM acore_characters.reserved_name WHERE name COLLATE utf8mb4_unicode_ci = @name COLLATE utf8mb4_unicode_ci)" } else { "" }; + line(format!("SET @rename := IF({collides}{reserved}, 1, 0);")); + line(format!("SET @final_name := IF(@rename = 1, CONCAT(LEFT(@name, {NAME_COLUMN_CHARS} - CHAR_LENGTH(HEX(@char))), HEX(@char)), @name);")); + line("SET @at_login := IF(@rename = 1, 1, 0);".into()); + + // leftovers of a deleted character that had this guid + for (table, column) in CHARACTER_KEYED.iter().filter(|(t, _)| has(t)) { + line(format!("DELETE FROM acore_characters.`{table}` WHERE `{column}` = @char;")); + } + + for insert in &inserts { + if let Some(sql) = insert.sql() { + line(sql); + } + } + + // the transaction checks its own result: a failed assertion raises an SQL error, so there is no COMMIT + line(assert_count("acore_characters.characters WHERE guid = @char", 1)); + line(assert_count("acore_characters.item_instance WHERE owner_guid = @char", counts.items)); + line(assert_count("acore_characters.character_inventory WHERE guid = @char", counts.inventory)); + line(assert_count("acore_characters.character_spell WHERE guid = @char", counts.spells)); + line(assert_count("acore_characters.character_talent WHERE guid = @char", counts.talents)); + line(assert_count("acore_characters.character_skills WHERE guid = @char", counts.skills)); + line(assert_count("acore_characters.character_reputation WHERE guid = @char", counts.reputation)); + line(assert_count("acore_characters.character_queststatus WHERE guid = @char", counts.quests)); + line(assert_count("acore_characters.character_queststatus_rewarded WHERE guid = @char", counts.rewarded)); + line(assert_count("acore_characters.character_action WHERE guid = @char", counts.actions)); + line(assert_count("acore_characters.character_pet WHERE owner = @char", counts.pets)); + line(assert_count("acore_characters.character_settings WHERE guid = @char", counts.settings + 1)); + // every inventory row points at an item of this character, every bag at an item of this character + line(format!( + "DO IF((SELECT COUNT(*) FROM acore_characters.character_inventory ci LEFT JOIN acore_characters.item_instance ii ON ii.guid = ci.item AND ii.owner_guid = @char WHERE ci.guid = @char AND ii.guid IS NULL) = 0, 1, {FAIL});" + )); + line(format!( + "DO IF((SELECT COUNT(*) FROM acore_characters.character_inventory ci LEFT JOIN acore_characters.character_inventory b ON b.item = ci.bag AND b.guid = @char WHERE ci.guid = @char AND ci.bag <> 0 AND b.item IS NULL) = 0, 1, {FAIL});" + )); + + line("SELECT '#R:alloc', @char, @item_base, @pet_base, @rename, CONCAT('x', HEX(@final_name));".into()); + line("COMMIT;".into()); + line("SELECT '#R:committed';".into()); + line(format!("DO RELEASE_LOCK('{IMPORT_LOCK}');")); + + Ok(ImportPlan { script: s, item_ids: model.items.iter().map(|i| i.id).collect(), pet_ids: model.pets.iter().map(|p| p.id).collect(), marker_data: marker, counts, not_applied_settings: not_applied }) +} + +/// What the realm answered after a successful import. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Allocation { + pub local_guid: u32, + pub item_base: u32, + pub pet_base: u32, + pub renamed: bool, + pub final_name: String, +} + +/// Parse the answer of the script. `None` for "committed" means the script did not get as far as the commit report. +pub fn parse_report(output: &str) -> Result<(Allocation, bool)> { + let mut alloc = None; + let mut committed = false; + for line in output.lines().filter(|l| !l.is_empty()) { + let cells: Vec<&str> = line.split('\t').collect(); + match cells[0] { + "#R:alloc" if cells.len() == 6 => { + let num = |i: usize| cells[i].parse::().map_err(|_| PortableError::CorruptSnapshot(format!("import report: {:?} is not a number", cells[i]))); + let name_hex = cells[5].strip_prefix('x').ok_or_else(|| PortableError::CorruptSnapshot("import report: name is not hex".into()))?; + let name = String::from_utf8(hex::decode(name_hex).map_err(|_| PortableError::CorruptSnapshot("import report: invalid hex".into()))?) + .map_err(|_| PortableError::CorruptSnapshot("import report: name is not UTF-8".into()))?; + alloc = Some(Allocation { + local_guid: to_u32("guid", num(1)?)?, + item_base: to_u32("item base", num(2)?)?, + pet_base: to_u32("pet base", num(3)?)?, + renamed: num(4)? != 0, + final_name: name, + }); + } + "#R:committed" => committed = true, + other => return Err(PortableError::CorruptSnapshot(format!("unexpected line in the import report: {other:?}"))), + } + } + let alloc = alloc.ok_or_else(|| PortableError::CorruptSnapshot("the import report has no allocation".into()))?; + Ok((alloc, committed)) +} diff --git a/crates/coa-core/src/portable/realm/script.rs b/crates/coa-core/src/portable/realm/script.rs index ee1cff8..e32491f 100644 --- a/crates/coa-core/src/portable/realm/script.rs +++ b/crates/coa-core/src/portable/realm/script.rs @@ -458,6 +458,11 @@ impl Row<'_> { .transpose() } + /// A plain (not hex) text cell such as a number or a keyword. + pub fn plain(&self, column: &str) -> Result { + Ok(self.cell(column)?.to_string()) + } + pub fn text(&self, column: &str) -> Result { Ok(self.opt_text(column)?.unwrap_or_default()) } diff --git a/crates/coa-core/src/portable/realm/sqlenc.rs b/crates/coa-core/src/portable/realm/sqlenc.rs new file mode 100644 index 0000000..a5b9ee7 --- /dev/null +++ b/crates/coa-core/src/portable/realm/sqlenc.rs @@ -0,0 +1,185 @@ +//! The typed SQL encoder: the **only** way data enters an import script. +//! +//! An untrusted string or blob (names, item texts, pet names, settings, macro blobs, ...) can only be a +//! [`Val::Text`] / [`Val::Bytes`], and those are rendered **exclusively as hex literals**: `_utf8mb4 0x48656C6C6F` / +//! `0xDEADBEEF`. Nothing is ever escaped, quoted or interpolated, so there is nothing to get wrong: no quote, backslash, +//! NUL, newline or `;` of any input can reach the SQL text. Numbers are rendered from typed integers. The only +//! free-form SQL is [`Val::Expr`], which takes a `&'static str`: it can only be a literal in this crate's source. +//! +//! Table and column names are `&'static str` for the same reason. + +use super::super::error::{PortableError, Result}; + +/// A value of one column. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Val { + Null, + Uint(u64), + Int(i64), + /// Character data, always rendered as `_utf8mb4 0x`. + Text(String), + /// Binary data, always rendered as `0x`. + Bytes(Vec), + /// The item at this index of the import plan (`@item_base + index`). + ItemRef(u32), + /// The pet at this index of the import plan (`@pet_base + index`). + PetRef(u32), + /// Fixed SQL written in this crate's source (a session variable such as `@char`, or `UNIX_TIMESTAMP()`). + Expr(&'static str), +} + +impl Val { + pub fn u>(v: T) -> Val { + Val::Uint(v.into()) + } + pub fn i>(v: T) -> Val { + Val::Int(v.into()) + } + pub fn text(s: impl Into) -> Val { + Val::Text(s.into()) + } + pub fn opt_text(s: Option<&str>) -> Val { + s.map_or(Val::Null, |s| Val::Text(s.to_string())) + } + + pub fn sql(&self) -> String { + match self { + Val::Null => "NULL".into(), + Val::Uint(v) => v.to_string(), + Val::Int(v) => v.to_string(), + Val::Text(s) if s.is_empty() => "_utf8mb4 X''".into(), + Val::Text(s) => format!("_utf8mb4 0x{}", hex::encode(s.as_bytes())), + Val::Bytes(b) if b.is_empty() => "X''".into(), + Val::Bytes(b) => format!("0x{}", hex::encode(b)), + Val::ItemRef(i) => format!("(@item_base + {i})"), + Val::PetRef(i) => format!("(@pet_base + {i})"), + Val::Expr(e) => (*e).to_string(), + } + } +} + +/// One `INSERT` with a fixed table and fixed columns. +#[derive(Clone, Debug)] +pub struct Insert { + table: &'static str, + columns: &'static [&'static str], + rows: Vec>, +} + +impl Insert { + pub fn new(table: &'static str, columns: &'static [&'static str]) -> Self { + Self { table, columns, rows: Vec::new() } + } + + /// A row must have exactly one value per column. + pub fn row(&mut self, values: Vec) -> Result<()> { + if values.len() != self.columns.len() { + return Err(PortableError::Invalid(format!("internal: {} values for {} columns of {}", values.len(), self.columns.len(), self.table))); + } + self.rows.push(values); + Ok(()) + } + + pub fn table(&self) -> &'static str { + self.table + } + pub fn len(&self) -> usize { + self.rows.len() + } + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + /// `None` when there is nothing to insert. + pub fn sql(&self) -> Option { + if self.rows.is_empty() { + return None; + } + let columns = self.columns.iter().map(|c| format!("`{c}`")).collect::>().join(", "); + let rows = self.rows.iter().map(|r| format!("({})", r.iter().map(Val::sql).collect::>().join(", "))).collect::>().join(",\n "); + Some(format!("INSERT INTO acore_characters.`{}` ({columns}) VALUES\n {rows};", self.table)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Characters that must never appear in generated SQL because of untrusted data. + const HOSTILE: &[char] = &['\'', '"', '\\', '\0', '\n', '\r', '\t', ';', '`', '-', '#', '/', '*', '%', ' ']; + + #[test] + fn text_is_a_hex_literal_whatever_it_contains() { + for s in [ + "plain", + "O'Brien", + "back\\slash", + "\"; DROP TABLE characters; --", + "'); DELETE FROM item_instance; --", + "line1\nline2\r\n\ttabbed", + "nul\0inside", + "emoji \u{1F409} and \u{4e2d}\u{6587} and \u{0414}", + "/* comment */ # hash", + "%_wildcards", + "", + ] { + let sql = Val::text(s).sql(); + let literal = sql.strip_prefix("_utf8mb4 ").unwrap(); + if s.is_empty() { + assert_eq!(literal, "X''"); + continue; + } + let hex = literal.strip_prefix("0x").expect("a hex literal"); + assert!(hex.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')), "{sql}"); + assert_eq!(hex::decode(hex).unwrap(), s.as_bytes(), "the hex decodes to exactly the input"); + assert!(!literal.contains(HOSTILE), "no hostile character may reach the SQL for {s:?}: {sql}"); + } + } + + #[test] + fn bytes_are_a_hex_literal_too() { + assert_eq!(Val::Bytes(vec![]).sql(), "X''"); + assert_eq!(Val::Bytes(vec![0, 255, 39, 92]).sql(), "0x00ff275c"); + let all: Vec = (0..=255).collect(); + let sql = Val::Bytes(all.clone()).sql(); + assert!(sql[2..].chars().all(|c| c.is_ascii_hexdigit())); + assert_eq!(hex::decode(&sql[2..]).unwrap(), all); + } + + #[test] + fn numbers_come_from_typed_integers() { + assert_eq!(Val::u(7u8).sql(), "7"); + assert_eq!(Val::u(u64::MAX).sql(), "18446744073709551615"); + assert_eq!(Val::i(-57i32).sql(), "-57"); + assert_eq!(Val::i(i64::MIN).sql(), "-9223372036854775808"); + assert_eq!(Val::Null.sql(), "NULL"); + assert_eq!(Val::ItemRef(3).sql(), "(@item_base + 3)"); + assert_eq!(Val::PetRef(0).sql(), "(@pet_base + 0)"); + assert_eq!(Val::Expr("@char").sql(), "@char"); + assert_eq!(Val::opt_text(None).sql(), "NULL"); + } + + #[test] + fn an_insert_is_fixed_columns_plus_encoded_values() { + let mut insert = Insert::new("character_settings", &["guid", "source", "data"]); + assert!(insert.sql().is_none(), "no rows, no statement"); + insert.row(vec![Val::Expr("@char"), Val::text("it's \"a\" source"), Val::text("1 2 3 ")]).unwrap(); + insert.row(vec![Val::Expr("@char"), Val::text("x"), Val::Null]).unwrap(); + let sql = insert.sql().unwrap(); + assert!(sql.starts_with("INSERT INTO acore_characters.`character_settings` (`guid`, `source`, `data`) VALUES\n (@char, _utf8mb4 0x")); + assert!(!sql.contains("it's") && !sql.contains("\"a\""), "{sql}"); + assert_eq!(sql.matches('\'').count(), 0, "the generated INSERT contains no quote at all"); + assert!(sql.ends_with(';')); + assert!(insert.row(vec![Val::Null]).is_err(), "arity is checked"); + } + + #[test] + fn the_whole_alphabet_of_a_generated_insert_is_small() { + // a hostile value produces nothing but hex digits, hex-literal syntax and the fixed scaffold + let mut insert = Insert::new("characters", &["name"]); + insert.row(vec![Val::text("Robert'); DROP TABLE characters;--\\\0\n")]).unwrap(); + let sql = insert.sql().unwrap(); + let allowed = |c: char| c.is_ascii_alphanumeric() || " _`(),.\n;".contains(c); + assert!(sql.chars().all(allowed), "{sql}"); + } +} diff --git a/crates/coa-core/src/portable/realm/testdata/realm-b-fixture.sql b/crates/coa-core/src/portable/realm/testdata/realm-b-fixture.sql new file mode 100644 index 0000000..28f0f79 --- /dev/null +++ b/crates/coa-core/src/portable/realm/testdata/realm-b-fixture.sql @@ -0,0 +1,23 @@ +-- Realm B: the disposable DESTINATION realm of the Phase 3 live tests. Applied to a copy of the schema fixture (which already +-- holds the CoA world data). Never applied to a real realm. +INSERT INTO acore_auth.account (id, username, salt, verifier) VALUES + (7001, 'PLAYERB', UNHEX(REPEAT('11',32)), UNHEX(REPEAT('22',32))), + (7002, 'COABOTHOST2', UNHEX(REPEAT('11',32)), UNHEX(REPEAT('22',32))), + (7003, 'FULLACCT', UNHEX(REPEAT('11',32)), UNHEX(REPEAT('22',32))); +INSERT INTO acore_characters.characters (`guid`, `account`, `name`, `race`, `class`, `gender`, `level`, `taximask`, `innTriggerId`) VALUES + (2001, 7001, _utf8mb4 0x4e414b4544, 1, 12, 0, 5, '', 0), + (2002, 7001, _utf8mb4 0x4578697374696e67, 1, 12, 0, 5, '', 0), + (3001, 7003, _utf8mb4 0x46756c6c30, 1, 12, 0, 1, '', 0), + (3002, 7003, _utf8mb4 0x46756c6c31, 1, 12, 0, 1, '', 0), + (3003, 7003, _utf8mb4 0x46756c6c32, 1, 12, 0, 1, '', 0), + (3004, 7003, _utf8mb4 0x46756c6c33, 1, 12, 0, 1, '', 0), + (3005, 7003, _utf8mb4 0x46756c6c34, 1, 12, 0, 1, '', 0), + (3006, 7003, _utf8mb4 0x46756c6c35, 1, 12, 0, 1, '', 0), + (3007, 7003, _utf8mb4 0x46756c6c36, 1, 12, 0, 1, '', 0), + (3008, 7003, _utf8mb4 0x46756c6c37, 1, 12, 0, 1, '', 0), + (3009, 7003, _utf8mb4 0x46756c6c38, 1, 12, 0, 1, '', 0), + (3010, 7003, _utf8mb4 0x46756c6c39, 1, 12, 0, 1, '', 0); +INSERT INTO acore_characters.reserved_name (name) VALUES (_utf8mb4 0x486973746f7279); +INSERT IGNORE INTO acore_world.creature_template (entry) VALUES (42), (43), (416); +-- the entries the fixture characters use that a real item_template may lack +INSERT IGNORE INTO acore_world.item_template (entry) VALUES (5042), (8000), (30000), (30017), (30034), (30051), (30068), (30085), (30102), (30119), (30136), (30153), (30170), (30187), (30204), (30221), (30238), (30255), (30272), (30289), (30306), (41019), (41020), (41021), (41022), (50000), (50001), (50002), (50003), (50004), (50005), (50010), (50011), (50012), (50013), (50014), (50015), (50020), (50021), (50022), (50023), (50024), (50025), (50030), (50031), (50032), (50033), (50034), (50035), (60023), (60024), (60025), (60026), (60027), (60028), (60029), (60030), (70001), (70002), (70003), (70004), (375250); diff --git a/crates/coa-core/src/portable/store.rs b/crates/coa-core/src/portable/store.rs index 4ca9c0e..2074276 100644 --- a/crates/coa-core/src/portable/store.rs +++ b/crates/coa-core/src/portable/store.rs @@ -18,7 +18,7 @@ use super::versions::{PORTABLE_COLLECTION_FORMAT_VERSION, SNAPSHOT_FORMAT_VERSIO pub const DATABASE_FILE: &str = "portable.db"; /// `PRAGMA application_id`: "COAP". Refuses to adopt an unrelated SQLite file. const APPLICATION_ID: i64 = 0x434F_4150; -const MIGRATIONS: &[&str] = &[include_str!("migrations/001_init.sql"), include_str!("migrations/002_item_mapping_lifecycle.sql")]; +const MIGRATIONS: &[&str] = &[include_str!("migrations/001_init.sql"), include_str!("migrations/002_item_mapping_lifecycle.sql"), include_str!("migrations/003_import_journal.sql")]; pub const DEFAULT_HISTORY_KEEP: u32 = 20; pub const MAX_HISTORY_KEEP: u32 = 1_000; @@ -269,6 +269,13 @@ impl Store { ids.iter().map(|text| read_character(&self.conn, text.parse()?)).collect() } + /// Every character of every profile (an overview for tools; normally the UI lists one profile). + pub fn list_characters_all(&self) -> Result> { + let mut stmt = self.conn.prepare("SELECT character_id FROM character ORDER BY character_id")?; + let ids: Vec = stmt.query_map([], |r| r.get(0))?.collect::>()?; + ids.iter().map(|text| read_character(&self.conn, text.parse()?)).collect() + } + pub fn set_archived(&mut self, id: CharacterId, archived: bool) -> Result<()> { let changed = self.conn.execute("UPDATE character SET archived = ?2 WHERE character_id = ?1", params![id.to_string(), archived])?; if changed == 0 { @@ -942,5 +949,8 @@ fn validate_observations(server_id: &str, observations: &[ItemObservation]) -> R Ok(()) } +mod journal; +pub use journal::*; + #[cfg(test)] mod tests; diff --git a/crates/coa-core/src/portable/store/journal.rs b/crates/coa-core/src/portable/store/journal.rs new file mode 100644 index 0000000..9e0d3f9 --- /dev/null +++ b/crates/coa-core/src/portable/store/journal.rs @@ -0,0 +1,229 @@ +//! The import journal: how a realm commit and a local commit stay consistent although they cannot be one transaction. +//! +//! ```text +//! begin_import local commit 1 journal row "prepared" (nonce, the plan: portable ids, entries, identities) +//! realm transaction inserts the character AND the marker `coa.portable.import = ` +//! finish_import local commit 2 mappings + binding + journal "committed", in ONE local transaction +//! ``` +//! +//! A crash can fall in two places. Before the realm commit: no marker exists, nothing happened in the realm; the entry is +//! aborted. After the realm commit but before `finish_import`: the marker exists in the realm, so the import *did* +//! happen and `finish_import` is simply run again from what the marker and the realm's rows say. The marker is part of +//! the realm transaction, so "marker present" and "import committed" are the same fact. + +use rusqlite::{params, OptionalExtension}; +use serde::{Deserialize, Serialize}; + +use super::*; +use crate::portable::ids::{ImportId, PortablePetId}; + +/// One item of the import plan, as far as the mapping needs to know it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PlannedItem { + pub id: PortableItemId, + pub entry: ContentId, + pub identity: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImportState { + /// Written before the realm transaction; the realm may or may not have committed. + Prepared, + Committed, + /// The realm did not commit: nothing of this import exists there. + Aborted, + /// The realm and the journal disagree in a way that must not be guessed about. + NeedsAttention, +} + +impl ImportState { + pub fn as_str(self) -> &'static str { + match self { + ImportState::Prepared => "prepared", + ImportState::Committed => "committed", + ImportState::Aborted => "aborted", + ImportState::NeedsAttention => "needs_attention", + } + } + fn parse(s: &str) -> Result { + Ok(match s { + "prepared" => ImportState::Prepared, + "committed" => ImportState::Committed, + "aborted" => ImportState::Aborted, + "needs_attention" => ImportState::NeedsAttention, + other => return Err(PortableError::Invalid(format!("unknown import state {other:?}"))), + }) + } +} + +/// What a begun import hands to the realm step. +#[derive(Debug, Clone)] +pub struct ImportTicket { + pub import_id: ImportId, + pub nonce: [u32; 4], + pub revision: u64, + pub marker: String, +} + +/// What the realm allocated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ImportAllocation { + pub local_guid: u32, + pub item_base: u32, + pub pet_base: u32, +} + +#[derive(Debug, Clone)] +pub struct JournalEntry { + pub import_id: ImportId, + pub character_id: CharacterId, + pub server_id: String, + pub revision: u64, + pub marker: String, + pub state: ImportState, + pub items: Vec, + pub pet_ids: Vec, + pub allocation: Option, + pub detail: Option, + pub created_at: String, + pub updated_at: String, +} + +fn nonce() -> [u32; 4] { + let bytes = uuid::Uuid::new_v4().into_bytes(); + let word = |i: usize| u32::from_be_bytes([bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3]]); + [word(0), word(4), word(8), word(12)] +} + +/// The marker text for a nonce and revision. Kept in step with `realm::plan::marker_data`. +pub fn marker_text(nonce: [u32; 4], revision: u64) -> String { + format!("{} {} {} {} {} ", nonce[0], nonce[1], nonce[2], nonce[3], revision) +} + +impl Store { + /// Record the intention to import `character` at its current canonical `revision` into `server_id`. + /// Refused when the revision is stale, when the character is already on that server, or when another import of the + /// same pair is unfinished. + pub fn begin_import(&mut self, character: CharacterId, server_id: &str, revision: u64, items: &[PlannedItem], pet_ids: &[PortablePetId]) -> Result { + check_server_id(server_id)?; + let tx = self.write_tx()?; + let record = read_character(&tx, character)?; + if record.revision != revision { + return Err(PortableError::StaleRevision { expected: revision, current: record.revision }); + } + let open: Option = tx + .query_row("SELECT import_id FROM import_journal WHERE character_id = ?1 AND server_id = ?2 AND state = 'prepared'", params![character.to_string(), server_id], |r| r.get(0)) + .optional()?; + if let Some(open) = open { + return Err(PortableError::ImportInProgress { import_id: open.parse()? }); + } + let bound: bool = tx.query_row( + "SELECT EXISTS(SELECT 1 FROM character_server_mapping WHERE character_id = ?1 AND server_id = ?2)", + params![character.to_string(), server_id], + |r| r.get(0), + )?; + if bound { + return Err(PortableError::AlreadyOnRealm { character, server_id: server_id.to_string() }); + } + let import_id = ImportId::new(); + let nonce = nonce(); + let marker = marker_text(nonce, revision); + let at = now(); + tx.execute( + "INSERT INTO import_journal(import_id, character_id, server_id, revision, marker, state, items, pet_ids, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, 'prepared', ?6, ?7, ?8, ?8)", + params![import_id.to_string(), character.to_string(), server_id, revision as i64, marker, serde_json::to_string(items)?, serde_json::to_string(pet_ids)?, at], + )?; + tx.commit()?; + Ok(ImportTicket { import_id, nonce, revision, marker }) + } + + /// The realm committed: record the binding and the item mappings and close the journal entry, in **one** local + /// transaction. Safe to call again for an entry that is already committed (it does nothing). + pub fn finish_import(&mut self, import_id: ImportId, allocation: ImportAllocation) -> Result<()> { + let tx = self.write_tx()?; + let entry = read_journal(&tx, import_id)?; + match entry.state { + ImportState::Committed => return Ok(()), + ImportState::Prepared => {} + other => return Err(PortableError::ImportState { import_id, state: other.as_str().into() }), + } + let mut observations = Vec::with_capacity(entry.items.len()); + for (i, planned) in entry.items.iter().enumerate() { + let guid = u32::try_from(allocation.item_base as u64 + i as u64).map_err(|_| PortableError::Invalid("item guid out of range".into()))?; + observations.push(ItemObservation { portable_item_id: planned.id, local_item_guid: guid, entry: planned.entry.clone(), identity: planned.identity.clone() }); + } + bind_in_tx(&tx, entry.character_id, &entry.server_id, allocation.local_guid, entry.revision, MappingState::Synced)?; + reconcile_in_tx(&tx, entry.character_id, &entry.server_id, entry.revision, &observations)?; + tx.execute( + "UPDATE import_journal SET state = 'committed', local_guid = ?2, item_base = ?3, pet_base = ?4, updated_at = ?5 WHERE import_id = ?1", + params![import_id.to_string(), allocation.local_guid, allocation.item_base, allocation.pet_base, now()], + )?; + tx.commit()?; + Ok(()) + } + + /// The realm did not commit. + pub fn abort_import(&mut self, import_id: ImportId, detail: &str) -> Result<()> { + self.close_import(import_id, ImportState::Aborted, detail) + } + + /// The realm and the journal disagree; a person has to look. The entry stays visible and blocks nothing else. + pub fn flag_import(&mut self, import_id: ImportId, detail: &str) -> Result<()> { + self.close_import(import_id, ImportState::NeedsAttention, detail) + } + + fn close_import(&mut self, import_id: ImportId, state: ImportState, detail: &str) -> Result<()> { + let tx = self.write_tx()?; + let entry = read_journal(&tx, import_id)?; + if entry.state != ImportState::Prepared { + return Err(PortableError::ImportState { import_id, state: entry.state.as_str().into() }); + } + tx.execute("UPDATE import_journal SET state = ?2, detail = ?3, updated_at = ?4 WHERE import_id = ?1", params![import_id.to_string(), state.as_str(), detail, now()])?; + tx.commit()?; + Ok(()) + } + + pub fn import_entry(&self, import_id: ImportId) -> Result { + read_journal(&self.conn, import_id) + } + + /// Unfinished imports into one server, oldest first. + pub fn open_imports(&self, server_id: &str) -> Result> { + let mut stmt = self.conn.prepare("SELECT import_id FROM import_journal WHERE server_id = ?1 AND state = 'prepared' ORDER BY import_id")?; + let ids: Vec = stmt.query_map([server_id], |r| r.get(0))?.collect::>()?; + ids.iter().map(|id| read_journal(&self.conn, id.parse()?)).collect() + } +} + +fn read_journal(conn: &Connection, import_id: ImportId) -> Result { + type Row = (String, String, i64, String, String, String, String, Option, Option, Option, Option, String, String); + let row: Option = conn + .query_row( + "SELECT character_id, server_id, revision, marker, state, items, pet_ids, local_guid, item_base, pet_base, detail, created_at, updated_at FROM import_journal WHERE import_id = ?1", + [import_id.to_string()], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?, r.get(7)?, r.get(8)?, r.get(9)?, r.get(10)?, r.get(11)?, r.get(12)?)), + ) + .optional()?; + let Some((character, server_id, revision, marker, state, items, pets, guid, item_base, pet_base, detail, created_at, updated_at)) = row else { + return Err(PortableError::UnknownImport(import_id)); + }; + let allocation = match (guid, item_base, pet_base) { + (Some(g), Some(i), Some(p)) => Some(ImportAllocation { local_guid: g as u32, item_base: i as u32, pet_base: p as u32 }), + _ => None, + }; + Ok(JournalEntry { + import_id, + character_id: character.parse()?, + server_id, + revision: revision as u64, + marker, + state: ImportState::parse(&state)?, + items: serde_json::from_str(&items)?, + pet_ids: serde_json::from_str(&pets)?, + allocation, + detail, + created_at, + updated_at, + }) +} diff --git a/crates/coa-core/src/portable/store/tests.rs b/crates/coa-core/src/portable/store/tests.rs index a6fed9c..a4f7ad0 100644 --- a/crates/coa-core/src/portable/store/tests.rs +++ b/crates/coa-core/src/portable/store/tests.rs @@ -492,7 +492,8 @@ fn schema_1_item_mappings_survive_the_migration_as_unverified() { item = store.load_current(character).unwrap().items[0].id; store.bind_server(character, "realm-a", 154, 1, MappingState::Active).unwrap(); store.conn.execute_batch( - "DROP TABLE item_mapping; + "DROP TABLE import_journal; + DROP TABLE item_mapping; CREATE TABLE item_mapping (character_id TEXT NOT NULL, server_id TEXT NOT NULL, portable_item_id TEXT NOT NULL, local_item_guid INTEGER NOT NULL, PRIMARY KEY (character_id, server_id, portable_item_id), UNIQUE (character_id, server_id, local_item_guid), FOREIGN KEY (character_id, server_id) REFERENCES character_server_mapping(character_id, server_id) ON DELETE CASCADE) STRICT; @@ -506,7 +507,7 @@ fn schema_1_item_mappings_survive_the_migration_as_unverified() { .unwrap(); } let store = Store::open_file(&file).unwrap(); - assert_eq!(store.schema_version().unwrap(), 2); + assert_eq!(store.schema_version().unwrap(), MIGRATIONS.len() as i64); let mappings = store.item_mappings(character, "realm-a").unwrap(); assert_eq!(mappings.len(), 1); assert_eq!((mappings[0].portable_item_id, mappings[0].local_item_guid, mappings[0].identity.as_str()), (item, 4242, "")); @@ -578,3 +579,118 @@ fn a_tampered_collection_is_detected() { store.conn.execute("UPDATE collection SET payload = ?1", [forged]).unwrap(); assert!(matches!(store.collection(profile, "coa:vanity"), Err(PortableError::CorruptSnapshot(_)))); } + +// ---- the import journal ------------------------------------------------------------------------------------------------- + +fn plan_of_current(store: &Store, id: CharacterId) -> Vec { + store + .load_current(id) + .unwrap() + .items + .iter() + .map(|i| PlannedItem { id: i.id, entry: i.entry.clone(), identity: crate::portable::identity::item_identity(&i.entry, i.random_property_id) }) + .collect() +} + +fn alloc(guid: u32) -> ImportAllocation { + ImportAllocation { local_guid: guid, item_base: 20_064, pet_base: 3_004 } +} + +#[test] +fn an_import_is_journaled_first_and_mapped_only_when_finished() { + let (mut store, profile) = fresh(); + let id = created(&mut store, profile); + let items = plan_of_current(&store, id); + let pets = store.load_current(id).unwrap().pets.iter().map(|p| p.id).collect::>(); + + let ticket = store.begin_import(id, "realm-b", 1, &items, &pets).unwrap(); + assert_eq!(ticket.marker, marker_text(ticket.nonce, 1)); + let entry = store.import_entry(ticket.import_id).unwrap(); + assert_eq!((entry.state, entry.revision, entry.items.len(), entry.pet_ids.len()), (ImportState::Prepared, 1, items.len(), pets.len())); + // nothing is mapped before the realm has committed + assert!(store.server_mappings(id).unwrap().is_empty()); + assert_eq!(store.find_by_local("realm-b", 1013).unwrap(), None); + assert_eq!(store.open_imports("realm-b").unwrap().len(), 1); + assert!(store.open_imports("realm-a").unwrap().is_empty()); + + store.finish_import(ticket.import_id, alloc(1013)).unwrap(); + + assert_eq!(store.find_by_local("realm-b", 1013).unwrap(), Some(id)); + let mapping = &store.server_mappings(id).unwrap()[0]; + assert_eq!((mapping.local_guid, mapping.last_revision, mapping.state), (1013, 1, MappingState::Synced)); + let mapped = store.item_mappings(id, "realm-b").unwrap(); + assert_eq!(mapped.len(), items.len()); + for (i, planned) in items.iter().enumerate() { + let m = mapped.iter().find(|m| m.portable_item_id == planned.id).unwrap(); + assert_eq!(m.local_item_guid, 20_064 + i as u32, "item {i} is item_base + {i}"); + assert_eq!((m.identity.as_str(), m.entry.as_str()), (planned.identity.as_str(), planned.entry.to_string().as_str())); + } + let done = store.import_entry(ticket.import_id).unwrap(); + assert_eq!((done.state, done.allocation), (ImportState::Committed, Some(alloc(1013)))); + assert!(store.open_imports("realm-b").unwrap().is_empty()); + + // finishing twice is harmless (recovery may run after a normal finish) + store.finish_import(ticket.import_id, alloc(1013)).unwrap(); + assert_eq!(store.item_mappings(id, "realm-b").unwrap().len(), items.len()); + // the character is on that realm now + assert!(matches!(store.begin_import(id, "realm-b", 1, &items, &pets), Err(PortableError::AlreadyOnRealm { .. }))); +} + +#[test] +fn begin_import_refuses_stale_revisions_and_a_second_unfinished_import() { + let (mut store, profile) = fresh(); + let id = created(&mut store, profile); + let items = plan_of_current(&store, id); + let model = store.load_current(id).unwrap(); + store.commit_snapshot(id, 1, model, "realm-a", None).unwrap(); + + assert!(matches!(store.begin_import(id, "realm-b", 1, &items, &[]), Err(PortableError::StaleRevision { expected: 1, current: 2 }))); + let first = store.begin_import(id, "realm-b", 2, &items, &[]).unwrap(); + assert!(matches!(store.begin_import(id, "realm-b", 2, &items, &[]), Err(PortableError::ImportInProgress { import_id }) if import_id == first.import_id)); + // another realm is independent + let other = store.begin_import(id, "realm-c", 2, &items, &[]).unwrap(); + assert_ne!(first.nonce, other.nonce, "every import has its own nonce"); + assert!(matches!(store.begin_import(CharacterId::new(), "realm-b", 1, &items, &[]), Err(PortableError::UnknownCharacter(_)))); + assert!(store.begin_import(id, "bad server!", 2, &items, &[]).is_err()); +} + +#[test] +fn an_aborted_import_can_be_retried_and_closed_entries_cannot_be_reopened() { + let (mut store, profile) = fresh(); + let id = created(&mut store, profile); + let items = plan_of_current(&store, id); + let first = store.begin_import(id, "realm-b", 1, &items, &[]).unwrap(); + store.abort_import(first.import_id, "the realm never committed").unwrap(); + let entry = store.import_entry(first.import_id).unwrap(); + assert_eq!((entry.state, entry.detail.as_deref()), (ImportState::Aborted, Some("the realm never committed"))); + assert!(store.server_mappings(id).unwrap().is_empty() && store.item_mappings(id, "realm-b").unwrap().is_empty()); + + // closed entries stay closed + assert!(matches!(store.finish_import(first.import_id, alloc(1)), Err(PortableError::ImportState { .. }))); + assert!(matches!(store.abort_import(first.import_id, "again"), Err(PortableError::ImportState { .. }))); + + let second = store.begin_import(id, "realm-b", 1, &items, &[]).unwrap(); + assert_ne!(first.import_id, second.import_id); + store.flag_import(second.import_id, "the realm has 3 items where 60 were planned").unwrap(); + assert_eq!(store.import_entry(second.import_id).unwrap().state, ImportState::NeedsAttention); + assert!(store.open_imports("realm-b").unwrap().is_empty(), "a flagged entry is not 'open'"); + assert!(matches!(store.finish_import(second.import_id, alloc(1)), Err(PortableError::ImportState { .. }))); + assert!(matches!(store.import_entry(crate::portable::ids::ImportId::new()), Err(PortableError::UnknownImport(_)))); +} + +#[test] +fn a_failing_finish_changes_nothing_locally() { + let (mut store, profile) = fresh(); + let id = created(&mut store, profile); + let other = store.create_character(profile, fixtures::naked_level_one(), "realm-a").unwrap(); + store.bind_server(other, "realm-b", 1013, 1, MappingState::Synced).unwrap(); + let items = plan_of_current(&store, id); + let ticket = store.begin_import(id, "realm-b", 1, &items, &[]).unwrap(); + + // the realm claims a local guid that already belongs to another portable character: the local step must roll back whole + let result = store.finish_import(ticket.import_id, alloc(1013)); + assert!(matches!(result, Err(PortableError::LocalGuidTaken { .. })), "{result:?}"); + assert_eq!(store.import_entry(ticket.import_id).unwrap().state, ImportState::Prepared, "still recoverable"); + assert!(store.server_mappings(id).unwrap().is_empty()); + assert!(store.item_mappings(id, "realm-b").unwrap().is_empty()); +} diff --git a/docs/PORTABLE_CHARACTERS_AUDIT.md b/docs/PORTABLE_CHARACTERS_AUDIT.md index 31415ef..f9cf02d 100644 --- a/docs/PORTABLE_CHARACTERS_AUDIT.md +++ b/docs/PORTABLE_CHARACTERS_AUDIT.md @@ -642,6 +642,12 @@ D1-D13 were approved as recommended in section 15, with these amendments: | D13 checkpoints | keyed **internally by the local guid**: checkpoints arrive as `(server_id, local_guid)` and are resolved to the portable character through `Store::find_by_local` | | item mapping | `UNIQUE(character_id, server_id, local_item_guid)` in addition to the primary key (implemented in Phase 1) | +**Phase 2.1 / Phase 3 amendments (2026-10-06):** `character_settings` carry list tightened to ten strict key patterns, everything +else quarantined (`PORTABLE_EXPORT.md`); Phase 3 is an offline importer into a stopped disposable realm that keeps the existing +`Db` abstraction, writes untrusted text only as hex literals from a typed encoder, imports in one MySQL transaction with +journaled crash recovery (`PORTABLE_IMPORT.md`); the production importer is an in-core `PortableImportService` triggered over RA by +an opaque job id (`PORTABLE_IMPORT_SERVICE.md`, design only). + --------------------------------------------------------------------------------------------------------- ## Appendix A. Phase 1 - exact proposed plan (not started) diff --git a/docs/PORTABLE_FORMAT.md b/docs/PORTABLE_FORMAT.md index a83f650..3648b41 100644 --- a/docs/PORTABLE_FORMAT.md +++ b/docs/PORTABLE_FORMAT.md @@ -76,7 +76,7 @@ Account-wide sets of numeric ids (`coa:wardrobe`, `coa:vanity`, ...), stored per ## Store (`%LOCALAPPDATA%\CoAServerManager\portable\portable.db`) SQLite, WAL, `synchronous=FULL`, `foreign_keys=ON`, `application_id = 0x434F4150`, forward-only migrations tracked in -`PRAGMA user_version` (currently 2). A database from a newer Manager, or a foreign SQLite file, is refused and left untouched. +`PRAGMA user_version` (currently 3). A database from a newer Manager, or a foreign SQLite file, is refused and left untouched. | Table | Purpose | |---|---| @@ -85,6 +85,7 @@ SQLite, WAL, `synchronous=FULL`, `foreign_keys=ON`, `application_id = 0x434F4150 | `snapshot` | `(character_id, revision)` -> compressed canonical payload + hash | | `character_server_mapping` | `(character_id, server_id)` -> local guid; `UNIQUE(server_id, local_guid)` | | `item_mapping` | one row per mapping with a lifecycle (schema 2, see below) | +| `import_journal` | schema 3: one row per import into a realm (`prepared` / `committed` / `aborted` / `needs_attention`) with the nonce/marker and the plan; see `PORTABLE_IMPORT.md` | | `collection` | per profile and kind: revision, hash, encoded set | | `setting` | `history_keep` | diff --git a/docs/PORTABLE_IMPORT.md b/docs/PORTABLE_IMPORT.md new file mode 100644 index 0000000..f8913f0 --- /dev/null +++ b/docs/PORTABLE_IMPORT.md @@ -0,0 +1,120 @@ +# Offline import into a stopped realm (Phase 3) + +Code: `crates/coa-core/src/portable/realm/{sqlenc,plan,import}.rs`, journal in `portable/store/journal.rs`. + +This is the **offline** importer: it writes a canonical snapshot into a **stopped** realm's characters database. It proves +the model end to end and serves realms that are stopped. It is **not** the production path for a running public host: +that is the in-core `PortableImportService` ([`PORTABLE_IMPORT_SERVICE.md`](PORTABLE_IMPORT_SERVICE.md)), which is designed but +not implemented. + +## Flow + +```text +preflight (read-only snapshot) -> journal "prepared" -> ONE realm transaction -> mappings + journal "committed" + (marker inside it) (ONE local transaction) +``` + +1. **Preflight** (`preflight`): a read-only consistent snapshot of the realm. Refuses (`ImportRefused`, nothing written, + no journal entry) when the character belongs to another ruleset (CoA and Wildcard never mix), an id is of a namespace + the importer does not apply, the realm is running (a session of the game server's database user exists), characters are + marked online, the account does not exist / is a bot or Manager account / is full, the realm has no start position for the + race and class, or `item_template` / `creature_template` do not know an entry the character carries (the realm would + delete such items on load). +2. **Journal**: `begin_import` writes an `import_journal` row `prepared` (nonce, the plan: portable ids, entries, + identities) *before* anything happens in the realm. +3. **Realm transaction**: one `mysql` session, one `START TRANSACTION ... COMMIT`, generated by `plan.rs`. +4. **Local commit**: `finish_import` records the realm binding, the item mappings and the journal state in **one** local + transaction. Manager mappings exist only after the realm has committed. + +No driver is added; everything goes through the existing `Db` abstraction. + +## The typed encoder (`sqlenc.rs`) + +Untrusted strings and blobs (names, item texts, pet names, settings, macro blobs) can only be `Val::Text` / `Val::Bytes` and +are rendered **only** as hex literals (`_utf8mb4 0x...` / `0x...`). Nothing is ever escaped or interpolated: a quote, +backslash, NUL, `;` or `--` in any input cannot reach the SQL text. Numbers come from typed integers; the only free-form SQL +is `Val::Expr(&'static str)`, i.e. literals in this crate's source. Tests prove this by building the script for a hostile +character and a quiet one and checking that the scaffolds are identical once hex literals are removed, and on a real MySQL by +importing hostile text and reading it back byte for byte. + +## What the transaction does + +* takes the named lock `coa_portable_import`; asserts the realm is stopped (no online character, no session of the + game server's database user), the account exists and is not internal, and has room; +* allocates `@char`, `@item_base`, `@pet_base` **inside the transaction from the realm's own tables**: above the highest + character guid, above every item guid that *anything* still names (inventory, mail, auction, guild bank, gifts, refund and + trade data, bank, loot, ...), above every pet number; +* remaps every reference through them: item *i* of the plan is `@item_base + i` in the item row, the inventory row and as a + bag's contents' `bag`; pet *i* is `@pet_base + i`; no guid of the source realm or of the snapshot is ever written + (crafter guids are zeroed); +* reads the **start position from the realm's own `playercreateinfo`** (map, zone, position, homebind); +* checks the name inside the transaction (case/accent-insensitively against characters and `reserved_name`); a taken or + reserved name becomes `` with `AT_LOGIN_RENAME`, as the core's `.pdump load` does; +* deletes **leftovers of a deleted character that had this guid** from a curated list of per-character tables (the realm's + own delete does not clean module tables); +* inserts the character and, for the approved v1 scope, its inventory/items/gifts, spells, skills, glyphs, reputation, quests, + action bars, pets (+ spells, declined names), carried settings and macros; +* **asserts its own result** (`DO IF(count = expected, 1, )` for every table, every inventory row resolves to an item of + this character, every bag to an inventory row of this character); a failed assertion is an SQL error, so there is no + `COMMIT` and everything rolls back; +* writes the **marker** `character_settings (@char, 'coa.portable.import', ' ')` in the same + transaction, then commits and reports `@char`, the bases and the final name. + +A fresh arrival is not online, has the intro marked as seen, no first-login flag, full health, mana and energy (the realm +clamps to the maximum). + +### What is not applied + +* **Stock talents (`character_talent`) are never written.** Found with a real worldserver: a row naming a spell that is not a + talent of the realm's DBC makes `Player::_LoadTalents` `ASSERT` and **kills the whole worldserver**; nothing in the + database can tell whether a spell id is a talent. CoA keeps talents as spells and builds in `character_settings`; real CoA + characters have no `character_talent` rows. The rows stay in the snapshot and are reported as not applied. +* Settings outside the carry list (policy v2) and all extensions stay in the snapshot only. +* Equipment sets, per-character wardrobe, personal bank, collections: deferred by decision. + +## Crash recovery (the gap between the two commits) + +The realm and the local store cannot share a transaction, so the **marker** (inside the realm transaction) decides what +happened: + +| Crash point | What exists | Recovery (`recover_imports`) | +|---|---|---| +| after `begin_import`, before the realm commit | journal `prepared`, no marker | marker absent -> `aborted` (after a grace period and after taking the import lock, so an orphaned client process that is still finishing the import is waited for) | +| after the realm commit, before `finish_import` | journal `prepared`, marker present | marker present -> item/pet bases and the guid are read from the realm, counts and contiguity are checked against the journal's plan, then `finish_import` records the mappings -> `committed` | +| realm does not match the plan | marker present, rows missing | `needs_attention`; **no mapping is recorded on a guess** | +| a lost answer (error after commit) | same as above | `import_character` itself resolves through the marker before reporting | + +`finish_import` is idempotent. The marker stays in the realm after a successful import; the export policy drops +`coa.portable.*` so it is never carried. + +## Verified against a real server + +On two disposable MySQL servers (source A with synthetic characters, destination B with the real CoA world data) and a +**real `worldserver.exe`** started on B: the six characters were imported while the realm was stopped; the server then started +(`Loaded Character Infos For 39 Characters`), four of them were logged in through the real `HandlePlayerLoginFromDB` / +`Player::LoadFromDB` (`botcmd spawnbot`), saved, and read back. + +Findings, all expected consequences of the realm's own rules, none of them import bugs: + +* the two characters imported under a temporary name (``, rename-at-login) cannot log in without a client: the + realm demands the rename first; +* on first login the realm **adds** defaults (class spells and skills, taxi mask, default reputations, `core.ascension_*` + settings), **resets** daily honor and an unknown chosen title, **removes** what its DBC does not know (action buttons and + spells of the synthetic fixture), and **mails** items it cannot place (31 of 60 synthetic items, whose entries do not fit the + slots they were put in); +* the import marker survives the realm's save. + +These are exactly the changes Phase 4's reconciliation has to treat as the realm's own, not as the player's. + +A real client login (character select, entering the world) was **not** performed: no game client is available to the +automated tests. + +## Tests + +* `realm/import_tests.rs` (no database): one transaction, hex-only text, remapping, counts and assertions, settings policy, + marker, guards, name logic, orphan cleanup list, refusals, the report parser. +* `store` tests: journal begin/finish/abort/flag, idempotence, rollback of a failing local step. +* `realm/live_import.rs` (`#[ignore]`, two real MySQL servers): a trip through a second realm for six characters with + read-back equality; GUID remapping and orphan cleanup; failure injected in the middle (all tables' row counts unchanged, + nothing mapped, retry works); every refusal; crash after the realm commit / before it / lock held / realm tampered; + hostile text; the realm's own start data. Steps `server_check_*` drive the real worldserver. diff --git a/docs/PORTABLE_IMPORT_SERVICE.md b/docs/PORTABLE_IMPORT_SERVICE.md new file mode 100644 index 0000000..4ce3dd0 --- /dev/null +++ b/docs/PORTABLE_IMPORT_SERVICE.md @@ -0,0 +1,77 @@ +# PortableImportService (production importer): design + +Status: **design only, approved, not implemented.** It is a core-side workstream (the fork's `src/server/game/Tools/`, next to +`PlayerDump`). The offline importer ([`PORTABLE_IMPORT.md`](PORTABLE_IMPORT.md)) stays the tool for stopped realms and for +tests. + +## Why in-core + +A running worldserver owns the id counters, the character name cache and the realm character count in memory +(`ObjectMgr::SetHighestGuids`, `CharacterCache`, `World::UpdateRealmCharCount`). Anything written from outside collides with +ids the server is about to hand out and is invisible to name lookups until a restart. `PlayerDumpReader::LoadDump` +(`PlayerDump.cpp:764`) already solves this inside the server; the service follows its runtime invariants and adds the CoA +tables and the portable model. + +## Trigger: an opaque job id, never a payload + +```text +Manager worldserver + 1. writes /.job (atomically) + 2. RA: `.portable import ` -------------> validates job_id, reads ONLY that file from the fixed JobDir + runs the import, writes /.result (atomically) + 3. RA reply: "OK " / "ERR " <-------- + 4. reads the .result file, deletes both files +``` + +* `PortableImport.JobDir` is a worldserver setting (default `/PortableImport/`). The command takes **only** `job_id`, a + UUIDv7 string matched against `^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`; the file name is + built by the service as `/.job`. No path, no separator, no payload, no SQL travels through RA, and RA's + typed-operation rule in `ra.rs` is kept (one new typed operation, `import_job(job_id)`). +* Console only, with its own RBAC permission; one job at a time; the reply never contains character data. +* The job file is the snapshot envelope of `PORTABLE_FORMAT.md` (compressed canonical JSON + SHA-256) plus a small header: + `{job_id, character_id, revision, account, nonce, expected_content_hash, ruleset}`. Size limits and the decompression cap are + enforced **before** parsing; the decoder is strict (unknown fields refused, bounded lists) and validates the same limits as + `PortableCharacter::validate`. + +## The runtime invariants (taken from `PlayerDumpReader`) + +All of these happen **inside the server, in one `CharacterDatabaseTransaction`**, on the world thread's DB path: + +1. Refuse when the account does not exist, is a bot/Manager account, is full (`CharactersPerAccount`), or a character with this + marker already exists (idempotency, below). +2. Allocate ids from the **live generators**: `GetGenerator().GetNextAfterMaxUsed()`, the item generator, + `_hiPetNumber`; remap every reference through those maps (the same remap rules as the offline importer). +3. Name check with `normalizePlayerName` / `CheckPlayerName` and the name cache; a taken or invalid name gets the temporary + name and `AT_LOGIN_RENAME` (as `.pdump load`). +4. Validate against the realm's *own* data, which the offline importer cannot see: `sObjectMgr->GetItemTemplate`, creature + templates, `sSpellMgr`, **`sTalentStore` (so stock talents can be validated instead of skipped)**, `playercreateinfo`. +5. Write with **prepared statements** (`CharacterDatabasePreparedStatement::SetData`), never string-built SQL; all rows of the + character (including the marker `coa.portable.import`) in the one transaction. +6. `CharacterDatabase.CommitTransaction(trans)`; **after** a successful commit only: + `sCharacterCache->AddCharacterCacheEntry(...)`, advance the generators/`_hiPetNumber` exactly as `LoadDump` does, + `sWorld->UpdateRealmCharCount(account)`. +7. Never touch a character that is online; never overwrite an existing one (replacing is the separate, backed-up operation of + Phase 4). + +## Idempotency and crash recovery + +The marker row is written in the same transaction as the character, with the job's nonce. Re-running the same job id finds the +marker, writes the same `.result` and does nothing else. The Manager keeps the same `import_journal` as the offline path, so +the recovery table of `PORTABLE_IMPORT.md` applies unchanged; only the "who writes the realm side" part differs. + +## Result file + +`.result` (JSON, written to a temp name and renamed): `{status, local_guid, item_base, pet_base, renamed, final_name, +counts, not_applied[], warnings[]}` or `{status:"refused", problems[]}` with the same problem codes as the offline preflight. The +Manager turns it into `finish_import`. + +## What it must also cover (beyond the offline importer) + +Online accounts (the character list on a connected client is stale until relog), the cache refresh above, per-character module +state through the module extension API of Phase 7, collections of the account (Phase 6, keyed by the destination account), and +the level-cap projection hooks of Phase 8. + +## Not part of this design + +Exports from a running realm (checkpoints need a per-character save command, `.character save `, tracked separately), +a payload over RA, a path argument, a remote caller of any kind.