From 35a2d5dea75ded5ee7650e15c4e900931cb519e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Wed, 7 Oct 2026 06:54:11 +0500 Subject: [PATCH] feat(portable): Phase 6.1 format v2 with strict v1 migration and job format; Phase 7 capability negotiation and extensions Co-Authored-By: Claude Sonnet 5.5 --- crates/coa-core/examples/portable_smoke.rs | 62 +- crates/coa-core/src/portable/capabilities.rs | 440 ++++++++++++++ crates/coa-core/src/portable/compat.rs | 418 +++++++++++++ crates/coa-core/src/portable/error.rs | 2 + crates/coa-core/src/portable/extension.rs | 571 ++++++++++++++++++ crates/coa-core/src/portable/merge.rs | 11 +- crates/coa-core/src/portable/merge/tests.rs | 50 ++ .../portable/migrations/007_capabilities.sql | 28 + crates/coa-core/src/portable/mod.rs | 3 + crates/coa-core/src/portable/realm/import.rs | 20 +- .../coa-core/src/portable/realm/knowledge.rs | 52 +- .../src/portable/realm/live_profile.rs | 167 +++++ crates/coa-core/src/portable/realm/mod.rs | 5 +- crates/coa-core/src/portable/realm/online.rs | 19 +- crates/coa-core/src/portable/realm/profile.rs | 187 ++++++ .../coa-core/src/portable/realm/reconcile.rs | 86 ++- .../src/portable/session/collection_tests.rs | 17 + crates/coa-core/src/portable/session/fake.rs | 10 + crates/coa-core/src/portable/session/host.rs | 18 +- crates/coa-core/src/portable/session/live.rs | 18 +- .../src/portable/session/live_session.rs | 93 +++ crates/coa-core/src/portable/session/mod.rs | 2 + .../src/portable/session/profile_tests.rs | 211 +++++++ crates/coa-core/src/portable/snapshot.rs | 123 +++- crates/coa-core/src/portable/store.rs | 26 +- crates/coa-core/src/portable/store/host.rs | 11 +- crates/coa-core/src/portable/store/owner.rs | 3 +- crates/coa-core/src/portable/store/profile.rs | 109 ++++ crates/coa-core/src/portable/store/sync.rs | 4 +- crates/coa-core/src/portable/store/tests.rs | 34 ++ crates/coa-core/src/portable/versions.rs | 14 +- crates/coa-core/src/ra.rs | 6 + docs/PORTABLE_CAPABILITIES.md | 124 ++++ docs/PORTABLE_FORMAT.md | 30 +- docs/PORTABLE_IMPORT_SERVICE.md | 3 +- 35 files changed, 2936 insertions(+), 41 deletions(-) create mode 100644 crates/coa-core/src/portable/capabilities.rs create mode 100644 crates/coa-core/src/portable/compat.rs create mode 100644 crates/coa-core/src/portable/extension.rs create mode 100644 crates/coa-core/src/portable/migrations/007_capabilities.sql create mode 100644 crates/coa-core/src/portable/realm/live_profile.rs create mode 100644 crates/coa-core/src/portable/realm/profile.rs create mode 100644 crates/coa-core/src/portable/session/profile_tests.rs create mode 100644 crates/coa-core/src/portable/store/profile.rs create mode 100644 docs/PORTABLE_CAPABILITIES.md diff --git a/crates/coa-core/examples/portable_smoke.rs b/crates/coa-core/examples/portable_smoke.rs index ba3c0c9..f07b5a9 100644 --- a/crates/coa-core/examples/portable_smoke.rs +++ b/crates/coa-core/examples/portable_smoke.rs @@ -23,6 +23,9 @@ //! dump the canonical JSON of the current revision (what a core import job carries) //! collection-states (owner store) the canonical account collections as messages //! collection-apply write the canonical collections to the realm account (known ids only, INSERT IGNORE) +//! capabilities the realm's content profile (from its core when --ra-port is given), remembered in the store +//! preflight [online|session] what would happen to the character on that realm, before anything is written +//! reevaluate look again at what was held back when the realm's content profile changed (realm stopped) //! collection-add ... (owner store) add ids to a profile collection by hand (a test aid) //! collection-show [] the Owner's collections (revision, count, hash) and, with an account, the realm's //! @@ -42,6 +45,20 @@ use coa_core::portable::session::{HostConfig, HostService, OwnerService}; use coa_core::portable::{CharacterId, Store}; use coa_core::realms::Mode; +/// The options of a command, with the realm's content profile: assembled from its schema, its client data and (given an RA port) its +/// core, and remembered in the store. Without `--data-dir` there is no profile and nothing is evaluated. +fn with_profile(db: &Db, store: &mut Store, base: &ImportOptions, data_dir: &str, ra_port: Option, ra_user: &str, server: &str) -> Result { + if data_dir.is_empty() { + return Ok(base.clone()); + } + let registry = coa_core::portable::extension::ExtensionRegistry::new(); + let mut ra = ra_port.and_then(|port| std::env::var("COA_RA_PASSWORD").ok().and_then(|pw| coa_core::ra::Ra::connect_to(port, ra_user, &pw).ok())); + let caps = realm::profile::probe_capabilities(db, Some(std::path::Path::new(data_dir)), ra.as_mut(), ®istry).map_err(|e| e.to_string())?; + let change = store.set_realm_profile(server, &caps, if ra.is_some() { "live" } else { "offline" }).map_err(|e| e.to_string())?; + eprintln!("realm {server}: content profile {}{}", &caps.content_profile_hash[..16], if change.changed() { " (new or changed)" } else { "" }); + Ok(ImportOptions { capabilities: Some(std::sync::Arc::new(caps)), extensions: Some(std::sync::Arc::new(registry)), ..base.clone() }) +} + fn usage() -> ! { eprintln!("{}", include_str!("portable_smoke.rs").lines().take_while(|l| l.starts_with("//!")).map(|l| l.trim_start_matches("//!").trim_start_matches(' ')).collect::>().join("\n")); std::process::exit(2) @@ -80,7 +97,8 @@ fn run() -> Result<(), String> { let user: &'static str = Box::leak(user.into_boxed_str()); let db = Db::with_tools(tools, port, user, &password, realm_mode); let mut store = Store::open(&store_dir).map_err(|e| e.to_string())?; - let knowledge = match take("--data-dir", Some(""))?.as_str() { + let data_dir = take("--data-dir", Some(""))?; + let knowledge = match data_dir.as_str() { "" => None, dir => Some(std::sync::Arc::new(coa_core::portable::realm::knowledge::RealmKnowledge::from_data_dir(std::path::Path::new(dir)).map_err(|e| e.to_string())?)), }; @@ -89,6 +107,7 @@ fn run() -> Result<(), String> { let ra_port: Option = take("--ra-port", Some("0"))?.parse().ok().filter(|p| *p != 0); let ra_user = take("--ra-user", Some("local"))?; let job_dir = take("--job-dir", Some(""))?; + let profile_for = |db: &Db, store: &mut Store, server: &str| -> Result { with_profile(db, store, &opts, &data_dir, ra_port, &ra_user, server) }; let interval: u64 = take("--interval", Some("60"))?.parse().map_err(|_| "--interval is not a number".to_string())?; let collection_interval: u64 = take("--collection-interval", Some("300"))?.parse().map_err(|_| "--collection-interval is not a number".to_string())?; let id = |s: &str| -> Result { s.parse().map_err(|_| format!("{s:?} is not a character id")) }; @@ -106,7 +125,11 @@ fn run() -> Result<(), String> { println!("portable character {} (revision {}) {}", made.character_id, made.revision, made.warnings.join("; ")); } ("import", [character, server, account]) => { + let opts = profile_for(&db, &mut store, server)?; let o = realm::import_character(&db, &mut store, id(character)?, server, account.parse().map_err(|_| "account")?, &opts).map_err(|e| e.to_string())?; + for line in &o.not_applied { + println!(" not applied: {line}"); + } println!("imported as local character {} \"{}\" ({} items, {} pets, renamed: {})", o.local_guid, o.final_name, o.items, o.pets, o.renamed); } ("begin-session", [character, server]) => { @@ -122,7 +145,11 @@ fn run() -> Result<(), String> { } } ("update", [character, server]) => { + let opts = profile_for(&db, &mut store, server)?; let o = realm::update_realm_character(&db, &mut store, id(character)?, server, &opts).map_err(|e| e.to_string())?; + for line in &o.warnings { + println!(" {line}"); + } println!("{} (revision {} -> {}): {:?}", if o.updated { "updated in place" } else { "already up to date" }, o.from_revision, o.to_revision, o.counts); } ("recover", [server]) => { @@ -141,6 +168,7 @@ fn run() -> Result<(), String> { let profile = host.default_profile().map_err(|e| e.to_string())?; HostService::new(&mut host, server, HostConfig::default()).accept_offer(profile, &offer).map_err(|e| e.to_string())?; let account: u32 = account.parse().map_err(|_| "account")?; + let opts = with_profile(&db, &mut host, &opts, &data_dir, ra_port, &ra_user, server)?; let outcome = match (ra_port, job_dir.is_empty()) { (Some(port), false) => { let mut ra = coa_core::ra::Ra::connect_to(port, &ra_user, &std::env::var("COA_RA_PASSWORD").map_err(|_| "set COA_RA_PASSWORD".to_string())?).map_err(|e| e.to_string())?; @@ -214,6 +242,38 @@ fn run() -> Result<(), String> { println!("{}: {:?}", state.kind, applied); } } + ("capabilities", [server]) => { + let caps = profile_for(&db, &mut store, server)?.capabilities.ok_or("give --data-dir to read the realm's client data")?; + println!("{}", serde_json::to_string_pretty(&*caps).map_err(|e| e.to_string())?); + } + ("preflight", [character, server, rest @ ..]) => { + let opts = profile_for(&db, &mut store, server)?; + let caps = opts.capabilities.clone().ok_or("give --data-dir to read the realm's client data")?; + let model = store.load_current(id(character)?).map_err(|e| e.to_string())?; + let operation = match rest.first().map(String::as_str) { + Some("online") => coa_core::portable::compat::Operation::OnlineImport, + Some("session") => coa_core::portable::compat::Operation::RuntimeSession, + _ if store.server_mappings(model.character_id).map_err(|e| e.to_string())?.iter().any(|m| &m.server_id == server) => coa_core::portable::compat::Operation::Update, + _ => coa_core::portable::compat::Operation::OfflineImport, + }; + let report = coa_core::portable::compat::evaluate(&coa_core::portable::compat::Inputs { operation, model: &model, capabilities: &caps, knowledge: opts.knowledge.as_deref(), collections: &[], extensions: opts.extensions.as_deref() }); + println!("{} on {server} (content profile {}): {:?}", report.operation, &caps.content_profile_hash[..16], report.verdict()); + for o in &report.outcomes { + println!(" {o}"); + } + } + ("reevaluate", [character, server]) => { + let opts = profile_for(&db, &mut store, server)?; + let r = realm::reevaluate_realm_character(&db, &mut store, id(character)?, server, &opts).map_err(|e| e.to_string())?; + if r.profile_unchanged { + println!("the realm's content profile is the one this character was synchronised under: nothing to look at"); + } else { + println!("{}", r.update.map(|u| format!("restored: {:?}", u.changes)).unwrap_or_else(|| "nothing new to show on this realm".into())); + for e in r.extensions { + println!(" {e}"); + } + } + } ("collection-add", [kind, ids @ ..]) => { let profile = store.default_profile().map_err(|e| e.to_string())?; let ids = ids.iter().map(|i| i.parse::().map_err(|_| format!("{i:?} is not an id"))).collect::, _>>()?; diff --git a/crates/coa-core/src/portable/capabilities.rs b/crates/coa-core/src/portable/capabilities.rs new file mode 100644 index 0000000..36e6d8e --- /dev/null +++ b/crates/coa-core/src/portable/capabilities.rs @@ -0,0 +1,440 @@ +//! What a realm can take (Phase 7): the **content profile** of a destination, transport-neutral and small, so that a Registry can +//! later publish it unchanged and a Manager can decide before it writes anything. +//! +//! ```text +//! RealmCapabilities +//! core who built the realm's core (commit, branch, date): identity, not content; not part of the hash +//! content ContentProfile: everything that decides what can be applied, and the only thing the hash covers +//! content_profile_hash SHA-256 of the canonical JSON of `content` +//! ``` +//! +//! The client-data catalog is carried as the **SHA-256 and record count of each client table** the portable features depend on +//! (`Appearances.dbc`, `ItemAppearances.dbc`, `VanityCollection.dbc`, `ItemSet.dbc`), never as the tens of thousands of ids in them: +//! two realms with the same hashes know exactly the same appearances. The ids themselves are read from the realm's own data directory +//! by the side that writes to the realm ([`RealmKnowledge`](super::realm::knowledge::RealmKnowledge)), and that read is checked +//! against these hashes. +//! +//! A change of anything in `content` changes `content_profile_hash`; the Manager remembers the hash a realm had when a character was last +//! synchronised with it and re-evaluates what it held back when the hash moves, even if the canonical revision did not. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use super::error::{PortableError, Result}; +use super::model::Ruleset; + +/// Version of this profile's shape. +pub const PROFILE_VERSION: u32 = 1; +/// The most extension namespaces a profile lists. +pub const MAX_PROFILE_EXTENSIONS: usize = 64; +/// The largest serialised profile accepted from outside. +pub const MAX_PROFILE_BYTES: usize = 256 * 1024; + +/// A portable feature a realm can support. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Feature { + /// A running session: the core holds the player until the baseline is taken, saves one character on request. + RuntimeSessions, + /// The selected appearances of a character (`PortableCharacter::wardrobe`). + Wardrobe, + /// Account collections (`coa:appearance`, `coa:vanity`). + Collections, +} + +impl Feature { + pub fn as_str(&self) -> &'static str { + match self { + Feature::RuntimeSessions => "runtime_sessions", + Feature::Wardrobe => "wardrobe", + Feature::Collections => "collections", + } + } +} + +impl std::fmt::Display for Feature { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// An inclusive range of format versions. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FormatRange { + pub min: u32, + pub max: u32, +} + +impl FormatRange { + pub fn new(min: u32, max: u32) -> Self { + Self { min, max } + } + pub fn single(v: u32) -> Self { + Self { min: v, max: v } + } + pub fn contains(&self, v: u32) -> bool { + self.min <= v && v <= self.max + } + fn valid(&self) -> bool { + self.min >= 1 && self.min <= self.max + } +} + +impl std::fmt::Display for FormatRange { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.min == self.max { + write!(f, "{}", self.min) + } else { + write!(f, "{}..{}", self.min, self.max) + } + } +} + +/// The portable character formats a side can read and write. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CharacterFormats { + pub readable: FormatRange, + pub writable: FormatRange, +} + +/// One client table: the SHA-256 of the file and its record count. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CatalogEntry { + pub sha256: String, + pub records: u32, +} + +/// The client tables the portable features depend on, by file name. A table the realm does not have is not listed. +pub type ClientCatalog = BTreeMap; + +/// The client tables a profile is about. +pub const CATALOG_TABLES: [&str; 4] = ["Appearances.dbc", "ItemAppearances.dbc", "VanityCollection.dbc", "ItemSet.dbc"]; + +/// An extension namespace a realm can apply and the format versions it understands. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ExtensionSupport { + pub namespace: String, + pub module_version: String, + pub formats: FormatRange, +} + +/// Who built the realm's core. Identity for logs and support; it does not decide what can be applied. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CoreIdentity { + pub commit: String, + pub branch: String, + pub date: String, +} + +/// Everything that decides what a realm can be given. Its hash is the `content_profile_hash`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContentProfile { + pub ruleset: Ruleset, + /// The character formats the Manager side that serves this realm reads and writes. + pub character_formats: CharacterFormats, + /// The job formats the realm's core accepts for an online import (empty: no online import, or the core was not asked). + pub online_import_job_formats: Vec, + /// Version of the session protocol (`PortableSessionStarted`, `PortableCheckpoint`, `OwnerAck`). + pub session_protocol: u32, + /// Version of the collection messages (`CollectionObserved`, `CollectionState`, `CollectionAck`). + pub collection_protocol: u32, + pub features: BTreeSet, + pub collection_kinds: BTreeSet, + /// Sorted by namespace. + pub extensions: Vec, + pub client_catalog: ClientCatalog, +} + +impl ContentProfile { + /// SHA-256 of the canonical JSON (fixed field order, sorted sets and maps). + pub fn hash(&self) -> Result<[u8; 32]> { + let mut canonical = self.clone(); + canonical.extensions.sort_by(|a, b| a.namespace.cmp(&b.namespace)); + let json = serde_json::to_vec(&canonical)?; + let mut h = Sha256::new(); + h.update(b"coa-content-profile-v1\0"); + h.update(json); + Ok(h.finalize().into()) + } + + pub fn supports(&self, feature: Feature) -> bool { + self.features.contains(&feature) + } + + pub fn extension(&self, namespace: &str) -> Option<&ExtensionSupport> { + self.extensions.iter().find(|e| e.namespace == namespace) + } + + /// The character formats a Manager of this version reads and writes. + pub fn manager_formats() -> CharacterFormats { + CharacterFormats { + readable: FormatRange::new(super::versions::PORTABLE_CHARACTER_MIN_READ_VERSION, super::versions::PORTABLE_CHARACTER_FORMAT_VERSION), + writable: FormatRange::single(super::versions::PORTABLE_CHARACTER_FORMAT_VERSION), + } + } +} + +/// A realm's capabilities as they are published and compared. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RealmCapabilities { + pub profile_version: u32, + pub core: Option, + pub content: ContentProfile, + /// Lower-case hex of [`ContentProfile::hash`]. + pub content_profile_hash: String, +} + +impl RealmCapabilities { + pub fn new(core: Option, mut content: ContentProfile) -> Result { + content.extensions.sort_by(|a, b| a.namespace.cmp(&b.namespace)); + let content_profile_hash = hex::encode(content.hash()?); + let caps = Self { profile_version: PROFILE_VERSION, core, content, content_profile_hash }; + caps.validate()?; + Ok(caps) + } + + pub fn hash(&self) -> &str { + &self.content_profile_hash + } + + /// Structural validation, and the hash must be the one of the content: a profile cannot claim another content's hash. + pub fn validate(&self) -> Result<()> { + let bad = |what: &str| PortableError::Invalid(format!("realm capabilities: {what}")); + if self.profile_version != PROFILE_VERSION { + return Err(PortableError::UnsupportedFormat { found: self.profile_version, supported: PROFILE_VERSION }); + } + let c = &self.content; + if !c.character_formats.readable.valid() || !c.character_formats.writable.valid() { + return Err(bad("a character format range is empty or starts below 1")); + } + if c.online_import_job_formats.iter().any(|v| *v == 0) || c.online_import_job_formats.windows(2).any(|w| w[0] >= w[1]) { + return Err(bad("the online import job formats must be ascending, unique and at least 1")); + } + if c.session_protocol == 0 || c.collection_protocol == 0 { + return Err(bad("a protocol version is 0")); + } + if c.extensions.len() > MAX_PROFILE_EXTENSIONS { + return Err(bad("too many extension namespaces")); + } + for (i, e) in c.extensions.iter().enumerate() { + if !super::ids::valid_namespace(&e.namespace) || !e.formats.valid() || e.module_version.is_empty() || e.module_version.len() > 64 { + return Err(bad(&format!("extension {:?} is not valid", e.namespace))); + } + if i > 0 && c.extensions[i - 1].namespace >= e.namespace { + return Err(bad("extensions must be sorted by namespace and unique")); + } + } + for kind in &c.collection_kinds { + if !super::collection::valid_kind(kind) { + return Err(bad(&format!("collection kind {kind:?} is not valid"))); + } + } + for (name, entry) in &c.client_catalog { + let known = CATALOG_TABLES.contains(&name.as_str()); + let hex_ok = entry.sha256.len() == 64 && entry.sha256.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + if !known || !hex_ok { + return Err(bad(&format!("client table {name:?} is not valid"))); + } + } + if hex::encode(c.hash()?) != self.content_profile_hash { + return Err(bad("the content profile hash does not match the content")); + } + Ok(()) + } + + pub fn to_json(&self) -> Result> { + let bytes = serde_json::to_vec(self)?; + if bytes.len() > MAX_PROFILE_BYTES { + return Err(PortableError::LimitExceeded(format!("a profile is limited to {MAX_PROFILE_BYTES} bytes"))); + } + Ok(bytes) + } + + /// Parse and validate a profile from outside: size first, version before anything else is trusted, unknown fields refused. + pub fn from_json(bytes: &[u8]) -> Result { + if bytes.len() > MAX_PROFILE_BYTES { + return Err(PortableError::LimitExceeded(format!("a profile is limited to {MAX_PROFILE_BYTES} bytes"))); + } + let value: serde_json::Value = serde_json::from_slice(bytes)?; + let version = value.get("profile_version").and_then(|v| v.as_u64()).ok_or_else(|| PortableError::Invalid("the profile has no version".into()))?; + if version != PROFILE_VERSION as u64 { + return Err(PortableError::UnsupportedFormat { found: version.min(u32::MAX as u64) as u32, supported: PROFILE_VERSION }); + } + let caps: RealmCapabilities = serde_json::from_value(value)?; + caps.validate()?; + Ok(caps) + } +} + +/// What the realm's core says about itself in answer to `portable capabilities` (RA). Strict: the Manager trusts nothing it does not +/// recognise. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CoreReport { + pub report_version: u32, + pub ruleset: String, + pub core: CoreIdentity, + pub portable: CorePortable, + pub extension_namespaces: Vec, + pub client_data: BTreeMap>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CorePortable { + pub job_formats: Vec, + pub character_formats: Vec, + pub session_marker_version: u32, + pub features: Vec, +} + +impl CoreReport { + /// Parse the single JSON line of the console's answer. Anything else in the answer is ignored; the JSON object must be there. + pub fn parse(answer: &str) -> Result { + let line = answer.lines().map(str::trim).find(|l| l.starts_with('{')).ok_or_else(|| PortableError::Invalid("the core did not answer with a capability report (an older core?)".into()))?; + if line.len() > MAX_PROFILE_BYTES { + return Err(PortableError::LimitExceeded("the core's capability report is too large".into())); + } + let report: CoreReport = serde_json::from_str(line).map_err(|e| PortableError::Invalid(format!("the core's capability report is not valid: {e}")))?; + if report.report_version != 1 { + return Err(PortableError::UnsupportedFormat { found: report.report_version, supported: 1 }); + } + if report.ruleset != "coa" { + return Err(PortableError::Invalid(format!("the core reports ruleset {:?}; only coa is carried", report.ruleset))); + } + for (name, entry) in &report.client_data { + if !CATALOG_TABLES.contains(&name.as_str()) { + return Err(PortableError::Invalid(format!("the core reports an unknown client table {name:?}"))); + } + if let Some(e) = entry { + if e.sha256.len() != 64 || !e.sha256.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { + return Err(PortableError::Invalid(format!("the hash of {name} is not a SHA-256"))); + } + } + } + Ok(report) + } + + pub fn has_feature(&self, name: &str) -> bool { + self.portable.features.iter().any(|f| f == name) + } + + /// The catalog as the core loaded it (tables it could not read are absent). + pub fn catalog(&self) -> ClientCatalog { + self.client_data.iter().filter_map(|(k, v)| v.clone().map(|e| (k.clone(), e))).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> ContentProfile { + let mut catalog = ClientCatalog::new(); + catalog.insert("Appearances.dbc".into(), CatalogEntry { sha256: "ab".repeat(32), records: 42_903 }); + catalog.insert("VanityCollection.dbc".into(), CatalogEntry { sha256: "cd".repeat(32), records: 10_764 }); + ContentProfile { + ruleset: Ruleset::Coa, + character_formats: CharacterFormats { readable: FormatRange::new(1, 2), writable: FormatRange::single(2) }, + online_import_job_formats: vec![2], + session_protocol: 1, + collection_protocol: 1, + features: [Feature::RuntimeSessions, Feature::Wardrobe, Feature::Collections].into_iter().collect(), + collection_kinds: ["coa:appearance".to_string(), "coa:vanity".to_string()].into_iter().collect(), + extensions: vec![ExtensionSupport { namespace: "mod:fake".into(), module_version: "1.2.0".into(), formats: FormatRange::new(1, 3) }], + client_catalog: catalog, + } + } + + #[test] + fn the_profile_round_trips_and_its_hash_covers_the_content_but_not_the_core_identity() { + let a = RealmCapabilities::new(Some(CoreIdentity { commit: "aaaa".into(), branch: "x".into(), date: "d1".into() }), sample()).unwrap(); + let b = RealmCapabilities::new(Some(CoreIdentity { commit: "bbbb".into(), branch: "y".into(), date: "d2".into() }), sample()).unwrap(); + assert_eq!(a.content_profile_hash, b.content_profile_hash, "a new core build with the same content is the same content profile"); + assert_ne!(a, b); + let back = RealmCapabilities::from_json(&a.to_json().unwrap()).unwrap(); + assert_eq!(back, a); + + // every part of the content moves the hash + let base = a.content_profile_hash.clone(); + let mut changed: Vec = Vec::new(); + let mut c = sample(); + c.features.remove(&Feature::Wardrobe); + changed.push(c); + let mut c = sample(); + c.client_catalog.get_mut("Appearances.dbc").unwrap().sha256 = "ee".repeat(32); + changed.push(c); + let mut c = sample(); + c.extensions[0].formats = FormatRange::new(1, 4); + changed.push(c); + let mut c = sample(); + c.online_import_job_formats = vec![2, 3]; + changed.push(c); + let mut c = sample(); + c.collection_kinds.insert("coa:mounts".into()); + changed.push(c); + let mut c = sample(); + c.session_protocol = 2; + changed.push(c); + for c in changed { + assert_ne!(RealmCapabilities::new(None, c).unwrap().content_profile_hash, base); + } + } + + #[test] + fn a_profile_from_outside_is_validated_and_a_forged_hash_is_refused() { + let caps = RealmCapabilities::new(None, sample()).unwrap(); + let mut value: serde_json::Value = serde_json::from_slice(&caps.to_json().unwrap()).unwrap(); + let bytes = |v: &serde_json::Value| serde_json::to_vec(v).unwrap(); + + let mut forged = value.clone(); + forged["content_profile_hash"] = "00".repeat(32).into(); + assert!(RealmCapabilities::from_json(&bytes(&forged)).is_err(), "the hash must be the content's"); + let mut edited = value.clone(); + edited["content"]["features"] = serde_json::json!(["wardrobe"]); + assert!(RealmCapabilities::from_json(&bytes(&edited)).is_err(), "content edited without its hash"); + let mut unknown = value.clone(); + unknown["extra"] = 1.into(); + assert!(RealmCapabilities::from_json(&bytes(&unknown)).is_err()); + let mut newer = value.clone(); + newer["profile_version"] = 2.into(); + assert!(matches!(RealmCapabilities::from_json(&bytes(&newer)), Err(PortableError::UnsupportedFormat { .. }))); + value["content"]["client_catalog"]["Appearances.dbc"]["sha256"] = "not hex".into(); + assert!(RealmCapabilities::from_json(&bytes(&value)).is_err()); + assert!(RealmCapabilities::from_json(&vec![b' '; MAX_PROFILE_BYTES + 1]).is_err()); + assert!(RealmCapabilities::from_json(b"[]").is_err()); + + let mut bad = sample(); + bad.extensions.push(ExtensionSupport { namespace: "mod:fake".into(), module_version: "1".into(), formats: FormatRange::single(1) }); + assert!(RealmCapabilities::new(None, bad).is_err(), "a namespace twice"); + let mut bad = sample(); + bad.character_formats.readable = FormatRange::new(3, 2); + assert!(RealmCapabilities::new(None, bad).is_err()); + let mut bad = sample(); + bad.client_catalog.insert("Whatever.dbc".into(), CatalogEntry { sha256: "ab".repeat(32), records: 1 }); + assert!(RealmCapabilities::new(None, bad).is_err()); + } + + #[test] + fn the_core_report_is_parsed_strictly() { + let line = r#"{"report_version":1,"ruleset":"coa","core":{"commit":"abc","branch":"feat/x","date":"2026-10-06"},"portable":{"job_formats":[2],"character_formats":[2],"session_marker_version":1,"features":["runtime_sessions","wardrobe"]},"extension_namespaces":[],"client_data":{"Appearances.dbc":{"sha256":"abababababababababababababababababababababababababababababababab","records":3},"ItemSet.dbc":null}}"#; + let report = CoreReport::parse(&format!("noise\n{line}\n")).unwrap(); + assert!(report.has_feature("wardrobe") && !report.has_feature("collections")); + assert_eq!(report.catalog().len(), 1, "a table the core could not read is not in the catalog"); + assert!(CoreReport::parse("Unknown command").is_err(), "an older core has no such command"); + assert!(CoreReport::parse(&line.replace("\"report_version\":1", "\"report_version\":2")).is_err()); + assert!(CoreReport::parse(&line.replace("\"coa\"", "\"wildcard\"")).is_err()); + assert!(CoreReport::parse(&line.replace("\"records\":3", "\"records\":3,\"x\":1")).is_err()); + assert!(CoreReport::parse(&line.replace("Appearances.dbc", "Secret.dbc")).is_err()); + assert!(CoreReport::parse(&line.replace("abababababababababababababababababababababababababababababababab", "zz")).is_err()); + } +} diff --git a/crates/coa-core/src/portable/compat.rs b/crates/coa-core/src/portable/compat.rs new file mode 100644 index 0000000..53312b6 --- /dev/null +++ b/crates/coa-core/src/portable/compat.rs @@ -0,0 +1,418 @@ +//! Compatibility (Phase 7): what would happen to a character if it were imported into, updated on, or run on a realm with a given +//! content profile, **decided before anything is written**. +//! +//! Every part of the character gets an explicit [`Outcome`]: +//! +//! * `Compatible`: applied completely; +//! * `Held`: some of it is not applied (the realm's client data does not know it, an extension is in another format) and it stays +//! in the canonical character for a realm that can take it; +//! * `Unsupported`: the realm lacks the feature altogether; nothing of it is applied and nothing of it is lost; +//! * `Blocking`: the operation itself cannot be done on this realm (another ruleset, a character format the realm cannot read, an +//! online import the core cannot take, a session it cannot run). It is refused before the first write. +//! +//! The report never changes anything. The operations that write call [`evaluate`] first and refuse on a blocking outcome. + +use super::capabilities::{Feature, RealmCapabilities}; +use super::collection::IdSet; +use super::extension::{ExtensionDisposition, ExtensionRegistry}; +use super::model::PortableCharacter; +use super::realm::knowledge::RealmKnowledge; +use super::versions::ONLINE_IMPORT_JOB_FORMAT_VERSION; + +/// What is about to be done. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Operation { + /// Import into a stopped realm (the Manager writes the rows itself). + OfflineImport, + /// Import into a running realm through the core's import service. + OnlineImport, + /// Bring the realm's own character to the canonical state, in place. + Update, + /// Run the character in a runtime portable session (the core holds the player until the baseline is taken). + RuntimeSession, +} + +impl std::fmt::Display for Operation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Operation::OfflineImport => "offline import", + Operation::OnlineImport => "online import", + Operation::Update => "update", + Operation::RuntimeSession => "runtime session", + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Topic { + Ruleset, + CharacterFormat, + OnlineImport, + RuntimeSessions, + ClientData, + Wardrobe, + Collection(String), + Extension(String), +} + +impl std::fmt::Display for Topic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Topic::Ruleset => f.write_str("ruleset"), + Topic::CharacterFormat => f.write_str("character format"), + Topic::OnlineImport => f.write_str("online import"), + Topic::RuntimeSessions => f.write_str("runtime sessions"), + Topic::ClientData => f.write_str("client data"), + Topic::Wardrobe => f.write_str("wardrobe"), + Topic::Collection(kind) => write!(f, "collection {kind}"), + Topic::Extension(ns) => write!(f, "extension {ns}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Outcome { + Compatible(Topic), + Held { topic: Topic, held: usize, applicable: usize, reason: String }, + Unsupported { topic: Topic, reason: String }, + Blocking { topic: Topic, reason: String }, +} + +impl Outcome { + pub fn topic(&self) -> &Topic { + match self { + Outcome::Compatible(t) | Outcome::Held { topic: t, .. } | Outcome::Unsupported { topic: t, .. } | Outcome::Blocking { topic: t, .. } => t, + } + } +} + +impl std::fmt::Display for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Outcome::Compatible(t) => write!(f, "{t}: compatible"), + Outcome::Held { topic, held, applicable, reason } => write!(f, "{topic}: {applicable} applied, {held} held back in the canonical character ({reason})"), + Outcome::Unsupported { topic, reason } => write!(f, "{topic}: not supported by this realm, nothing applied and nothing lost ({reason})"), + Outcome::Blocking { topic, reason } => write!(f, "{topic}: BLOCKS the operation ({reason})"), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// Everything is applied. + Compatible, + /// The operation can be done; some of the character stays canonical-only. + Degraded, + /// The operation cannot be done on this realm. + Incompatible, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompatibilityReport { + pub operation: Operation, + pub content_profile_hash: String, + pub outcomes: Vec, +} + +impl CompatibilityReport { + pub fn verdict(&self) -> Verdict { + if self.outcomes.iter().any(|o| matches!(o, Outcome::Blocking { .. })) { + Verdict::Incompatible + } else if self.outcomes.iter().any(|o| !matches!(o, Outcome::Compatible(_))) { + Verdict::Degraded + } else { + Verdict::Compatible + } + } + + pub fn blocking(&self) -> Vec<&Outcome> { + self.outcomes.iter().filter(|o| matches!(o, Outcome::Blocking { .. })).collect() + } + + pub fn outcome(&self, topic: &Topic) -> Option<&Outcome> { + self.outcomes.iter().find(|o| o.topic() == topic) + } + + /// What was not applied completely, one line each (what an operation reports back). + pub fn held_lines(&self) -> Vec { + self.outcomes.iter().filter(|o| !matches!(o, Outcome::Compatible(_))).map(|o| o.to_string()).collect() + } +} + +/// Everything the evaluation reads, and nothing it writes. +pub struct Inputs<'a> { + pub operation: Operation, + pub model: &'a PortableCharacter, + pub capabilities: &'a RealmCapabilities, + /// The ids the realm's client data knows (read from its data directory); `None` when the Manager has not read them. + pub knowledge: Option<&'a RealmKnowledge>, + /// The Owner's canonical collections that would be applied to the realm's account alongside. + pub collections: &'a [(&'a str, &'a IdSet)], + pub extensions: Option<&'a ExtensionRegistry>, +} + +pub fn evaluate(i: &Inputs<'_>) -> CompatibilityReport { + let content = &i.capabilities.content; + let mut out = Vec::new(); + + if i.model.ruleset == content.ruleset { + out.push(Outcome::Compatible(Topic::Ruleset)); + } else { + out.push(Outcome::Blocking { topic: Topic::Ruleset, reason: format!("the character is {} and the realm is {}", i.model.ruleset, content.ruleset) }); + } + + if content.character_formats.readable.contains(i.model.format_version) { + out.push(Outcome::Compatible(Topic::CharacterFormat)); + } else { + out.push(Outcome::Blocking { topic: Topic::CharacterFormat, reason: format!("the character is format {} and the realm reads {}", i.model.format_version, content.character_formats.readable) }); + } + + if i.operation == Operation::OnlineImport { + if content.online_import_job_formats.contains(&ONLINE_IMPORT_JOB_FORMAT_VERSION) { + out.push(Outcome::Compatible(Topic::OnlineImport)); + } else if content.online_import_job_formats.is_empty() { + out.push(Outcome::Blocking { topic: Topic::OnlineImport, reason: "the realm's core did not report an import job format (it was not asked, or it is too old to answer)".into() }); + } else { + out.push(Outcome::Blocking { topic: Topic::OnlineImport, reason: format!("the core reads job formats {:?}; this Manager writes {ONLINE_IMPORT_JOB_FORMAT_VERSION}", content.online_import_job_formats) }); + } + } + + if i.operation == Operation::RuntimeSession { + if content.supports(Feature::RuntimeSessions) { + out.push(Outcome::Compatible(Topic::RuntimeSessions)); + } else { + out.push(Outcome::Blocking { topic: Topic::RuntimeSessions, reason: "the realm has no portable session support (the core's session table or switch is missing)".into() }); + } + } + + if let Some(k) = i.knowledge { + if let Some(why) = catalog_mismatch(k, i.capabilities) { + out.push(Outcome::Blocking { topic: Topic::ClientData, reason: why }); + } + } + + out.push(wardrobe_outcome(i)); + for (kind, set) in i.collections { + out.push(collection_outcome(i, kind, set)); + } + if !i.model.extensions.is_empty() { + out.extend(extension_outcomes(i)); + } + + CompatibilityReport { operation: i.operation, content_profile_hash: i.capabilities.content_profile_hash.clone(), outcomes: out } +} + +/// The client tables the Manager read must be the ones the realm's profile describes. +fn catalog_mismatch(knowledge: &RealmKnowledge, caps: &RealmCapabilities) -> Option { + for (name, mine) in knowledge.catalog() { + match caps.content.client_catalog.get(name) { + Some(theirs) if theirs == mine => {} + Some(_) => return Some(format!("{name} as the Manager reads it is not the {name} the realm loaded")), + None => return Some(format!("the Manager read {name}, which the realm's profile does not list")), + } + } + None +} + +fn wardrobe_outcome(i: &Inputs<'_>) -> Outcome { + let ids = i.model.wardrobe.ids(); + if ids.is_empty() { + return Outcome::Compatible(Topic::Wardrobe); + } + if !i.capabilities.content.supports(Feature::Wardrobe) { + return Outcome::Unsupported { topic: Topic::Wardrobe, reason: "the realm has no appearance tables or no Appearances.dbc".into() }; + } + let Some(knowledge) = i.knowledge else { + return Outcome::Held { topic: Topic::Wardrobe, held: ids.len(), applicable: 0, reason: "the Manager has not read the realm's client data".into() }; + }; + let applicable = ids.iter().filter(|id| knowledge.knows_appearance(**id)).count(); + if applicable == ids.len() { + Outcome::Compatible(Topic::Wardrobe) + } else { + Outcome::Held { topic: Topic::Wardrobe, held: ids.len() - applicable, applicable, reason: "the realm's client data does not know these appearances".into() } + } +} + +fn collection_outcome(i: &Inputs<'_>, kind: &str, set: &IdSet) -> Outcome { + let topic = Topic::Collection(kind.to_string()); + let content = &i.capabilities.content; + if !content.supports(Feature::Collections) || !content.collection_kinds.contains(kind) { + return Outcome::Unsupported { topic, reason: "the realm does not carry this collection".into() }; + } + let Some(knowledge) = i.knowledge else { + return Outcome::Held { topic, held: set.len(), applicable: 0, reason: "the Manager has not read the realm's client data".into() }; + }; + let applicable = set + .ids() + .iter() + .filter(|id| match kind { + "coa:appearance" => knowledge.knows_appearance(**id), + _ => knowledge.knows_vanity(**id) && !super::realm::knowledge::is_bank_vanity_item(**id), + }) + .count(); + if applicable == set.len() { + Outcome::Compatible(topic) + } else { + Outcome::Held { topic, held: set.len() - applicable, applicable, reason: "the realm's client data does not know these ids".into() } + } +} + +fn extension_outcomes(i: &Inputs<'_>) -> Vec { + let content = &i.capabilities.content; + let empty = ExtensionRegistry::new(); + let registry = i.extensions.unwrap_or(&empty); + registry + .evaluate(&i.model.extensions, content) + .into_iter() + .map(|o| { + let topic = Topic::Extension(o.namespace.clone()); + match o.disposition { + ExtensionDisposition::Applied | ExtensionDisposition::AlreadyApplied => Outcome::Compatible(topic), + ExtensionDisposition::CanonicalOnly => Outcome::Compatible(topic), + ExtensionDisposition::Deferred(why) => Outcome::Held { topic, held: 1, applicable: 0, reason: why.to_string() }, + ExtensionDisposition::Failed(why) => Outcome::Held { topic, held: 1, applicable: 0, reason: why }, + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::portable::capabilities::*; + use crate::portable::extension::fake::{payload, FakeAdapter}; + use crate::portable::fixtures::geared_level_eighty; + use crate::portable::model::Ruleset; + + fn profile(features: &[Feature], jobs: &[u32]) -> RealmCapabilities { + let mut catalog = ClientCatalog::new(); + catalog.insert("Appearances.dbc".into(), CatalogEntry { sha256: "ab".repeat(32), records: 3 }); + RealmCapabilities::new( + None, + ContentProfile { + ruleset: Ruleset::Coa, + character_formats: ContentProfile::manager_formats(), + online_import_job_formats: jobs.to_vec(), + session_protocol: 1, + collection_protocol: 1, + features: features.iter().copied().collect(), + collection_kinds: ["coa:appearance".to_string(), "coa:vanity".to_string()].into_iter().collect(), + extensions: vec![], + client_catalog: catalog, + }, + ) + .unwrap() + } + + fn all() -> Vec { + vec![Feature::RuntimeSessions, Feature::Wardrobe, Feature::Collections] + } + + fn knows(a: &[u32]) -> RealmKnowledge { + RealmKnowledge::new(IdSet::from_ids(a.iter().copied()).unwrap(), IdSet::from_ids([50_001]).unwrap()) + } + + fn run(op: Operation, model: &PortableCharacter, caps: &RealmCapabilities, k: Option<&RealmKnowledge>, collections: &[(&str, &IdSet)], reg: Option<&ExtensionRegistry>) -> CompatibilityReport { + evaluate(&Inputs { operation: op, model, capabilities: caps, knowledge: k, collections, extensions: reg }) + } + + #[test] + fn a_plain_character_on_a_matching_realm_is_compatible() { + let r = run(Operation::OnlineImport, &geared_level_eighty(), &profile(&all(), &[2]), None, &[], None); + assert_eq!(r.verdict(), Verdict::Compatible, "{:#?}", r.outcomes); + assert!(r.held_lines().is_empty()); + } + + #[test] + fn what_cannot_be_done_on_a_realm_is_refused_with_a_reason_before_anything_is_written() { + let model = geared_level_eighty(); + // a core that reads another job format, one that was not asked, a realm without sessions, another ruleset + let r = run(Operation::OnlineImport, &model, &profile(&all(), &[3]), None, &[], None); + assert_eq!(r.verdict(), Verdict::Incompatible); + assert!(matches!(r.outcome(&Topic::OnlineImport), Some(Outcome::Blocking { reason, .. }) if reason.contains("[3]"))); + let r = run(Operation::OnlineImport, &model, &profile(&all(), &[]), None, &[], None); + assert!(matches!(r.outcome(&Topic::OnlineImport), Some(Outcome::Blocking { reason, .. }) if reason.contains("did not report"))); + // the same character through the offline importer does not need the core's job format + assert_eq!(run(Operation::OfflineImport, &model, &profile(&all(), &[]), None, &[], None).verdict(), Verdict::Compatible); + let r = run(Operation::RuntimeSession, &model, &profile(&[Feature::Wardrobe], &[2]), None, &[], None); + assert!(matches!(r.outcome(&Topic::RuntimeSessions), Some(Outcome::Blocking { .. }))); + let mut wildcard = model.clone(); + wildcard.ruleset = Ruleset::Wildcard; + assert!(matches!(run(Operation::Update, &wildcard, &profile(&all(), &[2]), None, &[], None).outcome(&Topic::Ruleset), Some(Outcome::Blocking { .. }))); + // a realm whose Manager side only reads format 1 cannot take a format 2 character + let mut old = profile(&all(), &[2]).content; + old.character_formats.readable = FormatRange::single(1); + let old = RealmCapabilities::new(None, old).unwrap(); + assert!(matches!(run(Operation::Update, &model, &old, None, &[], None).outcome(&Topic::CharacterFormat), Some(Outcome::Blocking { .. }))); + } + + #[test] + fn a_wardrobe_the_realm_cannot_fully_show_is_held_back_by_count_and_one_it_cannot_show_at_all_is_unsupported() { + let mut model = geared_level_eighty(); + model.wardrobe.active.insert(1, 100); + model.wardrobe.active.insert(2, 200); + model.wardrobe.outfits.insert("Set".into(), vec![100, 300, 0]); + let caps = profile(&all(), &[2]); + let r = run(Operation::Update, &model, &caps, Some(&knows(&[100, 200])), &[], None); + assert_eq!(r.verdict(), Verdict::Degraded); + assert_eq!(r.outcome(&Topic::Wardrobe), Some(&Outcome::Held { topic: Topic::Wardrobe, held: 1, applicable: 2, reason: "the realm's client data does not know these appearances".into() })); + assert_eq!(run(Operation::Update, &model, &caps, Some(&knows(&[100, 200, 300])), &[], None).verdict(), Verdict::Compatible); + // the Manager has not read the data: everything is held, nothing is guessed + assert!(matches!(run(Operation::Update, &model, &caps, None, &[], None).outcome(&Topic::Wardrobe), Some(Outcome::Held { applicable: 0, held: 3, .. }))); + // the realm has no wardrobe feature: nothing is applied, nothing is lost + let r = run(Operation::Update, &model, &profile(&[Feature::RuntimeSessions], &[2]), Some(&knows(&[100])), &[], None); + assert!(matches!(r.outcome(&Topic::Wardrobe), Some(Outcome::Unsupported { .. }))); + assert_eq!(r.verdict(), Verdict::Degraded, "an unsupported feature degrades; it does not block"); + // a character that wears nothing has nothing to hold + assert_eq!(run(Operation::Update, &geared_level_eighty(), &profile(&[], &[2]), None, &[], None).verdict(), Verdict::Compatible); + } + + #[test] + fn collections_are_counted_per_kind_and_the_bank_items_are_never_applicable() { + let appearances = IdSet::from_ids([100, 200, 300]).unwrap(); + let vanity = IdSet::from_ids([50_001, 50_002, 110_000]).unwrap(); + let caps = profile(&all(), &[2]); + let k = RealmKnowledge::new(IdSet::from_ids([100, 200]).unwrap(), IdSet::from_ids([50_001, 110_000]).unwrap()); + let r = run(Operation::Update, &geared_level_eighty(), &caps, Some(&k), &[("coa:appearance", &appearances), ("coa:vanity", &vanity)], None); + assert!(matches!(r.outcome(&Topic::Collection("coa:appearance".into())), Some(Outcome::Held { held: 1, applicable: 2, .. }))); + assert!(matches!(r.outcome(&Topic::Collection("coa:vanity".into())), Some(Outcome::Held { held: 2, applicable: 1, .. })), "50002 is unknown and the bank item is not carried"); + let r = run(Operation::Update, &geared_level_eighty(), &profile(&[Feature::Wardrobe], &[2]), Some(&k), &[("coa:appearance", &appearances)], None); + assert!(matches!(r.outcome(&Topic::Collection("coa:appearance".into())), Some(Outcome::Unsupported { .. }))); + } + + #[test] + fn extensions_are_applicable_deferred_or_canonical_only_and_never_block() { + let mut model = geared_level_eighty(); + model.extensions.insert("mod:fake".into(), payload(2, b"x")); + model.extensions.insert("mod:other".into(), payload(1, b"y")); + model.extensions.insert("coa:unlisted-settings".into(), payload(1, b"{}")); + let mut reg = ExtensionRegistry::new(); + reg.register(FakeAdapter::new(2, 3)).unwrap(); + let mut caps = profile(&all(), &[2]).content; + caps.extensions = vec![ExtensionSupport { namespace: "mod:fake".into(), module_version: "1.0.0".into(), formats: FormatRange::new(2, 3) }]; + let caps = RealmCapabilities::new(None, caps).unwrap(); + let r = run(Operation::Update, &model, &caps, None, &[], Some(®)); + assert_eq!(r.outcome(&Topic::Extension("mod:fake".into())), Some(&Outcome::Compatible(Topic::Extension("mod:fake".into())))); + assert!(matches!(r.outcome(&Topic::Extension("mod:other".into())), Some(Outcome::Held { reason, .. }) if reason.contains("does not have this module"))); + assert_eq!(r.verdict(), Verdict::Degraded); + assert!(r.blocking().is_empty()); + // the same payload in a format the realm does not read + model.extensions.insert("mod:fake".into(), payload(4, b"x")); + let r = run(Operation::Update, &model, &caps, None, &[], Some(®)); + assert!(matches!(r.outcome(&Topic::Extension("mod:fake".into())), Some(Outcome::Held { reason, .. }) if reason.contains("newer"))); + } + + #[test] + fn client_data_that_is_not_what_the_realm_loaded_blocks() { + let caps = profile(&all(), &[2]); + let mut k = knows(&[100]); + k.set_catalog_for_test("Appearances.dbc", &"ab".repeat(32), 3); + assert_eq!(run(Operation::Update, &geared_level_eighty(), &caps, Some(&k), &[], None).verdict(), Verdict::Compatible); + k.set_catalog_for_test("Appearances.dbc", &"cd".repeat(32), 3); + let r = run(Operation::Update, &geared_level_eighty(), &caps, Some(&k), &[], None); + assert!(matches!(r.outcome(&Topic::ClientData), Some(Outcome::Blocking { .. }))); + let mut k = knows(&[100]); + k.set_catalog_for_test("VanityCollection.dbc", &"ab".repeat(32), 3); + assert!(matches!(run(Operation::Update, &geared_level_eighty(), &caps, Some(&k), &[], None).outcome(&Topic::ClientData), Some(Outcome::Blocking { reason, .. }) if reason.contains("does not list"))); + } +} diff --git a/crates/coa-core/src/portable/error.rs b/crates/coa-core/src/portable/error.rs index 17c0a0a..f6bed61 100644 --- a/crates/coa-core/src/portable/error.rs +++ b/crates/coa-core/src/portable/error.rs @@ -52,6 +52,8 @@ pub enum PortableError { UnknownImport(ImportId), #[error("import {import_id} is {state}, which does not allow this")] ImportState { import_id: ImportId, state: String }, + #[error("this realm cannot take the {operation}; nothing was written: {}", .reasons.join("; "))] + Incompatible { operation: String, reasons: Vec }, #[error("the import was refused before anything was written: {}", .0.iter().map(|p| p.to_string()).collect::>().join("; "))] ImportRefused(Vec), #[error("import {import_id} needs attention: {detail}")] diff --git a/crates/coa-core/src/portable/extension.rs b/crates/coa-core/src/portable/extension.rs new file mode 100644 index 0000000..0ce0feb --- /dev/null +++ b/crates/coa-core/src/portable/extension.rs @@ -0,0 +1,571 @@ +//! Extensions (Phase 7): how data of a game module this Manager has no built-in knowledge of travels with a character. +//! +//! * In the **canonical character** an extension is an opaque, hash-protected blob (`PortableCharacter::extensions`). It is never +//! interpreted, edited or dropped by the Owner. A namespace nobody understands simply stays where it is. +//! * On a **realm**, an [`ExtensionAdapter`] is the only thing allowed to read the module's data out of the realm and to write the +//! blob into it. An adapter belongs to one namespace (`mod:`) and one range of format versions; whatever it does not +//! claim is never applied, so a blob is never applied "blindly". +//! * Nothing is destroyed on the way. A realm without the module, or with another format of it, gets nothing, and the canonical blob +//! stays for the next realm that can take it. +//! +//! No adapter for any real module exists yet; the framework is exercised by a fake test extension. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use super::capabilities::{ContentProfile, ExtensionSupport, FormatRange}; +use super::error::{PortableError, Result}; +use super::ids::valid_namespace; +use super::model::Extension; + +/// How an adapter reaches a realm: one character, a way to ask, a way to write. The adapter owns its own SQL; the framework owns +/// nothing but the order of calls. +pub trait ExtensionRealm { + fn local_guid(&self) -> u32; + fn has_table(&self, table: &str) -> bool; + fn query(&mut self, sql: &str) -> Result; + fn execute(&mut self, script: &str) -> Result<()>; +} + +/// Whether one extension payload can be applied to a realm, as an explicit answer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtensionCompatibility { + Compatible, + /// The realm does not have the module (or no adapter serves it): the payload stays canonical, the realm receives nothing. + MissingOnRealm, + /// The payload is older than the oldest format the realm understands. + FormatTooOld { payload: u32, realm: FormatRange }, + /// The payload is newer than the newest format the realm understands. + FormatTooNew { payload: u32, realm: FormatRange }, + /// The adapter refuses for its own reason. + Incompatible(String), +} + +impl ExtensionCompatibility { + pub fn is_compatible(&self) -> bool { + matches!(self, ExtensionCompatibility::Compatible) + } +} + +impl std::fmt::Display for ExtensionCompatibility { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExtensionCompatibility::Compatible => f.write_str("compatible"), + ExtensionCompatibility::MissingOnRealm => f.write_str("the realm does not have this module"), + ExtensionCompatibility::FormatTooOld { payload, realm } => write!(f, "the payload is format {payload}, older than the {realm} the realm understands"), + ExtensionCompatibility::FormatTooNew { payload, realm } => write!(f, "the payload is format {payload}, newer than the {realm} the realm understands"), + ExtensionCompatibility::Incompatible(why) => write!(f, "incompatible: {why}"), + } + } +} + +/// The adapter of one module. +pub trait ExtensionAdapter: Send + Sync { + /// `mod:`. + fn namespace(&self) -> &str; + fn module_version(&self) -> &str; + /// The format versions of the payload this adapter reads and writes. + fn supported_format_versions(&self) -> FormatRange; + /// Is the module present on this realm (its tables exist, its version is the one this adapter is for)? + fn available(&self, realm: &mut dyn ExtensionRealm) -> bool; + /// Is this payload well formed? Called on everything an adapter exports and before it applies anything. + fn validate(&self, extension: &Extension) -> Result<()>; + /// Read the module's data of the character out of the realm, or `None` when the character has none. + fn export(&self, realm: &mut dyn ExtensionRealm) -> Result>; + /// Write a validated, compatible payload into the realm. + fn apply(&self, realm: &mut dyn ExtensionRealm, extension: &Extension) -> Result<()>; + /// Can this payload be applied to a realm with this content profile? The default is the format check against what the profile + /// published; an adapter may be stricter. + fn compatibility(&self, extension: &Extension, realm: &ContentProfile) -> ExtensionCompatibility { + default_compatibility(self.namespace(), extension, realm) + } +} + +/// The format check every adapter starts from. +pub fn default_compatibility(namespace: &str, extension: &Extension, realm: &ContentProfile) -> ExtensionCompatibility { + let Some(support) = realm.extension(namespace) else { return ExtensionCompatibility::MissingOnRealm }; + if extension.format_version < support.formats.min { + ExtensionCompatibility::FormatTooOld { payload: extension.format_version, realm: support.formats } + } else if extension.format_version > support.formats.max { + ExtensionCompatibility::FormatTooNew { payload: extension.format_version, realm: support.formats } + } else { + ExtensionCompatibility::Compatible + } +} + +/// What happened to one canonical extension on one realm. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExtensionDisposition { + /// Written to the realm now. + Applied, + /// Written before and unchanged since. + AlreadyApplied, + /// Not written, and why. The canonical payload is untouched. + Deferred(ExtensionCompatibility), + /// The adapter said yes but failed; nothing is claimed, the payload stays canonical and the next attempt starts again. + Failed(String), + /// Not a module extension (`coa:*` data the Manager itself keeps aside): never applied to a realm. + CanonicalOnly, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtensionOutcome { + pub namespace: String, + pub disposition: ExtensionDisposition, +} + +impl std::fmt::Display for ExtensionOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.disposition { + ExtensionDisposition::Applied => write!(f, "{}: applied", self.namespace), + ExtensionDisposition::AlreadyApplied => write!(f, "{}: already applied", self.namespace), + ExtensionDisposition::Deferred(why) => write!(f, "{}: deferred, kept canonical ({why})", self.namespace), + ExtensionDisposition::Failed(why) => write!(f, "{}: not applied, kept canonical ({why})", self.namespace), + ExtensionDisposition::CanonicalOnly => write!(f, "{}: canonical only", self.namespace), + } + } +} + +/// Only module namespaces (`mod:...`) are ever handed to an adapter; the Manager's own `coa:*` blobs (quarantined settings) are +/// canonical-only. +pub fn is_module_namespace(namespace: &str) -> bool { + namespace.starts_with("mod:") && valid_namespace(namespace) +} + +#[derive(Default, Clone)] +pub struct ExtensionRegistry { + adapters: BTreeMap>, +} + +impl std::fmt::Debug for ExtensionRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_list().entries(self.adapters.keys()).finish() + } +} + +impl ExtensionRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn register(&mut self, adapter: Arc) -> Result<()> { + let ns = adapter.namespace().to_string(); + if !is_module_namespace(&ns) { + return Err(PortableError::Invalid(format!("an extension namespace is mod:, not {ns:?}"))); + } + let formats = adapter.supported_format_versions(); + if formats.min == 0 || formats.min > formats.max || adapter.module_version().is_empty() || adapter.module_version().len() > 64 { + return Err(PortableError::Invalid(format!("adapter {ns} declares an invalid version or format range"))); + } + if self.adapters.insert(ns.clone(), adapter).is_some() { + return Err(PortableError::Invalid(format!("two adapters for {ns}"))); + } + Ok(()) + } + + pub fn namespaces(&self) -> impl Iterator { + self.adapters.keys().map(String::as_str) + } + + pub fn get(&self, namespace: &str) -> Option<&Arc> { + self.adapters.get(namespace) + } + + /// The extensions this realm can apply: the adapters whose module is present on it. + pub fn supported_on(&self, realm: &mut dyn ExtensionRealm) -> Vec { + self.adapters + .values() + .filter(|a| a.available(realm)) + .map(|a| ExtensionSupport { namespace: a.namespace().to_string(), module_version: a.module_version().to_string(), formats: a.supported_format_versions() }) + .collect() + } + + /// The explicit outcome for each extension of a canonical character on a realm with `profile`, without touching anything. + pub fn evaluate(&self, extensions: &BTreeMap, profile: &ContentProfile) -> Vec { + extensions + .iter() + .map(|(namespace, ext)| { + let disposition = if !is_module_namespace(namespace) { + ExtensionDisposition::CanonicalOnly + } else { + match self.adapters.get(namespace) { + None => ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm), + Some(adapter) => match adapter.compatibility(ext, profile) { + ExtensionCompatibility::Compatible => ExtensionDisposition::Applied, + other => ExtensionDisposition::Deferred(other), + }, + } + }; + ExtensionOutcome { namespace: namespace.clone(), disposition } + }) + .collect() + } + + /// Read every supported extension of the character out of the realm. A payload an adapter produces is validated before it + /// leaves; an adapter that fails or produces rubbish contributes nothing and the canonical payload is left alone. + pub fn export_all(&self, realm: &mut dyn ExtensionRealm) -> (BTreeMap, Vec) { + let mut out = BTreeMap::new(); + let mut problems = Vec::new(); + for (namespace, adapter) in &self.adapters { + if !adapter.available(realm) { + continue; + } + match adapter.export(realm) { + Ok(Some(ext)) => match check_exported(adapter.as_ref(), &ext) { + Ok(()) => { + out.insert(namespace.clone(), ext); + } + Err(e) => problems.push(format!("{namespace}: the exported payload is not valid ({e})")), + }, + Ok(None) => {} + Err(e) => problems.push(format!("{namespace}: export failed ({e})")), + } + } + (out, problems) + } + + /// Apply what can be applied. `applied` is what was applied before (namespace -> content hash): an unchanged payload is not applied + /// twice. Returns one outcome per canonical extension, in namespace order. + pub fn apply_all(&self, realm: &mut dyn ExtensionRealm, extensions: &BTreeMap, profile: &ContentProfile, applied: &HashMap) -> Vec { + extensions + .iter() + .map(|(namespace, ext)| { + let disposition = if !is_module_namespace(namespace) { + ExtensionDisposition::CanonicalOnly + } else if let Some(adapter) = self.adapters.get(namespace) { + self.apply_one(adapter.as_ref(), realm, ext, profile, applied.get(namespace.as_str())) + } else { + ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm) + }; + ExtensionOutcome { namespace: namespace.clone(), disposition } + }) + .collect() + } + + fn apply_one(&self, adapter: &dyn ExtensionAdapter, realm: &mut dyn ExtensionRealm, ext: &Extension, profile: &ContentProfile, applied: Option<&String>) -> ExtensionDisposition { + if !adapter.available(realm) { + return ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm); + } + let compatibility = adapter.compatibility(ext, profile); + if !compatibility.is_compatible() { + return ExtensionDisposition::Deferred(compatibility); + } + if applied.is_some_and(|h| *h == ext.content_hash) { + return ExtensionDisposition::AlreadyApplied; + } + if !ext.is_intact() { + return ExtensionDisposition::Failed("the payload does not match its content hash".into()); + } + if let Err(e) = adapter.validate(ext) { + return ExtensionDisposition::Failed(format!("the payload is not valid: {e}")); + } + match adapter.apply(realm, ext) { + Ok(()) => ExtensionDisposition::Applied, + Err(e) => ExtensionDisposition::Failed(e.to_string()), + } + } +} + +fn check_exported(adapter: &dyn ExtensionAdapter, ext: &Extension) -> Result<()> { + if !ext.is_intact() { + return Err(PortableError::CorruptSnapshot("the payload does not match its content hash".into())); + } + if !adapter.supported_format_versions().contains(ext.format_version) { + return Err(PortableError::Invalid(format!("format {} is outside what the adapter declares", ext.format_version))); + } + adapter.validate(ext) +} + +/// An adapter's realm over the Manager's database connection: one character of one realm. +pub struct DbExtensionRealm<'a> { + db: &'a crate::db::Db, + guid: u32, + tables: &'a super::realm::SchemaProbe, +} + +impl<'a> DbExtensionRealm<'a> { + pub fn new(db: &'a crate::db::Db, guid: u32, tables: &'a super::realm::SchemaProbe) -> Self { + Self { db, guid, tables } + } +} + +impl ExtensionRealm for DbExtensionRealm<'_> { + fn local_guid(&self) -> u32 { + self.guid + } + fn has_table(&self, table: &str) -> bool { + self.tables.has(table) + } + fn query(&mut self, sql: &str) -> Result { + self.db.query(sql).map_err(|e| PortableError::RealmRead(e.to_string())) + } + fn execute(&mut self, script: &str) -> Result<()> { + self.db.query(script).map(|_| ()).map_err(|e| PortableError::RealmRead(e.to_string())) + } +} + +#[cfg(test)] +pub(crate) mod fake { + //! A fake module for the tests of the framework: its data is one blob per character, kept in a map. + + use std::sync::Mutex; + + use super::*; + + /// A fake realm: the module's table, per character. + #[derive(Default)] + pub struct FakeExtensionRealm { + pub guid: u32, + pub has_module: bool, + pub data: BTreeMap>, + pub writes: u32, + } + + impl ExtensionRealm for FakeExtensionRealm { + fn local_guid(&self) -> u32 { + self.guid + } + fn has_table(&self, table: &str) -> bool { + self.has_module && table == "mod_fake" + } + fn query(&mut self, sql: &str) -> Result { + assert_eq!(sql, "GET"); + Ok(self.data.get(&self.guid).map(hex::encode).unwrap_or_default()) + } + fn execute(&mut self, script: &str) -> Result<()> { + let hex = script.strip_prefix("PUT ").expect("a fake write"); + self.data.insert(self.guid, hex::decode(hex).unwrap()); + self.writes += 1; + Ok(()) + } + } + + /// The fake module's adapter: formats 2..=3, a payload is at least one byte and at most 64. + pub struct FakeAdapter { + pub formats: FormatRange, + pub version: &'static str, + pub fail_apply: Mutex, + } + + impl FakeAdapter { + pub fn new(min: u32, max: u32) -> Arc { + Arc::new(Self { formats: FormatRange::new(min, max), version: "1.0.0", fail_apply: Mutex::new(false) }) + } + } + + impl ExtensionAdapter for FakeAdapter { + fn namespace(&self) -> &str { + "mod:fake" + } + fn module_version(&self) -> &str { + self.version + } + fn supported_format_versions(&self) -> FormatRange { + self.formats + } + fn available(&self, realm: &mut dyn ExtensionRealm) -> bool { + realm.has_table("mod_fake") + } + fn validate(&self, extension: &Extension) -> Result<()> { + if extension.payload.0.is_empty() || extension.payload.0.len() > 64 { + return Err(PortableError::Invalid("a fake payload has 1 to 64 bytes".into())); + } + Ok(()) + } + fn export(&self, realm: &mut dyn ExtensionRealm) -> Result> { + let hex = realm.query("GET")?; + if hex.is_empty() { + return Ok(None); + } + Ok(Some(Extension::new(self.version, self.formats.max, hex::decode(hex).unwrap()))) + } + fn apply(&self, realm: &mut dyn ExtensionRealm, extension: &Extension) -> Result<()> { + if *self.fail_apply.lock().unwrap() { + return Err(PortableError::RealmRead("the module's table is locked".into())); + } + realm.execute(&format!("PUT {}", hex::encode(&extension.payload.0))) + } + } + + pub fn payload(format: u32, bytes: &[u8]) -> Extension { + Extension::new("1.0.0", format, bytes.to_vec()) + } +} + +#[cfg(test)] +mod tests { + use super::fake::*; + use super::*; + use crate::portable::capabilities::{CharacterFormats, ContentProfile}; + use crate::portable::model::Ruleset; + + fn profile_with(extensions: Vec) -> ContentProfile { + ContentProfile { + ruleset: Ruleset::Coa, + character_formats: CharacterFormats { readable: FormatRange::new(1, 2), writable: FormatRange::single(2) }, + online_import_job_formats: vec![2], + session_protocol: 1, + collection_protocol: 1, + features: Default::default(), + collection_kinds: Default::default(), + extensions, + client_catalog: Default::default(), + } + } + + fn registry(adapter: Arc) -> ExtensionRegistry { + let mut r = ExtensionRegistry::new(); + r.register(adapter).unwrap(); + r + } + + fn realm(has_module: bool) -> FakeExtensionRealm { + FakeExtensionRealm { guid: 7, has_module, ..Default::default() } + } + + #[test] + fn only_module_namespaces_with_a_sane_declaration_can_be_registered() { + struct Odd(&'static str, FormatRange); + impl ExtensionAdapter for Odd { + fn namespace(&self) -> &str { + self.0 + } + fn module_version(&self) -> &str { + "1" + } + fn supported_format_versions(&self) -> FormatRange { + self.1 + } + fn available(&self, _: &mut dyn ExtensionRealm) -> bool { + true + } + fn validate(&self, _: &Extension) -> Result<()> { + Ok(()) + } + fn export(&self, _: &mut dyn ExtensionRealm) -> Result> { + Ok(None) + } + fn apply(&self, _: &mut dyn ExtensionRealm, _: &Extension) -> Result<()> { + Ok(()) + } + } + let mut r = ExtensionRegistry::new(); + assert!(r.register(Arc::new(Odd("coa:unlisted-settings", FormatRange::single(1)))).is_err(), "the Manager's own namespaces are not adapters"); + assert!(r.register(Arc::new(Odd("fake", FormatRange::single(1)))).is_err()); + assert!(r.register(Arc::new(Odd("mod:zero", FormatRange::new(0, 1)))).is_err()); + assert!(r.register(Arc::new(Odd("mod:backwards", FormatRange::new(3, 2)))).is_err()); + r.register(Arc::new(Odd("mod:ok", FormatRange::new(1, 2)))).unwrap(); + assert!(r.register(Arc::new(Odd("mod:ok", FormatRange::new(1, 2)))).is_err(), "two adapters for one namespace"); + } + + #[test] + fn a_realm_without_the_module_gets_nothing_and_the_canonical_payload_is_retained_for_a_realm_that_has_it() { + let reg = registry(FakeAdapter::new(2, 3)); + let mut canonical: BTreeMap = BTreeMap::new(); + canonical.insert("mod:fake".into(), payload(2, b"fake-state")); + + // a realm without the module: the profile does not list it, so the payload is deferred and not one byte is written + let mut without = realm(false); + let profile = profile_with(reg.supported_on(&mut without)); + assert!(profile.extensions.is_empty()); + let out = reg.evaluate(&canonical, &profile); + assert_eq!(out[0].disposition, ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm)); + let applied = reg.apply_all(&mut without, &canonical, &profile, &HashMap::new()); + assert_eq!(applied[0].disposition, ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm)); + assert_eq!(without.writes, 0); + assert!(without.data.is_empty(), "the destination receives nothing"); + assert_eq!(canonical["mod:fake"].payload.0, b"fake-state", "the canonical payload is untouched"); + + // a later realm that has it receives it + let mut with = realm(true); + let profile = profile_with(reg.supported_on(&mut with)); + assert_eq!(profile.extensions.len(), 1); + let applied = reg.apply_all(&mut with, &canonical, &profile, &HashMap::new()); + assert_eq!(applied[0].disposition, ExtensionDisposition::Applied); + assert_eq!(with.data[&7], b"fake-state"); + // and applying again does not write again + let state: HashMap = [("mod:fake".to_string(), canonical["mod:fake"].content_hash.clone())].into_iter().collect(); + assert_eq!(reg.apply_all(&mut with, &canonical, &profile, &state)[0].disposition, ExtensionDisposition::AlreadyApplied); + assert_eq!(with.writes, 1); + // a changed payload is applied again + canonical.insert("mod:fake".into(), payload(2, b"newer-state")); + assert_eq!(reg.apply_all(&mut with, &canonical, &profile, &state)[0].disposition, ExtensionDisposition::Applied); + assert_eq!(with.data[&7], b"newer-state"); + } + + #[test] + fn a_payload_in_a_format_the_realm_does_not_understand_is_deferred_explicitly_older_or_newer() { + let reg = registry(FakeAdapter::new(2, 3)); + let mut with = realm(true); + let profile = profile_with(reg.supported_on(&mut with)); + for (format, expected) in [ + (1, ExtensionCompatibility::FormatTooOld { payload: 1, realm: FormatRange::new(2, 3) }), + (4, ExtensionCompatibility::FormatTooNew { payload: 4, realm: FormatRange::new(2, 3) }), + ] { + let mut canonical = BTreeMap::new(); + canonical.insert("mod:fake".to_string(), payload(format, b"x")); + assert_eq!(reg.evaluate(&canonical, &profile)[0].disposition, ExtensionDisposition::Deferred(expected.clone())); + let applied = reg.apply_all(&mut with, &canonical, &profile, &HashMap::new()); + assert_eq!(applied[0].disposition, ExtensionDisposition::Deferred(expected)); + assert!(with.data.is_empty(), "nothing is written for an incompatible format"); + assert_eq!(canonical["mod:fake"].payload.0, b"x"); + } + let mut ok = BTreeMap::new(); + ok.insert("mod:fake".to_string(), payload(3, b"x")); + assert_eq!(reg.evaluate(&ok, &profile)[0].disposition, ExtensionDisposition::Applied); + } + + #[test] + fn a_namespace_nobody_serves_and_the_managers_own_blobs_are_never_applied() { + let reg = registry(FakeAdapter::new(1, 1)); + let mut with = realm(true); + let profile = profile_with(reg.supported_on(&mut with)); + let mut canonical = BTreeMap::new(); + canonical.insert("mod:somebody-elses".to_string(), payload(1, b"opaque")); + canonical.insert("coa:unlisted-settings".to_string(), payload(1, b"{}")); + let out = reg.apply_all(&mut with, &canonical, &profile, &HashMap::new()); + assert_eq!(out.iter().find(|o| o.namespace == "mod:somebody-elses").unwrap().disposition, ExtensionDisposition::Deferred(ExtensionCompatibility::MissingOnRealm)); + assert_eq!(out.iter().find(|o| o.namespace == "coa:unlisted-settings").unwrap().disposition, ExtensionDisposition::CanonicalOnly); + assert_eq!(with.writes, 0); + } + + #[test] + fn an_adapter_that_fails_or_is_handed_rubbish_changes_nothing_and_claims_nothing() { + let adapter = FakeAdapter::new(2, 3); + let reg = registry(adapter.clone()); + let mut with = realm(true); + let profile = profile_with(reg.supported_on(&mut with)); + let mut canonical = BTreeMap::new(); + canonical.insert("mod:fake".to_string(), payload(2, b"state")); + *adapter.fail_apply.lock().unwrap() = true; + let out = reg.apply_all(&mut with, &canonical, &profile, &HashMap::new()); + assert!(matches!(out[0].disposition, ExtensionDisposition::Failed(_)), "{:?}", out[0]); + *adapter.fail_apply.lock().unwrap() = false; + + // a payload that fails the adapter's own validation (too large) and one whose content does not match its hash + canonical.insert("mod:fake".into(), payload(2, &[7u8; 65])); + assert!(matches!(reg.apply_all(&mut with, &canonical, &profile, &HashMap::new())[0].disposition, ExtensionDisposition::Failed(_))); + let mut corrupt = payload(2, b"state"); + corrupt.payload.0[0] ^= 1; + canonical.insert("mod:fake".into(), corrupt); + assert!(matches!(reg.apply_all(&mut with, &canonical, &profile, &HashMap::new())[0].disposition, ExtensionDisposition::Failed(_))); + assert_eq!(with.writes, 0); + } + + #[test] + fn exports_come_only_from_adapters_whose_module_is_present_and_are_validated() { + let reg = registry(FakeAdapter::new(2, 3)); + let mut without = realm(false); + without.data.insert(7, b"stray".to_vec()); + assert!(reg.export_all(&mut without).0.is_empty(), "no module, no export"); + let mut with = realm(true); + assert!(reg.export_all(&mut with).0.is_empty(), "a character without module data exports nothing"); + with.data.insert(7, b"state".to_vec()); + let (out, problems) = reg.export_all(&mut with); + assert!(problems.is_empty()); + assert_eq!((out["mod:fake"].payload.0.as_slice(), out["mod:fake"].format_version), (b"state".as_slice(), 3)); + with.data.insert(7, vec![1u8; 100]); + let (out, problems) = reg.export_all(&mut with); + assert!(out.is_empty() && problems.len() == 1, "a payload the adapter itself rejects contributes nothing: {problems:?}"); + } +} diff --git a/crates/coa-core/src/portable/merge.rs b/crates/coa-core/src/portable/merge.rs index 4faf4aa..d44be49 100644 --- a/crates/coa-core/src/portable/merge.rs +++ b/crates/coa-core/src/portable/merge.rs @@ -333,7 +333,16 @@ pub fn merge3(target: &PortableCharacter, base: &PortableCharacter, ours: &Porta cx.scalar("wardrobe.can_see_spell", &mut out.wardrobe.can_see_spell, &base.wardrobe.can_see_spell, &ours.wardrobe.can_see_spell); cx.map("client_data", &mut out.client_data, &base.client_data, &ours.client_data); } - // extensions are the canonical character's own (quarantined data); a realm never contributes to them + // Module extensions: a realm contributes only what an adapter of its own exported (its B0 and B1 hold nothing else), and only + // what changed between them; everything else of the canonical character, unknown namespaces included, is left exactly as it is. + // The Manager's own `coa:*` blobs (quarantined settings) are canonical-only: a realm never contributes to them. + { + let module = |m: &BTreeMap| -> BTreeMap { m.iter().filter(|(k, _)| super::extension::is_module_namespace(k)).map(|(k, v)| (k.clone(), v.clone())).collect() }; + let mut merged = module(&out.extensions); + cx.map("extensions", &mut merged, &module(&base.extensions), &module(&ours.extensions)); + out.extensions.retain(|k, _| !super::extension::is_module_namespace(k)); + out.extensions.extend(merged); + } // ---- items and pets ------------------------------------------------------------------------------------------------ let items = merge_items(&mut cx, &mut out, target, base, ours)?; diff --git a/crates/coa-core/src/portable/merge/tests.rs b/crates/coa-core/src/portable/merge/tests.rs index 41bb4e0..934f5db 100644 --- a/crates/coa-core/src/portable/merge/tests.rs +++ b/crates/coa-core/src/portable/merge/tests.rs @@ -513,3 +513,53 @@ fn a_character_without_a_wardrobe_has_the_bytes_it_had_before_the_section_existe let back: PortableCharacter = serde_json::from_str(&json).unwrap(); assert_eq!(back, worn); } + +// ---- Phase 7: module extensions ---------------------------------------------------------------------------------------------- + +fn ext(format: u32, bytes: &[u8]) -> Extension { + Extension::new("1.0", format, bytes.to_vec()) +} + +#[test] +fn an_extension_nobody_on_this_realm_understands_survives_any_session() { + let mut c0 = c0(); + c0.extensions.insert("mod:somebody-elses".into(), ext(5, b"opaque payload of another module")); + c0.extensions.insert("coa:unlisted-settings".into(), ext(1, b"{}")); + let c0 = c0.normalized(); + let (b0, _) = normalized_by_the_realm(&c0); + let mut b0 = b0; + b0.extensions.clear(); + let mut b1 = b0.clone(); + b1.progression.money += 10; + let merged = merge3(&c0, &b0, &b1, Mode::Lenient).unwrap(); + assert_eq!(merged.model.extensions, c0.extensions, "the realm exported none of them, so none of them is touched"); + assert_eq!(merged.model.progression.money, c0.progression.money + 10); +} + +#[test] +fn what_a_realms_adapter_exports_is_carried_and_a_change_replaces_it_but_the_managers_own_blobs_are_never_contributed() { + let c0 = c0(); + let mut b0 = c0.clone(); + b0.extensions.insert("mod:fake".into(), ext(2, b"one")); + b0.extensions.insert("coa:unlisted-settings".into(), ext(1, b"realm side")); + let mut b1 = b0.clone(); + b1.extensions.insert("mod:fake".into(), ext(2, b"two")); + b1.extensions.insert("coa:unlisted-settings".into(), ext(1, b"realm side changed")); + let merged = merge3(&c0, &b0, &b1, Mode::Lenient).unwrap(); + assert_eq!(merged.model.extensions["mod:fake"].payload.0, b"two"); + assert!(!merged.model.extensions.contains_key("coa:unlisted-settings"), "the Manager's own blobs are canonical-only"); + + // an unchanged export is not a change (the realm's own representation of what the canonical character already had) + let mut c0b = c0.clone(); + c0b.extensions.insert("mod:fake".into(), ext(2, b"canonical")); + let mut b0 = c0.clone(); + b0.extensions.insert("mod:fake".into(), ext(2, b"as the realm rewrote it")); + let merged = merge3(&c0b, &b0, &b0.clone(), Mode::Lenient).unwrap(); + assert_eq!(merged.model.extensions["mod:fake"].payload.0, b"canonical"); + + // a realm that deleted its module data says so: B0 had it, B1 does not + let mut b1 = b0.clone(); + b1.extensions.remove("mod:fake"); + let merged = merge3(&c0b, &b0, &b1, Mode::Lenient).unwrap(); + assert!(!merged.model.extensions.contains_key("mod:fake")); +} diff --git a/crates/coa-core/src/portable/migrations/007_capabilities.sql b/crates/coa-core/src/portable/migrations/007_capabilities.sql new file mode 100644 index 0000000..9a03800 --- /dev/null +++ b/crates/coa-core/src/portable/migrations/007_capabilities.sql @@ -0,0 +1,28 @@ +-- Schema 7: realm capabilities (Phase 7). +-- +-- realm_profile the last content profile known of each realm (live from its core, or assembled offline), so an offline +-- import or update can still be evaluated, and a change of it can be seen +-- character_server_mapping.content_profile_hash +-- the content profile hash the realm had when this character was last synchronised with it; a different +-- hash on the realm means what was held back for it must be looked at again, whatever the revision +-- realm_extension_state the extension payloads (by content hash) that were applied to a character on a realm, so a payload +-- is applied once and a changed one again + +ALTER TABLE character_server_mapping ADD COLUMN content_profile_hash TEXT; + +CREATE TABLE realm_profile ( + server_id TEXT PRIMARY KEY, + content_profile_hash TEXT NOT NULL, + profile TEXT NOT NULL, + source TEXT NOT NULL CHECK (source IN ('live', 'offline')), + fetched_at TEXT NOT NULL +) STRICT; + +CREATE TABLE realm_extension_state ( + character_id TEXT NOT NULL REFERENCES character(character_id) ON DELETE CASCADE, + server_id TEXT NOT NULL, + namespace TEXT NOT NULL, + applied_hash TEXT NOT NULL, + applied_at TEXT NOT NULL, + PRIMARY KEY (character_id, server_id, namespace) +) STRICT; diff --git a/crates/coa-core/src/portable/mod.rs b/crates/coa-core/src/portable/mod.rs index 7f7e9ce..0e152a6 100644 --- a/crates/coa-core/src/portable/mod.rs +++ b/crates/coa-core/src/portable/mod.rs @@ -6,8 +6,11 @@ //! //! See `docs/PORTABLE_CHARACTERS_AUDIT.md` for the reasoning and `docs/PORTABLE_FORMAT.md` for the format rules. +pub mod capabilities; pub mod collection; +pub mod compat; pub mod error; +pub mod extension; pub mod identity; pub mod ids; pub mod merge; diff --git a/crates/coa-core/src/portable/realm/import.rs b/crates/coa-core/src/portable/realm/import.rs index 223dbf6..ea3b745 100644 --- a/crates/coa-core/src/portable/realm/import.rs +++ b/crates/coa-core/src/portable/realm/import.rs @@ -44,11 +44,16 @@ pub struct ImportOptions { /// What the destination's client data knows (appearances, vanity items). Without it no appearance or collection is /// written to the realm (all of it stays canonical); see `PORTABLE_APPEARANCE.md`. pub knowledge: Option>, + /// The realm's content profile. With it, every import, update and session is evaluated against it **before** anything is + /// written (an incompatible one is refused, a partly compatible one reports what stays canonical); without it nothing is evaluated. + pub capabilities: Option>, + /// The extension adapters this Manager has. + pub extensions: Option>, } impl Default for ImportOptions { fn default() -> Self { - Self { max_characters_per_account: 10, game_server_users: vec!["acore".to_string()], recovery_grace: Duration::from_secs(30), lock_wait_seconds: 30, knowledge: None } + Self { max_characters_per_account: 10, game_server_users: vec!["acore".to_string()], recovery_grace: Duration::from_secs(30), lock_wait_seconds: 30, knowledge: None, capabilities: None, extensions: None } } } @@ -265,6 +270,11 @@ pub fn import_character_in_session(db: &Db, store: &mut Store, character_id: Cha return Err(PortableError::AlreadyOnRealm { character: character_id, server_id: server_id.to_string() }); } + let mut operations = vec![super::super::compat::Operation::OfflineImport]; + if session.is_some() { + operations.push(super::super::compat::Operation::RuntimeSession); + } + let compatibility = super::profile::gate(opts, &model, &operations)?; let report = preflight(db, &model, account, opts)?; if !report.problems.is_empty() { return Err(PortableError::ImportRefused(report.problems)); @@ -293,7 +303,9 @@ pub fn import_character_in_session(db: &Db, store: &mut Store, character_id: Cha Resolution::Committed(alloc) => { // the realm's own report was lost: read the final name back let (final_name, renamed) = read_name(db, alloc.local_guid)?; - Ok(ImportOutcome { import_id: ticket.import_id, local_guid: alloc.local_guid, final_name, renamed, items: plan.counts.items, pets: plan.counts.pets, not_applied: plan.not_applied_settings, warnings: report.warnings }) + let mut not_applied = plan.not_applied_settings; + not_applied.extend(super::profile::after_write(Some(db), store, character_id, server_id, opts, alloc.local_guid, &model, compatibility.as_ref())?); + Ok(ImportOutcome { import_id: ticket.import_id, local_guid: alloc.local_guid, final_name, renamed, items: plan.counts.items, pets: plan.counts.pets, not_applied, warnings: report.warnings }) } Resolution::Aborted => Err(original.unwrap_or_else(|| PortableError::RealmRead("the realm did not commit the import".into()))), Resolution::Pending(why) => Err(PortableError::ImportNeedsAttention { import_id: ticket.import_id, detail: format!("the outcome in the realm is not known yet ({why}); run recovery") }), @@ -302,6 +314,8 @@ pub fn import_character_in_session(db: &Db, store: &mut Store, character_id: Cha }; store.finish_import(ticket.import_id, ImportAllocation { local_guid: allocation.local_guid, item_base: allocation.item_base, pet_base: allocation.pet_base })?; + let mut not_applied = plan.not_applied_settings; + not_applied.extend(super::profile::after_write(Some(db), store, character_id, server_id, opts, allocation.local_guid, &model, compatibility.as_ref())?); Ok(ImportOutcome { import_id: ticket.import_id, local_guid: allocation.local_guid, @@ -309,7 +323,7 @@ pub fn import_character_in_session(db: &Db, store: &mut Store, character_id: Cha renamed: allocation.renamed, items: plan.counts.items, pets: plan.counts.pets, - not_applied: plan.not_applied_settings, + not_applied, warnings: report.warnings, }) } diff --git a/crates/coa-core/src/portable/realm/knowledge.rs b/crates/coa-core/src/portable/realm/knowledge.rs index 70b3fde..4aba031 100644 --- a/crates/coa-core/src/portable/realm/knowledge.rs +++ b/crates/coa-core/src/portable/realm/knowledge.rs @@ -11,6 +11,9 @@ use std::path::Path; +use sha2::{Digest, Sha256}; + +use super::super::capabilities::{CatalogEntry, ClientCatalog, CATALOG_TABLES}; use super::super::collection::IdSet; use super::super::error::{PortableError, Result}; @@ -30,6 +33,9 @@ const MAX_DBC_BYTES: u64 = 64 * 1024 * 1024; pub struct RealmKnowledge { appearances: IdSet, vanity: IdSet, + /// The SHA-256 and record count of the client tables this knowledge was read from (empty when it was built by hand): compared with + /// the catalog of the realm's content profile, so that what the Manager reads is what the realm loaded. + catalog: ClientCatalog, } impl std::fmt::Debug for RealmKnowledge { @@ -40,15 +46,34 @@ impl std::fmt::Debug for RealmKnowledge { impl RealmKnowledge { pub fn new(appearances: IdSet, vanity: IdSet) -> Self { - Self { appearances, vanity } + Self { appearances, vanity, catalog: ClientCatalog::new() } + } + + pub fn catalog(&self) -> &ClientCatalog { + &self.catalog + } + + #[cfg(test)] + pub(crate) fn set_catalog_for_test(&mut self, table: &str, sha256: &str, records: u32) { + self.catalog.insert(table.to_string(), CatalogEntry { sha256: sha256.to_string(), records }); } /// Read `/dbc/Appearances.dbc` and `VanityCollection.dbc`. pub fn from_data_dir(data_dir: &Path) -> Result { let dbc = data_dir.join("dbc"); - let appearances = read_ids(&dbc.join("Appearances.dbc"), 0, 9)?; - let vanity = read_ids(&dbc.join("VanityCollection.dbc"), 1, 77)?; - Ok(Self { appearances: IdSet::from_ids(appearances)?, vanity: IdSet::from_ids(vanity)? }) + let (appearances, appearances_entry) = read_table(&dbc.join("Appearances.dbc"), 0, 9)?; + let (vanity, vanity_entry) = read_table(&dbc.join("VanityCollection.dbc"), 1, 77)?; + let mut catalog = ClientCatalog::new(); + catalog.insert("Appearances.dbc".into(), appearances_entry); + catalog.insert("VanityCollection.dbc".into(), vanity_entry); + for name in CATALOG_TABLES { + if !catalog.contains_key(name) { + if let Some(entry) = table_entry(&dbc.join(name)) { + catalog.insert(name.to_string(), entry); + } + } + } + Ok(Self { appearances: IdSet::from_ids(appearances)?, vanity: IdSet::from_ids(vanity)?, catalog }) } pub fn knows_appearance(&self, id: u32) -> bool { @@ -68,6 +93,25 @@ impl RealmKnowledge { } } +/// The SHA-256 and record count of a client table, or `None` when the realm does not have it (or it is not a WDBC file). +pub fn table_entry(path: &Path) -> Option { + let size = std::fs::metadata(path).ok()?.len(); + if size > MAX_DBC_BYTES { + return None; + } + let bytes = std::fs::read(path).ok()?; + if bytes.len() < HEADER || &bytes[..4] != b"WDBC" { + return None; + } + Some(CatalogEntry { sha256: hex::encode(Sha256::digest(&bytes)), records: u32::from_le_bytes(bytes[4..8].try_into().ok()?) }) +} + +fn read_table(path: &Path, id_dword: usize, minimum_dwords: usize) -> Result<(Vec, CatalogEntry)> { + let ids = read_ids(path, id_dword, minimum_dwords)?; + let entry = table_entry(path).ok_or_else(|| PortableError::SchemaMismatch(format!("{}: is not a client table", path.display())))?; + Ok((ids, entry)) +} + /// The ids of one DWORD column of a WDBC file (`WDBC`, record count, field count, record size, string size, records, strings). /// Zero ids are not records of anything and are skipped, as the core does. fn read_ids(path: &Path, id_dword: usize, minimum_dwords: usize) -> Result> { diff --git a/crates/coa-core/src/portable/realm/live_profile.rs b/crates/coa-core/src/portable/realm/live_profile.rs new file mode 100644 index 0000000..bb1600e --- /dev/null +++ b/crates/coa-core/src/portable/realm/live_profile.rs @@ -0,0 +1,167 @@ +//! Live tests of Phase 7 against the two real disposable MySQL servers of `live_import.rs`: a realm's profile is assembled from what +//! can be seen of it, an operation is evaluated before it writes, and a change of the profile brings back what was held back. +//! `#[ignore]`d; see `live_import.rs` for how to run. `COA_PORTABLE_LIVE_DATA` is a `Data` directory with `dbc/Appearances.dbc` and +//! `dbc/VanityCollection.dbc` (default `C:/games/coa-fixture/Data`). + +use std::path::PathBuf; +use std::sync::Arc; + +use crate::portable::capabilities::*; +use crate::portable::collection::IdSet; +use crate::portable::extension::ExtensionRegistry; +use crate::portable::model::Ruleset; + +use super::knowledge::RealmKnowledge; +use super::live_import::{counts, fresh_store, make, opts_without_knowledge, realms, reset_b, ACCOUNT, B}; +use super::*; + +fn data_dir() -> Option { + let dir = PathBuf::from(std::env::var("COA_PORTABLE_LIVE_DATA").unwrap_or_else(|_| "C:/games/coa-fixture/Data".into())); + dir.join("dbc/Appearances.dbc").exists().then_some(dir) +} + +fn knows(appearances: &[u32]) -> Arc { + Arc::new(RealmKnowledge::new(IdSet::from_ids(appearances.iter().copied()).unwrap(), IdSet::new())) +} + +fn caps(marker: u8, features: &[Feature], jobs: &[u32], ruleset: Ruleset, readable: FormatRange) -> Arc { + let mut catalog = ClientCatalog::new(); + catalog.insert("Appearances.dbc".into(), CatalogEntry { sha256: format!("{marker:02x}").repeat(32), records: 9 }); + Arc::new( + RealmCapabilities::new( + None, + ContentProfile { + ruleset, + character_formats: CharacterFormats { readable, writable: FormatRange::single(2) }, + online_import_job_formats: jobs.to_vec(), + session_protocol: 1, + collection_protocol: 1, + features: features.iter().copied().collect(), + collection_kinds: ["coa:appearance".to_string(), "coa:vanity".to_string()].into_iter().collect(), + extensions: vec![], + client_catalog: catalog, + }, + ) + .unwrap(), + ) +} + +fn all() -> Vec { + vec![Feature::Wardrobe, Feature::Collections] +} + +fn selected(db: &crate::db::Db, guid: u32) -> Vec { + db.query(&format!("SELECT CONCAT(category_id, '=', appearance_id) FROM acore_characters.character_appearance WHERE guid = {guid} ORDER BY category_id")).unwrap().lines().map(|l| l.trim().to_string()).filter(|l| !l.is_empty()).collect() +} + +#[test] +#[ignore] +fn a_realms_profile_is_assembled_from_its_schema_and_client_data_and_is_stable() { + let Some(r) = realms() else { return }; + let Some(dir) = data_dir() else { return }; + let registry = ExtensionRegistry::new(); + let schema = probe(&r.b).unwrap(); + let a = super::profile::assemble(&r.b, &schema, Some(&dir), None, ®istry).unwrap(); + let again = super::profile::assemble(&r.b, &schema, Some(&dir), None, ®istry).unwrap(); + assert_eq!(a, again); + let c = &a.content; + assert_eq!(c.ruleset, Ruleset::Coa); + assert!(c.supports(Feature::Wardrobe) && c.supports(Feature::Collections)); + assert!(!c.supports(Feature::RuntimeSessions), "a session needs the core's own report; the Manager does not guess it"); + assert!(c.online_import_job_formats.is_empty(), "the job formats are the core's to say"); + assert_eq!(c.collection_kinds.len(), 2); + assert!(a.core.is_none()); + for table in ["Appearances.dbc", "VanityCollection.dbc", "ItemAppearances.dbc"] { + let e = &c.client_catalog[table]; + assert_eq!(e.sha256.len(), 64); + assert!(e.records > 1000, "{table}: {e:?}"); + } + // the ids the Manager reads are the tables the profile hashes: the same files + let knowledge = RealmKnowledge::from_data_dir(&dir).unwrap(); + for (name, entry) in knowledge.catalog() { + assert_eq!(&c.client_catalog[name], entry, "{name}"); + } + assert!(a.to_json().unwrap().len() < 4096, "no id list: a profile is a few hundred bytes of hashes"); + // another data directory is another content profile + let other = tempfile::tempdir().unwrap(); + std::fs::create_dir(other.path().join("dbc")).unwrap(); + std::fs::copy(dir.join("dbc/Appearances.dbc"), other.path().join("dbc/Appearances.dbc")).unwrap(); + let narrower = super::profile::assemble(&r.b, &schema, Some(other.path()), None, ®istry).unwrap(); + assert!(!narrower.content.supports(Feature::Collections), "without VanityCollection.dbc the collections cannot be carried"); + assert_ne!(narrower.content_profile_hash, a.content_profile_hash); +} + +#[test] +#[ignore] +fn an_operation_a_realm_cannot_take_is_refused_before_one_row_is_written() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1002); + let before = counts(&r.b); + + for (why, caps) in [ + ("another ruleset", caps(1, &all(), &[2], Ruleset::Wildcard, FormatRange::new(1, 2))), + ("a realm that only reads the old character format", caps(1, &all(), &[2], Ruleset::Coa, FormatRange::single(1))), + ] { + let opts = ImportOptions { capabilities: Some(caps), ..opts_without_knowledge() }; + let error = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts).expect_err(why); + assert!(matches!(error, PortableError::Incompatible { .. }), "{why}: {error}"); + assert!(error.to_string().contains("nothing was written"), "{error}"); + } + assert_eq!(counts(&r.b), before, "not one row of the realm changed"); + assert!(store.server_mappings(id).unwrap().iter().all(|m| m.server_id != B), "and no mapping was made"); + assert!(store.open_imports(B).unwrap().is_empty(), "nor a journal entry"); + + // a session on a realm without the session feature + let opts = ImportOptions { capabilities: Some(caps(1, &all(), &[2], Ruleset::Coa, FormatRange::new(1, 2))), ..opts_without_knowledge() }; + let session = crate::portable::ids::SessionId::new(); + assert!(matches!(import_character_in_session(&r.b, &mut store, id, B, ACCOUNT, &opts, Some(session)), Err(PortableError::Incompatible { .. }))); + assert_eq!(counts(&r.b), before); + + // a realm that can take it, but not all of it: the import goes ahead and says what stays canonical + let opts = ImportOptions { capabilities: Some(caps(1, &all(), &[2], Ruleset::Coa, FormatRange::new(1, 2))), knowledge: Some(knows(&[1101, 1104])), ..opts_without_knowledge() }; + let outcome = import_character(&r.b, &mut store, id, B, ACCOUNT, &opts).unwrap(); + assert!(outcome.not_applied.iter().any(|l| l.contains("wardrobe") && l.contains("1 held back")), "{:?}", outcome.not_applied); + assert_eq!(selected(&r.b, outcome.local_guid), ["1=1101", "4=1104"]); + assert_eq!(store.mapping_profile(id, B).unwrap().as_deref(), Some(opts.capabilities.as_ref().unwrap().content_profile_hash.as_str()), "the character remembers the profile it was synchronised under"); +} + +#[test] +#[ignore] +fn a_changed_profile_brings_back_what_was_held_back_at_the_same_revision_and_leaves_the_realms_own_choices_alone() { + let Some(r) = realms() else { return }; + reset_b(&r.b); + let (mut store, profile) = fresh_store(); + let id = make(&r, &mut store, profile, 1002); + let narrow = ImportOptions { capabilities: Some(caps(1, &all(), &[2], Ruleset::Coa, FormatRange::new(1, 2))), knowledge: Some(knows(&[1101, 1104])), ..opts_without_knowledge() }; + let g = import_character(&r.b, &mut store, id, B, ACCOUNT, &narrow).unwrap().local_guid; + assert_eq!(selected(&r.b, g), ["1=1101", "4=1104"], "1156 is held back"); + let revision = store.character(id).unwrap().revision; + + // the same profile: nothing is looked at + let same = reevaluate_realm_character(&r.b, &mut store, id, B, &narrow).unwrap(); + assert!(same.profile_unchanged && same.update.is_none()); + + // the player of that realm chooses something of its own in a category the canonical character has nothing in + r.b.query(&format!("INSERT INTO acore_characters.character_appearance (guid, category_id, appearance_id) VALUES ({g}, 9, 1104)")).unwrap(); + // and the realm's client data is replaced by a newer one that knows 1156 + let wide = ImportOptions { capabilities: Some(caps(2, &all(), &[2], Ruleset::Coa, FormatRange::new(1, 2))), knowledge: Some(knows(&[1101, 1104, 1156])), ..opts_without_knowledge() }; + let outcome = reevaluate_realm_character(&r.b, &mut store, id, B, &wide).unwrap(); + assert!(!outcome.profile_unchanged); + assert!(outcome.update.as_ref().is_some_and(|u| u.updated), "{outcome:?}"); + assert_eq!(selected(&r.b, g), ["1=1101", "4=1104", "9=1104", "56=1156"], "the held-back selection is written, the realm's own stays"); + assert_eq!(store.character(id).unwrap().revision, revision, "the canonical revision did not move"); + assert_eq!(store.mapping_profile(id, B).unwrap().as_deref(), Some(wide.capabilities.as_ref().unwrap().content_profile_hash.as_str())); + assert!(store.open_imports(B).unwrap().is_empty()); + + // and now it is quiet again + assert!(reevaluate_realm_character(&r.b, &mut store, id, B, &wide).unwrap().profile_unchanged); + // a profile that changed but has nothing new to show writes nothing + let third = ImportOptions { capabilities: Some(caps(3, &all(), &[2], Ruleset::Coa, FormatRange::new(1, 2))), knowledge: Some(knows(&[1101, 1104, 1156])), ..opts_without_knowledge() }; + let before = counts(&r.b); + let outcome = reevaluate_realm_character(&r.b, &mut store, id, B, &third).unwrap(); + assert!(!outcome.profile_unchanged && outcome.update.is_none()); + assert_eq!(counts(&r.b), before); + assert_eq!(store.mapping_profile(id, B).unwrap().as_deref(), Some(third.capabilities.as_ref().unwrap().content_profile_hash.as_str())); +} diff --git a/crates/coa-core/src/portable/realm/mod.rs b/crates/coa-core/src/portable/realm/mod.rs index 39a1d83..a0772b2 100644 --- a/crates/coa-core/src/portable/realm/mod.rs +++ b/crates/coa-core/src/portable/realm/mod.rs @@ -17,6 +17,7 @@ pub mod knowledge; pub mod online; pub mod plan; pub mod policy; +pub mod profile; pub mod reconcile; pub mod registry; pub mod script; @@ -37,7 +38,7 @@ use super::store::{RealmRegistration, Store}; pub use blockers::Blocker; pub use export::{build, session_row, ExportRequest, Exported}; pub use import::{import_character, import_character_in_session, preflight, recover_imports, resolve_import, ImportOptions, ImportOutcome, ImportProblem, PreflightReport, Resolution}; -pub use reconcile::{begin_session, reconcile_session, resolve_import_update, update_realm_character, update_realm_character_in_session, ReconcileOutcome, SessionStart, UpdateOutcome}; +pub use reconcile::{begin_session, reconcile_session, reevaluate_realm_character, resolve_import_update, update_realm_character, update_realm_character_in_session, Reevaluation, ReconcileOutcome, SessionStart, UpdateOutcome}; pub use script::SchemaProbe; /// The ruleset a database belongs to is decided by which realm profile it is (`realms::Mode`), never by the caller. @@ -210,6 +211,8 @@ pub(crate) mod live_import; #[cfg(test)] mod live_roundtrip; #[cfg(test)] +mod live_profile; +#[cfg(test)] mod live_update; #[cfg(test)] mod live_wardrobe; diff --git a/crates/coa-core/src/portable/realm/online.rs b/crates/coa-core/src/portable/realm/online.rs index 46e730b..bf2e645 100644 --- a/crates/coa-core/src/portable/realm/online.rs +++ b/crates/coa-core/src/portable/realm/online.rs @@ -28,6 +28,8 @@ use super::policy::{classify_setting, Disposition}; /// The header line of a job file. #[derive(Debug, Serialize)] struct JobHeader { + /// The job format: the core refuses a job whose format it does not read before it parses anything else. + job_format: u32, job_id: ImportId, nonce: [u32; 4], account: u32, @@ -62,6 +64,7 @@ pub fn job_model(model: &PortableCharacter, knowledge: Option<&super::knowledge: pub fn job_bytes(job_id: ImportId, nonce: [u32; 4], account: u32, revision: u64, max_characters: u32, model: &PortableCharacter, session: Option<(SessionId, u32)>, knowledge: Option<&super::knowledge::RealmKnowledge>) -> Result> { let snapshot_json = snapshot::canonical_json(&job_model(model, knowledge))?; let header = JobHeader { + job_format: super::super::versions::ONLINE_IMPORT_JOB_FORMAT_VERSION, job_id, nonce, account, @@ -115,11 +118,23 @@ struct JobProblem { /// Import the character's current canonical revision into a **running** realm through the core's import service. pub fn import_character_online(ra: &mut Ra, store: &mut Store, character_id: CharacterId, server_id: &str, account: u32, opts: &ImportOptions, job_dir: &Path, session: Option) -> Result { + import_character_online_on(None, ra, store, character_id, server_id, account, opts, job_dir, session) +} + +/// The same, with the realm's database connection: the extensions of the character are then applied by their adapters after the core +/// committed the import (without it they stay canonical-only and are reported as not applied). +#[allow(clippy::too_many_arguments)] +pub fn import_character_online_on(db: Option<&crate::db::Db>, ra: &mut Ra, store: &mut Store, character_id: CharacterId, server_id: &str, account: u32, opts: &ImportOptions, job_dir: &Path, session: Option) -> Result { let record = store.character(character_id)?; let model = store.load_snapshot(character_id, record.revision)?; if store.server_mappings(character_id)?.iter().any(|m| m.server_id == server_id) { return Err(PortableError::AlreadyOnRealm { character: character_id, server_id: server_id.to_string() }); } + let mut operations = vec![super::super::compat::Operation::OnlineImport]; + if session.is_some() { + operations.push(super::super::compat::Operation::RuntimeSession); + } + let compatibility = super::profile::gate(opts, &model, &operations)?; let ticket = store.begin_import(character_id, server_id, record.revision, &planned_items(&model), &planned_pets(&model))?; let (job_file, result_file) = job_paths(job_dir, ticket.import_id); let write = || -> Result<()> { @@ -150,7 +165,9 @@ pub fn import_character_online(ra: &mut Ra, store: &mut Store, character_id: Cha } store.finish_import(ticket.import_id, allocation)?; cleanup(); - Ok(ImportOutcome { import_id: ticket.import_id, local_guid: r.local_guid, final_name: r.final_name, renamed: r.renamed, items: r.items, pets: r.pets, not_applied: r.not_applied, warnings: vec![] }) + let mut not_applied = r.not_applied; + not_applied.extend(super::profile::after_write(db, store, character_id, server_id, opts, r.local_guid, &model, compatibility.as_ref())?); + Ok(ImportOutcome { import_id: ticket.import_id, local_guid: r.local_guid, final_name: r.final_name, renamed: r.renamed, items: r.items, pets: r.pets, not_applied, warnings: vec![] }) } Some(r) if r.status == "refused" => { let detail = r.problems.iter().map(|p| format!("{}: {}", p.code, p.detail)).collect::>().join("; "); diff --git a/crates/coa-core/src/portable/realm/profile.rs b/crates/coa-core/src/portable/realm/profile.rs new file mode 100644 index 0000000..753f5fd --- /dev/null +++ b/crates/coa-core/src/portable/realm/profile.rs @@ -0,0 +1,187 @@ +//! The realm side of Phase 7: building a realm's capabilities, gating an operation on them, and what follows a successful write +//! (the profile hash the character was synchronised under, the extensions an adapter can apply). + +use std::path::Path; + +use crate::db::Db; +use crate::ra::Ra; + +use super::super::capabilities::*; +use super::super::compat::{self, CompatibilityReport, Inputs, Operation, Verdict}; +use super::super::error::{PortableError, Result}; +use super::super::extension::{DbExtensionRealm, ExtensionDisposition, ExtensionOutcome, ExtensionRegistry}; +use super::super::ids::CharacterId; +use super::super::model::PortableCharacter; +use super::super::session::protocol::{COLLECTION_KINDS, PROTOCOL_VERSION}; +use super::super::store::Store; +use super::import::ImportOptions; +use super::script::SchemaProbe; +use super::wardrobe; +use super::{probe, ruleset_of}; + +/// Ask the realm's core for its capability report (RA `portable capabilities`). +pub fn core_report(ra: &mut Ra) -> Result { + ra.portable_capabilities().map_err(|e| PortableError::RealmRead(e.to_string())) +} + +/// Build a realm's capabilities from what can be seen of it. +/// +/// * `probe`: the realm's characters schema (which module tables exist); +/// * `data_dir`: the realm's `Data` directory, whose client tables are hashed (the catalog) and must match what the core reports it +/// loaded; +/// * `core`: the core's own report, when it could be asked. Without it the identity and the online-import job formats are unknown and +/// the runtime-session feature cannot be claimed; +/// * `registry`: the extension adapters this Manager has, listed when their module is on the realm. +pub fn assemble(db: &Db, probe: &SchemaProbe, data_dir: Option<&Path>, core: Option<&CoreReport>, registry: &ExtensionRegistry) -> Result { + let mut catalog = ClientCatalog::new(); + if let Some(dir) = data_dir { + for name in CATALOG_TABLES { + if let Some(entry) = super::knowledge::table_entry(&dir.join("dbc").join(name)) { + catalog.insert(name.to_string(), entry); + } + } + } + if let Some(report) = core { + // what the core loaded must be what the Manager sees in the data directory, table by table + for (name, theirs) in report.catalog() { + if let Some(mine) = catalog.get(&name) { + if *mine != theirs { + return Err(PortableError::SchemaMismatch(format!("{name}: the realm's core loaded another file than the one in {}", data_dir.map(|d| d.display().to_string()).unwrap_or_default()))); + } + } else if data_dir.is_none() { + catalog.insert(name, theirs); + } + } + } + + let mut features = std::collections::BTreeSet::new(); + let have_catalog = |name: &str| catalog.contains_key(name); + if wardrobe::tables_present(probe) && have_catalog("Appearances.dbc") { + features.insert(Feature::Wardrobe); + } + if probe.has("account_appearance_collection") && probe.has("account_vanity_collection") && have_catalog("Appearances.dbc") && have_catalog("VanityCollection.dbc") { + features.insert(Feature::Collections); + } + if probe.has("coa_portable_session") && core.is_some_and(|c| c.has_feature("runtime_sessions")) { + features.insert(Feature::RuntimeSessions); + } + + let mut extension_realm = NoRealm { probe }; + let extensions = registry.supported_on(&mut extension_realm); + let content = ContentProfile { + ruleset: ruleset_of(db), + character_formats: ContentProfile::manager_formats(), + online_import_job_formats: core.map(|c| c.portable.job_formats.clone()).unwrap_or_default(), + session_protocol: PROTOCOL_VERSION, + collection_protocol: PROTOCOL_VERSION, + features, + collection_kinds: if probe.has("account_appearance_collection") && probe.has("account_vanity_collection") { COLLECTION_KINDS.iter().map(|k| k.to_string()).collect() } else { Default::default() }, + extensions, + client_catalog: catalog, + }; + RealmCapabilities::new(core.map(|c| c.core.clone()), content) +} + +/// Probe a running or stopped realm: the schema from the database, the core from RA when a console is given. +pub fn probe_capabilities(db: &Db, data_dir: Option<&Path>, ra: Option<&mut Ra>, registry: &ExtensionRegistry) -> Result { + let schema = probe(db)?; + let report = match ra { + Some(ra) => Some(core_report(ra)?), + None => None, + }; + assemble(db, &schema, data_dir, report.as_ref(), registry) +} + +/// The table-level view an adapter needs to say whether its module is present, before any character is involved. +struct NoRealm<'a> { + probe: &'a SchemaProbe, +} + +impl super::super::extension::ExtensionRealm for NoRealm<'_> { + fn local_guid(&self) -> u32 { + 0 + } + fn has_table(&self, table: &str) -> bool { + self.probe.has(table) + } + fn query(&mut self, _: &str) -> Result { + Err(PortableError::Invalid("no character is selected: this view only tells which tables exist".into())) + } + fn execute(&mut self, _: &str) -> Result<()> { + Err(PortableError::Invalid("no character is selected: this view only tells which tables exist".into())) + } +} + +/// Evaluate `model` for the operations about to be done, and refuse before anything is written when one of them cannot be done. +/// Without capabilities in the options nothing is evaluated (the lower layers' own tests, and tools that do not know the realm). +pub fn gate(opts: &ImportOptions, model: &PortableCharacter, operations: &[Operation]) -> Result> { + let Some(caps) = opts.capabilities.as_deref() else { return Ok(None) }; + let mut merged: Option = None; + for op in operations { + let report = compat::evaluate(&Inputs { operation: *op, model, capabilities: caps, knowledge: opts.knowledge.as_deref(), collections: &[], extensions: opts.extensions.as_deref() }); + merged = Some(match merged { + None => report, + Some(mut m) => { + for o in report.outcomes { + if !m.outcomes.iter().any(|e| e.topic() == o.topic()) { + m.outcomes.push(o); + } + } + m + } + }); + } + let report = merged.ok_or_else(|| PortableError::Invalid("an operation is needed to evaluate".into()))?; + if report.verdict() == Verdict::Incompatible { + return Err(PortableError::Incompatible { operation: report.operation.to_string(), reasons: report.blocking().iter().map(|o| o.to_string()).collect() }); + } + Ok(Some(report)) +} + +/// After a character was written to a realm: remember the profile it was synchronised under, apply the extensions an adapter can +/// take, and say in words everything that was not applied completely. +pub fn after_write(db: Option<&Db>, store: &mut Store, id: CharacterId, server_id: &str, opts: &ImportOptions, guid: u32, model: &PortableCharacter, report: Option<&CompatibilityReport>) -> Result> { + let Some(caps) = opts.capabilities.as_deref() else { return Ok(vec![]) }; + let mut lines: Vec = report.map(|r| r.held_lines().into_iter().filter(|l| !l.starts_with("extension ")).collect()).unwrap_or_default(); + let outcomes = match db { + Some(db) => apply_extensions(db, store, id, server_id, opts, guid, model)?, + None if model.extensions.keys().any(|k| super::super::extension::is_module_namespace(k)) && opts.extensions.is_some() => { + lines.push("module extensions were not applied: the online import was not given the realm's database connection (they stay canonical)".to_string()); + vec![] + } + None => vec![], + }; + lines.extend(outcomes.iter().filter(|o| !matches!(o.disposition, ExtensionDisposition::Applied | ExtensionDisposition::AlreadyApplied | ExtensionDisposition::CanonicalOnly)).map(|o| o.to_string())); + store.set_mapping_profile(id, server_id, &caps.content_profile_hash)?; + Ok(lines) +} + +/// Apply the canonical extensions to the character on the realm, once per payload, and record what was applied. +pub fn apply_extensions(db: &Db, store: &mut Store, id: CharacterId, server_id: &str, opts: &ImportOptions, guid: u32, model: &PortableCharacter) -> Result> { + let (Some(caps), Some(registry)) = (opts.capabilities.as_deref(), opts.extensions.as_deref()) else { return Ok(vec![]) }; + if model.extensions.is_empty() { + return Ok(vec![]); + } + let schema = probe(db)?; + let applied = store.extension_state(id, server_id)?; + let mut realm = DbExtensionRealm::new(db, guid, &schema); + let outcomes = registry.apply_all(&mut realm, &model.extensions, &caps.content, &applied); + for o in &outcomes { + if o.disposition == ExtensionDisposition::Applied { + if let Some(ext) = model.extensions.get(&o.namespace) { + store.set_extension_applied(id, server_id, &o.namespace, &ext.content_hash)?; + } + } + } + Ok(outcomes) +} + +/// The extensions of the character that the realm's adapters can read, to be merged as the realm's side (B0, B1). A problem of an +/// adapter is returned as text and contributes nothing. +pub fn export_extensions(db: &Db, schema: &SchemaProbe, guid: u32, registry: &ExtensionRegistry, model: &mut PortableCharacter) -> Vec { + let mut realm = DbExtensionRealm::new(db, guid, schema); + let (exported, problems) = registry.export_all(&mut realm); + model.extensions.retain(|k, _| !super::super::extension::is_module_namespace(k)); + model.extensions.extend(exported); + problems +} diff --git a/crates/coa-core/src/portable/realm/reconcile.rs b/crates/coa-core/src/portable/realm/reconcile.rs index 0c4f550..96bb5c9 100644 --- a/crates/coa-core/src/portable/realm/reconcile.rs +++ b/crates/coa-core/src/portable/realm/reconcile.rs @@ -214,6 +214,56 @@ pub fn update_realm_character(db: &Db, store: &mut Store, id: CharacterId, serve /// The same, arming the runtime portable session `session` on the updated character in the same realm transaction. pub fn update_realm_character_in_session(db: &Db, store: &mut Store, id: CharacterId, server_id: &str, opts: &ImportOptions, session: Option) -> Result { + update_inner(db, store, id, server_id, opts, session, false) +} + +/// What a re-evaluation did. +#[derive(Debug)] +pub struct Reevaluation { + /// The realm's content profile is the one the character was last synchronised under: nothing was looked at. + pub profile_unchanged: bool, + /// The realm's character was written (held-back appearances that the realm can now show). + pub update: Option, + pub extensions: Vec, +} + +/// The realm's content profile changed since this character was synchronised with it (a newer client data directory, a module added or +/// removed): look again at what was held back **even though the canonical revision did not change**. Only additions are made: an +/// appearance the realm can now show and does not have is written, a module payload that now has an adapter is applied; nothing the +/// realm itself has or changed since is touched. Afterwards the character carries the new profile hash. +/// +/// The realm must be stopped, as for an update. Without capabilities in the options there is nothing to compare and it is refused. +pub fn reevaluate_realm_character(db: &Db, store: &mut Store, id: CharacterId, server_id: &str, opts: &ImportOptions) -> Result { + let caps = opts.capabilities.clone().ok_or_else(|| PortableError::Invalid("re-evaluation needs the realm's content profile".into()))?; + if store.mapping_profile(id, server_id)?.as_deref() == Some(caps.content_profile_hash.as_str()) { + return Ok(Reevaluation { profile_unchanged: true, update: None, extensions: vec![] }); + } + let update = update_inner(db, store, id, server_id, opts, None, true)?; + let (guid, _) = local_guid(store, id, server_id)?; + let canonical = store.load_current(id)?; + let extensions = super::profile::apply_extensions(db, store, id, server_id, opts, guid, &canonical)?; + store.set_mapping_profile(id, server_id, &caps.content_profile_hash)?; + Ok(Reevaluation { profile_unchanged: false, update: update.updated.then_some(update), extensions }) +} + +/// The realm's character with the appearances the canonical character has, the realm can now show and the realm lacks added: a +/// category without a selection gets the canonical one, an outfit that is not there is added. What the realm has stays. +fn restored_wardrobe(current: &PortableCharacter, canonical: &PortableCharacter, knows: &dyn Fn(u32) -> bool) -> PortableCharacter { + let mut out = current.clone(); + for (category, appearance) in &canonical.wardrobe.active { + if knows(*appearance) && !out.wardrobe.active.contains_key(category) { + out.wardrobe.active.insert(*category, *appearance); + } + } + for (name, ids) in &canonical.wardrobe.outfits { + if !out.wardrobe.outfits.contains_key(name) && ids.iter().all(|id| *id == 0 || knows(*id)) { + out.wardrobe.outfits.insert(name.clone(), ids.clone()); + } + } + out +} + +fn update_inner(db: &Db, store: &mut Store, id: CharacterId, server_id: &str, opts: &ImportOptions, session: Option, reevaluate: bool) -> Result { let (guid, last_revision) = local_guid(store, id, server_id)?; if store.open_baseline(id, server_id)?.is_some() { return Err(PortableError::SessionOpen); @@ -222,7 +272,11 @@ pub fn update_realm_character_in_session(db: &Db, store: &mut Store, id: Charact return Err(PortableError::ImportInProgress { import_id: open.import_id }); } let record = store.character(id)?; - if record.revision == last_revision { + if reevaluate { + if record.revision != last_revision { + return Err(PortableError::Invalid("the realm is not at the canonical revision: update it first, then re-evaluate".into())); + } + } else if record.revision == last_revision { return Ok(UpdateOutcome { import_id: None, from_revision: last_revision, to_revision: last_revision, updated: false, counts: UpdateCounts::default(), changes: vec![], left_alone: vec![], warnings: vec![] }); } if record.revision < last_revision { @@ -230,12 +284,30 @@ pub fn update_realm_character_in_session(db: &Db, store: &mut Store, id: Charact } let synced = store.synced_model(id, server_id)?.ok_or_else(|| PortableError::Invalid("the realm has no synchronised snapshot to update from".into()))?; let canonical = store.load_current(id)?; + let mut operations = vec![super::super::compat::Operation::Update]; + if session.is_some() { + operations.push(super::super::compat::Operation::RuntimeSession); + } + let compatibility = super::profile::gate(opts, &canonical, &operations)?; let view = read_realm(db, store, id, server_id, guid)?; - let merged = merge3(&view.exported.model, &synced, &canonical, Mode::Strict)?; - if !merged.conflicts.is_empty() { - return Err(PortableError::UpdateConflicts(merged.conflicts.iter().map(|c| format!("{}: {}", c.path, c.detail)).collect())); - } + let merged = if reevaluate { + let model = match opts.knowledge.as_deref() { + Some(k) => restored_wardrobe(&view.exported.model, &canonical, &|id| k.knows_appearance(id)), + None => view.exported.model.clone(), + }; + let changes = if model.wardrobe != view.exported.model.wardrobe { vec!["wardrobe: appearances the realm can now show were restored".to_string()] } else { vec![] }; + if changes.is_empty() { + return Ok(UpdateOutcome { import_id: None, from_revision: last_revision, to_revision: last_revision, updated: false, counts: UpdateCounts::default(), changes: vec![], left_alone: vec![], warnings: vec![] }); + } + super::super::merge::Merged { model, changes, left_alone: vec![], conflicts: vec![], items: Default::default(), pets: Default::default() } + } else { + let merged = merge3(&view.exported.model, &synced, &canonical, Mode::Strict)?; + if !merged.conflicts.is_empty() { + return Err(PortableError::UpdateConflicts(merged.conflicts.iter().map(|c| format!("{}: {}", c.path, c.detail)).collect())); + } + merged + }; let schema = probe(db)?; let context = |nonce| UpdateContext { ruleset: ruleset_of(db), @@ -291,6 +363,8 @@ pub fn update_realm_character_in_session(db: &Db, store: &mut Store, id: Charact } } } + let mut warnings = view.exported.warnings; + warnings.extend(super::profile::after_write(Some(db), store, id, server_id, opts, guid, &canonical, compatibility.as_ref())?); Ok(UpdateOutcome { import_id: Some(ticket.import_id), from_revision: last_revision, @@ -299,7 +373,7 @@ pub fn update_realm_character_in_session(db: &Db, store: &mut Store, id: Charact counts: script.counts, changes: merged.changes, left_alone: merged.left_alone, - warnings: view.exported.warnings, + warnings, }) } diff --git a/crates/coa-core/src/portable/session/collection_tests.rs b/crates/coa-core/src/portable/session/collection_tests.rs index 1744722..081b1a4 100644 --- a/crates/coa-core/src/portable/session/collection_tests.rs +++ b/crates/coa-core/src/portable/session/collection_tests.rs @@ -414,3 +414,20 @@ fn what_the_host_has_not_had_acknowledged_survives_a_restart_and_what_it_has_ack assert!(queued(&events).is_empty() && h.collection_outbox().unwrap().is_empty()); assert_eq!(realm.full_reads, reads, "nothing was read again after the first look of the two kinds"); } + +#[test] +fn a_session_that_changes_nothing_on_a_format_1_store_makes_no_new_revision() { + let Rig { mut owner, mut host, mut realm, id, .. } = rig(|_| {}); + owner.downgrade_head_to_format_1(id).unwrap(); + let before = owner.character(id).unwrap().revision; + let mut h = HostService::new(&mut host, SERVER, HostConfig::default()); + let mut o = OwnerService::new(&mut owner); + start(&mut h, &mut o, &mut realm); + h.tick(&mut realm, 100).unwrap(); + h.tick(&mut realm, 101).unwrap(); + deliver_characters(&mut h, &mut o, &mut realm); + realm.logout(); + h.tick(&mut realm, 200).unwrap(); + deliver_characters(&mut h, &mut o, &mut realm); + assert_eq!(o.store().character(id).unwrap().revision, before, "the same character in a newer format is not a change"); +} diff --git a/crates/coa-core/src/portable/session/fake.rs b/crates/coa-core/src/portable/session/fake.rs index dfa5fb3..fc96c11 100644 --- a/crates/coa-core/src/portable/session/fake.rs +++ b/crates/coa-core/src/portable/session/fake.rs @@ -43,6 +43,9 @@ pub struct FakeRealm { pub fingerprints: u32, pub full_reads: u32, pub collection_writes: u32, + /// The fake module of the realm and the adapters that read it into every read of the character. + pub extension_realm: super::super::extension::fake::FakeExtensionRealm, + pub registry: Option, } pub fn new_internal_item(n: u64, slot: u8, entry: u64) -> PortableItem { @@ -90,6 +93,8 @@ impl FakeRealm { fingerprints: 0, full_reads: 0, collection_writes: 0, + extension_realm: super::super::extension::fake::FakeExtensionRealm { guid, has_module: false, ..Default::default() }, + registry: None, } } @@ -233,6 +238,11 @@ impl RealmBridge for FakeRealm { pet.id = id; pet_observations.push(PetObservation { portable_pet_id: id, local_pet_number: number, identity }); } + if let Some(registry) = &self.registry { + model.extensions.retain(|k, _| !super::super::extension::is_module_namespace(k)); + let (exported, _) = registry.export_all(&mut self.extension_realm); + model.extensions.extend(exported); + } // the realm's tables are keyed by guid: a later read finds the same rows under the same internal ids let model = model.normalized(); Ok(RealmRead { exported: Exported { model, local_guid: self.guid, account: 1, observations, pet_observations, session: self.row.clone(), warnings: vec![] }, session: self.row.clone(), online: self.online }) diff --git a/crates/coa-core/src/portable/session/host.rs b/crates/coa-core/src/portable/session/host.rs index bb8a06d..1b56d01 100644 --- a/crates/coa-core/src/portable/session/host.rs +++ b/crates/coa-core/src/portable/session/host.rs @@ -4,10 +4,11 @@ use std::collections::{HashMap, HashSet}; use super::super::collection::IdSet; +use super::super::capabilities::RealmCapabilities; use super::super::error::{PortableError, Result}; use super::super::realm::collections::Applied; use super::super::ids::{CharacterId, PortableItemId, PortablePetId, ProfileId, SessionId}; -use super::super::store::{AckEffect, CollectionOutboxMessage, HostSession, HostState, OutboxMessage, Store}; +use super::super::store::{AckEffect, CollectionOutboxMessage, HostSession, HostState, OutboxMessage, ProfileChange, StaleMapping, Store}; use super::bridge::{CheckpointReply, RealmBridge, RealmRead, RowState}; use super::protocol::*; @@ -304,4 +305,19 @@ impl<'a> HostService<'a> { self.store.host_collection_acknowledge(&self.server_id, account, &state.kind, Some(&fingerprint), Some(&now_hash), &now_hash, state.collection_revision, Some(&canonical_hash))?; Ok(Some(applied)) } + + // ---- the realm's content profile (Phase 7) ------------------------------------------------------------------------------ + + /// The realm's content profile as it is now. When it differs from the one remembered, what the Host sent and received of the account + /// collections is forgotten so that the next look reports every account again (the realm may now know ids it did not, and the Owner + /// answers with what it holds); the characters whose profile is stale are returned for a re-evaluation (whatever their revision). + pub fn observe_profile(&mut self, caps: &RealmCapabilities, source: &str) -> Result<(ProfileChange, Vec)> { + let change = self.store.set_realm_profile(&self.server_id, caps, source)?; + if change.changed() { + self.store.host_collection_reset(&self.server_id)?; + self.last_collection.clear(); + } + let stale = self.store.mappings_with_other_profile(&self.server_id, &caps.content_profile_hash)?; + Ok((change, stale)) + } } diff --git a/crates/coa-core/src/portable/session/live.rs b/crates/coa-core/src/portable/session/live.rs index 030fc0d..a80372a 100644 --- a/crates/coa-core/src/portable/session/live.rs +++ b/crates/coa-core/src/portable/session/live.rs @@ -17,15 +17,16 @@ pub struct LiveBridge<'a> { db: &'a Db, ra: Option, knowledge: Option>, + extensions: Option>, } impl<'a> LiveBridge<'a> { pub fn new(db: &'a Db, ra: Ra) -> Self { - Self { db, ra: Some(ra), knowledge: None } + Self { db, ra: Some(ra), knowledge: None, extensions: None } } pub fn without_console(db: &'a Db) -> Self { - Self { db, ra: None, knowledge: None } + Self { db, ra: None, knowledge: None, extensions: None } } /// What the realm's client data knows: without it no account collection is ever written to the realm. @@ -34,6 +35,12 @@ impl<'a> LiveBridge<'a> { self } + /// The extension adapters that read their module's data of a character into every baseline and checkpoint. + pub fn with_extensions(mut self, extensions: Option>) -> Self { + self.extensions = extensions; + self + } + fn console(&mut self) -> Result<&mut Ra> { self.ra.as_mut().ok_or_else(|| PortableError::RealmRead("the realm's console is not connected".into())) } @@ -79,8 +86,11 @@ impl RealmBridge for LiveBridge<'_> { fn read(&mut self, local_guid: u32, prior_items: &HashMap, prior_pets: &HashMap) -> Result { let row = self.session_row(local_guid)?; let character = row.as_ref().map(|r| r.character_id).ok_or_else(|| PortableError::Invalid("the character has no portable session row".into()))?; - let _ = probe(self.db)?; - let exported = read_session_character(self.db, local_guid, character, prior_items, prior_pets)?; + let schema = probe(self.db)?; + let mut exported = read_session_character(self.db, local_guid, character, prior_items, prior_pets)?; + if let Some(registry) = &self.extensions { + exported.warnings.extend(super::super::realm::profile::export_extensions(self.db, &schema, local_guid, registry, &mut exported.model)); + } let session = exported.session.clone(); Ok(RealmRead { exported, session, online: true }) } diff --git a/crates/coa-core/src/portable/session/live_session.rs b/crates/coa-core/src/portable/session/live_session.rs index 744cf1b..9f77324 100644 --- a/crates/coa-core/src/portable/session/live_session.rs +++ b/crates/coa-core/src/portable/session/live_session.rs @@ -410,3 +410,96 @@ fn the_core_imports_characters_into_a_running_realm_like_the_offline_importer_do refused(b"{\"job_id\":\"x\"}\nnot json".to_vec(), "garbage"); server.stop(); } + +#[test] +#[ignore] +fn the_core_reports_what_it_is_the_manager_evaluates_before_it_writes_and_the_core_refuses_a_job_format_it_does_not_read() { + use crate::portable::capabilities::*; + use crate::portable::extension::ExtensionRegistry; + use crate::portable::realm::profile::{core_report, probe_capabilities}; + let (Some(r), Some(spec), Some(jobs)) = (realms(), spec(), job_dir()) else { return }; + reset_b(&r.b); + std::fs::create_dir_all(&jobs).unwrap(); + let data = std::path::Path::new("C:/games/coa-schema-fixture-20261005/Data"); + let dir = tempfile::tempdir().unwrap(); + let mut store = Store::open_file(&dir.path().join("portable.db")).unwrap(); + let profile = store.default_profile().unwrap(); + let server = Server::start(spec); + let mut ra = server.ra(); + + // what the core says of itself, and the profile assembled from it + let report = core_report(&mut ra).unwrap(); + assert_eq!(report.portable.job_formats, [2]); + assert_eq!(report.portable.character_formats, [2]); + assert!(report.has_feature("runtime_sessions") && report.has_feature("wardrobe")); + assert!(report.core.commit.len() >= 7 && !report.core.branch.is_empty()); + assert!(report.catalog().contains_key("Appearances.dbc") && report.catalog().contains_key("VanityCollection.dbc")); + let registry = ExtensionRegistry::new(); + let caps = probe_capabilities(&r.b, Some(data), Some(&mut ra), ®istry).unwrap(); + assert!(caps.content.supports(Feature::RuntimeSessions) && caps.content.supports(Feature::Wardrobe) && caps.content.supports(Feature::Collections)); + assert_eq!(caps.content.online_import_job_formats, [2]); + assert_eq!(caps.core.as_ref().unwrap().commit, report.core.commit); + assert_eq!(probe_capabilities(&r.b, Some(data), Some(&mut ra), ®istry).unwrap(), caps, "asking twice gives the same profile"); + assert!(caps.to_json().unwrap().len() < 4096, "a few hundred bytes of hashes, no id lists"); + // the catalog the Manager hashed from the data directory is the one the core loaded: a different directory is refused outright + let elsewhere = tempfile::tempdir().unwrap(); + std::fs::create_dir(elsewhere.path().join("dbc")).unwrap(); + std::fs::copy(data.join("dbc/Appearances.dbc"), elsewhere.path().join("dbc/Appearances.dbc")).unwrap(); + let mut changed = std::fs::read(elsewhere.path().join("dbc/Appearances.dbc")).unwrap(); + let last = changed.len() - 2; + changed[last] ^= 1; + std::fs::write(elsewhere.path().join("dbc/Appearances.dbc"), changed).unwrap(); + assert!(probe_capabilities(&r.b, Some(elsewhere.path()), Some(&mut ra), ®istry).is_err(), "another Appearances.dbc than the one the core loaded"); + + // an online import is evaluated before the job file is written: a profile whose core reads another job format + let id = make(&r, &mut store, profile, 1005); + let mut other = caps.content.clone(); + other.online_import_job_formats = vec![3]; + let other = RealmCapabilities::new(None, other).unwrap(); + let mut o = opts(); + o.capabilities = Some(std::sync::Arc::new(other)); + let before = number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters"); + let error = crate::portable::realm::online::import_character_online(&mut ra, &mut store, id, B, ACCOUNT, &o, &jobs, None).expect_err("the core reads job format 3 only according to this profile"); + assert!(matches!(error, crate::portable::PortableError::Incompatible { .. }), "{error}"); + assert!(std::fs::read_dir(&jobs).unwrap().next().is_none(), "no job file was written"); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters"), before); + assert!(store.open_imports(B).unwrap().is_empty()); + + // with the real profile it goes through, and the character remembers the profile it was synchronised under + o.capabilities = Some(std::sync::Arc::new(caps.clone())); + let imported = crate::portable::realm::online::import_character_online(&mut ra, &mut store, id, B, ACCOUNT, &o, &jobs, None).unwrap(); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters"), before + 1); + assert_eq!(store.mapping_profile(id, B).unwrap().as_deref(), Some(caps.content_profile_hash.as_str())); + let _ = imported; + + // the core itself refuses a job whose format it does not read, before it looks at the body: a newer one, one without the field + // (what Phase 5 wrote), one that is not a number + let id = make(&r, &mut store, profile, 1003); + let record = store.character(id).unwrap(); + let model = store.load_snapshot(id, record.revision).unwrap(); + let ticket = store.begin_import(id, B, record.revision, &crate::portable::realm::import::planned_items(&model), &crate::portable::realm::import::planned_pets(&model)).unwrap(); + let good = crate::portable::realm::online::job_bytes(ticket.import_id, ticket.nonce, ACCOUNT, record.revision, 10, &model, None, None).unwrap(); + let text = String::from_utf8(good).unwrap(); + assert!(text.starts_with("{\"job_format\":2,"), "{}", &text[..60]); + let chars = number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters") + number(&r.b, "SELECT COUNT(*) FROM acore_characters.item_instance"); + for (why, edited) in [ + ("a newer job format", text.replacen("\"job_format\":2", "\"job_format\":99", 1)), + ("a job without a format", text.replacen("\"job_format\":2,", "", 1)), + ("a job whose format is not a number", text.replacen("\"job_format\":2", "\"job_format\":\"2\"", 1)), + ("the format of Phase 5", text.replacen("\"job_format\":2", "\"job_format\":1", 1)), + ("a body that is garbage behind a format the core does not read", format!("{{\"job_format\":7}}\n{{{{ not json at all")), + ] { + let job = ImportId::new(); + let edited = edited.replacen(&ticket.import_id.to_string(), &job.to_string(), 1); + std::fs::write(jobs.join(format!("{job}.job")), edited).unwrap(); + let reply = ra.portable_import(job); + assert!(reply.as_ref().is_err_and(|e| e.to_string().contains("unsupported_job_format")), "{why}: {reply:?}"); + let result = std::fs::read_to_string(jobs.join(format!("{job}.result"))).unwrap(); + assert!(result.contains("unsupported_job_format") && result.contains("\"supported_job_formats\":[2]"), "{why}: {result}"); + assert_eq!(number(&r.b, "SELECT COUNT(*) FROM acore_characters.characters") + number(&r.b, "SELECT COUNT(*) FROM acore_characters.item_instance"), chars, "{why}: nothing was written"); + let _ = std::fs::remove_file(jobs.join(format!("{job}.job"))); + let _ = std::fs::remove_file(jobs.join(format!("{job}.result"))); + } + store.abort_import(ticket.import_id, "test").ok(); + server.stop(); +} diff --git a/crates/coa-core/src/portable/session/mod.rs b/crates/coa-core/src/portable/session/mod.rs index d286d69..33b8ac2 100644 --- a/crates/coa-core/src/portable/session/mod.rs +++ b/crates/coa-core/src/portable/session/mod.rs @@ -12,6 +12,8 @@ mod fake; #[cfg(test)] mod collection_tests; #[cfg(test)] +mod profile_tests; +#[cfg(test)] mod tests; #[cfg(test)] mod live_session; diff --git a/crates/coa-core/src/portable/session/profile_tests.rs b/crates/coa-core/src/portable/session/profile_tests.rs new file mode 100644 index 0000000..6b45b3e --- /dev/null +++ b/crates/coa-core/src/portable/session/profile_tests.rs @@ -0,0 +1,211 @@ +//! Phase 7 on the simulated realm: a change of the realm's content profile is noticed and what was held back is looked at again, and +//! module extensions travel opaque through the Owner, are exported only by an adapter of the realm, and are never destroyed. + +use super::fake::*; +use super::*; +use crate::portable::capabilities::*; +use crate::portable::collection::IdSet; +use crate::portable::extension::fake::{payload, FakeAdapter}; +use crate::portable::extension::ExtensionRegistry; +use crate::portable::fixtures::geared_level_eighty; +use crate::portable::ids::CharacterId; +use crate::portable::model::{PortableCharacter, Ruleset}; +use crate::portable::realm::knowledge::RealmKnowledge; +use crate::portable::store::Store; + +const SERVER: &str = "realm-b"; +const GUID: u32 = 4242; +const APPEARANCE: &str = "coa:appearance"; + +fn caps(catalog_marker: u8) -> RealmCapabilities { + let mut catalog = ClientCatalog::new(); + catalog.insert("Appearances.dbc".into(), CatalogEntry { sha256: format!("{catalog_marker:02x}").repeat(32), records: 500 }); + RealmCapabilities::new( + None, + ContentProfile { + ruleset: Ruleset::Coa, + character_formats: ContentProfile::manager_formats(), + online_import_job_formats: vec![2], + session_protocol: 1, + collection_protocol: 1, + features: [Feature::RuntimeSessions, Feature::Wardrobe, Feature::Collections].into_iter().collect(), + collection_kinds: ["coa:appearance".to_string(), "coa:vanity".to_string()].into_iter().collect(), + extensions: vec![], + client_catalog: catalog, + }, + ) + .unwrap() +} + +struct Rig { + owner: Store, + host: Store, + realm: FakeRealm, + id: CharacterId, +} + +fn rig(model: impl FnOnce(&mut PortableCharacter)) -> Rig { + let mut owner = Store::open_in_memory().unwrap(); + let mut host = Store::open_in_memory().unwrap(); + let profile = owner.default_profile().unwrap(); + let mut m = geared_level_eighty(); + model(&mut m); + let id = owner.create_character(profile, m, "realm-a").unwrap(); + let offer = OwnerService::new(&mut owner).offer(id, SERVER).unwrap(); + let host_profile = host.default_profile().unwrap(); + let mut realm = FakeRealm::new(GUID); + HostService::new(&mut host, SERVER, HostConfig::default()).accept_offer(host_profile, &offer).unwrap(); + realm.import(&mut host, id, SERVER, offer.canonical_revision, offer.session_id).unwrap(); + HostService::new(&mut host, SERVER, HostConfig::default()).bind(offer.session_id, GUID).unwrap(); + Rig { owner, host, realm, id } +} + +fn deliver_all(h: &mut HostService, o: &mut OwnerService, realm: &mut FakeRealm) { + for m in h.outbox().unwrap() { + let ack = if m.started { o.handle_started(&m.bytes) } else { o.handle_checkpoint(&m.bytes) }.unwrap(); + h.receive_ack(realm, &ack).unwrap(); + } + for m in h.collection_outbox().unwrap() { + let ack = o.handle_collection(&m.bytes).unwrap(); + h.receive_collection_ack(realm, m.account, &ack).unwrap(); + } +} + +fn start(h: &mut HostService, o: &mut OwnerService, realm: &mut FakeRealm) { + realm.login(); + h.tick(realm, 0).unwrap(); + deliver_all(h, o, realm); +} + +fn ids(range: std::ops::RangeInclusive) -> IdSet { + IdSet::from_ids(range).unwrap() +} + +#[test] +fn a_changed_content_profile_makes_the_realm_report_its_collections_again_and_receive_what_it_can_now_show_without_a_new_revision() { + let Rig { mut owner, mut host, mut realm, id } = rig(|_| {}); + let profile = owner.default_profile().unwrap(); + owner.merge_collection(profile, APPEARANCE, &ids(1..=500)).unwrap(); + // the realm's client data knows 1..=300 of the canonical 500 + realm.knowledge = RealmKnowledge::new(ids(1..=300), IdSet::new()); + let mut h = HostService::new(&mut host, SERVER, HostConfig::default()); + let mut o = OwnerService::new(&mut owner); + let first = caps(1); + let (change, stale) = h.observe_profile(&first, "live").unwrap(); + assert!(change.changed() && change.previous.is_none()); + assert_eq!(stale.len(), 1, "a character synchronised before profiles existed is stale"); + start(&mut h, &mut o, &mut realm); + assert_eq!(realm.appearance_rows.len(), 300, "what the realm knows was written, the rest stays canonical"); + let revision = o.store().character(id).unwrap().revision; + let reads = realm.full_reads; + + // the same profile again: nothing is forgotten, nothing is read + let (change, _) = h.observe_profile(&first, "live").unwrap(); + assert!(!change.changed()); + h.tick(&mut realm, 10).unwrap(); + assert_eq!(realm.full_reads, reads); + + // the realm gets newer client data: another catalog hash, and it now knows all 500 + realm.knowledge = RealmKnowledge::new(ids(1..=500), IdSet::new()); + let second = caps(2); + assert_ne!(first.content_profile_hash, second.content_profile_hash); + let (change, _) = h.observe_profile(&second, "live").unwrap(); + assert_eq!(change.previous.as_deref(), Some(first.content_profile_hash.as_str())); + let events = h.tick(&mut realm, 20).unwrap(); + assert!(events.iter().any(|e| matches!(e, HostEvent::CollectionQueued { kind, .. } if kind == APPEARANCE)), "the account is reported again: {events:?}"); + deliver_all(&mut h, &mut o, &mut realm); + assert_eq!(realm.appearance_rows.len(), 500, "the 200 that were held back are written now"); + assert_eq!(o.store().character(id).unwrap().revision, revision, "the canonical revision did not change"); + assert_eq!(o.store().collection(profile, APPEARANCE).unwrap().unwrap().1.len(), 500, "and nothing was lost or doubled in the Owner"); +} + +#[test] +fn the_profile_hash_is_remembered_per_realm_and_per_character() { + let Rig { mut host, id, .. } = rig(|_| {}); + let a = caps(1); + let change = host.set_realm_profile(SERVER, &a, "live").unwrap(); + assert!(change.changed()); + assert_eq!(host.realm_profile(SERVER).unwrap().unwrap(), a); + assert!(host.realm_profile("realm-z").unwrap().is_none()); + assert!(!host.set_realm_profile(SERVER, &a, "offline").unwrap().changed(), "the same content, whatever its source"); + assert_eq!(host.mapping_profile(id, SERVER).unwrap(), None); + host.set_mapping_profile(id, SERVER, &a.content_profile_hash).unwrap(); + assert!(host.mappings_with_other_profile(SERVER, &a.content_profile_hash).unwrap().is_empty()); + let b = caps(2); + assert_eq!(host.mappings_with_other_profile(SERVER, &b.content_profile_hash).unwrap().len(), 1); + assert!(host.set_mapping_profile(id, "realm-z", "x").is_err(), "a character that is not on the realm has no profile to record"); + assert!(host.set_realm_profile("not a server id!", &a, "live").is_err()); +} + +fn fake_registry() -> (ExtensionRegistry, std::sync::Arc) { + let adapter = FakeAdapter::new(2, 3); + let mut r = ExtensionRegistry::new(); + r.register(adapter.clone()).unwrap(); + (r, adapter) +} + +#[test] +fn an_extension_the_realm_cannot_read_survives_its_sessions_and_a_realm_with_the_module_gets_it_and_hands_changes_back() { + let (registry, _adapter) = fake_registry(); + // the canonical character carries a module payload from an earlier realm + let Rig { mut owner, mut host, mut realm, id } = rig(|m| { + m.extensions.insert("mod:fake".into(), payload(2, b"from the first realm")); + m.extensions.insert("mod:unknown-module".into(), payload(9, b"opaque, nobody here understands it")); + }); + let original = owner.load_current(id).unwrap().extensions.clone(); + realm.db.extensions.clear(); + + // this realm has no module: B0 and B1 hold no module data, the session changes the money and nothing else + { + let mut h = HostService::new(&mut host, SERVER, HostConfig::default()); + let mut o = OwnerService::new(&mut owner); + start(&mut h, &mut o, &mut realm); + realm.play(|m| m.progression.money += 5); + h.tick(&mut realm, 100).unwrap(); + h.tick(&mut realm, 101).unwrap(); + deliver_all(&mut h, &mut o, &mut realm); + realm.logout(); + h.tick(&mut realm, 200).unwrap(); + deliver_all(&mut h, &mut o, &mut realm); + } + let after = owner.load_current(id).unwrap(); + assert_eq!(after.extensions, original, "no payload was destroyed, changed or applied by a realm that does not understand them"); + assert_eq!(after.progression.money, geared_level_eighty().progression.money + 5); + + // a later realm that has the module: the payload is applied, and its own changes come back as a new canonical payload + let (mut owner2, mut host2) = (owner, Store::open_in_memory().unwrap()); + let offer = OwnerService::new(&mut owner2).offer(id, "realm-c").unwrap(); + let hp = host2.default_profile().unwrap(); + let mut realm2 = FakeRealm::new(GUID); + realm2.extension_realm.has_module = true; + realm2.registry = Some(registry.clone()); + HostService::new(&mut host2, "realm-c", HostConfig::default()).accept_offer(hp, &offer).unwrap(); + realm2.import(&mut host2, id, "realm-c", offer.canonical_revision, offer.session_id).unwrap(); + HostService::new(&mut host2, "realm-c", HostConfig::default()).bind(offer.session_id, GUID).unwrap(); + let mut with_module = caps(1).content; + with_module.extensions = { + let mut r = crate::portable::extension::fake::FakeExtensionRealm { guid: GUID, has_module: true, ..Default::default() }; + registry.supported_on(&mut r) + }; + let profile = RealmCapabilities::new(None, with_module).unwrap(); + let canonical = owner2.load_current(id).unwrap(); + let applied = registry.apply_all(&mut realm2.extension_realm, &canonical.extensions, &profile.content, &Default::default()); + assert_eq!(applied.iter().find(|o| o.namespace == "mod:fake").unwrap().disposition, crate::portable::extension::ExtensionDisposition::Applied); + assert!(matches!(applied.iter().find(|o| o.namespace == "mod:unknown-module").unwrap().disposition, crate::portable::extension::ExtensionDisposition::Deferred(_))); + assert_eq!(realm2.extension_realm.data[&GUID], b"from the first realm"); + + { + let mut h = HostService::new(&mut host2, "realm-c", HostConfig::default()); + let mut o = OwnerService::new(&mut owner2); + start(&mut h, &mut o, &mut realm2); + // the player changes the module's data on this realm + realm2.extension_realm.data.insert(GUID, b"changed on the second realm".to_vec()); + realm2.play(|m| m.progression.money += 1); + h.tick(&mut realm2, 100).unwrap(); + h.tick(&mut realm2, 101).unwrap(); + deliver_all(&mut h, &mut o, &mut realm2); + } + let after = owner2.load_current(id).unwrap(); + assert_eq!(after.extensions["mod:fake"].payload.0, b"changed on the second realm"); + assert_eq!(after.extensions["mod:unknown-module"], original["mod:unknown-module"], "the module nobody here has is still exactly what it was"); +} diff --git a/crates/coa-core/src/portable/snapshot.rs b/crates/coa-core/src/portable/snapshot.rs index e22fe29..f927df5 100644 --- a/crates/coa-core/src/portable/snapshot.rs +++ b/crates/coa-core/src/portable/snapshot.rs @@ -12,7 +12,7 @@ use sha2::{Digest, Sha256}; use super::error::{PortableError, Result}; use super::model::{limits::*, PortableCharacter}; -use super::versions::PORTABLE_CHARACTER_FORMAT_VERSION; +use super::versions::{PORTABLE_CHARACTER_FORMAT_VERSION, PORTABLE_CHARACTER_MIN_READ_VERSION}; const ZSTD_LEVEL: i32 = 3; @@ -54,8 +54,33 @@ pub fn encode(model: &PortableCharacter) -> Result { Ok(EncodedSnapshot { content_hash, uncompressed_size: json.len() as u64, payload }) } +/// A decoded snapshot with where it came from: the character is always in the current format, `source_version` and +/// `source_hash` say what was stored (the hash of the payload as it was written, verified before any migration). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decoded { + pub model: PortableCharacter, + pub source_version: u32, + pub source_hash: [u8; 32], +} + /// Decode and fully verify. `expected_hash` is the hash recorded next to the payload. pub fn decode(payload: &[u8], expected_hash: Option<&[u8; 32]>) -> Result { + decode_verified(payload, expected_hash).map(|d| d.model) +} + +/// The hash of the character **as the current format writes it**, for a stored payload of any readable version. A payload that was +/// written in format 1 has another hash than the same character in format 2; whoever asks "is this the same character?" compares +/// these, never the stored hash with a freshly encoded one. +pub fn semantic_hash(payload: &[u8], stored_hash: &[u8; 32]) -> Result<[u8; 32]> { + let decoded = decode_verified(payload, Some(stored_hash))?; + if decoded.source_version == PORTABLE_CHARACTER_FORMAT_VERSION { + return Ok(decoded.source_hash); + } + content_hash(&decoded.model) +} + +/// [`decode`] that also reports the stored version and the stored hash. +pub fn decode_verified(payload: &[u8], expected_hash: Option<&[u8; 32]>) -> Result { if payload.len() > MAX_COMPRESSED_BYTES { return Err(PortableError::LimitExceeded(format!("the compressed snapshot is larger than {MAX_COMPRESSED_BYTES} bytes"))); } @@ -69,18 +94,29 @@ pub fn decode(payload: &[u8], expected_hash: Option<&[u8; 32]>) -> Result MAX_SNAPSHOT_BYTES { return Err(PortableError::LimitExceeded(format!("the snapshot inflates beyond {MAX_SNAPSHOT_BYTES} bytes"))); } + // the hash of the payload as it was written is verified BEFORE anything is migrated or interpreted + let source_hash: [u8; 32] = Sha256::digest(&json).into(); if let Some(expected) = expected_hash { - let actual: [u8; 32] = Sha256::digest(&json).into(); - if &actual != expected { + if &source_hash != expected { return Err(PortableError::CorruptSnapshot("content hash mismatch".into())); } } - decode_json(&json) + let (model, source_version) = decode_json_versioned(&json)?; + Ok(Decoded { model, source_version, source_hash }) } /// Decode canonical JSON (already decompressed and, if needed, hash-checked). pub fn decode_json(json: &[u8]) -> Result { - let value: serde_json::Value = serde_json::from_slice(json)?; + decode_json_versioned(json).map(|(model, _)| model) +} + +/// Decode canonical JSON of any readable version into the current model, and say which version it was. +/// +/// The version is read and checked before the shape. A newer one is refused. Version 1 is migrated strictly: it is the current +/// shape without `wardrobe`, so a version 1 payload that carries a `wardrobe` is not a version 1 payload and is refused, and a +/// genuine one becomes a character with an empty wardrobe. Every other field is still checked by `deny_unknown_fields`. +pub fn decode_json_versioned(json: &[u8]) -> Result<(PortableCharacter, u32)> { + let mut value: serde_json::Value = serde_json::from_slice(json)?; let version = value .get("format_version") .and_then(|v| v.as_u64()) @@ -88,9 +124,30 @@ pub fn decode_json(json: &[u8]) -> Result { if version > PORTABLE_CHARACTER_FORMAT_VERSION as u64 { return Err(PortableError::UnsupportedFormat { found: version.min(u32::MAX as u64) as u32, supported: PORTABLE_CHARACTER_FORMAT_VERSION }); } + if version < PORTABLE_CHARACTER_MIN_READ_VERSION as u64 { + return Err(PortableError::UnsupportedFormat { found: version as u32, supported: PORTABLE_CHARACTER_FORMAT_VERSION }); + } + if version == 1 { + let object = value.as_object_mut().ok_or_else(|| PortableError::CorruptSnapshot("a snapshot is not an object".into()))?; + if object.contains_key("wardrobe") { + return Err(PortableError::CorruptSnapshot("a format 1 snapshot cannot contain a wardrobe (it was added in format 2)".into())); + } + object.insert("format_version".into(), serde_json::json!(PORTABLE_CHARACTER_FORMAT_VERSION)); + } let model: PortableCharacter = serde_json::from_value(value)?; model.validate()?; - Ok(model) + Ok((model, version as u32)) +} + +/// What format 1 wrote for a character that has no wardrobe (the one thing the migration needs to read): the same canonical JSON with +/// `format_version` 1. Used by the tests of the migration and of the stores that still hold such snapshots. +#[cfg(test)] +pub(crate) fn encode_as_format_1(model: &PortableCharacter) -> EncodedSnapshot { + assert!(model.wardrobe.is_empty(), "format 1 has no wardrobe"); + let mut value: serde_json::Value = serde_json::from_slice(&canonical_json(model).unwrap()).unwrap(); + value["format_version"] = serde_json::json!(1); + let json = serde_json::to_vec(&value).unwrap(); + EncodedSnapshot { content_hash: Sha256::digest(&json).into(), uncompressed_size: json.len() as u64, payload: zstd::stream::encode_all(json.as_slice(), ZSTD_LEVEL).unwrap() } } #[cfg(test)] @@ -227,4 +284,58 @@ mod tests { let encoded = encode(&fixtures::geared_level_eighty()).unwrap(); assert!(encoded.payload.len() < 64 * 1024, "{} bytes", encoded.payload.len()); } + + #[test] + fn a_genuine_format_1_snapshot_migrates_with_an_empty_wardrobe_and_its_original_hash_is_verified_first() { + let model = fixtures::geared_level_eighty(); + let v1 = encode_as_format_1(&model); + let decoded = decode_verified(&v1.payload, Some(&v1.content_hash)).unwrap(); + assert_eq!((decoded.source_version, decoded.source_hash), (1, v1.content_hash)); + assert_eq!(decoded.model.format_version, PORTABLE_CHARACTER_FORMAT_VERSION); + assert!(decoded.model.wardrobe.is_empty()); + assert_eq!(decoded.model, model.clone().normalized(), "nothing but the version changed"); + assert_ne!(v1.content_hash, content_hash(&decoded.model).unwrap(), "the same character has another hash in format 2"); + assert_eq!(semantic_hash(&v1.payload, &v1.content_hash).unwrap(), content_hash(&decoded.model).unwrap()); + let v2 = encode(&model).unwrap(); + assert_eq!(semantic_hash(&v2.payload, &v2.content_hash).unwrap(), v2.content_hash); + + // the stored hash is checked against the payload as it was written, before the migration can hide a difference + let mut wrong = v1.content_hash; + wrong[0] ^= 1; + assert!(matches!(decode(&v1.payload, Some(&wrong)), Err(PortableError::CorruptSnapshot(m)) if m.contains("hash"))); + // the migrated form's hash is not accepted for the old payload + assert!(decode(&v1.payload, Some(&content_hash(&decoded.model).unwrap())).is_err()); + } + + #[test] + fn a_format_1_payload_with_a_wardrobe_is_not_a_format_1_payload() { + let mut model = fixtures::geared_level_eighty(); + model.wardrobe.active.insert(1, 100); + let mut value: serde_json::Value = serde_json::from_slice(&canonical_json(&model).unwrap()).unwrap(); + value["format_version"] = serde_json::json!(1); + let json = serde_json::to_vec(&value).unwrap(); + let payload = zstd::stream::encode_all(json.as_slice(), ZSTD_LEVEL).unwrap(); + let hash: [u8; 32] = Sha256::digest(&json).into(); + // even with a correct hash, an old version cannot carry what the old version did not have + assert!(matches!(decode(&payload, Some(&hash)), Err(PortableError::CorruptSnapshot(m)) if m.contains("wardrobe"))); + assert!(decode_json(&json).is_err()); + // an empty wardrobe object written by hand is still a wardrobe in a version that had none + let mut value: serde_json::Value = serde_json::from_slice(&canonical_json(&fixtures::geared_level_eighty()).unwrap()).unwrap(); + value["format_version"] = serde_json::json!(1); + value["wardrobe"] = serde_json::json!({}); + assert!(decode_json(&serde_json::to_vec(&value).unwrap()).is_err()); + } + + #[test] + fn format_1_keeps_every_other_strictness_and_version_0_is_not_a_version() { + let mut value: serde_json::Value = serde_json::from_slice(&canonical_json(&fixtures::geared_level_eighty()).unwrap()).unwrap(); + value["format_version"] = serde_json::json!(1); + value["identity"]["secret_flag"] = serde_json::json!(1); + assert!(decode_json(&serde_json::to_vec(&value).unwrap()).is_err(), "unknown fields are refused in the old format too"); + let mut value: serde_json::Value = serde_json::from_slice(&canonical_json(&fixtures::geared_level_eighty()).unwrap()).unwrap(); + value["format_version"] = serde_json::json!(0); + assert!(matches!(decode_json(&serde_json::to_vec(&value).unwrap()), Err(PortableError::UnsupportedFormat { .. }))); + value["format_version"] = serde_json::json!("1"); + assert!(decode_json(&serde_json::to_vec(&value).unwrap()).is_err()); + } } diff --git a/crates/coa-core/src/portable/store.rs b/crates/coa-core/src/portable/store.rs index 507c130..e15b8f4 100644 --- a/crates/coa-core/src/portable/store.rs +++ b/crates/coa-core/src/portable/store.rs @@ -18,7 +18,7 @@ use super::versions::{PORTABLE_COLLECTION_FORMAT_VERSION, SNAPSHOT_FORMAT_VERSIO pub const DATABASE_FILE: &str = "portable.db"; /// `PRAGMA application_id`: "COAP". Refuses to adopt an unrelated SQLite file. const APPLICATION_ID: i64 = 0x434F_4150; -const MIGRATIONS: &[&str] = &[include_str!("migrations/001_init.sql"), include_str!("migrations/002_item_mapping_lifecycle.sql"), include_str!("migrations/003_import_journal.sql"), include_str!("migrations/004_reconciliation.sql"), include_str!("migrations/005_sessions.sql"), include_str!("migrations/006_collections.sql")]; +const MIGRATIONS: &[&str] = &[include_str!("migrations/001_init.sql"), include_str!("migrations/002_item_mapping_lifecycle.sql"), include_str!("migrations/003_import_journal.sql"), include_str!("migrations/004_reconciliation.sql"), include_str!("migrations/005_sessions.sql"), include_str!("migrations/006_collections.sql"), include_str!("migrations/007_capabilities.sql")]; pub const DEFAULT_HISTORY_KEEP: u32 = 20; pub const MAX_HISTORY_KEEP: u32 = 1_000; @@ -545,6 +545,19 @@ impl Store { read_collection(&self.conn, profile, kind) } + /// Rewrite the current revision exactly as format 1 stored it (the stores of Phases 1-5 hold such snapshots). Tests only. + #[cfg(test)] + pub(crate) fn downgrade_head_to_format_1(&mut self, id: CharacterId) -> Result<()> { + let record = read_character(&self.conn, id)?; + let model = self.load_current(id)?; + let v1 = snapshot::encode_as_format_1(&model); + self.conn.execute( + "UPDATE snapshot SET content_hash = ?3, uncompressed_size = ?4, payload = ?5 WHERE character_id = ?1 AND revision = ?2", + params![id.to_string(), record.revision as i64, v1.content_hash.as_slice(), v1.uncompressed_size as i64, v1.payload], + )?; + Ok(()) + } + pub fn path_of(dir: &Path) -> PathBuf { dir.join(DATABASE_FILE) } @@ -636,6 +649,15 @@ fn prune_tx(tx: &Transaction<'_>, id: CharacterId, current: u64, keep: u32) -> R /// Read one revision and verify it against its recorded hash. Also returns the re-encoded form so a rollback can store /// exactly the same payload. +/// The hash a revision has in the current character format (see [`snapshot::semantic_hash`]): what to compare with a freshly +/// encoded character, whatever version the stored snapshot was written in. +pub(super) fn head_semantic_hash(conn: &Connection, id: CharacterId, revision: u64) -> Result<[u8; 32]> { + let (blob, payload): (Vec, Vec) = conn + .query_row("SELECT content_hash, payload FROM snapshot WHERE character_id = ?1 AND revision = ?2", params![id.to_string(), revision as i64], |r| Ok((r.get(0)?, r.get(1)?))) + .map_err(|_| PortableError::UnknownRevision { character: id, revision })?; + snapshot::semantic_hash(&payload, &hash_from_blob(blob)?) +} + fn read_snapshot_row(conn: &Connection, id: CharacterId, revision: u64) -> Result<(PortableCharacter, EncodedSnapshot)> { let row = conn .query_row( @@ -1019,10 +1041,12 @@ fn validate_observations(server_id: &str, observations: &[ItemObservation]) -> R } mod host; +mod profile; mod journal; mod owner; mod sync; pub use host::*; +pub use profile::*; pub use journal::*; pub use sync::*; diff --git a/crates/coa-core/src/portable/store/host.rs b/crates/coa-core/src/portable/store/host.rs index a60849b..5499ec9 100644 --- a/crates/coa-core/src/portable/store/host.rs +++ b/crates/coa-core/src/portable/store/host.rs @@ -128,7 +128,7 @@ fn install_copy_in_tx(tx: &Transaction<'_>, profile: ProfileId, model: &Portable } if revision == record.revision { let (_, have) = read_snapshot_row(tx, id, revision)?; - if have.content_hash != encoded.content_hash { + if snapshot::semantic_hash(&have.payload, &have.content_hash)? != encoded.content_hash { return Err(PortableError::Invalid("the Host already holds a different canonical state at this revision".into())); } return Ok(()); @@ -418,4 +418,13 @@ impl Store { let rows = stmt.query_map([server_id], |r| Ok(CollectionOutboxMessage { account: r.get::<_, i64>(0)? as u32, kind: r.get(1)?, bytes: r.get(2)? }))?; Ok(rows.collect::, _>>()?) } + + /// Forget what was read and acknowledged of every account collection of a realm: the next look reads and reports them again. Used + /// when the realm's content profile changed (it may now know ids it did not). + pub fn host_collection_reset(&mut self, server_id: &str) -> Result { + let tx = self.write_tx()?; + let n = tx.execute("UPDATE host_collection SET fingerprint = '', acked_hash = NULL, pending = NULL, updated_at = ?2 WHERE server_id = ?1", params![server_id, now()])?; + tx.commit()?; + Ok(n) + } } diff --git a/crates/coa-core/src/portable/store/owner.rs b/crates/coa-core/src/portable/store/owner.rs index 51a8743..87a198f 100644 --- a/crates/coa-core/src/portable/store/owner.rs +++ b/crates/coa-core/src/portable/store/owner.rs @@ -53,8 +53,7 @@ fn read_owner_row(conn: &Connection, session: SessionId) -> Result Result<[u8; 32]> { - let blob: Vec = conn.query_row("SELECT content_hash FROM snapshot WHERE character_id = ?1 AND revision = ?2", params![id.to_string(), revision as i64], |r| r.get(0)).map_err(|_| PortableError::UnknownRevision { character: id, revision })?; - hash_from_blob(blob) + head_semantic_hash(conn, id, revision) } fn ack(session: SessionId, sequence: u64, outcome: AckOutcome, revision: u64, hash: [u8; 32]) -> OwnerAck { diff --git a/crates/coa-core/src/portable/store/profile.rs b/crates/coa-core/src/portable/store/profile.rs new file mode 100644 index 0000000..13794f0 --- /dev/null +++ b/crates/coa-core/src/portable/store/profile.rs @@ -0,0 +1,109 @@ +//! What the Manager remembers of the realms' content profiles (Phase 7): the last profile of each realm, the profile hash each +//! character was last synchronised under, and which extension payloads were applied where. + +use std::collections::HashMap; + +use rusqlite::{params, OptionalExtension}; + +use super::*; +use crate::portable::capabilities::RealmCapabilities; + +/// Whether a stored realm profile changed when a new one was recorded. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProfileChange { + pub previous: Option, + pub current: String, +} + +impl ProfileChange { + pub fn changed(&self) -> bool { + self.previous.as_deref() != Some(self.current.as_str()) + } +} + +/// A character whose synchronised profile is not the realm's current one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StaleMapping { + pub character_id: CharacterId, + pub local_guid: u32, + /// `None`: synchronised before profiles existed. + pub synced_profile: Option, +} + +impl Store { + /// Remember a realm's profile. The previous hash is returned so the caller can tell that the realm changed. + pub fn set_realm_profile(&mut self, server_id: &str, caps: &RealmCapabilities, source: &str) -> Result { + check_server_id(server_id)?; + if !matches!(source, "live" | "offline") { + return Err(PortableError::Invalid("a profile source is live or offline".into())); + } + caps.validate()?; + let json = String::from_utf8(caps.to_json()?).map_err(|_| PortableError::Invalid("the profile is not UTF-8".into()))?; + let tx = self.write_tx()?; + let previous: Option = tx.query_row("SELECT content_profile_hash FROM realm_profile WHERE server_id = ?1", [server_id], |r| r.get(0)).optional()?; + tx.execute( + "INSERT INTO realm_profile(server_id, content_profile_hash, profile, source, fetched_at) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(server_id) DO UPDATE SET content_profile_hash = excluded.content_profile_hash, profile = excluded.profile, source = excluded.source, fetched_at = excluded.fetched_at", + params![server_id, caps.content_profile_hash, json, source, now()], + )?; + tx.commit()?; + Ok(ProfileChange { previous, current: caps.content_profile_hash.clone() }) + } + + pub fn realm_profile(&self, server_id: &str) -> Result> { + let json: Option = self.conn.query_row("SELECT profile FROM realm_profile WHERE server_id = ?1", [server_id], |r| r.get(0)).optional()?; + json.map(|j| RealmCapabilities::from_json(j.as_bytes())).transpose() + } + + /// Record the profile hash a character was synchronised under on a realm. + pub fn set_mapping_profile(&mut self, id: CharacterId, server_id: &str, hash: &str) -> Result<()> { + let n = self.conn.execute("UPDATE character_server_mapping SET content_profile_hash = ?3 WHERE character_id = ?1 AND server_id = ?2", params![id.to_string(), server_id, hash])?; + if n == 0 { + return Err(PortableError::Invalid(format!("character {id} is not on realm {server_id}"))); + } + Ok(()) + } + + pub fn mapping_profile(&self, id: CharacterId, server_id: &str) -> Result> { + Ok(self + .conn + .query_row("SELECT content_profile_hash FROM character_server_mapping WHERE character_id = ?1 AND server_id = ?2", params![id.to_string(), server_id], |r| r.get::<_, Option>(0)) + .optional()? + .flatten()) + } + + /// The characters of a realm whose synchronised profile is not `hash`. + pub fn mappings_with_other_profile(&self, server_id: &str, hash: &str) -> Result> { + let mut stmt = self + .conn + .prepare("SELECT character_id, local_guid, content_profile_hash FROM character_server_mapping WHERE server_id = ?1 AND (content_profile_hash IS NULL OR content_profile_hash <> ?2) ORDER BY character_id")?; + let rows = stmt.query_map(params![server_id, hash], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?, r.get::<_, Option>(2)?)))?; + rows.map(|r| { + let (id, guid, profile) = r?; + Ok(StaleMapping { character_id: id.parse()?, local_guid: guid as u32, synced_profile: profile }) + }) + .collect() + } + + /// The extension payloads applied to a character on a realm: namespace -> content hash of what was applied. + pub fn extension_state(&self, id: CharacterId, server_id: &str) -> Result> { + let mut stmt = self.conn.prepare("SELECT namespace, applied_hash FROM realm_extension_state WHERE character_id = ?1 AND server_id = ?2")?; + let rows = stmt.query_map(params![id.to_string(), server_id], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))?; + Ok(rows.collect::, _>>()?) + } + + pub fn set_extension_applied(&mut self, id: CharacterId, server_id: &str, namespace: &str, hash: &str) -> Result<()> { + self.conn.execute( + "INSERT INTO realm_extension_state(character_id, server_id, namespace, applied_hash, applied_at) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(character_id, server_id, namespace) DO UPDATE SET applied_hash = excluded.applied_hash, applied_at = excluded.applied_at", + params![id.to_string(), server_id, namespace, hash, now()], + )?; + Ok(()) + } + + /// Forget that a namespace was applied (the payload left the character, or the realm lost the module). + pub fn clear_extension_applied(&mut self, id: CharacterId, server_id: &str, namespace: &str) -> Result<()> { + self.conn.execute("DELETE FROM realm_extension_state WHERE character_id = ?1 AND server_id = ?2 AND namespace = ?3", params![id.to_string(), server_id, namespace])?; + Ok(()) + } +} diff --git a/crates/coa-core/src/portable/store/sync.rs b/crates/coa-core/src/portable/store/sync.rs index f9207ba..faf2e41 100644 --- a/crates/coa-core/src/portable/store/sync.rs +++ b/crates/coa-core/src/portable/store/sync.rs @@ -349,8 +349,8 @@ impl Store { } let at = now(); // a session that changed nothing (or only what the realm normalised) makes no new revision - let head_hash: Vec = tx.query_row("SELECT content_hash FROM snapshot WHERE character_id = ?1 AND revision = ?2", params![id.to_string(), record.revision as i64], |r| r.get(0))?; - let revision = if head_hash.as_slice() == encoded.content_hash.as_slice() { + let head_hash = head_semantic_hash(&tx, id, record.revision)?; + let revision = if head_hash == encoded.content_hash { record.revision } else { let revision = record.revision + 1; diff --git a/crates/coa-core/src/portable/store/tests.rs b/crates/coa-core/src/portable/store/tests.rs index 2df0ca6..b899686 100644 --- a/crates/coa-core/src/portable/store/tests.rs +++ b/crates/coa-core/src/portable/store/tests.rs @@ -992,3 +992,37 @@ fn baselines_mappings_and_the_synced_snapshot_survive_reopening_the_file() { drop(store); let _ = std::fs::remove_dir_all(&dir); } + +// ---- Phase 6.1: stores written by format 1 stay readable and comparable -------------------------------------------------------- + +#[test] +fn a_store_that_holds_format_1_snapshots_reads_them_and_compares_them_by_what_they_say_not_by_their_format() { + let mut owner = Store::open_in_memory().unwrap(); + let profile = owner.default_profile().unwrap(); + let model = crate::portable::fixtures::geared_level_eighty(); + let id = owner.create_character(profile, model.clone(), "realm-a").unwrap(); + owner.downgrade_head_to_format_1(id).unwrap(); + + // read as the current format, with an empty wardrobe, hash verified against the stored (format 1) hash + let current = owner.load_current(id).unwrap(); + let mut expected = model.clone().normalized(); + expected.character_id = id; + assert_eq!(current, expected); + assert!(current.wardrobe.is_empty()); + + // a Host that is handed the same revision in the current format accepts it, one that is handed another character state refuses + let mut host = Store::open_in_memory().unwrap(); + let host_profile = host.default_profile().unwrap(); + host.host_install_copy(host_profile, ¤t, 1, "owner").unwrap(); + host.downgrade_head_to_format_1(id).unwrap(); + host.host_install_copy(host_profile, ¤t, 1, "owner").expect("the same character in another format is the same character"); + let mut other = current.clone(); + other.progression.money += 1; + assert!(host.host_install_copy(host_profile, &other, 1, "owner").is_err()); + + // a new revision is made on top of the old one without touching it + let rev = owner.commit_snapshot(id, 1, other.clone(), "realm-a", None).unwrap(); + assert_eq!(rev, 2); + assert_eq!(owner.load_snapshot(id, 1).unwrap(), current, "the old revision is still readable as it was"); + assert_eq!(owner.load_current(id).unwrap(), other); +} diff --git a/crates/coa-core/src/portable/versions.rs b/crates/coa-core/src/portable/versions.rs index ef9b732..4d9f19f 100644 --- a/crates/coa-core/src/portable/versions.rs +++ b/crates/coa-core/src/portable/versions.rs @@ -4,8 +4,18 @@ //! The registry and relay versions are not used by anything yet (Phases 10 and 13); they live here so there is one //! place that lists every version the portable system speaks. -/// Version of the serialised [`PortableCharacter`](super::model::PortableCharacter). -pub const PORTABLE_CHARACTER_FORMAT_VERSION: u32 = 1; +/// Version of the serialised [`PortableCharacter`](super::model::PortableCharacter) this build **writes**. +/// +/// * 1: Phases 1-5. +/// * 2: Phase 6 added the `wardrobe` section. A version 1 payload is read through a strict migration (see `snapshot.rs`): it must +/// not contain a `wardrobe`, its original hash is verified before it is migrated, and it becomes a character with an empty one. +pub const PORTABLE_CHARACTER_FORMAT_VERSION: u32 = 2; +/// The oldest character format this build can still **read**. +pub const PORTABLE_CHARACTER_MIN_READ_VERSION: u32 = 1; +/// Version of the header + body of an online import job (`.job`), checked by the core before it parses the body. +/// * 1: Phase 5 (no version field; never accepted by a core that knows versions). +/// * 2: the explicit `job_format` field, body = character format 2 (optional `wardrobe`). +pub const ONLINE_IMPORT_JOB_FORMAT_VERSION: u32 = 2; /// Version of the serialised collection set ([`IdSet`](super::collection::IdSet)). pub const PORTABLE_COLLECTION_FORMAT_VERSION: u32 = 1; /// Version of the extension container ([`Extension`](super::model::Extension)). diff --git a/crates/coa-core/src/ra.rs b/crates/coa-core/src/ra.rs index 1f269b4..7dea55b 100644 --- a/crates/coa-core/src/ra.rs +++ b/crates/coa-core/src/ra.rs @@ -118,6 +118,12 @@ impl Ra { } } + /// `portable capabilities`: the realm's core says what it is and what portable features it has. An older core does not know the command. + pub fn portable_capabilities(&mut self) -> Result { + let out = self.command("portable capabilities")?; + crate::portable::capabilities::CoreReport::parse(&out).map_err(|e| Error::Invalid(e.to_string())) + } + /// `portable release `: lets a held portable character play. Sent only after its baseline was persisted. pub fn portable_release(&mut self, session: crate::portable::SessionId) -> Result<()> { let out = self.command(&format!("portable release {session}"))?; diff --git a/docs/PORTABLE_CAPABILITIES.md b/docs/PORTABLE_CAPABILITIES.md new file mode 100644 index 0000000..13f20f8 --- /dev/null +++ b/docs/PORTABLE_CAPABILITIES.md @@ -0,0 +1,124 @@ +# Capability negotiation and extensions (Phase 7) + +Code: `portable/capabilities.rs`, `compat.rs`, `extension.rs`, `realm/profile.rs`, `store/profile.rs`, `realm/reconcile.rs` +(`reevaluate_realm_character`). Not in this phase: Registry, Relay, any network transport (the profile is plain data a Registry can +publish later), Wildcard, the personal bank, equipment sets, level-cap projection, and an adapter for any real module. + +## 1. The content profile + +`RealmCapabilities` is what a destination can take, transport-neutral and small (a few hundred bytes of hashes, no id lists): + +```text +RealmCapabilities + profile_version 1 (a newer one is refused before the shape is read) + core { commit, branch, date } identity of the realm's core build; for logs and support, not part of the hash; absent when the core was not asked + content: ContentProfile everything that decides what can be applied + ruleset coa (Wildcard transfer is disabled) + character_formats readable 1..2, writable 2 (the Manager side that serves the realm) + online_import_job_formats what the realm's core reads for an online import (empty: not asked / too old to answer) + session_protocol 1 (PortableSessionStarted / PortableCheckpoint / OwnerAck) + collection_protocol 1 (CollectionObserved / CollectionState / CollectionAck) + features runtime_sessions, wardrobe, collections + collection_kinds coa:appearance, coa:vanity + extensions [{namespace, module_version, formats: {min, max}}] sorted by namespace + client_catalog {Appearances.dbc, ItemAppearances.dbc, VanityCollection.dbc, ItemSet.dbc} -> {sha256, records} + content_profile_hash SHA-256 of the canonical JSON of `content` +``` + +* **Hash, not lists.** Two realms with the same `client_catalog` hashes know exactly the same appearances and vanity items. The ids + themselves are read, where they are needed (the side that writes to the realm), from the realm's own data directory + (`RealmKnowledge::from_data_dir`), and that read carries the same hashes: an evaluation blocks when the files the Manager read are not + the ones the profile (and the core) describe. +* **The hash covers content only.** A new core build with the same content has the same `content_profile_hash`; a different client + table, a feature switched off, another extension range, another job format or protocol version moves it. A profile from outside is + size-checked, version-checked, strict (`deny_unknown_fields`) and its hash must be the content's: a forged or edited one is refused. +* **Sources.** The core (RA `portable capabilities`: commit, branch, date, the job formats it reads, the session marker version, its + features, and the SHA-256 and record count of the client tables **it loaded**), the realm's schema (which module tables exist), the + data directory (the catalog), and the registered extension adapters. `probe_capabilities()` assembles them; when the core cannot be asked + (a stopped realm) the profile says so by the absence of `core`, empty job formats and no `runtime_sessions`: the Manager never guesses a + capability it did not see. When the core was asked, its catalog must equal the one hashed from the data directory, table by table. +* **Remembered.** `realm_profile` (schema 7) keeps the last profile of every realm and whether it came `live` or `offline`. + +## 2. Preflight: explicit outcomes before anything is written + +`compat::evaluate` returns, for a character and an operation (`OfflineImport`, `OnlineImport`, `Update`, `RuntimeSession`), one `Outcome` per +topic and an overall `Verdict`: + +| Outcome | Meaning | +|---|---| +| `Compatible` | applied completely | +| `Held` | partly or wholly **not applied** (ids the realm's client data does not know, an extension in another format); stays in the canonical character, with counts and the reason | +| `Unsupported` | the realm lacks the feature; nothing applied, nothing lost | +| `Blocking` | the operation cannot be done on this realm: another ruleset, a character format the realm cannot read, an online import the core cannot take (another job format, or the core was not asked), a session it cannot run, client data that is not what the realm loaded | + +`Verdict`: `Incompatible` (any blocking), `Degraded` (anything held or unsupported), `Compatible`. The operations that write +(`import_character*`, `import_character_online[_on]`, `update_realm_character*`) call it **first** when the options carry the profile +(`ImportOptions::capabilities`) and refuse with `PortableError::Incompatible { operation, reasons }` before the first row, journal entry or job file; +what is held back is returned in `not_applied` / `warnings`. Without a profile nothing is evaluated (the lower layers' own tests, tools +that do not know the realm); the harness always assembles one (`--data-dir`, plus `--ra-port` for the core). + +## 3. Extensions + +In the canonical character an extension (`mod:`) is an **opaque, hash-protected blob**: the Owner never interprets, edits, drops or applies it. +`ExtensionAdapter` is the only thing allowed to touch a module's data on a realm: + +```text +namespace() mod: +module_version() +supported_format_versions() {min, max} +available(realm) is the module on this realm +validate(extension) well formed? (also run on everything an adapter exports) +export(realm) the module's data of this character, or none +apply(realm, extension) write a validated, compatible payload +compatibility(extension, profile) default: the format check against what the profile published; an adapter may be stricter +``` + +`ExtensionRegistry` registers adapters (only `mod:` namespaces, one per namespace, a sane range), lists what a realm supports (`supported_on`), +evaluates (`evaluate`), exports (`export_all`, validated) and applies (`apply_all`). Rules, each covered by a test with the fake extension: + +* **A realm without the extension** (no module, or no adapter): the profile does not list the namespace, the outcome is + `Deferred(MissingOnRealm)`, the destination receives **nothing**, the canonical payload is untouched, and a later realm that has the module + receives it. +* **An incompatible format** (older or newer than the realm's range): `Deferred(FormatTooOld | FormatTooNew)`; nothing is written; the payload stays. +* **A failing or lying adapter** (`apply` fails, a payload fails its own validation or its content hash): `Failed`, nothing is claimed, the next attempt starts again. +* **Applied once.** What was applied is remembered by content hash (`realm_extension_state`); an unchanged payload is not written again, a changed one is. +* **The Manager's own blobs** (`coa:*`, e.g. quarantined settings) are canonical-only and never handed to an adapter. +* **Merge.** A realm contributes only what its own adapter exported (its `B0` and `B1` hold nothing else) and only what changed between them; + a namespace nobody on that realm understands is not in `B0`/`B1`, so it is not in the delta and survives every session unchanged. + The Host reads adapter exports into baselines and checkpoints (`LiveBridge::with_extensions`). + +No adapter for a real module exists; the framework is exercised by a fake module only. + +## 4. The profile follows the mapping + +`character_server_mapping.content_profile_hash` (schema 7) is the hash the realm had when the character was last synchronised with it (set +after every import, update and re-evaluation). When the realm's profile changes: + +* `HostService::observe_profile` records the new profile; if the hash moved, what the Host sent and received of the **account collections** is forgotten + so the next look reports every account again and the Owner answers with its canonical sets (the realm may now know ids it did not), and the characters + whose synchronised hash is not the new one are returned (`mappings_with_other_profile`); +* `reevaluate_realm_character` looks again at what was held back for such a character **at the same canonical revision**: appearances (selections, outfits) + the realm can now show and does not have are written, extension payloads that now have an adapter are applied; **only additions**, nothing the realm + has or changed since is touched; the new hash is recorded. With the same hash it does nothing; with a new hash and nothing new to show it writes nothing. + +## 5. Verification (Phase 6.1 and 7 gate) + +| What | Result | +|---|---| +| `cargo test --workspace --locked` | see section 6 | +| Format 1 -> 2 | genuine v1 migrates with an empty wardrobe and its original hash is verified first; v1 with a wardrobe is rejected; wrong hash / unknown field / version 0 / non-numeric refused; stores holding v1 snapshots read them and compare by `semantic_hash` (no spurious revision, the Host accepts the same revision in the new format) | +| Job format | the core refuses `job_format` 99, absent, non-numeric and 1, with `supported_job_formats`, before parsing the body, nothing written (real worldserver) | +| Profile | stable, strict, forged hash refused; assembled from a real schema + data directory; the core's report cross-checked with the data directory (a different `Appearances.dbc` is refused); a profile is under 4 KB | +| Preflight | another ruleset, a reader of format 1 only, a core that reads another job format, no session feature: refused with the reason and **not one row, mapping or journal entry written** (real MySQL, real worldserver); a partly compatible one proceeds and reports what is held | +| Profile change | simulated: collections reported again and the 200 held-back ids written without a new revision; real MySQL: held-back selection restored at the same revision, the realm's own choice untouched, a second call and a call with nothing new write nothing | +| Extensions | fake module: absent on realm / incompatible old and new / failing adapter / rubbish payload / unknown namespace / `coa:*`; opaque through sessions and restored on a later realm that has the module | + +## 6. Limits + +* The adapter exports are read by the Host's baselines and checkpoints (`LiveBridge`, the simulated realm); the Phase 4 offline `begin_session` / `reconcile_session` + functions do not take adapters. +* The online import applies extensions only when it is given the realm's database connection (`import_character_online_on`); `import_character_online` without it + reports them as not applied. +* A held-back **selection** is restored by a re-evaluation (a profile change); a held-back **extension** is applied by it; nothing is re-applied when the profile did not change. +* Whether the profile of a running realm still matches (a core upgraded in place) is noticed only when it is read again; there is no push. +* The Host driver is still a library plus the harness; the profile is not yet fetched by the Manager application on its own. diff --git a/docs/PORTABLE_FORMAT.md b/docs/PORTABLE_FORMAT.md index 3648b41..d2406b5 100644 --- a/docs/PORTABLE_FORMAT.md +++ b/docs/PORTABLE_FORMAT.md @@ -3,15 +3,16 @@ What is stored locally and how it is serialised. Code: `crates/coa-core/src/portable/`. Reasoning: `PORTABLE_CHARACTERS_AUDIT.md`. -**Status:** format version 1 is *provisional* until the Phase 3 gate (Realm A -> Manager -> Realm B) passes. Until then -a change of shape may be made without a migration because nothing outside development machines holds version 1 data. -After that gate every change bumps the version. +**Status:** the Phase 3 gate passed with format 1; since then **every change of shape bumps the version**. Phase 6 added the optional +`wardrobe` section, so the character format is now **2** (Phase 6.1); version 1 is still read, through the strict migration below. ## Versions (`versions.rs`) | Constant | Value | Meaning | |---|---|---| -| `PORTABLE_CHARACTER_FORMAT_VERSION` | 1 | the serialised `PortableCharacter` | +| `PORTABLE_CHARACTER_FORMAT_VERSION` | 2 | the serialised `PortableCharacter` this build writes | +| `PORTABLE_CHARACTER_MIN_READ_VERSION` | 1 | the oldest character format this build still reads | +| `ONLINE_IMPORT_JOB_FORMAT_VERSION` | 2 | the online import job (`.job`) the core reads; checked by the core before it parses the body | | `PORTABLE_COLLECTION_FORMAT_VERSION` | 1 | the encoded id set of a collection | | `EXTENSION_FORMAT_VERSION` | 1 | the extension container | | `SNAPSHOT_FORMAT_VERSION` | 1 | the stored snapshot envelope (compressed canonical JSON) | @@ -21,6 +22,27 @@ After that gate every change bumps the version. A reader **refuses** data whose version is newer than it knows (checked before the shape is parsed), and reads older versions through a migration. +### Character format 1 -> 2 (Phase 6.1) + +Format 2 is format 1 plus the optional `wardrobe` section (absent when empty, so an empty one serialises to the same bytes as before apart from +`format_version`). The migration is strict and lives in `snapshot.rs`: + +1. the payload is decompressed under the size caps and its SHA-256 is verified against the **stored** hash **before** anything is + interpreted or migrated (the original format 1 hash is what was recorded; a payload that does not match it is corrupt); +2. `format_version` is read first; newer than 2 is refused, 0 or non-numeric is refused; +3. a format 1 payload that contains a `wardrobe` (even an empty object) is **rejected**: format 1 had none, so it is not a format 1 payload; +4. a genuine format 1 payload becomes a format 2 character with an empty wardrobe; `deny_unknown_fields` still applies to everything else. + +Stored snapshots are never rewritten. The same character has a different hash in the two formats, so anything that asks "is this the same +character?" compares `snapshot::semantic_hash` (the hash of the character as format 2 writes it, for a stored payload of either version) and +never a stored hash with a freshly encoded one (the owner's checkpoint head check, the reconcile head check, the Host's same-revision check). + +### Online import job format 2 + +The header gained `job_format` (first field). The core reads the header line first and refuses a job whose `job_format` it does not read, or that +has none (what Phase 5 wrote), with `unsupported_job_format` and the list of formats it reads, **before** it computes the body hash or parses +the body; the body must be character format 2 (`wardrobe` optional). + ## Identity * `CharacterId`, `ProfileId`, `PortableItemId`, `PortablePetId` are **UUIDv7**, generated locally. Only version-7 ids diff --git a/docs/PORTABLE_IMPORT_SERVICE.md b/docs/PORTABLE_IMPORT_SERVICE.md index 96e6584..379e469 100644 --- a/docs/PORTABLE_IMPORT_SERVICE.md +++ b/docs/PORTABLE_IMPORT_SERVICE.md @@ -83,7 +83,8 @@ a payload over RA, a path argument, a remote caller of any kind. `, not zstd: the core has neither a JSON nor a zstd dependency, and a small strict parser (`CoAPortableJson`: bounded depth/nodes/string size, integers only, duplicate and unknown fields refused, UTF-8 checked) is easier to audit than a new dependency. The header carries `snapshot_sha256` of the exact bytes of the - second line; the file size is capped at 24 MiB before it is read. The Manager ships the character **without extensions** and without + second line; the header's first field is `job_format` (Phase 6.1: the core refuses a job whose format it does not read before it looks at + anything else); the file size is capped at 24 MiB before it is read. The Manager ships the character **without extensions** and without settings the policy does not carry. * **Writes.** Every row is written by a prepared `INSERT` (`CHAR_INS_PORTABLE_*`); the only string-built statements are the `DELETE ... WHERE = ` that clear leftovers of a deleted character with the same guid.