From 9153e3ece52a9ea0faf6d5bbfe488193398350e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Thu, 5 Nov 2026 15:48:23 +0500 Subject: [PATCH] Install the modules a plugin-host core recommends before its first start A core that loads Content Scaling, Companions and SQUID as plugins ships Core/plugins/recommended.json with the module repositories and commits. During the update (the move from a monolithic stable server) the Manager downloads each module as GitHub's archive of that commit (no git needed), builds it with its bundled trees, installs and enables it, before the trial start. Rules keep the bot choice: SQUID only if it was switched on in playerbots.conf, otherwise Companions. Handled modules are remembered so a removed module is not reinstalled; a failed one is retried at the next core change. Verified on a CI core: Content Scaling (+packs) and Companions downloaded, built, installed and written to the plugin lock in 225 s. Co-Authored-By: Claude Opus 5.5 --- ...105-recommended-modules-on-plugin-cores.md | 7 + crates/coa-core/src/lib.rs | 1 + crates/coa-core/src/module_sources.rs | 127 ++++++++++ crates/coa-core/src/rebuild.rs | 229 +++++++++++++++++- src-tauri/src/lib.rs | 2 + src/lib/api.ts | 2 +- src/lib/serverUpdate.ts | 2 +- 7 files changed, 366 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20261105-recommended-modules-on-plugin-cores.md create mode 100644 crates/coa-core/src/module_sources.rs diff --git a/changelog.d/20261105-recommended-modules-on-plugin-cores.md b/changelog.d/20261105-recommended-modules-on-plugin-cores.md new file mode 100644 index 0000000..9be8ef2 --- /dev/null +++ b/changelog.d/20261105-recommended-modules-on-plugin-cores.md @@ -0,0 +1,7 @@ +--- +area: manager +type: added +audience: admins +title: Moving to a core with modules installs the modules you used +--- +When a server moves to a core that loads Content Scaling, Companions or SQUID's Playerbots as modules, the Manager downloads them at the versions the core names, builds them and switches them on before the first start. Bots follow what the server used before: SQUID's Playerbots if it was switched on, otherwise Companions. A module you remove later is not installed again. diff --git a/crates/coa-core/src/lib.rs b/crates/coa-core/src/lib.rs index beac5d8..e393ceb 100644 --- a/crates/coa-core/src/lib.rs +++ b/crates/coa-core/src/lib.rs @@ -43,6 +43,7 @@ pub mod logging; pub mod manifest; pub mod migrations; pub mod modules; +pub mod module_sources; pub mod msvc_setup; pub mod multiworld; mod mysql_snapshot; diff --git a/crates/coa-core/src/module_sources.rs b/crates/coa-core/src/module_sources.rs new file mode 100644 index 0000000..f2e6fe6 --- /dev/null +++ b/crates/coa-core/src/module_sources.rs @@ -0,0 +1,127 @@ +//! Sources of a module at an exact commit, downloaded as GitHub's archive of that commit (no git needed). +//! +//! The cache layout is the one the rest of the Manager uses, `/github///`, so a +//! module identifies itself by its repository name and a later rebuild finds the same tree. + +use std::fs; +use std::io::Read; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +use crate::error::{Error, Result}; + +const MAX_ARCHIVE_BYTES: u64 = 512 * 1024 * 1024; + +fn valid_name(s: &str) -> bool { + !s.is_empty() && s.len() <= 100 && s != "." && s != ".." + && s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) +} + +/// `/github///`, downloading it when it is not there yet. +pub fn fetch_pinned(owner: &str, repo: &str, sha: &str, cache_root: &Path) -> Result { + if !valid_name(owner) || !valid_name(repo) || sha.len() != 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(Error::Invalid(format!("Invalid module source {owner}/{repo}@{sha}"))); + } + let sha = sha.to_ascii_lowercase(); + let dest = cache_root.join("github").join(owner).join(repo).join(&sha); + if dest.join(".coa_tree_digest").is_file() { + return Ok(dest); + } + let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{sha}"); + let client = reqwest::blocking::Client::builder() + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(600)) + .build() + .map_err(|e| Error::Invalid(e.to_string()))?; + let reply = client.get(&url).send().map_err(|e| Error::NetworkUnreachable(format!("{url}: {e}")))?; + if !reply.status().is_success() { + return Err(Error::Invalid(format!("GitHub answered {} for {owner}/{repo} at {sha}", reply.status()))); + } + let staging = cache_root.join("staging").join(format!("{owner}_{repo}_{sha}_{}", std::process::id())); + if staging.exists() { + fs::remove_dir_all(&staging)?; + } + fs::create_dir_all(&staging)?; + let result = (|| -> Result<()> { + unpack(reply.take(MAX_ARCHIVE_BYTES + 1), &staging)?; + crate::analyzer::validate_source_tree_safety(&staging)?; + let digest = crate::analyzer::compute_tree_digest(&staging)?; + fs::write(staging.join(".coa_tree_digest"), digest)?; + Ok(()) + })(); + if let Err(e) = result { + let _ = fs::remove_dir_all(&staging); + return Err(e); + } + fs::create_dir_all(dest.parent().unwrap())?; + if dest.exists() { + fs::remove_dir_all(&dest)?; + } + fs::rename(&staging, &dest)?; + Ok(dest) +} + +/// Extracts a GitHub tar.gz (one top-level `-/` folder) into `dest`, files and folders only. +fn unpack(reader: impl Read, dest: &Path) -> Result<()> { + let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(reader)); + let mut total = 0u64; + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + let rel: PathBuf = path.components().skip(1).collect(); + if rel.as_os_str().is_empty() { + continue; + } + if rel.components().any(|c| !matches!(c, Component::Normal(_))) { + return Err(Error::PathRejected(format!("archive entry {}", path.display()))); + } + let kind = entry.header().entry_type(); + let target = dest.join(&rel); + if kind.is_dir() { + fs::create_dir_all(&target)?; + } else if kind.is_file() { + total += entry.header().size()?; + if total > MAX_ARCHIVE_BYTES { + return Err(Error::Invalid("The module archive is too large".into())); + } + fs::create_dir_all(target.parent().unwrap())?; + let mut out = fs::File::create(&target)?; + std::io::copy(&mut entry, &mut out)?; + } + // Links and special entries are not part of a module's sources: skipped. + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tar_gz(entries: &[(&str, &[u8])]) -> Vec { + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast())); + for (name, data) in entries { + let mut h = tar::Header::new_gnu(); + h.set_size(data.len() as u64); + h.set_mode(0o644); + h.set_entry_type(tar::EntryType::Regular); + builder.append_data(&mut h, name, *data).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap() + } + + #[test] + fn archive_is_unpacked_without_its_top_folder() { + let dir = tempfile::tempdir().unwrap(); + let gz = tar_gz(&[("mod-x-abc/src/a.cpp", b"int a;"), ("mod-x-abc/conf/x.conf.dist", b"X = 1")]); + unpack(&gz[..], dir.path()).unwrap(); + assert_eq!(fs::read(dir.path().join("src/a.cpp")).unwrap(), b"int a;"); + assert!(dir.path().join("conf/x.conf.dist").is_file()); + } + + #[test] + fn bad_names_are_refused_before_any_download() { + let dir = tempfile::tempdir().unwrap(); + assert!(fetch_pinned("../x", "repo", &"a".repeat(40), dir.path()).is_err()); + assert!(fetch_pinned("owner", "repo", "main", dir.path()).is_err()); + } +} diff --git a/crates/coa-core/src/rebuild.rs b/crates/coa-core/src/rebuild.rs index 7ac963e..6b0e78a 100644 --- a/crates/coa-core/src/rebuild.rs +++ b/crates/coa-core/src/rebuild.rs @@ -9,7 +9,11 @@ use std::fs; use std::path::Path; -use serde::Serialize; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::OnceLock; + +use serde::{Deserialize, Serialize}; use crate::build_engine; use crate::error::Result; @@ -23,6 +27,9 @@ pub enum ModuleRebuild { /// An earlier build for this core was still in the store (a rollback to a core seen before). Reused { id: String }, Rebuilt { id: String, seconds: u64 }, + /// A module the core recommends, built and switched on for the first time (the move from a core + /// with the module built in to a core that loads it as a plugin). + Installed { id: String, seconds: u64 }, /// Switched off; `also` lists the plugins that depend on it and were switched off with it. Disabled { id: String, reason: String, also: Vec }, } @@ -41,9 +48,12 @@ pub fn rebuild_for_current_core( let Some(host) = crate::provenance::realm_host_fingerprint(realm_root) else { return Ok(Vec::new()); }; + let mut out = install_recommended(realm_root, progress); let registry = plugins::load_registry(realm_root)?; - let mut out = Vec::new(); for (id, entry) in registry.plugins.iter().filter(|(_, e)| wanted(e)) { + if out.iter().any(|o| matches!(o, ModuleRebuild::Installed { id: done, .. } if done == id)) { + continue; + } if entry.enabled && plugins::installed_plugin_fingerprint(realm_root, id).as_deref() == Some(host.as_str()) { out.push(ModuleRebuild::UpToDate { id: id.clone() }); continue; @@ -126,6 +136,172 @@ fn reuse_stored(realm_root: &Path, id: &str, sha: &str) -> Result<()> { plugins::update(realm_root, meta, &dll, None, &confs) } +// --------------------------------------------------------------------------------------------------- +// Modules the core recommends +// --------------------------------------------------------------------------------------------------- + +static SOURCE_CACHE: OnceLock = OnceLock::new(); + +/// Where module sources are downloaded (the app's own cache); set once at startup. +pub fn set_source_cache(dir: PathBuf) { + let _ = SOURCE_CACHE.set(dir); +} + +fn source_cache() -> PathBuf { + SOURCE_CACHE.get().cloned().unwrap_or_else(|| std::env::temp_dir().join("coa_github_cache")) +} + +/// `Core/plugins/recommended.json`, shipped with a plugin-host core: the modules a server of the +/// monolithic release had built in, at the commits this core was qualified with. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Recommended { + pub schema: u32, + pub modules: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct RecommendedModule { + pub id: String, + /// `owner/name` on GitHub. + pub repo: String, + pub commit: String, + /// Source trees compiled into the same plugin: a folder of this repository, or of another one. + #[serde(default)] + pub bundled: Vec, + /// When to install it: "always" (default), "config::=" (the owner had it switched + /// on in that module config), or "unless:" (a module that excludes it is switched on). + #[serde(default)] + pub when: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BundledSource { + #[serde(default)] + pub repo: Option, + #[serde(default)] + pub commit: Option, + #[serde(default)] + pub path: String, +} + +fn recommended(realm_root: &Path) -> Option { + let path = plugins::plugins_install_dir(realm_root).join("recommended.json"); + let r: Recommended = serde_json::from_slice(&fs::read(path).ok()?).ok()?; + (r.schema == 1).then_some(r) +} + +/// `.manager/recommended-modules.json`: recommended modules already dealt with (installed, or not wanted +/// by their rule), so a module the owner later removes is not installed again. +fn handled_path(realm_root: &Path) -> PathBuf { + plugins::meta_dir(realm_root).join("recommended-modules.json") +} + +fn load_handled(realm_root: &Path) -> BTreeMap { + fs::read(handled_path(realm_root)).ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .and_then(|v| serde_json::from_value(v.get("handled")?.clone()).ok()) + .unwrap_or_default() +} + +fn save_handled(realm_root: &Path, handled: &BTreeMap) { + let body = serde_json::json!({ "schema": 1, "handled": handled }); + let _ = fs::create_dir_all(plugins::meta_dir(realm_root)); + let text = serde_json::to_string_pretty(&body).unwrap_or_default(); + let _ = crate::fsx::atomic_write(&handled_path(realm_root), text.as_bytes()); +} + +/// Whether `rule` asks for the module on this server, given the plugins already switched on. +fn rule_allows(realm_root: &Path, rule: Option<&str>, enabled: &[String]) -> bool { + let rule = rule.unwrap_or("always"); + if rule == "always" { + return true; + } + if let Some(other) = rule.strip_prefix("unless:") { + return !enabled.iter().any(|e| e == other); + } + if let Some(spec) = rule.strip_prefix("config:") { + let Some((file, kv)) = spec.split_once(':') else { return false }; + let Some((key, value)) = kv.split_once('=') else { return false }; + if file.contains('/') || file.contains('\\') || file.contains("..") { + return false; + } + let path = plugins::modules_config_dir(realm_root).join(file); + return fs::read(&path).ok() + .and_then(|b| crate::config::parser::ConfFile::parse_bytes(&b).ok()) + .and_then(|c| c.get(key).map(|v| v.trim().trim_matches('"') == value)) + .unwrap_or(false); + } + false +} + +fn split_repo(repo: &str) -> std::result::Result<(&str, &str), String> { + repo.split_once('/') + .filter(|(o, r)| !o.is_empty() && !r.is_empty() && !r.contains('/')) + .ok_or_else(|| format!("invalid repository '{repo}'")) +} + +fn install_recommended(realm_root: &Path, progress: &dyn Fn(&str, &str)) -> Vec { + let Some(rec) = recommended(realm_root) else { return Vec::new() }; + let mut handled = load_handled(realm_root); + let mut out = Vec::new(); + for m in &rec.modules { + let registry = plugins::load_registry(realm_root).unwrap_or_default(); + if handled.contains_key(&m.id) || registry.plugins.contains_key(&m.id) { + continue; + } + let enabled: Vec = registry.plugins.iter().filter(|(_, e)| e.enabled).map(|(k, _)| k.clone()).collect(); + if !rule_allows(realm_root, m.when.as_deref(), &enabled) { + handled.insert(m.id.clone(), format!("not wanted ({})", m.when.as_deref().unwrap_or(""))); + continue; + } + progress(&m.id, "Downloading and building the module for this core"); + let started = std::time::Instant::now(); + match install_one(realm_root, m) { + Ok(()) => { + handled.insert(m.id.clone(), m.commit.clone()); + out.push(ModuleRebuild::Installed { id: m.id.clone(), seconds: started.elapsed().as_secs() }); + } + // Not marked as handled: the next core change tries again. + Err(reason) => out.push(ModuleRebuild::Disabled { id: m.id.clone(), reason: first_line(&reason), also: Vec::new() }), + } + } + save_handled(realm_root, &handled); + out +} + +fn install_one(realm_root: &Path, m: &RecommendedModule) -> std::result::Result<(), String> { + let cache = source_cache(); + let (owner, name) = split_repo(&m.repo)?; + let primary = crate::module_sources::fetch_pinned(owner, name, &m.commit, &cache).map_err(|e| e.to_string())?; + let mut bundled = Vec::new(); + for b in &m.bundled { + let base = match (&b.repo, &b.commit) { + (Some(repo), Some(commit)) => { + let (o, n) = split_repo(repo)?; + crate::module_sources::fetch_pinned(o, n, commit, &cache).map_err(|e| e.to_string())? + } + (None, None) => primary.clone(), + _ => return Err(format!("a bundled source of '{}' needs both repo and commit", m.id)), + }; + let dir = if b.path.is_empty() { base } else { crate::fsx::safe_join(&base, &b.path).map_err(|e| e.to_string())? }; + if !dir.is_dir() { + return Err(format!("bundled source '{}' is missing", b.path)); + } + bundled.push(dir); + } + let built = build_engine::build_module_with_bundle(&primary, &bundled, realm_root, None, None).map_err(|e| e.to_string())?; + if built.module_id != m.id { + return Err(format!("the sources build module '{}', not '{}'", built.module_id, m.id)); + } + let (meta, dll, confs) = crate::preview::built_package(realm_root, &m.id, &built.version).map_err(|e| e.to_string())?; + let confs: Vec<(String, &[u8])> = confs.iter().map(|(n, b)| (n.clone(), b.as_slice())).collect(); + plugins::install(realm_root, meta, &dll, None, &confs, true).map_err(|e| e.to_string()) +} + + fn first_line(s: &str) -> String { let line = s.lines().find(|l| !l.trim().is_empty()).unwrap_or(s).trim(); if line.chars().count() > 300 { line.chars().take(300).collect::() + "…" } else { line.to_string() } @@ -231,6 +407,55 @@ mod tests { assert!(reg.plugins["lost"].disabled_reason.is_none()); } + #[test] + fn recommended_modules_follow_what_the_owner_used() { + let dir = tempfile::tempdir().unwrap(); + let server = dir.path(); + fs::create_dir_all(server.join("Core/configs/modules")).unwrap(); + fs::create_dir_all(server.join("Core/plugins")).unwrap(); + fs::write(server.join("Core/worldserver.exe"), image("worldserver", &"c".repeat(64))).unwrap(); + let conf = server.join("Core/configs/modules/playerbots.conf"); + + fs::write(&conf, "AiPlayerbot.Enabled = 1\n").unwrap(); + assert!(rule_allows(server, Some("config:playerbots.conf:AiPlayerbot.Enabled=1"), &[])); + fs::write(&conf, "AiPlayerbot.Enabled = 0\n").unwrap(); + assert!(!rule_allows(server, Some("config:playerbots.conf:AiPlayerbot.Enabled=1"), &[])); + assert!(!rule_allows(server, Some("config:../x.conf:A=1"), &[])); + assert!(!rule_allows(server, Some("unless:playerbots"), &["playerbots".into()])); + assert!(rule_allows(server, Some("unless:playerbots"), &["companions".into()])); + assert!(rule_allows(server, None, &[])); + assert!(!rule_allows(server, Some("sometimes"), &[])); + + // SQUID was off on the old server: it is not installed, and is not offered again later. A module + // whose sources cannot be fetched is reported and retried at the next core change. + fs::write(server.join("Core/plugins/recommended.json"), r#"{"schema":1,"modules":[ + {"id":"playerbots","repo":"Zyth45/mod-playerbots","commit":"0000000000000000000000000000000000000000","when":"config:playerbots.conf:AiPlayerbot.Enabled=1"}, + {"id":"companions","repo":"not-a-repo","commit":"0000000000000000000000000000000000000000","when":"unless:playerbots"}]}"#).unwrap(); + let out = rebuild_for_current_core(server, &|_, _| {}).unwrap(); + assert_eq!(out.len(), 1); + assert!(matches!(&out[0], ModuleRebuild::Disabled { id, reason, .. } if id == "companions" && reason.contains("invalid repository"))); + let handled = load_handled(server); + assert!(handled["playerbots"].starts_with("not wanted")); + assert!(!handled.contains_key("companions")); + } + + /// Real run: COA_TEST_RECOMMENDED_REALM is a plugin-host realm with Core/plugins/recommended.json (or + /// plugins/recommended.json); the modules are downloaded from GitHub, built and switched on. + #[test] + fn recommended_modules_install_on_a_real_realm() { + let Ok(realm) = std::env::var("COA_TEST_RECOMMENDED_REALM") else { return }; + let realm = Path::new(&realm); + let out = rebuild_for_current_core(realm, &|id, what| println!("{id}: {what}")).unwrap(); + println!("{out:#?}"); + let host = crate::provenance::realm_host_fingerprint(realm).unwrap(); + for o in &out { + let ModuleRebuild::Installed { id, .. } = o else { panic!("not installed: {o:?}") }; + assert_eq!(plugins::installed_plugin_fingerprint(realm, id).as_deref(), Some(host.as_str())); + } + let lock = fs::read_to_string(plugins::lockfile_path(realm)).unwrap(); + println!("{lock}"); + } + #[test] fn a_monolithic_core_needs_no_rebuild() { let dir = tempfile::tempdir().unwrap(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7c6a9cd..36bcf09 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -3380,6 +3380,8 @@ static QUIT_REQUESTED: std::sync::atomic::AtomicBool = std::sync::atomic::Atomic pub fn run() { let dir = data_dir(); let _ = coa_core::logging::init(&dir.join("logs").join("manager.log")); + // Module sources fetched during updates land in the same cache as those the Modules page fetches. + coa_core::rebuild::set_source_cache(dir.join("github_cache")); tauri::Builder::default() .plugin(tauri_plugin_single_instance::init(|app, _, _| { show_main_window(app) diff --git a/src/lib/api.ts b/src/lib/api.ts index 51bc12a..77a9e54 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -9,7 +9,7 @@ export interface RemoteConnection { host: string; client_path: string | null } /** What an update did to an installed plugin on the new core (coa-core rebuild::ModuleRebuild). */ export type ModuleRebuild = | { result: "upToDate" | "reused"; id: string } - | { result: "rebuilt"; id: string; seconds: number } + | { result: "rebuilt" | "installed"; id: string; seconds: number } | { result: "disabled"; id: string; reason: string; also: string[] }; export type Classification = "healthy" | "partial" | "unknown-custom" | "incompatible"; diff --git a/src/lib/serverUpdate.ts b/src/lib/serverUpdate.ts index fca9e28..7f718f6 100644 --- a/src/lib/serverUpdate.ts +++ b/src/lib/serverUpdate.ts @@ -106,7 +106,7 @@ export async function applyServerUpdate(id: string, opts?: { choices: Record m.result === "rebuilt" || m.result === "reused").map((m) => m.id); + const rebuilt = mods.filter((m) => m.result === "rebuilt" || m.result === "reused" || m.result === "installed").map((m) => m.id); const off = mods.flatMap((m) => m.result === "disabled" ? [{ id: m.id, reason: m.reason }, ...m.also.map((a) => ({ id: a, reason: m.reason }))] : []); set(id, { result: out.txn.state === "committed" ? { committed: out.txn.to_version ?? "", ...(out.preexisting_character_count ? { preexistingCharacters: out.preexisting_character_count } : {}), ...(rebuilt.length ? { modulesRebuilt: rebuilt } : {}), ...(off.length ? { modulesOff: off } : {}) } : { pending: out.txn } }); if (out.txn.state === "committed" && !wasRunning && !startAfterUpdate("server", id)) await api.stop(id).catch(() => undefined);