Install the modules a plugin-host core recommends before its first start

A core that loads Content Scaling, Companions and SQUID as plugins ships
Core/plugins/recommended.json with the module repositories and commits. During
the update (the move from a monolithic stable server) the Manager downloads
each module as GitHub's archive of that commit (no git needed), builds it with
its bundled trees, installs and enables it, before the trial start. Rules keep
the bot choice: SQUID only if it was switched on in playerbots.conf, otherwise
Companions. Handled modules are remembered so a removed module is not
reinstalled; a failed one is retried at the next core change.

Verified on a CI core: Content Scaling (+packs) and Companions downloaded,
built, installed and written to the plugin lock in 225 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ДмитрийandClaude Opus 5.5 committed 2026-11-05 15:48:23 +05:00
1 parent 3be5fe9609
commit 9153e3ece5
7 files changed
+366 -4

No files matched your search

+1
View File
@@ -43,6 +43,7 @@ pub mod logging;
pub mod manifest;
pub mod migrations;
pub mod modules;
pub mod module_sources;
pub mod msvc_setup;
pub mod multiworld;
mod mysql_snapshot;
+127
View File
@@ -0,0 +1,127 @@
//! Sources of a module at an exact commit, downloaded as GitHub's archive of that commit (no git needed).
//!
//! The cache layout is the one the rest of the Manager uses, `<cache>/github/<owner>/<repo>/<sha>`, so a
//! module identifies itself by its repository name and a later rebuild finds the same tree.
use std::fs;
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::time::Duration;
use crate::error::{Error, Result};
const MAX_ARCHIVE_BYTES: u64 = 512 * 1024 * 1024;
fn valid_name(s: &str) -> bool {
!s.is_empty() && s.len() <= 100 && s != "." && s != ".."
&& s.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))
}
/// `<cache>/github/<owner>/<repo>/<sha>`, downloading it when it is not there yet.
pub fn fetch_pinned(owner: &str, repo: &str, sha: &str, cache_root: &Path) -> Result<PathBuf> {
if !valid_name(owner) || !valid_name(repo) || sha.len() != 40 || !sha.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(Error::Invalid(format!("Invalid module source {owner}/{repo}@{sha}")));
}
let sha = sha.to_ascii_lowercase();
let dest = cache_root.join("github").join(owner).join(repo).join(&sha);
if dest.join(".coa_tree_digest").is_file() {
return Ok(dest);
}
let url = format!("https://codeload.github.com/{owner}/{repo}/tar.gz/{sha}");
let client = reqwest::blocking::Client::builder()
.connect_timeout(Duration::from_secs(15))
.timeout(Duration::from_secs(600))
.build()
.map_err(|e| Error::Invalid(e.to_string()))?;
let reply = client.get(&url).send().map_err(|e| Error::NetworkUnreachable(format!("{url}: {e}")))?;
if !reply.status().is_success() {
return Err(Error::Invalid(format!("GitHub answered {} for {owner}/{repo} at {sha}", reply.status())));
}
let staging = cache_root.join("staging").join(format!("{owner}_{repo}_{sha}_{}", std::process::id()));
if staging.exists() {
fs::remove_dir_all(&staging)?;
}
fs::create_dir_all(&staging)?;
let result = (|| -> Result<()> {
unpack(reply.take(MAX_ARCHIVE_BYTES + 1), &staging)?;
crate::analyzer::validate_source_tree_safety(&staging)?;
let digest = crate::analyzer::compute_tree_digest(&staging)?;
fs::write(staging.join(".coa_tree_digest"), digest)?;
Ok(())
})();
if let Err(e) = result {
let _ = fs::remove_dir_all(&staging);
return Err(e);
}
fs::create_dir_all(dest.parent().unwrap())?;
if dest.exists() {
fs::remove_dir_all(&dest)?;
}
fs::rename(&staging, &dest)?;
Ok(dest)
}
/// Extracts a GitHub tar.gz (one top-level `<repo>-<sha>/` folder) into `dest`, files and folders only.
fn unpack(reader: impl Read, dest: &Path) -> Result<()> {
let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(reader));
let mut total = 0u64;
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?.into_owned();
let rel: PathBuf = path.components().skip(1).collect();
if rel.as_os_str().is_empty() {
continue;
}
if rel.components().any(|c| !matches!(c, Component::Normal(_))) {
return Err(Error::PathRejected(format!("archive entry {}", path.display())));
}
let kind = entry.header().entry_type();
let target = dest.join(&rel);
if kind.is_dir() {
fs::create_dir_all(&target)?;
} else if kind.is_file() {
total += entry.header().size()?;
if total > MAX_ARCHIVE_BYTES {
return Err(Error::Invalid("The module archive is too large".into()));
}
fs::create_dir_all(target.parent().unwrap())?;
let mut out = fs::File::create(&target)?;
std::io::copy(&mut entry, &mut out)?;
}
// Links and special entries are not part of a module's sources: skipped.
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn tar_gz(entries: &[(&str, &[u8])]) -> Vec<u8> {
let mut builder = tar::Builder::new(flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()));
for (name, data) in entries {
let mut h = tar::Header::new_gnu();
h.set_size(data.len() as u64);
h.set_mode(0o644);
h.set_entry_type(tar::EntryType::Regular);
builder.append_data(&mut h, name, *data).unwrap();
}
builder.into_inner().unwrap().finish().unwrap()
}
#[test]
fn archive_is_unpacked_without_its_top_folder() {
let dir = tempfile::tempdir().unwrap();
let gz = tar_gz(&[("mod-x-abc/src/a.cpp", b"int a;"), ("mod-x-abc/conf/x.conf.dist", b"X = 1")]);
unpack(&gz[..], dir.path()).unwrap();
assert_eq!(fs::read(dir.path().join("src/a.cpp")).unwrap(), b"int a;");
assert!(dir.path().join("conf/x.conf.dist").is_file());
}
#[test]
fn bad_names_are_refused_before_any_download() {
let dir = tempfile::tempdir().unwrap();
assert!(fetch_pinned("../x", "repo", &"a".repeat(40), dir.path()).is_err());
assert!(fetch_pinned("owner", "repo", "main", dir.path()).is_err());
}
}
+227 -2
View File
@@ -9,7 +9,11 @@
use std::fs;
use std::path::Path;
use serde::Serialize;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::OnceLock;
use serde::{Deserialize, Serialize};
use crate::build_engine;
use crate::error::Result;
@@ -23,6 +27,9 @@ pub enum ModuleRebuild {
/// An earlier build for this core was still in the store (a rollback to a core seen before).
Reused { id: String },
Rebuilt { id: String, seconds: u64 },
/// A module the core recommends, built and switched on for the first time (the move from a core
/// with the module built in to a core that loads it as a plugin).
Installed { id: String, seconds: u64 },
/// Switched off; `also` lists the plugins that depend on it and were switched off with it.
Disabled { id: String, reason: String, also: Vec<String> },
}
@@ -41,9 +48,12 @@ pub fn rebuild_for_current_core(
let Some(host) = crate::provenance::realm_host_fingerprint(realm_root) else {
return Ok(Vec::new());
};
let mut out = install_recommended(realm_root, progress);
let registry = plugins::load_registry(realm_root)?;
let mut out = Vec::new();
for (id, entry) in registry.plugins.iter().filter(|(_, e)| wanted(e)) {
if out.iter().any(|o| matches!(o, ModuleRebuild::Installed { id: done, .. } if done == id)) {
continue;
}
if entry.enabled && plugins::installed_plugin_fingerprint(realm_root, id).as_deref() == Some(host.as_str()) {
out.push(ModuleRebuild::UpToDate { id: id.clone() });
continue;
@@ -126,6 +136,172 @@ fn reuse_stored(realm_root: &Path, id: &str, sha: &str) -> Result<()> {
plugins::update(realm_root, meta, &dll, None, &confs)
}
// ---------------------------------------------------------------------------------------------------
// Modules the core recommends
// ---------------------------------------------------------------------------------------------------
static SOURCE_CACHE: OnceLock<PathBuf> = OnceLock::new();
/// Where module sources are downloaded (the app's own cache); set once at startup.
pub fn set_source_cache(dir: PathBuf) {
let _ = SOURCE_CACHE.set(dir);
}
fn source_cache() -> PathBuf {
SOURCE_CACHE.get().cloned().unwrap_or_else(|| std::env::temp_dir().join("coa_github_cache"))
}
/// `Core/plugins/recommended.json`, shipped with a plugin-host core: the modules a server of the
/// monolithic release had built in, at the commits this core was qualified with.
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct Recommended {
pub schema: u32,
pub modules: Vec<RecommendedModule>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct RecommendedModule {
pub id: String,
/// `owner/name` on GitHub.
pub repo: String,
pub commit: String,
/// Source trees compiled into the same plugin: a folder of this repository, or of another one.
#[serde(default)]
pub bundled: Vec<BundledSource>,
/// When to install it: "always" (default), "config:<file>:<key>=<value>" (the owner had it switched
/// on in that module config), or "unless:<id>" (a module that excludes it is switched on).
#[serde(default)]
pub when: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
pub struct BundledSource {
#[serde(default)]
pub repo: Option<String>,
#[serde(default)]
pub commit: Option<String>,
#[serde(default)]
pub path: String,
}
fn recommended(realm_root: &Path) -> Option<Recommended> {
let path = plugins::plugins_install_dir(realm_root).join("recommended.json");
let r: Recommended = serde_json::from_slice(&fs::read(path).ok()?).ok()?;
(r.schema == 1).then_some(r)
}
/// `.manager/recommended-modules.json`: recommended modules already dealt with (installed, or not wanted
/// by their rule), so a module the owner later removes is not installed again.
fn handled_path(realm_root: &Path) -> PathBuf {
plugins::meta_dir(realm_root).join("recommended-modules.json")
}
fn load_handled(realm_root: &Path) -> BTreeMap<String, String> {
fs::read(handled_path(realm_root)).ok()
.and_then(|b| serde_json::from_slice::<serde_json::Value>(&b).ok())
.and_then(|v| serde_json::from_value(v.get("handled")?.clone()).ok())
.unwrap_or_default()
}
fn save_handled(realm_root: &Path, handled: &BTreeMap<String, String>) {
let body = serde_json::json!({ "schema": 1, "handled": handled });
let _ = fs::create_dir_all(plugins::meta_dir(realm_root));
let text = serde_json::to_string_pretty(&body).unwrap_or_default();
let _ = crate::fsx::atomic_write(&handled_path(realm_root), text.as_bytes());
}
/// Whether `rule` asks for the module on this server, given the plugins already switched on.
fn rule_allows(realm_root: &Path, rule: Option<&str>, enabled: &[String]) -> bool {
let rule = rule.unwrap_or("always");
if rule == "always" {
return true;
}
if let Some(other) = rule.strip_prefix("unless:") {
return !enabled.iter().any(|e| e == other);
}
if let Some(spec) = rule.strip_prefix("config:") {
let Some((file, kv)) = spec.split_once(':') else { return false };
let Some((key, value)) = kv.split_once('=') else { return false };
if file.contains('/') || file.contains('\\') || file.contains("..") {
return false;
}
let path = plugins::modules_config_dir(realm_root).join(file);
return fs::read(&path).ok()
.and_then(|b| crate::config::parser::ConfFile::parse_bytes(&b).ok())
.and_then(|c| c.get(key).map(|v| v.trim().trim_matches('"') == value))
.unwrap_or(false);
}
false
}
fn split_repo(repo: &str) -> std::result::Result<(&str, &str), String> {
repo.split_once('/')
.filter(|(o, r)| !o.is_empty() && !r.is_empty() && !r.contains('/'))
.ok_or_else(|| format!("invalid repository '{repo}'"))
}
fn install_recommended(realm_root: &Path, progress: &dyn Fn(&str, &str)) -> Vec<ModuleRebuild> {
let Some(rec) = recommended(realm_root) else { return Vec::new() };
let mut handled = load_handled(realm_root);
let mut out = Vec::new();
for m in &rec.modules {
let registry = plugins::load_registry(realm_root).unwrap_or_default();
if handled.contains_key(&m.id) || registry.plugins.contains_key(&m.id) {
continue;
}
let enabled: Vec<String> = registry.plugins.iter().filter(|(_, e)| e.enabled).map(|(k, _)| k.clone()).collect();
if !rule_allows(realm_root, m.when.as_deref(), &enabled) {
handled.insert(m.id.clone(), format!("not wanted ({})", m.when.as_deref().unwrap_or("")));
continue;
}
progress(&m.id, "Downloading and building the module for this core");
let started = std::time::Instant::now();
match install_one(realm_root, m) {
Ok(()) => {
handled.insert(m.id.clone(), m.commit.clone());
out.push(ModuleRebuild::Installed { id: m.id.clone(), seconds: started.elapsed().as_secs() });
}
// Not marked as handled: the next core change tries again.
Err(reason) => out.push(ModuleRebuild::Disabled { id: m.id.clone(), reason: first_line(&reason), also: Vec::new() }),
}
}
save_handled(realm_root, &handled);
out
}
fn install_one(realm_root: &Path, m: &RecommendedModule) -> std::result::Result<(), String> {
let cache = source_cache();
let (owner, name) = split_repo(&m.repo)?;
let primary = crate::module_sources::fetch_pinned(owner, name, &m.commit, &cache).map_err(|e| e.to_string())?;
let mut bundled = Vec::new();
for b in &m.bundled {
let base = match (&b.repo, &b.commit) {
(Some(repo), Some(commit)) => {
let (o, n) = split_repo(repo)?;
crate::module_sources::fetch_pinned(o, n, commit, &cache).map_err(|e| e.to_string())?
}
(None, None) => primary.clone(),
_ => return Err(format!("a bundled source of '{}' needs both repo and commit", m.id)),
};
let dir = if b.path.is_empty() { base } else { crate::fsx::safe_join(&base, &b.path).map_err(|e| e.to_string())? };
if !dir.is_dir() {
return Err(format!("bundled source '{}' is missing", b.path));
}
bundled.push(dir);
}
let built = build_engine::build_module_with_bundle(&primary, &bundled, realm_root, None, None).map_err(|e| e.to_string())?;
if built.module_id != m.id {
return Err(format!("the sources build module '{}', not '{}'", built.module_id, m.id));
}
let (meta, dll, confs) = crate::preview::built_package(realm_root, &m.id, &built.version).map_err(|e| e.to_string())?;
let confs: Vec<(String, &[u8])> = confs.iter().map(|(n, b)| (n.clone(), b.as_slice())).collect();
plugins::install(realm_root, meta, &dll, None, &confs, true).map_err(|e| e.to_string())
}
fn first_line(s: &str) -> String {
let line = s.lines().find(|l| !l.trim().is_empty()).unwrap_or(s).trim();
if line.chars().count() > 300 { line.chars().take(300).collect::<String>() + "…" } else { line.to_string() }
@@ -231,6 +407,55 @@ mod tests {
assert!(reg.plugins["lost"].disabled_reason.is_none());
}
#[test]
fn recommended_modules_follow_what_the_owner_used() {
let dir = tempfile::tempdir().unwrap();
let server = dir.path();
fs::create_dir_all(server.join("Core/configs/modules")).unwrap();
fs::create_dir_all(server.join("Core/plugins")).unwrap();
fs::write(server.join("Core/worldserver.exe"), image("worldserver", &"c".repeat(64))).unwrap();
let conf = server.join("Core/configs/modules/playerbots.conf");
fs::write(&conf, "AiPlayerbot.Enabled = 1\n").unwrap();
assert!(rule_allows(server, Some("config:playerbots.conf:AiPlayerbot.Enabled=1"), &[]));
fs::write(&conf, "AiPlayerbot.Enabled = 0\n").unwrap();
assert!(!rule_allows(server, Some("config:playerbots.conf:AiPlayerbot.Enabled=1"), &[]));
assert!(!rule_allows(server, Some("config:../x.conf:A=1"), &[]));
assert!(!rule_allows(server, Some("unless:playerbots"), &["playerbots".into()]));
assert!(rule_allows(server, Some("unless:playerbots"), &["companions".into()]));
assert!(rule_allows(server, None, &[]));
assert!(!rule_allows(server, Some("sometimes"), &[]));
// SQUID was off on the old server: it is not installed, and is not offered again later. A module
// whose sources cannot be fetched is reported and retried at the next core change.
fs::write(server.join("Core/plugins/recommended.json"), r#"{"schema":1,"modules":[
{"id":"playerbots","repo":"Zyth45/mod-playerbots","commit":"0000000000000000000000000000000000000000","when":"config:playerbots.conf:AiPlayerbot.Enabled=1"},
{"id":"companions","repo":"not-a-repo","commit":"0000000000000000000000000000000000000000","when":"unless:playerbots"}]}"#).unwrap();
let out = rebuild_for_current_core(server, &|_, _| {}).unwrap();
assert_eq!(out.len(), 1);
assert!(matches!(&out[0], ModuleRebuild::Disabled { id, reason, .. } if id == "companions" && reason.contains("invalid repository")));
let handled = load_handled(server);
assert!(handled["playerbots"].starts_with("not wanted"));
assert!(!handled.contains_key("companions"));
}
/// Real run: COA_TEST_RECOMMENDED_REALM is a plugin-host realm with Core/plugins/recommended.json (or
/// plugins/recommended.json); the modules are downloaded from GitHub, built and switched on.
#[test]
fn recommended_modules_install_on_a_real_realm() {
let Ok(realm) = std::env::var("COA_TEST_RECOMMENDED_REALM") else { return };
let realm = Path::new(&realm);
let out = rebuild_for_current_core(realm, &|id, what| println!("{id}: {what}")).unwrap();
println!("{out:#?}");
let host = crate::provenance::realm_host_fingerprint(realm).unwrap();
for o in &out {
let ModuleRebuild::Installed { id, .. } = o else { panic!("not installed: {o:?}") };
assert_eq!(plugins::installed_plugin_fingerprint(realm, id).as_deref(), Some(host.as_str()));
}
let lock = fs::read_to_string(plugins::lockfile_path(realm)).unwrap();
println!("{lock}");
}
#[test]
fn a_monolithic_core_needs_no_rebuild() {
let dir = tempfile::tempdir().unwrap();