From 899b7c661710d4647fcbe5d68c2da9034803ad5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Sat, 7 Nov 2026 11:29:26 +0500 Subject: [PATCH] preview: compressed module build cache with a clear dialog; module sources without client files - Build cache (build_cache.rs): the build folder of a module is unpacked for a build and packed into .tar.zst afterwards (object files ~4-6x smaller), modification times kept so Ninja recompiles only what changed. A cache made for another core is dropped instead of unpacked; .ilk and .map files are not kept; total limit 2 GB, least recently built first. The build log moves next to the archive (build-cache/.log). - "Build cache" in the Modules menu: size per module (packed, built when, for this core or not), remove one, remove all, remove downloaded sources. Builds now hold the server's busy guard, so the cache is never cleared under a running build. - Module sources: "Add from a link" downloads GitHub's archive of the commit instead of a full git clone, and game-client files (.mpq, .dbc, maps, textures, archives) are left out and listed in .coa_skipped_assets. mod-bg-battle-for-gilneas: 365 MB repository, 3.1 MB kept (was refused at the 300 MB limit). - Overlay: Helias's battleground modules (Gilneas, Twin Peaks, Slavery Valley) are Incompatible: they need the server's DBC files replaced (CoA ships its own) and a client patch. Checked: build_cache unit tests; examples/build_cache_ninja_check with real MSVC + CMake + Ninja: after pack/unpack nothing is recompiled, after one edit one file is; coa-core lib tests 865 passed. Co-Authored-By: Claude Opus 5.5 --- crates/coa-core/examples/analyze_url.rs | 7 + .../examples/build_cache_ninja_check.rs | 58 +++ crates/coa-core/src/analyzer.rs | 5 + crates/coa-core/src/build_cache.rs | 355 ++++++++++++++++++ crates/coa-core/src/build_engine.rs | 25 +- crates/coa-core/src/coa-modules-overlay.json | 18 + crates/coa-core/src/lib.rs | 1 + crates/coa-core/src/module_sources.rs | 39 +- src-tauri/src/lib.rs | 40 ++ src/lib/api.ts | 19 + src/screens/BuildCacheDialog.tsx | 121 ++++++ src/screens/ModulesPage.tsx | 22 ++ 12 files changed, 705 insertions(+), 5 deletions(-) create mode 100644 crates/coa-core/examples/analyze_url.rs create mode 100644 crates/coa-core/examples/build_cache_ninja_check.rs create mode 100644 crates/coa-core/src/build_cache.rs create mode 100644 src/screens/BuildCacheDialog.tsx diff --git a/crates/coa-core/examples/analyze_url.rs b/crates/coa-core/examples/analyze_url.rs new file mode 100644 index 0000000..f0fc4bf --- /dev/null +++ b/crates/coa-core/examples/analyze_url.rs @@ -0,0 +1,7 @@ +//! Analyze a module by its GitHub link, as "Add from a link" does. usage: analyze_url +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + let r = coa_core::analyzer::analyze_github_repository(&a[0], Some(std::path::Path::new(&a[1]))).expect("analysis"); + println!("{} @ {} cache_hit={:?}", r.repository_name, r.commit, r.cache_hit); + println!("classification {:?}; hooks {}; sql {}; cmake {}", r.classification, r.hooks.len(), r.sql_files.len(), r.has_cmakelists); +} diff --git a/crates/coa-core/examples/build_cache_ninja_check.rs b/crates/coa-core/examples/build_cache_ninja_check.rs new file mode 100644 index 0000000..0db0102 --- /dev/null +++ b/crates/coa-core/examples/build_cache_ninja_check.rs @@ -0,0 +1,58 @@ +//! Real MSVC + CMake + Ninja: after the build cache is packed and unpacked, a rebuild with nothing changed +//! compiles nothing, and a rebuild after one source changed compiles only that source. +//! usage: build_cache_ninja_check +use std::path::Path; +use std::process::Command; + +fn build(root: &Path, dir: &Path, tc: &coa_core::build_engine::ToolchainStatus) -> String { + let bat = root.join("run.bat"); + let ninja = Path::new(tc.ninja_path.as_ref().unwrap()).parent().unwrap().display().to_string(); + std::fs::write(&bat, format!( + "@echo off\r\ncall \"{}\" > nul 2>&1\r\nset \"PATH={};%PATH%\"\r\n\"{}\" -G Ninja -S \"{}\" -B \"{}\\build\" -DCMAKE_BUILD_TYPE=RelWithDebInfo > nul\r\n\"{}\" --build \"{}\\build\" -- -d explain\r\n", + tc.vcvars64_path.as_ref().unwrap(), ninja, tc.cmake_path.as_ref().unwrap(), dir.display(), dir.display(), tc.cmake_path.as_ref().unwrap(), dir.display() + )).unwrap(); + let out = Command::new("cmd.exe").args(["/c", &bat.to_string_lossy()]).output().unwrap(); + assert!(out.status.success(), "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr) +} + +fn compiled(log: &str) -> usize { + log.lines().filter(|l| l.contains("Building CXX object")).count() +} + +fn main() { + let root = std::path::PathBuf::from(std::env::args().nth(1).expect("usage: build_cache_ninja_check ")); + assert!(!root.exists() || std::fs::read_dir(&root).unwrap().next().is_none(), "folder must be new or empty"); + std::fs::create_dir_all(root.join(".manager")).unwrap(); + let tc = coa_core::build_engine::detect_toolchain(); + assert!(tc.msvc_available && tc.cmake_available && tc.ninja_available, "toolchain: {tc:?}"); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + std::fs::write(dir.join("CMakeLists.txt"), "cmake_minimum_required(VERSION 3.22)\nproject(p CXX)\nadd_library(p SHARED a.cpp b.cpp c.cpp)\n").unwrap(); + for f in ["a", "b", "c"] { + std::fs::write(dir.join(format!("{f}.cpp")), format!("#include \n#include \n__declspec(dllexport) int {f}() {{ return (int)std::vector(3).size(); }}\n")).unwrap(); + } + let first = build(&root, &dir, &tc); + println!("first build compiled {}", compiled(&first)); + assert_eq!(compiled(&first), 3); + + coa_core::build_cache::finish(&root, "probe"); + let s = coa_core::build_cache::summary(&root, None); + println!("packed: {} bytes from {} unpacked", s.total_bytes, s.modules[0].unpacked_bytes); + assert!(!dir.exists()); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + let again = build(&root, &dir, &tc); + println!("after unpack, nothing changed: compiled {}\n{}", compiled(&again), again.lines().filter(|l| l.contains("explain")).take(8).collect::>().join("\n")); + assert_eq!(compiled(&again), 0, "nothing should be recompiled"); + coa_core::build_cache::finish(&root, "probe"); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + std::thread::sleep(std::time::Duration::from_secs(2)); + std::fs::write(dir.join("b.cpp"), "__declspec(dllexport) int b() { return 2; }\n").unwrap(); + let one = build(&root, &dir, &tc); + println!("after one change: compiled {}", compiled(&one)); + assert_eq!(compiled(&one), 1); + coa_core::build_cache::finish(&root, "probe"); + println!("PASS"); +} diff --git a/crates/coa-core/src/analyzer.rs b/crates/coa-core/src/analyzer.rs index 327278b..df9ab97 100644 --- a/crates/coa-core/src/analyzer.rs +++ b/crates/coa-core/src/analyzer.rs @@ -1032,8 +1032,13 @@ pub fn analyze_github_repository(url_str: &str, cache_base_dir: Option<&Path>) - let marker_file = commit_cache_dir.join(".coa_tree_digest"); let cache_hit = commit_cache_dir.exists() && marker_file.exists(); + let pinned = resolved_commit.len() == 40 && resolved_commit.bytes().all(|b| b.is_ascii_hexdigit()); let target_dir = if cache_hit { commit_cache_dir + } else if pinned { + // GitHub's archive of the exact commit, without the game-client files some repositories carry (they + // are not needed to build a server module and used to push such repositories over the size limit). + crate::module_sources::fetch_pinned(&owner, &repo, &resolved_commit, &base_cache)? } else { // Clone into temporary staging directory let staging_id = format!("{}_{}_{}", owner, repo, std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()); diff --git a/crates/coa-core/src/build_cache.rs b/crates/coa-core/src/build_cache.rs new file mode 100644 index 0000000..a3ea028 --- /dev/null +++ b/crates/coa-core/src/build_cache.rs @@ -0,0 +1,355 @@ +//! The compiler's work kept between builds of a module, small on disk. +//! +//! A build works in `/build-cache//` (sources, CMake/Ninja state, object files). Between builds +//! that folder is packed into `.tar.zst` (object files compress about 4x) with a small record of the +//! core it was made for. Before the next build it is unpacked again, so Ninja recompiles only what changed. +//! +//! Object files made for another core are useless (every one depends on the core's headers), so a cache for +//! another core is dropped instead of unpacked. The whole cache is kept under a size limit: the modules built +//! longest ago go first. Incremental-link files (`.ilk`) and linker maps are not kept. + +use std::fs; +use std::io::{BufReader, BufWriter}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::error::{Error, Result}; + +/// Packed caches above this total are removed, the least recently built first. +pub const LIMIT_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Record { + /// ABI fingerprint of the core the objects were compiled for. + fingerprint: String, + /// Unix seconds of the last build. + last_used: i64, + unpacked_bytes: u64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CacheEntry { + pub module: String, + /// Bytes on disk now (packed, or unpacked while a build runs). + pub bytes: u64, + pub unpacked_bytes: u64, + pub last_used: i64, + /// Made for the core this server runs now (otherwise it is dropped at the next build). + pub current_core: bool, + pub packed: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CacheSummary { + pub modules: Vec, + pub total_bytes: u64, + /// Downloaded module sources (shared by all servers of this Manager). + pub sources_bytes: u64, + pub limit_bytes: u64, +} + +pub fn root(realm_root: &Path) -> PathBuf { + crate::plugins::meta_dir(realm_root).join("build-cache") +} + +fn valid_id(id: &str) -> bool { + !id.is_empty() && id.len() <= 100 && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) && id != "." && id != ".." +} + +fn paths(realm_root: &Path, id: &str) -> Result<(PathBuf, PathBuf, PathBuf)> { + if !valid_id(id) { + return Err(Error::Invalid(format!("Invalid module id for the build cache: {id}"))); + } + let r = root(realm_root); + Ok((r.join(id), r.join(format!("{id}.tar.zst")), r.join(format!("{id}.json")))) +} + +fn now() -> i64 { + chrono::Utc::now().timestamp() +} + +pub(crate) fn dir_bytes(dir: &Path) -> u64 { + fs::read_dir(dir) + .map(|rd| { + rd.flatten() + .map(|e| match e.metadata() { + Ok(m) if m.is_dir() => dir_bytes(&e.path()), + Ok(m) => m.len(), + Err(_) => 0, + }) + .sum() + }) + .unwrap_or(0) +} + +fn read_record(path: &Path) -> Option { + serde_json::from_slice(&fs::read(path).ok()?).ok() +} + +/// The working folder for a build of `id` for the core `fingerprint`, with the earlier work of a build for the +/// same core unpacked into it. A cache for another core is dropped. +pub fn prepare(realm_root: &Path, id: &str, fingerprint: &str) -> Result { + let (dir, archive, record) = paths(realm_root, id)?; + let rec = read_record(&record); + let same_core = rec.as_ref().is_some_and(|r| r.fingerprint == fingerprint); + if !same_core { + // made for another core (or unknown): its objects would all be rebuilt anyway + if dir.exists() { + fs::remove_dir_all(&dir)?; + } + let _ = fs::remove_file(&archive); + let _ = fs::remove_file(&record); + } else if archive.is_file() { + if dir.exists() { + fs::remove_dir_all(&dir)?; // an older, interrupted working folder: the archive is the newer state + } + unpack(&archive, &dir).map_err(|e| { + let _ = fs::remove_dir_all(&dir); + Error::Invalid(format!("The build cache of {id} could not be unpacked: {e}")) + })?; + fs::remove_file(&archive)?; + } + fs::create_dir_all(&dir)?; + let unpacked_bytes = dir_bytes(&dir); + crate::fsx::atomic_write_json(&record, &Record { fingerprint: fingerprint.into(), last_used: now(), unpacked_bytes })?; + Ok(dir) +} + +/// After a build (successful or not): drop what does not speed up the next one, pack the rest, keep the total +/// under the limit. Never fails the build: problems only cost the cache. +pub fn finish(realm_root: &Path, id: &str) { + let Ok((dir, archive, record)) = paths(realm_root, id) else { return }; + if !dir.is_dir() { + return; + } + remove_by_ext(&dir, &["ilk", "map"]); + let unpacked_bytes = dir_bytes(&dir); + let tmp = archive.with_extension("zst.partial"); + match pack(&dir, &tmp).and_then(|_| fs::rename(&tmp, &archive).map_err(Into::into)) { + Ok(()) => { + let _ = fs::remove_dir_all(&dir); + if let Some(mut r) = read_record(&record) { + r.last_used = now(); + r.unpacked_bytes = unpacked_bytes; + let _ = crate::fsx::atomic_write_json(&record, &r); + } + } + Err(e) => { + tracing::warn!(module = id, error = %e, "build cache not packed; it is kept unpacked"); + let _ = fs::remove_file(&tmp); + } + } + enforce_limit(realm_root, LIMIT_BYTES, Some(id)); +} + +fn remove_by_ext(dir: &Path, exts: &[&str]) { + for e in fs::read_dir(dir).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + remove_by_ext(&p, exts); + } else if p.extension().and_then(|x| x.to_str()).is_some_and(|x| exts.iter().any(|w| w.eq_ignore_ascii_case(x))) { + let _ = fs::remove_file(p); + } + } +} + +fn pack(dir: &Path, out: &Path) -> Result<()> { + let file = BufWriter::new(fs::File::create(out)?); + let mut enc = zstd::Encoder::new(file, 3)?; + let _ = enc.multithread(std::thread::available_parallelism().map(|n| n.get() as u32).unwrap_or(1).min(8)); + { + let mut tar = tar::Builder::new(&mut enc); + tar.follow_symlinks(false); + // modification times are kept: Ninja decides from them what is up to date + tar.mode(tar::HeaderMode::Complete); + tar.append_dir_all(".", dir)?; + tar.finish()?; + } + enc.finish()?; + Ok(()) +} + +fn unpack(archive: &Path, dir: &Path) -> Result<()> { + fs::create_dir_all(dir)?; + let dec = zstd::Decoder::new(BufReader::new(fs::File::open(archive)?))?; + let mut tar = tar::Archive::new(dec); + tar.set_preserve_mtime(true); + for entry in tar.entries()? { + let mut entry = entry?; + // unpack_in refuses paths that would leave `dir` + if !entry.unpack_in(dir)? { + return Err(Error::PathRejected(format!("build cache entry {}", entry.path()?.display()))); + } + } + Ok(()) +} + +/// Everything in the build cache of this server, largest first. +pub fn summary(realm_root: &Path, sources_dir: Option<&Path>) -> CacheSummary { + let current = crate::provenance::realm_host_fingerprint(realm_root); + let r = root(realm_root); + let mut modules = Vec::new(); + for e in fs::read_dir(&r).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let Some(id) = name.strip_suffix(".json") else { continue }; + let Ok((dir, archive, record)) = paths(realm_root, id) else { continue }; + let Some(rec) = read_record(&record) else { continue }; + let packed = archive.is_file(); + let bytes = if packed { fs::metadata(&archive).map(|m| m.len()).unwrap_or(0) } else { dir_bytes(&dir) }; + modules.push(CacheEntry { + module: id.to_string(), + bytes, + unpacked_bytes: rec.unpacked_bytes, + last_used: rec.last_used, + current_core: current.as_deref() == Some(rec.fingerprint.as_str()), + packed, + }); + } + // working folders of an older Manager, without a record + for e in fs::read_dir(&r).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + if e.path().is_dir() && valid_id(&name) && !modules.iter().any(|m| m.module == name) { + let bytes = dir_bytes(&e.path()); + modules.push(CacheEntry { module: name, bytes, unpacked_bytes: bytes, last_used: 0, current_core: false, packed: false }); + } + } + modules.sort_by(|a, b| b.bytes.cmp(&a.bytes)); + CacheSummary { + total_bytes: modules.iter().map(|m| m.bytes).sum(), + sources_bytes: sources_dir.map(dir_bytes).unwrap_or(0), + limit_bytes: LIMIT_BYTES, + modules, + } +} + +/// Remove the cache of one module, or of all. The next build of a module compiles it in full. +pub fn clear(realm_root: &Path, module: Option<&str>) -> Result { + let before = summary(realm_root, None).total_bytes; + match module { + Some(id) => { + let (dir, archive, record) = paths(realm_root, id)?; + if dir.exists() { + fs::remove_dir_all(dir)?; + } + let _ = fs::remove_file(archive); + let _ = fs::remove_file(record); + } + None => { + let r = root(realm_root); + if r.exists() { + fs::remove_dir_all(&r)?; + } + } + } + Ok(before.saturating_sub(summary(realm_root, None).total_bytes)) +} + +/// Keep the packed caches under `limit`, dropping the least recently built (never `keep`). +fn enforce_limit(realm_root: &Path, limit: u64, keep: Option<&str>) { + let mut s = summary(realm_root, None); + s.modules.sort_by_key(|m| m.last_used); + let mut total = s.total_bytes; + for m in &s.modules { + if total <= limit { + break; + } + if Some(m.module.as_str()) == keep { + continue; + } + if clear(realm_root, Some(&m.module)).is_ok() { + tracing::info!(module = %m.module, bytes = m.bytes, "build cache over its limit: oldest removed"); + total = total.saturating_sub(m.bytes); + } + } +} + +/// Downloaded module sources the Manager keeps (`/github`, `/staging`). +pub fn clear_sources(sources_dir: &Path) -> Result { + let before = dir_bytes(sources_dir); + for sub in ["github", "staging"] { + let p = sources_dir.join(sub); + if p.exists() { + fs::remove_dir_all(p)?; + } + } + Ok(before.saturating_sub(dir_bytes(sources_dir))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, SystemTime}; + + fn server() -> tempfile::TempDir { + let d = tempfile::tempdir().unwrap(); + fs::create_dir_all(d.path().join(".manager")).unwrap(); + d + } + + #[test] + fn a_cache_is_packed_between_builds_and_comes_back_with_its_times() { + let s = server(); + let dir = prepare(s.path(), "companions", "fp1").unwrap(); + fs::create_dir_all(dir.join("build/obj")).unwrap(); + fs::write(dir.join("build/obj/a.obj"), vec![7u8; 100_000]).unwrap(); + fs::write(dir.join("build/x.ilk"), b"incremental link").unwrap(); + let old = SystemTime::now() - Duration::from_secs(3600); + fs::File::options().write(true).open(dir.join("build/obj/a.obj")).unwrap().set_modified(old).unwrap(); + finish(s.path(), "companions"); + assert!(!dir.exists(), "packed between builds"); + let sum = summary(s.path(), None); + assert_eq!(sum.modules.len(), 1); + assert!(sum.modules[0].packed && sum.modules[0].bytes < 100_000 / 4, "compressed: {}", sum.modules[0].bytes); + + let dir = prepare(s.path(), "companions", "fp1").unwrap(); + assert_eq!(fs::read(dir.join("build/obj/a.obj")).unwrap().len(), 100_000); + assert!(!dir.join("build/x.ilk").exists(), "incremental-link files are not kept"); + let mtime = fs::metadata(dir.join("build/obj/a.obj")).unwrap().modified().unwrap(); + let drift = mtime.duration_since(old).unwrap_or_else(|e| e.duration()); + assert!(drift < Duration::from_secs(2), "modification time kept (drift {drift:?})"); + } + + #[test] + fn a_cache_for_another_core_is_dropped() { + let s = server(); + let dir = prepare(s.path(), "scaling", "old-core").unwrap(); + fs::write(dir.join("a.obj"), b"old").unwrap(); + finish(s.path(), "scaling"); + let dir = prepare(s.path(), "scaling", "new-core").unwrap(); + assert!(!dir.join("a.obj").exists()); + } + + #[test] + fn clear_removes_one_module_or_all_and_bad_ids_are_refused() { + let s = server(); + for id in ["a", "b"] { + let d = prepare(s.path(), id, "fp").unwrap(); + fs::write(d.join("x.obj"), b"x").unwrap(); + finish(s.path(), id); + } + clear(s.path(), Some("a")).unwrap(); + assert_eq!(summary(s.path(), None).modules.iter().map(|m| m.module.as_str()).collect::>(), ["b"]); + clear(s.path(), None).unwrap(); + assert!(summary(s.path(), None).modules.is_empty()); + assert!(clear(s.path(), Some("../x")).is_err()); + } + + #[test] + fn the_least_recently_built_go_first_over_the_limit() { + let s = server(); + for (n, id) in ["old", "new"].iter().enumerate() { + let d = prepare(s.path(), id, "fp").unwrap(); + fs::write(d.join("x.obj"), vec![n as u8; 50_000]).unwrap(); + finish(s.path(), id); + let rec = root(s.path()).join(format!("{id}.json")); + let mut r = read_record(&rec).unwrap(); + r.last_used = 1000 + n as i64; + crate::fsx::atomic_write_json(&rec, &r).unwrap(); + } + let total = summary(s.path(), None).total_bytes; + enforce_limit(s.path(), total - 1, None); + let left: Vec = summary(s.path(), None).modules.into_iter().map(|m| m.module).collect(); + assert_eq!(left, ["new"]); + } +} diff --git a/crates/coa-core/src/build_engine.rs b/crates/coa-core/src/build_engine.rs index 83e3b62..dd1072b 100644 --- a/crates/coa-core/src/build_engine.rs +++ b/crates/coa-core/src/build_engine.rs @@ -977,8 +977,25 @@ pub fn build_module_with_bundle( .unwrap_or_else(|| with_plugin_id(&adapter_source, &plugin_id)); // 2. Prepare the persistent staging directory - let default_staging = realm_root.join(".manager").join("build-cache").join(&module_id); - let staging_dir = custom_staging_dir.map(PathBuf::from).unwrap_or(default_staging); + // The Manager's own builds use the module's build cache (unpacked for the build, packed again afterwards); + // a caller-chosen folder is used as it is. + let staging_dir = match custom_staging_dir { + Some(p) => PathBuf::from(p), + None => crate::build_cache::prepare(realm_root, &module_id, &sdk_abi_fingerprint(Path::new(&sdk_dir))?)?, + }; + struct PackAfter<'a>(Option<(&'a Path, String)>); + impl Drop for PackAfter<'_> { + fn drop(&mut self) { + if let Some((root, id)) = self.0.take() { + crate::build_cache::finish(root, &id); + } + } + } + let _pack_after = PackAfter(custom_staging_dir.is_none().then(|| (realm_root, module_id.clone()))); + let log_path = match custom_staging_dir { + Some(_) => staging_dir.join("build.log"), + None => crate::build_cache::root(realm_root).join(format!("{module_id}.log")), + }; fs::create_dir_all(&staging_dir)?; let staging_src = staging_dir.join("src"); fs::create_dir_all(&staging_src)?; @@ -1142,11 +1159,11 @@ if errorlevel 1 exit /b 103 } // The full log stays next to the build, so a failure can always be read afterwards. - let _ = fs::write(staging_dir.join("build.log"), all_logs.join(" + let _ = fs::write(&log_path, all_logs.join(" ")); let mut errors = parse_msvc_errors(&all_logs); if exit_code != 0 && errors.is_empty() { - errors.push(unparsed_failure(exit_code, &all_logs, &staging_dir.join("build.log"))); + errors.push(unparsed_failure(exit_code, &all_logs, &log_path)); } let expected_dll_path = build_output_dir.join(format!("{}.dll", safe_target_name)); diff --git a/crates/coa-core/src/coa-modules-overlay.json b/crates/coa-core/src/coa-modules-overlay.json index b401041..1dae9f5 100644 --- a/crates/coa-core/src/coa-modules-overlay.json +++ b/crates/coa-core/src/coa-modules-overlay.json @@ -62,6 +62,24 @@ "is_recommended": false, "reason": "Already built into the CoA core (cross-faction battlegrounds); a separate DLL would load it a second time." }, + { + "repo_slug": "helias/mod-bg-battle-for-gilneas", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (including Spell.dbc, which CoA ships its own of), a client patch and a modified wow.exe; written for AzerothCore 2019." + }, + { + "repo_slug": "helias/mod-bg-twinpeaks", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (CoA ships its own), a client patch and a modified wow.exe." + }, + { + "repo_slug": "helias/mod-bg-slaveryvalley", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (CoA ships its own) and a client patch." + }, { "repo_slug": "azerothcore/mod-learn-spells", "status": "Incompatible", diff --git a/crates/coa-core/src/lib.rs b/crates/coa-core/src/lib.rs index c07e8bb..80625e8 100644 --- a/crates/coa-core/src/lib.rs +++ b/crates/coa-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod accounts; pub mod allsettings; pub mod analyzer; pub mod backup; +pub mod build_cache; pub mod build_engine; pub mod catalogue; pub mod channels; diff --git a/crates/coa-core/src/module_sources.rs b/crates/coa-core/src/module_sources.rs index f2e6fe6..7476c29 100644 --- a/crates/coa-core/src/module_sources.rs +++ b/crates/coa-core/src/module_sources.rs @@ -61,10 +61,24 @@ pub fn fetch_pinned(owner: &str, repo: &str, sha: &str, cache_root: &Path) -> Re Ok(dest) } +/// Game-client and media files a server module's build never reads (client patches, DBC/map data, textures, +/// archives). Some repositories carry hundreds of MB of them next to a few KB of server code; they are left out +/// of the sources, and their names listed in `.coa_skipped_assets`. +fn is_client_asset(rel: &Path) -> bool { + const EXT: [&str; 22] = [ + "mpq", "dbc", "db2", "adt", "wdt", "wdl", "wmo", "m2", "skin", "anim", "blp", "tga", "trs", "vmtile", "vmtree", + "mmap", "mmtile", "zip", "7z", "rar", "mp3", "ogg", + ]; + rel.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| EXT.iter().any(|x| x.eq_ignore_ascii_case(e))) +} + /// Extracts a GitHub tar.gz (one top-level `-/` folder) into `dest`, files and folders only. -fn unpack(reader: impl Read, dest: &Path) -> Result<()> { +pub(crate) fn unpack(reader: impl Read, dest: &Path) -> Result<()> { let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(reader)); let mut total = 0u64; + let mut skipped = Vec::new(); for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?.into_owned(); @@ -79,6 +93,8 @@ fn unpack(reader: impl Read, dest: &Path) -> Result<()> { let target = dest.join(&rel); if kind.is_dir() { fs::create_dir_all(&target)?; + } else if kind.is_file() && is_client_asset(&rel) { + skipped.push(rel.to_string_lossy().replace('\\', "/")); } else if kind.is_file() { total += entry.header().size()?; if total > MAX_ARCHIVE_BYTES { @@ -90,6 +106,9 @@ fn unpack(reader: impl Read, dest: &Path) -> Result<()> { } // Links and special entries are not part of a module's sources: skipped. } + if !skipped.is_empty() { + fs::write(dest.join(".coa_skipped_assets"), skipped.join("\n"))?; + } Ok(()) } @@ -118,6 +137,24 @@ mod tests { assert!(dir.path().join("conf/x.conf.dist").is_file()); } + #[test] + fn client_assets_are_left_out_and_listed() { + let dir = tempfile::tempdir().unwrap(); + let gz = tar_gz(&[ + ("mod-bg-abc/server-side/src/BG.cpp", b"int a;"), + ("mod-bg-abc/client-side/DBFilesClient/Spell.dbc", b"dbc"), + ("mod-bg-abc/client-data.zip", b"zip"), + ("mod-bg-abc/data/sql/db-world/bg.sql", b"SELECT 1;"), + ]); + unpack(&gz[..], dir.path()).unwrap(); + assert!(dir.path().join("server-side/src/BG.cpp").is_file()); + assert!(dir.path().join("data/sql/db-world/bg.sql").is_file()); + assert!(!dir.path().join("client-side/DBFilesClient/Spell.dbc").exists()); + assert!(!dir.path().join("client-data.zip").exists()); + let listed = fs::read_to_string(dir.path().join(".coa_skipped_assets")).unwrap(); + assert!(listed.contains("Spell.dbc") && listed.contains("client-data.zip")); + } + #[test] fn bad_names_are_refused_before_any_download() { let dir = tempfile::tempdir().unwrap(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 4829739..4929f0c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -3083,6 +3083,42 @@ async fn analyze_github_module( .await } +/// What the module build cache of a server (and the shared source downloads) take on disk. +#[tauri::command] +async fn build_cache_summary( + state: State<'_, AppState>, + id: String, +) -> std::result::Result { + let root = path_of(&state, &id)?; + let sources = data_dir().join("github_cache"); + blocking(move || Ok(coa_core::build_cache::summary(&root, Some(&sources)))).await +} + +/// Remove the build cache of one module (`module`), of all modules (no `module`), and optionally the downloaded +/// sources. Refused while a build runs on this server. Returns the bytes freed. +#[tauri::command] +async fn build_cache_clear( + state: State<'_, AppState>, + id: String, + module: Option, + sources: bool, +) -> std::result::Result { + let root = path_of(&state, &id)?; + let _guard = BusyGuard::acquire(&state, &id)?; + let sources_dir = data_dir().join("github_cache"); + blocking(move || { + let mut freed = 0; + if module.is_some() || !sources { + freed += coa_core::build_cache::clear(&root, module.as_deref())?; + } + if sources { + freed += coa_core::build_cache::clear_sources(&sources_dir)?; + } + Ok(freed) + }) + .await +} + #[tauri::command] async fn get_public_module_catalog( force_refresh: Option, @@ -3131,6 +3167,8 @@ async fn preview_build_module( let realm_root = path_of(&state, &realm_id)?; let cache_dir = data_dir().join("github_cache"); + // The build works in the module's build cache: clearing the cache waits for it. + let _guard = BusyGuard::acquire(&state, &realm_id)?; let app_clone = app.clone(); let progress_cb: Box = @@ -3594,6 +3632,8 @@ pub fn run() { account_delete, install_new, is_classic_server, + build_cache_summary, + build_cache_clear, cancel_install, create_account, check_update, diff --git a/src/lib/api.ts b/src/lib/api.ts index d9c5d66..2494f0b 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -479,6 +479,22 @@ export interface AccountInfo { characters: number; } +export interface BuildCacheEntry { + module: string; + bytes: number; + unpacked_bytes: number; + last_used: number; + current_core: boolean; + packed: boolean; +} + +export interface BuildCacheSummary { + modules: BuildCacheEntry[]; + total_bytes: number; + sources_bytes: number; + limit_bytes: number; +} + export interface CopyReport { schemas: string[]; accounts: number; @@ -818,6 +834,9 @@ export const api = { installNew: (dest: string, pkg?: string, gameData?: string, copyFrom?: string) => invoke("install_new", { dest, package: pkg ?? null, gameData: gameData ?? null, copyFrom: copyFrom ?? null }), isClassicServer: (path: string) => invoke("is_classic_server", { path }), + buildCacheSummary: (id: string) => invoke("build_cache_summary", { id }), + buildCacheClear: (id: string, module: string | null, sources: boolean) => + invoke("build_cache_clear", { id, module, sources }), cancelInstall: () => invoke("cancel_install"), createAccount: (id: string, username: string, password: string, administrator: boolean) => invoke("create_account", { id, username, password, administrator }), diff --git a/src/screens/BuildCacheDialog.tsx b/src/screens/BuildCacheDialog.tsx new file mode 100644 index 0000000..1aaf7f5 --- /dev/null +++ b/src/screens/BuildCacheDialog.tsx @@ -0,0 +1,121 @@ +import { useCallback, useEffect, useState } from "react"; +import { Loader2, Trash2, X } from "lucide-react"; +import { api, asUiError, type BuildCacheSummary } from "@/lib/api"; +import { formatBytes } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; + +/** What the module build cache takes, with removal of one module, all modules, or the downloaded sources. */ +export function BuildCacheDialog(props: { serverId: string; locale: string; onClose: () => void }) { + const ru = props.locale === "ru"; + const [summary, setSummary] = useState(null); + const [busy, setBusy] = useState(null); + const [message, setMessage] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(() => { + void api + .buildCacheSummary(props.serverId) + .then(setSummary) + .catch((e) => setError(asUiError(e).technical)); + }, [props.serverId]); + useEffect(load, [load]); + + async function clear(key: string, module: string | null, sources: boolean) { + setBusy(key); + setError(null); + try { + const freed = await api.buildCacheClear(props.serverId, module, sources); + setMessage(ru ? `Освобождено ${formatBytes(freed)}` : `Freed ${formatBytes(freed)}`); + load(); + } catch (e) { + setError(asUiError(e).technical); + } finally { + setBusy(null); + } + } + + const when = (secs: number) => + secs > 0 ? new Date(secs * 1000).toLocaleDateString(ru ? "ru-RU" : undefined, { day: "numeric", month: "short" }) : "—"; + + return ( +
+ +
+
+

{ru ? "Кэш сборок" : "Build cache"}

+

+ {ru + ? "Скомпилированные части модулей хранятся сжатыми и ускоряют повторную сборку. После обновления ядра старый кэш удаляется сам. Удаление ничего не ломает: следующая сборка модуля будет полной." + : "Compiled parts of modules are kept compressed and speed up the next build. After a core update the old cache goes by itself. Removing it breaks nothing: the next build of a module is a full one."} +

+
+ +
+ + {!summary && !error && } + + {summary && ( + <> +
+ {summary.modules.length === 0 &&

{ru ? "Кэш пуст." : "The cache is empty."}

} + {summary.modules.map((m) => ( +
+
+

{m.module}

+

+ {m.packed + ? ru ? `сжато из ${formatBytes(m.unpacked_bytes)}` : `packed from ${formatBytes(m.unpacked_bytes)}` + : ru ? "не сжато" : "not packed"} + {" · "} + {ru ? "сборка " : "built "} + {when(m.last_used)} + {!m.current_core && (ru ? " · для другого ядра" : " · for another core")} +

+
+ {formatBytes(m.bytes)} + +
+ ))} +
+ +
+ + {ru ? "Скачанные исходники модулей" : "Downloaded module sources"}: {formatBytes(summary.sources_bytes)} + + +
+ +
+ + {ru ? "Всего" : "Total"}: {formatBytes(summary.total_bytes)} + {ru ? `, предел ${formatBytes(summary.limit_bytes)}` : `, limit ${formatBytes(summary.limit_bytes)}`} + + +
+ + )} + + {message &&

{message}

} + {error &&

{error}

} +
+
+ ); +} diff --git a/src/screens/ModulesPage.tsx b/src/screens/ModulesPage.tsx index ee0af4e..eb17604 100644 --- a/src/screens/ModulesPage.tsx +++ b/src/screens/ModulesPage.tsx @@ -57,6 +57,8 @@ import { useHuman, useI18n, useSchemaText, type Key } from "@/i18n"; import { auctionWeightKey, canonicalModuleValue, moduleEnableKey, moduleField, retiredModuleKeys } from "@/lib/moduleSettings"; import { AuctionTypes } from "@/screens/AuctionTypes"; import { RaceAvailabilitySettings } from "@/screens/RaceAvailabilitySettings"; +import { BuildCacheDialog } from "@/screens/BuildCacheDialog"; +import { formatBytes } from "@/lib/utils"; import type { UnsavedChangesGuard } from "@/lib/unsavedGuard"; /** The GitHub mark */ @@ -639,6 +641,13 @@ export function ModulesPage({ // "Add from a link" stays folded away until asked for; the menu holds the rarely used build tools. const [showUrlBar, setShowUrlBar] = useState(false); const [menuOpen, setMenuOpen] = useState(false); + const [cacheOpen, setCacheOpen] = useState(false); + const [cacheBytes, setCacheBytes] = useState(null); + useEffect(() => { + if (menuOpen) { + void api.buildCacheSummary(serverId).then((s) => setCacheBytes(s.total_bytes + s.sources_bytes)).catch(() => setCacheBytes(null)); + } + }, [menuOpen, serverId]); const [analyzing, setAnalyzing] = useState(false); const [analysisError, setAnalysisError] = useState(null); const [report, setReport] = useState(null); @@ -1234,6 +1243,17 @@ export function ModulesPage({ {tab === "builds" && building && } ))} +