diff --git a/crates/coa-core/examples/analyze_url.rs b/crates/coa-core/examples/analyze_url.rs new file mode 100644 index 0000000..f0fc4bf --- /dev/null +++ b/crates/coa-core/examples/analyze_url.rs @@ -0,0 +1,7 @@ +//! Analyze a module by its GitHub link, as "Add from a link" does. usage: analyze_url +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + let r = coa_core::analyzer::analyze_github_repository(&a[0], Some(std::path::Path::new(&a[1]))).expect("analysis"); + println!("{} @ {} cache_hit={:?}", r.repository_name, r.commit, r.cache_hit); + println!("classification {:?}; hooks {}; sql {}; cmake {}", r.classification, r.hooks.len(), r.sql_files.len(), r.has_cmakelists); +} diff --git a/crates/coa-core/examples/build_cache_ninja_check.rs b/crates/coa-core/examples/build_cache_ninja_check.rs new file mode 100644 index 0000000..0db0102 --- /dev/null +++ b/crates/coa-core/examples/build_cache_ninja_check.rs @@ -0,0 +1,58 @@ +//! Real MSVC + CMake + Ninja: after the build cache is packed and unpacked, a rebuild with nothing changed +//! compiles nothing, and a rebuild after one source changed compiles only that source. +//! usage: build_cache_ninja_check +use std::path::Path; +use std::process::Command; + +fn build(root: &Path, dir: &Path, tc: &coa_core::build_engine::ToolchainStatus) -> String { + let bat = root.join("run.bat"); + let ninja = Path::new(tc.ninja_path.as_ref().unwrap()).parent().unwrap().display().to_string(); + std::fs::write(&bat, format!( + "@echo off\r\ncall \"{}\" > nul 2>&1\r\nset \"PATH={};%PATH%\"\r\n\"{}\" -G Ninja -S \"{}\" -B \"{}\\build\" -DCMAKE_BUILD_TYPE=RelWithDebInfo > nul\r\n\"{}\" --build \"{}\\build\" -- -d explain\r\n", + tc.vcvars64_path.as_ref().unwrap(), ninja, tc.cmake_path.as_ref().unwrap(), dir.display(), dir.display(), tc.cmake_path.as_ref().unwrap(), dir.display() + )).unwrap(); + let out = Command::new("cmd.exe").args(["/c", &bat.to_string_lossy()]).output().unwrap(); + assert!(out.status.success(), "{}{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr)); + String::from_utf8_lossy(&out.stdout).into_owned() + &String::from_utf8_lossy(&out.stderr) +} + +fn compiled(log: &str) -> usize { + log.lines().filter(|l| l.contains("Building CXX object")).count() +} + +fn main() { + let root = std::path::PathBuf::from(std::env::args().nth(1).expect("usage: build_cache_ninja_check ")); + assert!(!root.exists() || std::fs::read_dir(&root).unwrap().next().is_none(), "folder must be new or empty"); + std::fs::create_dir_all(root.join(".manager")).unwrap(); + let tc = coa_core::build_engine::detect_toolchain(); + assert!(tc.msvc_available && tc.cmake_available && tc.ninja_available, "toolchain: {tc:?}"); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + std::fs::write(dir.join("CMakeLists.txt"), "cmake_minimum_required(VERSION 3.22)\nproject(p CXX)\nadd_library(p SHARED a.cpp b.cpp c.cpp)\n").unwrap(); + for f in ["a", "b", "c"] { + std::fs::write(dir.join(format!("{f}.cpp")), format!("#include \n#include \n__declspec(dllexport) int {f}() {{ return (int)std::vector(3).size(); }}\n")).unwrap(); + } + let first = build(&root, &dir, &tc); + println!("first build compiled {}", compiled(&first)); + assert_eq!(compiled(&first), 3); + + coa_core::build_cache::finish(&root, "probe"); + let s = coa_core::build_cache::summary(&root, None); + println!("packed: {} bytes from {} unpacked", s.total_bytes, s.modules[0].unpacked_bytes); + assert!(!dir.exists()); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + let again = build(&root, &dir, &tc); + println!("after unpack, nothing changed: compiled {}\n{}", compiled(&again), again.lines().filter(|l| l.contains("explain")).take(8).collect::>().join("\n")); + assert_eq!(compiled(&again), 0, "nothing should be recompiled"); + coa_core::build_cache::finish(&root, "probe"); + + let dir = coa_core::build_cache::prepare(&root, "probe", "fp").unwrap(); + std::thread::sleep(std::time::Duration::from_secs(2)); + std::fs::write(dir.join("b.cpp"), "__declspec(dllexport) int b() { return 2; }\n").unwrap(); + let one = build(&root, &dir, &tc); + println!("after one change: compiled {}", compiled(&one)); + assert_eq!(compiled(&one), 1); + coa_core::build_cache::finish(&root, "probe"); + println!("PASS"); +} diff --git a/crates/coa-core/src/analyzer.rs b/crates/coa-core/src/analyzer.rs index 327278b..df9ab97 100644 --- a/crates/coa-core/src/analyzer.rs +++ b/crates/coa-core/src/analyzer.rs @@ -1032,8 +1032,13 @@ pub fn analyze_github_repository(url_str: &str, cache_base_dir: Option<&Path>) - let marker_file = commit_cache_dir.join(".coa_tree_digest"); let cache_hit = commit_cache_dir.exists() && marker_file.exists(); + let pinned = resolved_commit.len() == 40 && resolved_commit.bytes().all(|b| b.is_ascii_hexdigit()); let target_dir = if cache_hit { commit_cache_dir + } else if pinned { + // GitHub's archive of the exact commit, without the game-client files some repositories carry (they + // are not needed to build a server module and used to push such repositories over the size limit). + crate::module_sources::fetch_pinned(&owner, &repo, &resolved_commit, &base_cache)? } else { // Clone into temporary staging directory let staging_id = format!("{}_{}_{}", owner, repo, std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis()); diff --git a/crates/coa-core/src/build_cache.rs b/crates/coa-core/src/build_cache.rs new file mode 100644 index 0000000..a3ea028 --- /dev/null +++ b/crates/coa-core/src/build_cache.rs @@ -0,0 +1,355 @@ +//! The compiler's work kept between builds of a module, small on disk. +//! +//! A build works in `/build-cache//` (sources, CMake/Ninja state, object files). Between builds +//! that folder is packed into `.tar.zst` (object files compress about 4x) with a small record of the +//! core it was made for. Before the next build it is unpacked again, so Ninja recompiles only what changed. +//! +//! Object files made for another core are useless (every one depends on the core's headers), so a cache for +//! another core is dropped instead of unpacked. The whole cache is kept under a size limit: the modules built +//! longest ago go first. Incremental-link files (`.ilk`) and linker maps are not kept. + +use std::fs; +use std::io::{BufReader, BufWriter}; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::error::{Error, Result}; + +/// Packed caches above this total are removed, the least recently built first. +pub const LIMIT_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Record { + /// ABI fingerprint of the core the objects were compiled for. + fingerprint: String, + /// Unix seconds of the last build. + last_used: i64, + unpacked_bytes: u64, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CacheEntry { + pub module: String, + /// Bytes on disk now (packed, or unpacked while a build runs). + pub bytes: u64, + pub unpacked_bytes: u64, + pub last_used: i64, + /// Made for the core this server runs now (otherwise it is dropped at the next build). + pub current_core: bool, + pub packed: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CacheSummary { + pub modules: Vec, + pub total_bytes: u64, + /// Downloaded module sources (shared by all servers of this Manager). + pub sources_bytes: u64, + pub limit_bytes: u64, +} + +pub fn root(realm_root: &Path) -> PathBuf { + crate::plugins::meta_dir(realm_root).join("build-cache") +} + +fn valid_id(id: &str) -> bool { + !id.is_empty() && id.len() <= 100 && id.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) && id != "." && id != ".." +} + +fn paths(realm_root: &Path, id: &str) -> Result<(PathBuf, PathBuf, PathBuf)> { + if !valid_id(id) { + return Err(Error::Invalid(format!("Invalid module id for the build cache: {id}"))); + } + let r = root(realm_root); + Ok((r.join(id), r.join(format!("{id}.tar.zst")), r.join(format!("{id}.json")))) +} + +fn now() -> i64 { + chrono::Utc::now().timestamp() +} + +pub(crate) fn dir_bytes(dir: &Path) -> u64 { + fs::read_dir(dir) + .map(|rd| { + rd.flatten() + .map(|e| match e.metadata() { + Ok(m) if m.is_dir() => dir_bytes(&e.path()), + Ok(m) => m.len(), + Err(_) => 0, + }) + .sum() + }) + .unwrap_or(0) +} + +fn read_record(path: &Path) -> Option { + serde_json::from_slice(&fs::read(path).ok()?).ok() +} + +/// The working folder for a build of `id` for the core `fingerprint`, with the earlier work of a build for the +/// same core unpacked into it. A cache for another core is dropped. +pub fn prepare(realm_root: &Path, id: &str, fingerprint: &str) -> Result { + let (dir, archive, record) = paths(realm_root, id)?; + let rec = read_record(&record); + let same_core = rec.as_ref().is_some_and(|r| r.fingerprint == fingerprint); + if !same_core { + // made for another core (or unknown): its objects would all be rebuilt anyway + if dir.exists() { + fs::remove_dir_all(&dir)?; + } + let _ = fs::remove_file(&archive); + let _ = fs::remove_file(&record); + } else if archive.is_file() { + if dir.exists() { + fs::remove_dir_all(&dir)?; // an older, interrupted working folder: the archive is the newer state + } + unpack(&archive, &dir).map_err(|e| { + let _ = fs::remove_dir_all(&dir); + Error::Invalid(format!("The build cache of {id} could not be unpacked: {e}")) + })?; + fs::remove_file(&archive)?; + } + fs::create_dir_all(&dir)?; + let unpacked_bytes = dir_bytes(&dir); + crate::fsx::atomic_write_json(&record, &Record { fingerprint: fingerprint.into(), last_used: now(), unpacked_bytes })?; + Ok(dir) +} + +/// After a build (successful or not): drop what does not speed up the next one, pack the rest, keep the total +/// under the limit. Never fails the build: problems only cost the cache. +pub fn finish(realm_root: &Path, id: &str) { + let Ok((dir, archive, record)) = paths(realm_root, id) else { return }; + if !dir.is_dir() { + return; + } + remove_by_ext(&dir, &["ilk", "map"]); + let unpacked_bytes = dir_bytes(&dir); + let tmp = archive.with_extension("zst.partial"); + match pack(&dir, &tmp).and_then(|_| fs::rename(&tmp, &archive).map_err(Into::into)) { + Ok(()) => { + let _ = fs::remove_dir_all(&dir); + if let Some(mut r) = read_record(&record) { + r.last_used = now(); + r.unpacked_bytes = unpacked_bytes; + let _ = crate::fsx::atomic_write_json(&record, &r); + } + } + Err(e) => { + tracing::warn!(module = id, error = %e, "build cache not packed; it is kept unpacked"); + let _ = fs::remove_file(&tmp); + } + } + enforce_limit(realm_root, LIMIT_BYTES, Some(id)); +} + +fn remove_by_ext(dir: &Path, exts: &[&str]) { + for e in fs::read_dir(dir).into_iter().flatten().flatten() { + let p = e.path(); + if p.is_dir() { + remove_by_ext(&p, exts); + } else if p.extension().and_then(|x| x.to_str()).is_some_and(|x| exts.iter().any(|w| w.eq_ignore_ascii_case(x))) { + let _ = fs::remove_file(p); + } + } +} + +fn pack(dir: &Path, out: &Path) -> Result<()> { + let file = BufWriter::new(fs::File::create(out)?); + let mut enc = zstd::Encoder::new(file, 3)?; + let _ = enc.multithread(std::thread::available_parallelism().map(|n| n.get() as u32).unwrap_or(1).min(8)); + { + let mut tar = tar::Builder::new(&mut enc); + tar.follow_symlinks(false); + // modification times are kept: Ninja decides from them what is up to date + tar.mode(tar::HeaderMode::Complete); + tar.append_dir_all(".", dir)?; + tar.finish()?; + } + enc.finish()?; + Ok(()) +} + +fn unpack(archive: &Path, dir: &Path) -> Result<()> { + fs::create_dir_all(dir)?; + let dec = zstd::Decoder::new(BufReader::new(fs::File::open(archive)?))?; + let mut tar = tar::Archive::new(dec); + tar.set_preserve_mtime(true); + for entry in tar.entries()? { + let mut entry = entry?; + // unpack_in refuses paths that would leave `dir` + if !entry.unpack_in(dir)? { + return Err(Error::PathRejected(format!("build cache entry {}", entry.path()?.display()))); + } + } + Ok(()) +} + +/// Everything in the build cache of this server, largest first. +pub fn summary(realm_root: &Path, sources_dir: Option<&Path>) -> CacheSummary { + let current = crate::provenance::realm_host_fingerprint(realm_root); + let r = root(realm_root); + let mut modules = Vec::new(); + for e in fs::read_dir(&r).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let Some(id) = name.strip_suffix(".json") else { continue }; + let Ok((dir, archive, record)) = paths(realm_root, id) else { continue }; + let Some(rec) = read_record(&record) else { continue }; + let packed = archive.is_file(); + let bytes = if packed { fs::metadata(&archive).map(|m| m.len()).unwrap_or(0) } else { dir_bytes(&dir) }; + modules.push(CacheEntry { + module: id.to_string(), + bytes, + unpacked_bytes: rec.unpacked_bytes, + last_used: rec.last_used, + current_core: current.as_deref() == Some(rec.fingerprint.as_str()), + packed, + }); + } + // working folders of an older Manager, without a record + for e in fs::read_dir(&r).into_iter().flatten().flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + if e.path().is_dir() && valid_id(&name) && !modules.iter().any(|m| m.module == name) { + let bytes = dir_bytes(&e.path()); + modules.push(CacheEntry { module: name, bytes, unpacked_bytes: bytes, last_used: 0, current_core: false, packed: false }); + } + } + modules.sort_by(|a, b| b.bytes.cmp(&a.bytes)); + CacheSummary { + total_bytes: modules.iter().map(|m| m.bytes).sum(), + sources_bytes: sources_dir.map(dir_bytes).unwrap_or(0), + limit_bytes: LIMIT_BYTES, + modules, + } +} + +/// Remove the cache of one module, or of all. The next build of a module compiles it in full. +pub fn clear(realm_root: &Path, module: Option<&str>) -> Result { + let before = summary(realm_root, None).total_bytes; + match module { + Some(id) => { + let (dir, archive, record) = paths(realm_root, id)?; + if dir.exists() { + fs::remove_dir_all(dir)?; + } + let _ = fs::remove_file(archive); + let _ = fs::remove_file(record); + } + None => { + let r = root(realm_root); + if r.exists() { + fs::remove_dir_all(&r)?; + } + } + } + Ok(before.saturating_sub(summary(realm_root, None).total_bytes)) +} + +/// Keep the packed caches under `limit`, dropping the least recently built (never `keep`). +fn enforce_limit(realm_root: &Path, limit: u64, keep: Option<&str>) { + let mut s = summary(realm_root, None); + s.modules.sort_by_key(|m| m.last_used); + let mut total = s.total_bytes; + for m in &s.modules { + if total <= limit { + break; + } + if Some(m.module.as_str()) == keep { + continue; + } + if clear(realm_root, Some(&m.module)).is_ok() { + tracing::info!(module = %m.module, bytes = m.bytes, "build cache over its limit: oldest removed"); + total = total.saturating_sub(m.bytes); + } + } +} + +/// Downloaded module sources the Manager keeps (`/github`, `/staging`). +pub fn clear_sources(sources_dir: &Path) -> Result { + let before = dir_bytes(sources_dir); + for sub in ["github", "staging"] { + let p = sources_dir.join(sub); + if p.exists() { + fs::remove_dir_all(p)?; + } + } + Ok(before.saturating_sub(dir_bytes(sources_dir))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, SystemTime}; + + fn server() -> tempfile::TempDir { + let d = tempfile::tempdir().unwrap(); + fs::create_dir_all(d.path().join(".manager")).unwrap(); + d + } + + #[test] + fn a_cache_is_packed_between_builds_and_comes_back_with_its_times() { + let s = server(); + let dir = prepare(s.path(), "companions", "fp1").unwrap(); + fs::create_dir_all(dir.join("build/obj")).unwrap(); + fs::write(dir.join("build/obj/a.obj"), vec![7u8; 100_000]).unwrap(); + fs::write(dir.join("build/x.ilk"), b"incremental link").unwrap(); + let old = SystemTime::now() - Duration::from_secs(3600); + fs::File::options().write(true).open(dir.join("build/obj/a.obj")).unwrap().set_modified(old).unwrap(); + finish(s.path(), "companions"); + assert!(!dir.exists(), "packed between builds"); + let sum = summary(s.path(), None); + assert_eq!(sum.modules.len(), 1); + assert!(sum.modules[0].packed && sum.modules[0].bytes < 100_000 / 4, "compressed: {}", sum.modules[0].bytes); + + let dir = prepare(s.path(), "companions", "fp1").unwrap(); + assert_eq!(fs::read(dir.join("build/obj/a.obj")).unwrap().len(), 100_000); + assert!(!dir.join("build/x.ilk").exists(), "incremental-link files are not kept"); + let mtime = fs::metadata(dir.join("build/obj/a.obj")).unwrap().modified().unwrap(); + let drift = mtime.duration_since(old).unwrap_or_else(|e| e.duration()); + assert!(drift < Duration::from_secs(2), "modification time kept (drift {drift:?})"); + } + + #[test] + fn a_cache_for_another_core_is_dropped() { + let s = server(); + let dir = prepare(s.path(), "scaling", "old-core").unwrap(); + fs::write(dir.join("a.obj"), b"old").unwrap(); + finish(s.path(), "scaling"); + let dir = prepare(s.path(), "scaling", "new-core").unwrap(); + assert!(!dir.join("a.obj").exists()); + } + + #[test] + fn clear_removes_one_module_or_all_and_bad_ids_are_refused() { + let s = server(); + for id in ["a", "b"] { + let d = prepare(s.path(), id, "fp").unwrap(); + fs::write(d.join("x.obj"), b"x").unwrap(); + finish(s.path(), id); + } + clear(s.path(), Some("a")).unwrap(); + assert_eq!(summary(s.path(), None).modules.iter().map(|m| m.module.as_str()).collect::>(), ["b"]); + clear(s.path(), None).unwrap(); + assert!(summary(s.path(), None).modules.is_empty()); + assert!(clear(s.path(), Some("../x")).is_err()); + } + + #[test] + fn the_least_recently_built_go_first_over_the_limit() { + let s = server(); + for (n, id) in ["old", "new"].iter().enumerate() { + let d = prepare(s.path(), id, "fp").unwrap(); + fs::write(d.join("x.obj"), vec![n as u8; 50_000]).unwrap(); + finish(s.path(), id); + let rec = root(s.path()).join(format!("{id}.json")); + let mut r = read_record(&rec).unwrap(); + r.last_used = 1000 + n as i64; + crate::fsx::atomic_write_json(&rec, &r).unwrap(); + } + let total = summary(s.path(), None).total_bytes; + enforce_limit(s.path(), total - 1, None); + let left: Vec = summary(s.path(), None).modules.into_iter().map(|m| m.module).collect(); + assert_eq!(left, ["new"]); + } +} diff --git a/crates/coa-core/src/build_engine.rs b/crates/coa-core/src/build_engine.rs index 83e3b62..dd1072b 100644 --- a/crates/coa-core/src/build_engine.rs +++ b/crates/coa-core/src/build_engine.rs @@ -977,8 +977,25 @@ pub fn build_module_with_bundle( .unwrap_or_else(|| with_plugin_id(&adapter_source, &plugin_id)); // 2. Prepare the persistent staging directory - let default_staging = realm_root.join(".manager").join("build-cache").join(&module_id); - let staging_dir = custom_staging_dir.map(PathBuf::from).unwrap_or(default_staging); + // The Manager's own builds use the module's build cache (unpacked for the build, packed again afterwards); + // a caller-chosen folder is used as it is. + let staging_dir = match custom_staging_dir { + Some(p) => PathBuf::from(p), + None => crate::build_cache::prepare(realm_root, &module_id, &sdk_abi_fingerprint(Path::new(&sdk_dir))?)?, + }; + struct PackAfter<'a>(Option<(&'a Path, String)>); + impl Drop for PackAfter<'_> { + fn drop(&mut self) { + if let Some((root, id)) = self.0.take() { + crate::build_cache::finish(root, &id); + } + } + } + let _pack_after = PackAfter(custom_staging_dir.is_none().then(|| (realm_root, module_id.clone()))); + let log_path = match custom_staging_dir { + Some(_) => staging_dir.join("build.log"), + None => crate::build_cache::root(realm_root).join(format!("{module_id}.log")), + }; fs::create_dir_all(&staging_dir)?; let staging_src = staging_dir.join("src"); fs::create_dir_all(&staging_src)?; @@ -1142,11 +1159,11 @@ if errorlevel 1 exit /b 103 } // The full log stays next to the build, so a failure can always be read afterwards. - let _ = fs::write(staging_dir.join("build.log"), all_logs.join(" + let _ = fs::write(&log_path, all_logs.join(" ")); let mut errors = parse_msvc_errors(&all_logs); if exit_code != 0 && errors.is_empty() { - errors.push(unparsed_failure(exit_code, &all_logs, &staging_dir.join("build.log"))); + errors.push(unparsed_failure(exit_code, &all_logs, &log_path)); } let expected_dll_path = build_output_dir.join(format!("{}.dll", safe_target_name)); diff --git a/crates/coa-core/src/coa-modules-overlay.json b/crates/coa-core/src/coa-modules-overlay.json index b401041..1dae9f5 100644 --- a/crates/coa-core/src/coa-modules-overlay.json +++ b/crates/coa-core/src/coa-modules-overlay.json @@ -62,6 +62,24 @@ "is_recommended": false, "reason": "Already built into the CoA core (cross-faction battlegrounds); a separate DLL would load it a second time." }, + { + "repo_slug": "helias/mod-bg-battle-for-gilneas", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (including Spell.dbc, which CoA ships its own of), a client patch and a modified wow.exe; written for AzerothCore 2019." + }, + { + "repo_slug": "helias/mod-bg-twinpeaks", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (CoA ships its own), a client patch and a modified wow.exe." + }, + { + "repo_slug": "helias/mod-bg-slaveryvalley", + "status": "Incompatible", + "is_recommended": false, + "reason": "Needs the server's DBC files replaced (CoA ships its own) and a client patch." + }, { "repo_slug": "azerothcore/mod-learn-spells", "status": "Incompatible", diff --git a/crates/coa-core/src/lib.rs b/crates/coa-core/src/lib.rs index c07e8bb..80625e8 100644 --- a/crates/coa-core/src/lib.rs +++ b/crates/coa-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod accounts; pub mod allsettings; pub mod analyzer; pub mod backup; +pub mod build_cache; pub mod build_engine; pub mod catalogue; pub mod channels; diff --git a/crates/coa-core/src/module_sources.rs b/crates/coa-core/src/module_sources.rs index f2e6fe6..7476c29 100644 --- a/crates/coa-core/src/module_sources.rs +++ b/crates/coa-core/src/module_sources.rs @@ -61,10 +61,24 @@ pub fn fetch_pinned(owner: &str, repo: &str, sha: &str, cache_root: &Path) -> Re Ok(dest) } +/// Game-client and media files a server module's build never reads (client patches, DBC/map data, textures, +/// archives). Some repositories carry hundreds of MB of them next to a few KB of server code; they are left out +/// of the sources, and their names listed in `.coa_skipped_assets`. +fn is_client_asset(rel: &Path) -> bool { + const EXT: [&str; 22] = [ + "mpq", "dbc", "db2", "adt", "wdt", "wdl", "wmo", "m2", "skin", "anim", "blp", "tga", "trs", "vmtile", "vmtree", + "mmap", "mmtile", "zip", "7z", "rar", "mp3", "ogg", + ]; + rel.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| EXT.iter().any(|x| x.eq_ignore_ascii_case(e))) +} + /// Extracts a GitHub tar.gz (one top-level `-/` folder) into `dest`, files and folders only. -fn unpack(reader: impl Read, dest: &Path) -> Result<()> { +pub(crate) fn unpack(reader: impl Read, dest: &Path) -> Result<()> { let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(reader)); let mut total = 0u64; + let mut skipped = Vec::new(); for entry in archive.entries()? { let mut entry = entry?; let path = entry.path()?.into_owned(); @@ -79,6 +93,8 @@ fn unpack(reader: impl Read, dest: &Path) -> Result<()> { let target = dest.join(&rel); if kind.is_dir() { fs::create_dir_all(&target)?; + } else if kind.is_file() && is_client_asset(&rel) { + skipped.push(rel.to_string_lossy().replace('\\', "/")); } else if kind.is_file() { total += entry.header().size()?; if total > MAX_ARCHIVE_BYTES { @@ -90,6 +106,9 @@ fn unpack(reader: impl Read, dest: &Path) -> Result<()> { } // Links and special entries are not part of a module's sources: skipped. } + if !skipped.is_empty() { + fs::write(dest.join(".coa_skipped_assets"), skipped.join("\n"))?; + } Ok(()) } @@ -118,6 +137,24 @@ mod tests { assert!(dir.path().join("conf/x.conf.dist").is_file()); } + #[test] + fn client_assets_are_left_out_and_listed() { + let dir = tempfile::tempdir().unwrap(); + let gz = tar_gz(&[ + ("mod-bg-abc/server-side/src/BG.cpp", b"int a;"), + ("mod-bg-abc/client-side/DBFilesClient/Spell.dbc", b"dbc"), + ("mod-bg-abc/client-data.zip", b"zip"), + ("mod-bg-abc/data/sql/db-world/bg.sql", b"SELECT 1;"), + ]); + unpack(&gz[..], dir.path()).unwrap(); + assert!(dir.path().join("server-side/src/BG.cpp").is_file()); + assert!(dir.path().join("data/sql/db-world/bg.sql").is_file()); + assert!(!dir.path().join("client-side/DBFilesClient/Spell.dbc").exists()); + assert!(!dir.path().join("client-data.zip").exists()); + let listed = fs::read_to_string(dir.path().join(".coa_skipped_assets")).unwrap(); + assert!(listed.contains("Spell.dbc") && listed.contains("client-data.zip")); + } + #[test] fn bad_names_are_refused_before_any_download() { let dir = tempfile::tempdir().unwrap(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 4829739..4929f0c 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -3083,6 +3083,42 @@ async fn analyze_github_module( .await } +/// What the module build cache of a server (and the shared source downloads) take on disk. +#[tauri::command] +async fn build_cache_summary( + state: State<'_, AppState>, + id: String, +) -> std::result::Result { + let root = path_of(&state, &id)?; + let sources = data_dir().join("github_cache"); + blocking(move || Ok(coa_core::build_cache::summary(&root, Some(&sources)))).await +} + +/// Remove the build cache of one module (`module`), of all modules (no `module`), and optionally the downloaded +/// sources. Refused while a build runs on this server. Returns the bytes freed. +#[tauri::command] +async fn build_cache_clear( + state: State<'_, AppState>, + id: String, + module: Option, + sources: bool, +) -> std::result::Result { + let root = path_of(&state, &id)?; + let _guard = BusyGuard::acquire(&state, &id)?; + let sources_dir = data_dir().join("github_cache"); + blocking(move || { + let mut freed = 0; + if module.is_some() || !sources { + freed += coa_core::build_cache::clear(&root, module.as_deref())?; + } + if sources { + freed += coa_core::build_cache::clear_sources(&sources_dir)?; + } + Ok(freed) + }) + .await +} + #[tauri::command] async fn get_public_module_catalog( force_refresh: Option, @@ -3131,6 +3167,8 @@ async fn preview_build_module( let realm_root = path_of(&state, &realm_id)?; let cache_dir = data_dir().join("github_cache"); + // The build works in the module's build cache: clearing the cache waits for it. + let _guard = BusyGuard::acquire(&state, &realm_id)?; let app_clone = app.clone(); let progress_cb: Box = @@ -3594,6 +3632,8 @@ pub fn run() { account_delete, install_new, is_classic_server, + build_cache_summary, + build_cache_clear, cancel_install, create_account, check_update, diff --git a/src/lib/api.ts b/src/lib/api.ts index d9c5d66..2494f0b 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -479,6 +479,22 @@ export interface AccountInfo { characters: number; } +export interface BuildCacheEntry { + module: string; + bytes: number; + unpacked_bytes: number; + last_used: number; + current_core: boolean; + packed: boolean; +} + +export interface BuildCacheSummary { + modules: BuildCacheEntry[]; + total_bytes: number; + sources_bytes: number; + limit_bytes: number; +} + export interface CopyReport { schemas: string[]; accounts: number; @@ -818,6 +834,9 @@ export const api = { installNew: (dest: string, pkg?: string, gameData?: string, copyFrom?: string) => invoke("install_new", { dest, package: pkg ?? null, gameData: gameData ?? null, copyFrom: copyFrom ?? null }), isClassicServer: (path: string) => invoke("is_classic_server", { path }), + buildCacheSummary: (id: string) => invoke("build_cache_summary", { id }), + buildCacheClear: (id: string, module: string | null, sources: boolean) => + invoke("build_cache_clear", { id, module, sources }), cancelInstall: () => invoke("cancel_install"), createAccount: (id: string, username: string, password: string, administrator: boolean) => invoke("create_account", { id, username, password, administrator }), diff --git a/src/screens/BuildCacheDialog.tsx b/src/screens/BuildCacheDialog.tsx new file mode 100644 index 0000000..1aaf7f5 --- /dev/null +++ b/src/screens/BuildCacheDialog.tsx @@ -0,0 +1,121 @@ +import { useCallback, useEffect, useState } from "react"; +import { Loader2, Trash2, X } from "lucide-react"; +import { api, asUiError, type BuildCacheSummary } from "@/lib/api"; +import { formatBytes } from "@/lib/utils"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; + +/** What the module build cache takes, with removal of one module, all modules, or the downloaded sources. */ +export function BuildCacheDialog(props: { serverId: string; locale: string; onClose: () => void }) { + const ru = props.locale === "ru"; + const [summary, setSummary] = useState(null); + const [busy, setBusy] = useState(null); + const [message, setMessage] = useState(null); + const [error, setError] = useState(null); + + const load = useCallback(() => { + void api + .buildCacheSummary(props.serverId) + .then(setSummary) + .catch((e) => setError(asUiError(e).technical)); + }, [props.serverId]); + useEffect(load, [load]); + + async function clear(key: string, module: string | null, sources: boolean) { + setBusy(key); + setError(null); + try { + const freed = await api.buildCacheClear(props.serverId, module, sources); + setMessage(ru ? `Освобождено ${formatBytes(freed)}` : `Freed ${formatBytes(freed)}`); + load(); + } catch (e) { + setError(asUiError(e).technical); + } finally { + setBusy(null); + } + } + + const when = (secs: number) => + secs > 0 ? new Date(secs * 1000).toLocaleDateString(ru ? "ru-RU" : undefined, { day: "numeric", month: "short" }) : "—"; + + return ( +
+ +
+
+

{ru ? "Кэш сборок" : "Build cache"}

+

+ {ru + ? "Скомпилированные части модулей хранятся сжатыми и ускоряют повторную сборку. После обновления ядра старый кэш удаляется сам. Удаление ничего не ломает: следующая сборка модуля будет полной." + : "Compiled parts of modules are kept compressed and speed up the next build. After a core update the old cache goes by itself. Removing it breaks nothing: the next build of a module is a full one."} +

+
+ +
+ + {!summary && !error && } + + {summary && ( + <> +
+ {summary.modules.length === 0 &&

{ru ? "Кэш пуст." : "The cache is empty."}

} + {summary.modules.map((m) => ( +
+
+

{m.module}

+

+ {m.packed + ? ru ? `сжато из ${formatBytes(m.unpacked_bytes)}` : `packed from ${formatBytes(m.unpacked_bytes)}` + : ru ? "не сжато" : "not packed"} + {" · "} + {ru ? "сборка " : "built "} + {when(m.last_used)} + {!m.current_core && (ru ? " · для другого ядра" : " · for another core")} +

+
+ {formatBytes(m.bytes)} + +
+ ))} +
+ +
+ + {ru ? "Скачанные исходники модулей" : "Downloaded module sources"}: {formatBytes(summary.sources_bytes)} + + +
+ +
+ + {ru ? "Всего" : "Total"}: {formatBytes(summary.total_bytes)} + {ru ? `, предел ${formatBytes(summary.limit_bytes)}` : `, limit ${formatBytes(summary.limit_bytes)}`} + + +
+ + )} + + {message &&

{message}

} + {error &&

{error}

} +
+
+ ); +} diff --git a/src/screens/ModulesPage.tsx b/src/screens/ModulesPage.tsx index ee0af4e..eb17604 100644 --- a/src/screens/ModulesPage.tsx +++ b/src/screens/ModulesPage.tsx @@ -57,6 +57,8 @@ import { useHuman, useI18n, useSchemaText, type Key } from "@/i18n"; import { auctionWeightKey, canonicalModuleValue, moduleEnableKey, moduleField, retiredModuleKeys } from "@/lib/moduleSettings"; import { AuctionTypes } from "@/screens/AuctionTypes"; import { RaceAvailabilitySettings } from "@/screens/RaceAvailabilitySettings"; +import { BuildCacheDialog } from "@/screens/BuildCacheDialog"; +import { formatBytes } from "@/lib/utils"; import type { UnsavedChangesGuard } from "@/lib/unsavedGuard"; /** The GitHub mark */ @@ -639,6 +641,13 @@ export function ModulesPage({ // "Add from a link" stays folded away until asked for; the menu holds the rarely used build tools. const [showUrlBar, setShowUrlBar] = useState(false); const [menuOpen, setMenuOpen] = useState(false); + const [cacheOpen, setCacheOpen] = useState(false); + const [cacheBytes, setCacheBytes] = useState(null); + useEffect(() => { + if (menuOpen) { + void api.buildCacheSummary(serverId).then((s) => setCacheBytes(s.total_bytes + s.sources_bytes)).catch(() => setCacheBytes(null)); + } + }, [menuOpen, serverId]); const [analyzing, setAnalyzing] = useState(false); const [analysisError, setAnalysisError] = useState(null); const [report, setReport] = useState(null); @@ -1234,6 +1243,17 @@ export function ModulesPage({ {tab === "builds" && building && } ))} +