Phase 12: control protocol crate (Noise channel, coordinator frames, app messages) and Coordinator service

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
ДмитрийandClaude Sonnet 5.5 committed 2026-10-07 22:12:57 +05:00
1 parent 95ab5dd9ef
commit 6fa3c9ccf4
14 files changed
+3982 -2

No files matched your search

+1873
View File
File diff suppressed because it is too large. Load diff
+36
View File
@@ -0,0 +1,36 @@
[package]
name = "coa-coordinator"
description = "CoA control-plane Coordinator (Phase 12): routes end-to-end encrypted control frames between a Host Manager and Player Managers. Never reads them."
version = "0.1.0"
edition = "2021"
license = "AGPL-3.0-only"
publish = false
[workspace]
[dependencies]
coa-control-proto = { path = "../../crates/coa-control-proto" }
coa-registry-proto = { path = "../../crates/coa-registry-proto" }
axum = { version = "0.8", default-features = false, features = ["http1", "json", "tokio", "ws", "query"] }
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal", "time", "net", "sync"] }
tower = { version = "0.5", features = ["limit", "util"] }
tokio-postgres = "0.7"
deadpool-postgres = "0.14"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
ed25519-dalek = "2"
uuid = { version = "1", features = ["v4"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
hex = "0.4"
pin-project-lite = "0.2"
[dev-dependencies]
tungstenite = "0.26"
tempfile = "3"
uuid = { version = "1", features = ["v4", "v7"] }
[profile.release]
lto = "thin"
codegen-units = 4
strip = true
+469
View File
@@ -0,0 +1,469 @@
//! The Coordinator: control-plane routing between Host Managers and Player Managers (Phase 12). **It never reads what it routes.**
//!
//! A Host Manager keeps one outbound WebSocket to `/coord/v1/host` and proves it holds the key the Registry publishes for its realm. A Player Manager opens
//! `/coord/v1/player?realm=<id>` and proves it holds its own player key. After that both sides run a Noise channel through the Coordinator; the Coordinator only
//! sees the realm id, a connection number, sizes and times. Everything it could be asked to store is in memory, bounded, and gone when the connection ends.
//! It is not the Relay: it carries no game traffic.
pub mod limits;
pub mod listener;
use std::collections::HashMap;
use std::future::Future;
use std::net::{IpAddr, SocketAddr};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
use axum::extract::{ConnectInfo, Query, State};
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::{Json, Router};
use coa_control_proto::coord::{self, limits as lim, ErrorCode, Frame};
use coa_registry_proto::RealmId;
use ed25519_dalek::VerifyingKey;
use serde::Deserialize;
use tokio::sync::mpsc;
use limits::{Limiter, Rate};
use listener::Peer;
/// Where a realm's Host key comes from: the Registry's table (the key it was registered with).
pub trait KeyLookup: Send + Sync + 'static {
fn host_key(&self, realm: &RealmId) -> impl Future<Output = Option<VerifyingKey>> + Send;
}
#[derive(Default)]
pub struct MemoryKeys(Mutex<HashMap<RealmId, [u8; 32]>>);
impl MemoryKeys {
pub fn set(&self, realm: RealmId, key: &VerifyingKey) {
self.0.lock().unwrap().insert(realm, key.to_bytes());
}
}
impl KeyLookup for MemoryKeys {
async fn host_key(&self, realm: &RealmId) -> Option<VerifyingKey> {
self.0.lock().unwrap().get(realm).and_then(|b| VerifyingKey::from_bytes(b).ok())
}
}
impl<T: KeyLookup> KeyLookup for Arc<T> {
fn host_key(&self, realm: &RealmId) -> impl Future<Output = Option<VerifyingKey>> + Send {
(**self).host_key(realm)
}
}
pub struct PgKeys {
pool: deadpool_postgres::Pool,
}
impl PgKeys {
pub fn new(pool: deadpool_postgres::Pool) -> Self {
Self { pool }
}
}
impl KeyLookup for PgKeys {
async fn host_key(&self, realm: &RealmId) -> Option<VerifyingKey> {
let c = self.pool.get().await.ok()?;
let row = c.query_opt("SELECT public_key FROM realms WHERE realm_id = $1::text::uuid AND published AND advert_version = 2", &[&realm.to_string()]).await.ok()??;
let bytes: Vec<u8> = row.get(0);
VerifyingKey::from_bytes(&bytes.try_into().ok()?).ok()
}
}
#[derive(Clone, Debug)]
pub struct Config {
pub trust_proxy: bool,
pub per_host_connections: usize,
pub per_ip_connections: usize,
pub per_realm: Rate,
pub per_ip: Rate,
pub bytes_per_connection: usize,
pub lifetime: Duration,
pub idle: Duration,
pub host_answer: Duration,
pub hello_timeout: Duration,
}
impl Default for Config {
fn default() -> Self {
Self {
trust_proxy: false,
per_host_connections: lim::PER_HOST_CONNECTIONS,
per_ip_connections: lim::PER_IP_CONNECTIONS,
per_realm: Rate::per_minute(lim::PER_REALM_PER_MINUTE, f64::from(lim::PER_REALM_PER_MINUTE)),
per_ip: Rate::per_minute(lim::PER_IP_PER_MINUTE, f64::from(lim::PER_IP_PER_MINUTE)),
bytes_per_connection: lim::BYTES_PER_CONNECTION,
lifetime: Duration::from_secs(lim::CONNECTION_LIFETIME_SECS),
idle: Duration::from_secs(lim::IDLE_SECS),
host_answer: Duration::from_secs(lim::HOST_ANSWER_SECS),
hello_timeout: Duration::from_secs(coord::HELLO_TIMEOUT_SECS),
}
}
}
enum HostMsg {
Text(String),
Binary(Vec<u8>),
Stop,
}
enum PlayerMsg {
Binary(Vec<u8>),
Close(Option<String>),
}
struct Link {
tx: mpsc::Sender<PlayerMsg>,
started: Instant,
last: Mutex<Instant>,
bytes: AtomicUsize,
answered: AtomicBool,
}
struct HostEntry {
generation: u64,
tx: mpsc::Sender<HostMsg>,
links: Mutex<HashMap<u32, Arc<Link>>>,
next_conn: AtomicU32,
}
pub struct Hub<K: KeyLookup> {
keys: K,
cfg: Config,
clock: Arc<dyn Fn() -> i64 + Send + Sync>,
hosts: Mutex<HashMap<RealmId, Arc<HostEntry>>>,
ip_open: Mutex<HashMap<IpAddr, usize>>,
ip_rate: Limiter<IpAddr>,
ip_hello: Limiter<IpAddr>,
realm_rate: Limiter<RealmId>,
generation: AtomicU64,
started: Instant,
}
impl<K: KeyLookup> Hub<K> {
pub fn new(keys: K, cfg: Config, clock: Arc<dyn Fn() -> i64 + Send + Sync>) -> Arc<Self> {
Arc::new(Self {
keys,
ip_rate: Limiter::new(cfg.per_ip, 100_000),
ip_hello: Limiter::new(Rate::per_minute(60, 60.0), 100_000),
realm_rate: Limiter::new(cfg.per_realm, 100_000),
cfg,
clock,
hosts: Mutex::new(HashMap::new()),
ip_open: Mutex::new(HashMap::new()),
generation: AtomicU64::new(1),
started: Instant::now(),
})
}
fn mono(&self) -> f64 {
self.started.elapsed().as_secs_f64()
}
pub fn connected_hosts(&self) -> usize {
self.hosts.lock().map(|h| h.len()).unwrap_or(0)
}
}
/// Counts a connection against its address for as long as it lives.
struct IpGuard<K: KeyLookup> {
hub: Arc<Hub<K>>,
ip: IpAddr,
}
impl<K: KeyLookup> IpGuard<K> {
fn take(hub: &Arc<Hub<K>>, ip: IpAddr) -> Option<Self> {
let mut open = hub.ip_open.lock().ok()?;
let n = open.entry(ip).or_insert(0);
if *n >= hub.cfg.per_ip_connections {
return None;
}
*n += 1;
Some(Self { hub: hub.clone(), ip })
}
}
impl<K: KeyLookup> Drop for IpGuard<K> {
fn drop(&mut self) {
if let Ok(mut open) = self.hub.ip_open.lock() {
if let Some(n) = open.get_mut(&self.ip) {
*n = n.saturating_sub(1);
if *n == 0 {
open.remove(&self.ip);
}
}
}
}
}
fn client_ip(headers: &HeaderMap, peer: SocketAddr, trust_proxy: bool) -> IpAddr {
if trust_proxy {
if let Some(ip) = headers.get("x-forwarded-for").and_then(|v| v.to_str().ok()).and_then(|v| v.rsplit(',').next()).and_then(|v| v.trim().parse::<IpAddr>().ok()) {
return ip;
}
}
peer.ip()
}
pub fn router<K: KeyLookup>(hub: Arc<Hub<K>>) -> Router {
Router::new()
.route("/coord/v1/health", get(health::<K>))
.route("/coord/v1/host", get(host_ws::<K>))
.route("/coord/v1/player", get(player_ws::<K>))
.fallback(|| async { (StatusCode::NOT_FOUND, "not found") })
.with_state(hub)
}
async fn health<K: KeyLookup>(State(hub): State<Arc<Hub<K>>>) -> Response {
Json(serde_json::json!({ "status": "ok", "protocol_version": coa_control_proto::CONTROL_PROTOCOL_VERSION, "hosts": hub.connected_hosts() })).into_response()
}
async fn host_ws<K: KeyLookup>(State(hub): State<Arc<Hub<K>>>, ConnectInfo(Peer(peer)): ConnectInfo<Peer>, headers: HeaderMap, ws: WebSocketUpgrade) -> Response {
let ip = client_ip(&headers, peer, hub.cfg.trust_proxy);
if !hub.ip_hello.take(&ip, 1.0, hub.mono()) {
return (StatusCode::TOO_MANY_REQUESTS, "too many connections").into_response();
}
let Some(guard) = IpGuard::take(&hub, ip) else { return (StatusCode::TOO_MANY_REQUESTS, "too many connections").into_response() };
ws.max_message_size(coord::CONN_PREFIX + coord::MAX_FRAME_BYTES).max_frame_size(coord::CONN_PREFIX + coord::MAX_FRAME_BYTES).on_upgrade(move |socket| async move {
run_host(hub, socket, ip).await;
drop(guard);
})
}
#[derive(Deserialize)]
struct PlayerQuery {
realm: String,
}
async fn player_ws<K: KeyLookup>(State(hub): State<Arc<Hub<K>>>, ConnectInfo(Peer(peer)): ConnectInfo<Peer>, headers: HeaderMap, Query(q): Query<PlayerQuery>, ws: WebSocketUpgrade) -> Response {
let ip = client_ip(&headers, peer, hub.cfg.trust_proxy);
let Ok(realm) = RealmId::parse(&q.realm) else { return (StatusCode::BAD_REQUEST, "bad realm").into_response() };
if !hub.ip_hello.take(&ip, 1.0, hub.mono()) {
return (StatusCode::TOO_MANY_REQUESTS, "too many connections").into_response();
}
let Some(guard) = IpGuard::take(&hub, ip) else { return (StatusCode::TOO_MANY_REQUESTS, "too many connections").into_response() };
ws.max_message_size(coord::CONN_PREFIX + coord::MAX_FRAME_BYTES).max_frame_size(coord::CONN_PREFIX + coord::MAX_FRAME_BYTES).on_upgrade(move |socket| async move {
run_player(hub, socket, ip, realm).await;
drop(guard);
})
}
async fn send_frame(sock: &mut WebSocket, f: &Frame) -> bool {
sock.send(Message::Text(coord::to_text(f).into())).await.is_ok()
}
async fn refuse(sock: &mut WebSocket, code: ErrorCode, message: &str) {
let _ = send_frame(sock, &Frame::Error { code, message: message.to_string() }).await;
let _ = sock.send(Message::Close(None)).await;
// let the peer read the error and answer the close; dropping at once can reset the connection and discard the error before it is read
let _ = tokio::time::timeout(Duration::from_millis(500), async { while let Some(Ok(_)) = sock.recv().await {} }).await;
}
/// The next text frame within `timeout` (pings and pongs are the transport's business).
async fn next_text(sock: &mut WebSocket, timeout: Duration) -> Option<Frame> {
let end = Instant::now() + timeout;
loop {
let left = end.checked_duration_since(Instant::now())?;
match tokio::time::timeout(left, sock.recv()).await {
Ok(Some(Ok(Message::Text(t)))) => return coord::parse_text(t.as_str()).ok(),
Ok(Some(Ok(Message::Ping(_) | Message::Pong(_)))) => continue,
_ => return None,
}
}
}
async fn run_host<K: KeyLookup>(hub: Arc<Hub<K>>, mut sock: WebSocket, ip: IpAddr) {
let nonce = coord::challenge_nonce();
if !send_frame(&mut sock, &Frame::Challenge { protocol: coa_control_proto::CONTROL_PROTOCOL_VERSION, nonce: nonce.clone() }).await {
return;
}
let Some(hello) = next_text(&mut sock, hub.cfg.hello_timeout).await else { return refuse(&mut sock, ErrorCode::Timeout, "no valid hello").await };
let Frame::HostHello { realm_id: realm, .. } = &hello else { return refuse(&mut sock, ErrorCode::BadHello, "a host hello was expected").await };
let realm = *realm;
let Some(key) = hub.keys.host_key(&realm).await else { return refuse(&mut sock, ErrorCode::UnknownRealm, "this realm is not published").await };
if coord::verify_host_hello(&hello, &key, None, &nonce, (hub.clock)()).is_err() {
tracing::warn!(%realm, %ip, "host hello refused");
return refuse(&mut sock, ErrorCode::BadHello, "the hello does not prove the realm key").await;
}
let (tx, mut rx) = mpsc::channel::<HostMsg>(256);
let entry = Arc::new(HostEntry { generation: hub.generation.fetch_add(1, Ordering::SeqCst), tx, links: Mutex::new(HashMap::new()), next_conn: AtomicU32::new(1) });
if let Some(old) = hub.hosts.lock().ok().and_then(|mut h| h.insert(realm, entry.clone())) {
let _ = old.tx.try_send(HostMsg::Stop);
}
if !send_frame(&mut sock, &Frame::HostReady { max_connections: hub.cfg.per_host_connections as u32, max_frame: coord::MAX_FRAME_BYTES as u32 }).await {
remove_host(&hub, &realm, &entry);
return;
}
tracing::info!(%realm, %ip, "host connected");
let mut tick = tokio::time::interval(Duration::from_secs(5));
loop {
tokio::select! {
incoming = sock.recv() => match incoming {
Some(Ok(Message::Binary(b))) => {
let Ok((conn, frame)) = coord::split_binary(&b) else { break };
let link = entry.links.lock().ok().and_then(|l| l.get(&conn).cloned());
let Some(link) = link else { continue };
if link.bytes.fetch_add(frame.len(), Ordering::Relaxed) + frame.len() > hub.cfg.bytes_per_connection {
let _ = link.tx.try_send(PlayerMsg::Close(Some("byte limit".into())));
continue;
}
link.answered.store(true, Ordering::Relaxed);
if let Ok(mut l) = link.last.lock() { *l = Instant::now(); }
if link.tx.send(PlayerMsg::Binary(frame.to_vec())).await.is_err() {
if let Ok(mut l) = entry.links.lock() { l.remove(&conn); }
}
}
Some(Ok(Message::Text(t))) => match coord::parse_text(t.as_str()) {
Ok(Frame::Close { conn, reason }) => {
let link = entry.links.lock().ok().and_then(|mut l| l.remove(&conn));
if let Some(link) = link { let _ = link.tx.try_send(PlayerMsg::Close(reason)); }
}
_ => break,
},
Some(Ok(Message::Ping(_) | Message::Pong(_))) => {}
_ => break,
},
out = rx.recv() => match out {
Some(HostMsg::Text(t)) => if sock.send(Message::Text(t.into())).await.is_err() { break },
Some(HostMsg::Binary(b)) => if sock.send(Message::Binary(b.into())).await.is_err() { break },
Some(HostMsg::Stop) | None => break,
},
_ = tick.tick() => {
if sock.send(Message::Ping(Vec::new().into())).await.is_err() { break }
expire(&hub, &entry);
}
}
}
remove_host(&hub, &realm, &entry);
tracing::info!(%realm, "host disconnected");
}
fn remove_host<K: KeyLookup>(hub: &Hub<K>, realm: &RealmId, entry: &Arc<HostEntry>) {
if let Ok(mut hosts) = hub.hosts.lock() {
if hosts.get(realm).is_some_and(|e| e.generation == entry.generation) {
hosts.remove(realm);
}
}
let links: Vec<Arc<Link>> = entry.links.lock().map(|mut l| l.drain().map(|(_, v)| v).collect()).unwrap_or_default();
for l in links {
let _ = l.tx.try_send(PlayerMsg::Close(Some("host offline".into())));
}
}
/// Close connections that are too old, silent too long, or whose Host never answered.
fn expire<K: KeyLookup>(hub: &Hub<K>, entry: &HostEntry) {
let now = Instant::now();
let dead: Vec<(u32, &'static str)> = entry
.links
.lock()
.map(|l| {
l.iter()
.filter_map(|(conn, link)| {
let last = link.last.lock().map(|t| *t).unwrap_or(now);
if now.duration_since(link.started) > hub.cfg.lifetime {
Some((*conn, "lifetime"))
} else if now.duration_since(last) > hub.cfg.idle {
Some((*conn, "idle"))
} else if !link.answered.load(Ordering::Relaxed) && now.duration_since(link.started) > hub.cfg.host_answer {
Some((*conn, "host did not answer"))
} else {
None
}
})
.collect()
})
.unwrap_or_default();
for (conn, why) in dead {
let link = entry.links.lock().ok().and_then(|mut l| l.remove(&conn));
if let Some(link) = link {
let _ = link.tx.try_send(PlayerMsg::Close(Some(why.into())));
}
let _ = entry.tx.try_send(HostMsg::Text(coord::to_text(&Frame::Close { conn, reason: Some(why.into()) })));
}
}
async fn run_player<K: KeyLookup>(hub: Arc<Hub<K>>, mut sock: WebSocket, ip: IpAddr, realm: RealmId) {
let nonce = coord::challenge_nonce();
if !send_frame(&mut sock, &Frame::Challenge { protocol: coa_control_proto::CONTROL_PROTOCOL_VERSION, nonce: nonce.clone() }).await {
return;
}
let Some(hello) = next_text(&mut sock, hub.cfg.hello_timeout).await else { return refuse(&mut sock, ErrorCode::Timeout, "no valid hello").await };
let Ok((player, hello_realm, _key)) = coord::verify_player_hello(&hello, &nonce, (hub.clock)()) else { return refuse(&mut sock, ErrorCode::BadHello, "the hello does not prove the player key").await };
if hello_realm != realm {
return refuse(&mut sock, ErrorCode::BadHello, "the hello is for another realm").await;
}
if !hub.ip_rate.take(&ip, 1.0, hub.mono()) || !hub.realm_rate.take(&realm, 1.0, hub.mono()) {
return refuse(&mut sock, ErrorCode::RateLimited, "too many connections").await;
}
let entry = hub.hosts.lock().ok().and_then(|h| h.get(&realm).cloned());
let Some(entry) = entry else { return refuse(&mut sock, ErrorCode::HostOffline, "the realm's Manager is not connected").await };
let (ptx, mut prx) = mpsc::channel::<PlayerMsg>(64);
let link = Arc::new(Link { tx: ptx, started: Instant::now(), last: Mutex::new(Instant::now()), bytes: AtomicUsize::new(0), answered: AtomicBool::new(false) });
let slot = take_slot(&hub, &entry, &link);
let Some(conn) = slot else { return refuse(&mut sock, ErrorCode::HostBusy, "the realm's Manager is busy").await };
if entry.tx.send(HostMsg::Text(coord::to_text(&Frame::Open { conn }))).await.is_err() {
if let Ok(mut l) = entry.links.lock() {
l.remove(&conn);
}
return refuse(&mut sock, ErrorCode::HostOffline, "the realm's Manager is not connected").await;
}
tracing::info!(%realm, conn, %player, %ip, "player connected");
if !send_frame(&mut sock, &Frame::PlayerReady { conn, max_frame: coord::MAX_FRAME_BYTES as u32 }).await {
end_link(&entry, conn).await;
return;
}
loop {
tokio::select! {
incoming = sock.recv() => match incoming {
Some(Ok(Message::Binary(b))) => {
if b.len() > coord::MAX_FRAME_BYTES || link.bytes.fetch_add(b.len(), Ordering::Relaxed) + b.len() > hub.cfg.bytes_per_connection {
refuse(&mut sock, ErrorCode::TooLarge, "a limit was exceeded").await;
break;
}
if let Ok(mut l) = link.last.lock() { *l = Instant::now(); }
if entry.tx.send(HostMsg::Binary(coord::binary(conn, &b))).await.is_err() { break }
}
Some(Ok(Message::Ping(_) | Message::Pong(_))) => {}
Some(Ok(Message::Text(t))) => {
if !matches!(coord::parse_text(t.as_str()), Ok(Frame::Close { .. })) { break }
break;
}
_ => break,
},
out = prx.recv() => match out {
Some(PlayerMsg::Binary(b)) => if sock.send(Message::Binary(b.into())).await.is_err() { break },
Some(PlayerMsg::Close(reason)) => {
let code = if reason.as_deref() == Some("host offline") { ErrorCode::HostOffline } else { ErrorCode::Timeout };
refuse(&mut sock, code, reason.as_deref().unwrap_or("closed")).await;
break;
}
None => break,
},
}
}
end_link(&entry, conn).await;
tracing::info!(%realm, conn, bytes = link.bytes.load(Ordering::Relaxed), seconds = link.started.elapsed().as_secs(), "player disconnected");
}
/// A connection number on this Host, or `None` when it is at its limit.
fn take_slot<K: KeyLookup>(hub: &Hub<K>, entry: &HostEntry, link: &Arc<Link>) -> Option<u32> {
let mut links = entry.links.lock().ok()?;
if links.len() >= hub.cfg.per_host_connections {
return None;
}
let conn = entry.next_conn.fetch_add(1, Ordering::SeqCst);
links.insert(conn, link.clone());
Some(conn)
}
async fn end_link(entry: &HostEntry, conn: u32) {
let was = entry.links.lock().ok().and_then(|mut l| l.remove(&conn));
if was.is_some() {
let _ = entry.tx.try_send(HostMsg::Text(coord::to_text(&Frame::Close { conn, reason: None })));
}
}
+88
View File
@@ -0,0 +1,88 @@
//! In-memory token buckets for abuse control. One process, one node: no shared store is needed.
use std::collections::HashMap;
use std::hash::Hash;
use std::sync::Mutex;
#[derive(Clone, Copy, Debug)]
pub struct Rate {
pub burst: f64,
/// Tokens per second.
pub refill: f64,
}
impl Rate {
pub fn per_minute(burst: u32, per_minute: f64) -> Self {
Self { burst: burst as f64, refill: per_minute / 60.0 }
}
pub fn per_hour(burst: u32, per_hour: f64) -> Self {
Self { burst: burst as f64, refill: per_hour / 3600.0 }
}
}
struct Bucket {
tokens: f64,
at: f64,
}
pub struct Limiter<K: Eq + Hash + Clone> {
rate: Rate,
max_keys: usize,
buckets: Mutex<HashMap<K, Bucket>>,
}
impl<K: Eq + Hash + Clone> Limiter<K> {
pub fn new(rate: Rate, max_keys: usize) -> Self {
Self { rate, max_keys, buckets: Mutex::new(HashMap::new()) }
}
/// Take `cost` tokens at time `now` (seconds, any monotonic origin). `false`: refused, nothing taken.
pub fn take(&self, key: &K, cost: f64, now: f64) -> bool {
let mut map = self.buckets.lock().unwrap();
if map.len() >= self.max_keys && !map.contains_key(key) {
// a full table is cleaned of buckets that have refilled completely (they carry no information)
let full = self.rate.burst;
let refill = self.rate.refill;
map.retain(|_, b| b.tokens + (now - b.at).max(0.0) * refill < full);
if map.len() >= self.max_keys {
return false;
}
}
let b = map.entry(key.clone()).or_insert(Bucket { tokens: self.rate.burst, at: now });
b.tokens = (b.tokens + (now - b.at).max(0.0) * self.rate.refill).min(self.rate.burst);
b.at = now;
if b.tokens >= cost {
b.tokens -= cost;
true
} else {
false
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_bucket_allows_a_burst_then_refills() {
let l: Limiter<&str> = Limiter::new(Rate { burst: 3.0, refill: 0.5 }, 100);
assert!(l.take(&"a", 1.0, 0.0) && l.take(&"a", 1.0, 0.0) && l.take(&"a", 1.0, 0.0));
assert!(!l.take(&"a", 1.0, 0.0), "the burst is used up");
assert!(l.take(&"b", 1.0, 0.0), "another key is unaffected");
assert!(!l.take(&"a", 1.0, 1.0), "half a token is not enough");
assert!(l.take(&"a", 1.0, 2.0), "one token after two seconds");
assert!(l.take(&"a", 3.0, 1000.0), "a long pause refills to the burst");
assert!(!l.take(&"a", 1.0, 1000.0), "and never beyond it");
}
#[test]
fn the_table_is_bounded() {
let l: Limiter<u32> = Limiter::new(Rate { burst: 1.0, refill: 0.0 }, 4);
for k in 0..4 {
assert!(l.take(&k, 1.0, 0.0));
}
assert!(!l.take(&99, 1.0, 0.0), "a new key is refused when the table is full of live buckets");
assert!(!l.take(&0, 1.0, 0.0), "an existing one keeps its state");
}
}
+87
View File
@@ -0,0 +1,87 @@
//! A TCP listener that never has more than a fixed number of connections open: a new one waits in the kernel backlog until a slot is free.
use std::io;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use pin_project_lite::pin_project;
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
pub struct LimitedListener {
inner: TcpListener,
slots: Arc<Semaphore>,
}
impl LimitedListener {
pub fn new(inner: TcpListener, max_connections: usize) -> Self {
Self { inner, slots: Arc::new(Semaphore::new(max_connections)) }
}
}
pin_project! {
pub struct LimitedIo {
#[pin]
stream: TcpStream,
_slot: OwnedSemaphorePermit,
}
}
impl AsyncRead for LimitedIo {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
self.project().stream.poll_read(cx, buf)
}
}
impl AsyncWrite for LimitedIo {
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
self.project().stream.poll_write(cx, buf)
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
self.project().stream.poll_flush(cx)
}
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
self.project().stream.poll_shutdown(cx)
}
}
impl axum::serve::Listener for LimitedListener {
type Io = LimitedIo;
type Addr = SocketAddr;
async fn accept(&mut self) -> (Self::Io, Self::Addr) {
loop {
let slot = self.slots.clone().acquire_owned().await.expect("the semaphore is never closed");
match self.inner.accept().await {
Ok((stream, addr)) => return (LimitedIo { stream, _slot: slot }, addr),
Err(e) => {
tracing::warn!(error = %e, "accept failed");
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
}
}
}
}
fn local_addr(&self) -> io::Result<Self::Addr> {
self.inner.local_addr()
}
}
/// The address of the other end of a connection, whichever listener accepted it.
#[derive(Clone, Copy, Debug)]
pub struct Peer(pub SocketAddr);
impl axum::extract::connect_info::Connected<axum::serve::IncomingStream<'_, LimitedListener>> for Peer {
fn connect_info(stream: axum::serve::IncomingStream<'_, LimitedListener>) -> Self {
Peer(*stream.remote_addr())
}
}
impl axum::extract::connect_info::Connected<axum::serve::IncomingStream<'_, TcpListener>> for Peer {
fn connect_info(stream: axum::serve::IncomingStream<'_, TcpListener>) -> Self {
Peer(*stream.remote_addr())
}
}
+81
View File
@@ -0,0 +1,81 @@
use std::sync::Arc;
use std::time::Duration;
use coa_coordinator::listener::{LimitedListener, Peer};
use coa_coordinator::{router, Config, Hub, PgKeys};
use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod, Runtime};
use tokio_postgres::NoTls;
use tower::limit::ConcurrencyLimitLayer;
fn var(name: &str, default: &str) -> String {
std::env::var(name).ok().filter(|v| !v.is_empty()).unwrap_or_else(|| default.to_string())
}
fn pool() -> Result<Pool, String> {
let file = std::env::var("COORD_DB_PASSWORD_FILE").map_err(|_| "COORD_DB_PASSWORD_FILE is required".to_string())?;
let password = std::fs::read_to_string(&file).map_err(|e| format!("cannot read the database password file: {e}"))?.trim().to_string();
let mut pg = tokio_postgres::Config::new();
pg.host(&var("COORD_DB_HOST", "coa-postgres")).port(var("COORD_DB_PORT", "5432").parse().map_err(|_| "COORD_DB_PORT")?).dbname(&var("COORD_DB_NAME", "coa_registry")).user(&var("COORD_DB_USER", "coa_coordinator")).password(&password);
pg.connect_timeout(Duration::from_secs(5)).application_name("coa-coordinator").options("-c statement_timeout=3000 -c default_transaction_read_only=on");
Pool::builder(Manager::from_config(pg, NoTls, ManagerConfig { recycling_method: RecyclingMethod::Fast })).max_size(4).wait_timeout(Some(Duration::from_secs(5))).create_timeout(Some(Duration::from_secs(5))).runtime(Runtime::Tokio1).build().map_err(|e| e.to_string())
}
/// `coa-coordinator --healthcheck`: ask the local server and exit 0 or 1 (the image has no curl).
fn healthcheck() -> i32 {
use std::io::{Read, Write};
let port = var("COORD_LISTEN", "0.0.0.0:8081").rsplit(':').next().unwrap_or("8081").to_string();
let Ok(mut s) = std::net::TcpStream::connect(format!("127.0.0.1:{port}")) else { return 1 };
let _ = s.set_read_timeout(Some(Duration::from_secs(3)));
if s.write_all(b"GET /coord/v1/health HTTP/1.1\r\nHost: localhost\r\n\r\n").is_err() {
return 1;
}
// the status line is all that is needed
let mut buf = [0u8; 16];
i32::from(!matches!(s.read(&mut buf), Ok(n) if n >= 12 && buf.starts_with(b"HTTP/1.1 200")))
}
async fn shutdown() {
let ctrl_c = tokio::signal::ctrl_c();
#[cfg(unix)]
{
let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("a SIGTERM handler");
tokio::select! { _ = ctrl_c => {}, _ = term.recv() => {} }
}
#[cfg(not(unix))]
{
let _ = ctrl_c.await;
}
}
#[tokio::main]
async fn main() {
if std::env::args().nth(1).as_deref() == Some("--healthcheck") {
std::process::exit(healthcheck());
}
tracing_subscriber::fmt().json().with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into())).with_current_span(false).init();
let pool = match pool() {
Ok(p) => p,
Err(e) => {
tracing::error!(error = %e, "configuration");
std::process::exit(2);
}
};
let cfg = Config { trust_proxy: var("COORD_TRUST_PROXY", "0") == "1", ..Config::default() };
let clock = Arc::new(|| std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0));
let hub = Hub::new(PgKeys::new(pool), cfg, clock);
let app = router(hub).layer(ConcurrencyLimitLayer::new(512));
let listen = var("COORD_LISTEN", "0.0.0.0:8081");
let tcp = match tokio::net::TcpListener::bind(&listen).await {
Ok(l) => l,
Err(e) => {
tracing::error!(error = %e, %listen, "bind");
std::process::exit(2);
}
};
tracing::info!(%listen, "coordinator started");
let serve = axum::serve(LimitedListener::new(tcp, 512), app.into_make_service_with_connect_info::<Peer>()).with_graceful_shutdown(shutdown());
if let Err(e) = serve.await {
tracing::error!(error = %e, "server");
std::process::exit(1);
}
}
+381
View File
@@ -0,0 +1,381 @@
//! The Coordinator end to end over real WebSockets: authentication of both sides, routing by connection number, limits, and that it forwards bytes it cannot read.
use std::net::TcpStream;
use std::sync::Arc;
use std::time::Duration;
use coa_control_proto::coord::{self, ErrorCode, Frame};
use coa_coordinator::listener::{LimitedListener, Peer};
use coa_coordinator::{router, Config, Hub, MemoryKeys};
use coa_registry_proto::RealmId;
use ed25519_dalek::SigningKey;
use tungstenite::stream::MaybeTlsStream;
use tungstenite::{Message, WebSocket};
use uuid::Uuid;
type Ws = WebSocket<MaybeTlsStream<TcpStream>>;
struct Server {
addr: std::net::SocketAddr,
keys: Arc<MemoryKeys>,
hub: Arc<Hub<Arc<MemoryKeys>>>,
}
fn now() -> i64 {
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64
}
fn start(cfg: Config) -> Server {
let keys = Arc::new(MemoryKeys::default());
let hub = Hub::new(keys.clone(), cfg, Arc::new(now));
let app = router(hub.clone());
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || {
let rt = tokio::runtime::Builder::new_multi_thread().worker_threads(2).enable_all().build().unwrap();
rt.block_on(async move {
let tcp = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
tx.send(tcp.local_addr().unwrap()).unwrap();
axum::serve(LimitedListener::new(tcp, 64), app.into_make_service_with_connect_info::<Peer>()).await.unwrap();
});
});
Server { addr: rx.recv().unwrap(), keys, hub }
}
fn quick() -> Config {
Config { per_ip_connections: 64, per_ip: coa_coordinator::limits::Rate::per_minute(1000, 1000.0), per_realm: coa_coordinator::limits::Rate::per_minute(1000, 1000.0), ..Config::default() }
}
fn connect(s: &Server, path: &str) -> Ws {
let (ws, _) = tungstenite::connect(format!("ws://{}{path}", s.addr)).expect("connect");
if let MaybeTlsStream::Plain(t) = ws.get_ref() {
t.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
}
ws
}
fn text(ws: &mut Ws) -> Frame {
loop {
match ws.read().expect("a frame") {
Message::Text(t) => return coord::parse_text(t.as_str()).expect("a valid frame"),
Message::Ping(_) | Message::Pong(_) => continue,
other => panic!("expected text, got {other:?}"),
}
}
}
fn nonce(ws: &mut Ws) -> String {
match text(ws) {
Frame::Challenge { nonce, .. } => nonce,
f => panic!("{f:?}"),
}
}
fn send(ws: &mut Ws, f: &Frame) {
ws.send(Message::Text(coord::to_text(f).into())).unwrap();
}
fn expect_error(ws: &mut Ws, code: ErrorCode) {
match text(ws) {
Frame::Error { code: c, .. } => assert_eq!(c, code),
f => panic!("expected {code:?}, got {f:?}"),
}
}
fn host(s: &Server, realm: RealmId, key: &SigningKey) -> Ws {
s.keys.set(realm, &key.verifying_key());
let mut ws = connect(s, "/coord/v1/host");
let n = nonce(&mut ws);
send(&mut ws, &coord::sign_host_hello(key, &realm, &n, now()));
assert!(matches!(text(&mut ws), Frame::HostReady { .. }));
ws
}
fn player(s: &Server, realm: RealmId, key: &SigningKey) -> Ws {
let mut ws = connect(s, &format!("/coord/v1/player?realm={realm}"));
let n = nonce(&mut ws);
send(&mut ws, &coord::sign_player_hello(key, &Uuid::now_v7(), &realm, &n, now()));
ws
}
fn key(n: u8) -> SigningKey {
SigningKey::from_bytes(&[n; 32])
}
#[test]
fn routes_both_ways_by_connection_number_and_never_interprets() {
let s = start(quick());
let (realm, hk) = (RealmId::new(), key(1));
let mut h = host(&s, realm, &hk);
let mut p1 = player(&s, realm, &key(10));
let mut p2 = player(&s, realm, &key(11));
let Frame::Open { conn: c1 } = text(&mut h) else { panic!("open 1") };
let Frame::Open { conn: c2 } = text(&mut h) else { panic!("open 2") };
assert_ne!(c1, c2);
assert!(matches!(text(&mut p1), Frame::PlayerReady { conn, .. } if conn == c1));
assert!(matches!(text(&mut p2), Frame::PlayerReady { conn, .. } if conn == c2));
// bytes that are not a Noise message at all are forwarded untouched: the Coordinator has nothing to parse
let blob = vec![0xA5u8; 300];
p1.send(Message::Binary(blob.clone().into())).unwrap();
p2.send(Message::Binary(b"second".to_vec().into())).unwrap();
let mut got = std::collections::HashMap::new();
for _ in 0..2 {
match h.read().unwrap() {
Message::Binary(b) => {
let (c, f) = coord::split_binary(&b).unwrap();
got.insert(c, f.to_vec());
}
m => panic!("{m:?}"),
}
}
assert_eq!(got[&c1], blob);
assert_eq!(got[&c2], b"second");
h.send(Message::Binary(coord::binary(c2, b"to-two").into())).unwrap();
h.send(Message::Binary(coord::binary(c1, b"to-one").into())).unwrap();
assert_eq!(p1.read().unwrap(), Message::Binary(b"to-one".to_vec().into()));
assert_eq!(p2.read().unwrap(), Message::Binary(b"to-two".to_vec().into()));
// the player hangs up; the host is told
p1.close(None).ok();
loop {
if let Frame::Close { conn, .. } = text(&mut h) {
assert_eq!(conn, c1);
break;
}
}
// the host closes one; that player is told and the other is unaffected
send(&mut h, &Frame::Close { conn: c2, reason: None });
assert!(matches!(p2.read(), Ok(Message::Text(_) | Message::Close(_)) | Err(_)));
}
#[test]
fn a_host_must_hold_the_registered_key() {
let s = start(quick());
let realm = RealmId::new();
s.keys.set(realm, &key(1).verifying_key());
let mut ws = connect(&s, "/coord/v1/host");
let n = nonce(&mut ws);
send(&mut ws, &coord::sign_host_hello(&key(2), &realm, &n, now()));
expect_error(&mut ws, ErrorCode::BadHello);
let mut ws = connect(&s, "/coord/v1/host");
let n = nonce(&mut ws);
send(&mut ws, &coord::sign_host_hello(&key(1), &RealmId::new(), &n, now()));
expect_error(&mut ws, ErrorCode::UnknownRealm);
// a hello signed for another challenge is a replay
let mut a = connect(&s, "/coord/v1/host");
let na = nonce(&mut a);
let hello = coord::sign_host_hello(&key(1), &realm, &na, now());
let mut b = connect(&s, "/coord/v1/host");
let _ = nonce(&mut b);
send(&mut b, &hello);
expect_error(&mut b, ErrorCode::BadHello);
// a stale timestamp
let mut c = connect(&s, "/coord/v1/host");
let nc = nonce(&mut c);
send(&mut c, &coord::sign_host_hello(&key(1), &realm, &nc, now() - 3600));
expect_error(&mut c, ErrorCode::BadHello);
assert_eq!(s.hub.connected_hosts(), 0);
}
#[test]
fn a_player_needs_a_signature_the_right_realm_and_an_online_host() {
let s = start(quick());
let realm = RealmId::new();
// host offline
let mut p = player(&s, realm, &key(5));
expect_error(&mut p, ErrorCode::HostOffline);
// a hello for another realm than the URL's
let mut p = connect(&s, &format!("/coord/v1/player?realm={realm}"));
let n = nonce(&mut p);
send(&mut p, &coord::sign_player_hello(&key(5), &Uuid::now_v7(), &RealmId::new(), &n, now()));
expect_error(&mut p, ErrorCode::BadHello);
// a key that did not sign
let mut p = connect(&s, &format!("/coord/v1/player?realm={realm}"));
let n = nonce(&mut p);
let Frame::PlayerHello { player_id, realm_id, ts, sig, .. } = coord::sign_player_hello(&key(5), &Uuid::now_v7(), &realm, &n, now()) else { unreachable!() };
send(&mut p, &Frame::PlayerHello { player_id, public_key: coord::encode_public_key(&key(6).verifying_key()), realm_id, ts, sig });
expect_error(&mut p, ErrorCode::BadHello);
// garbage and the wrong frame
let mut p = connect(&s, &format!("/coord/v1/player?realm={realm}"));
let _ = nonce(&mut p);
p.send(Message::Text("not json".into())).unwrap();
assert!(!matches!(p.read(), Ok(Message::Text(t)) if matches!(coord::parse_text(t.as_str()), Ok(Frame::PlayerReady { .. }))));
assert!(tungstenite::connect(format!("ws://{}/coord/v1/player?realm=nope", s.addr)).is_err());
}
#[test]
fn a_host_serves_a_bounded_number_of_players() {
let s = start(Config { per_host_connections: 2, ..quick() });
let realm = RealmId::new();
let mut h = host(&s, realm, &key(1));
let mut a = player(&s, realm, &key(2));
let mut b = player(&s, realm, &key(3));
assert!(matches!(text(&mut a), Frame::PlayerReady { .. }));
assert!(matches!(text(&mut b), Frame::PlayerReady { .. }));
let mut c = player(&s, realm, &key(4));
expect_error(&mut c, ErrorCode::HostBusy);
// when one leaves there is room again
a.close(None).ok();
loop {
if matches!(text(&mut h), Frame::Close { .. }) {
break;
}
}
let mut d = player(&s, realm, &key(7));
assert!(matches!(text(&mut d), Frame::PlayerReady { .. }));
}
#[test]
fn a_connection_that_exceeds_its_byte_budget_is_cut() {
let s = start(Config { bytes_per_connection: 1000, ..quick() });
let realm = RealmId::new();
let mut h = host(&s, realm, &key(1));
let mut p = player(&s, realm, &key(2));
assert!(matches!(text(&mut p), Frame::PlayerReady { .. }));
let Frame::Open { .. } = text(&mut h) else { panic!() };
for _ in 0..5 {
if p.send(Message::Binary(vec![1u8; 400].into())).is_err() {
break;
}
}
let mut cut = false;
for _ in 0..10 {
match p.read() {
Ok(Message::Text(t)) if matches!(coord::parse_text(t.as_str()), Ok(Frame::Error { code: ErrorCode::TooLarge, .. })) => {
cut = true;
break;
}
Ok(Message::Close(_)) | Err(_) => {
cut = true;
break;
}
_ => {}
}
}
assert!(cut, "the budget is enforced");
}
#[test]
fn an_oversized_frame_is_refused() {
let s = start(quick());
let realm = RealmId::new();
let _h = host(&s, realm, &key(1));
let mut p = player(&s, realm, &key(2));
assert!(matches!(text(&mut p), Frame::PlayerReady { .. }));
let _ = p.send(Message::Binary(vec![0u8; coord::MAX_FRAME_BYTES + 100].into()));
let mut closed = false;
for _ in 0..5 {
match p.read() {
Ok(Message::Close(_)) | Err(_) => {
closed = true;
break;
}
Ok(Message::Text(t)) if matches!(coord::parse_text(t.as_str()), Ok(Frame::Error { .. })) => {
closed = true;
break;
}
_ => {}
}
}
assert!(closed);
}
#[test]
fn a_reconnecting_host_replaces_the_old_connection_and_its_players_are_dropped() {
let s = start(quick());
let realm = RealmId::new();
let mut h1 = host(&s, realm, &key(1));
let mut p = player(&s, realm, &key(2));
assert!(matches!(text(&mut p), Frame::PlayerReady { .. }));
let Frame::Open { .. } = text(&mut h1) else { panic!() };
let mut h2 = host(&s, realm, &key(1));
// the old host connection ends, and its player is told the host went away
assert!(h1.read().is_err() || matches!(h1.read(), Ok(Message::Close(_)) | Err(_)));
let mut told = false;
for _ in 0..5 {
match p.read() {
Ok(Message::Text(t)) if matches!(coord::parse_text(t.as_str()), Ok(Frame::Error { code: ErrorCode::HostOffline, .. })) => {
told = true;
break;
}
Ok(Message::Close(_)) | Err(_) => {
told = true;
break;
}
_ => {}
}
}
assert!(told);
// a new player reaches the new host
let mut p2 = player(&s, realm, &key(3));
assert!(matches!(text(&mut p2), Frame::PlayerReady { .. }));
assert!(matches!(text(&mut h2), Frame::Open { .. }));
assert_eq!(s.hub.connected_hosts(), 1);
}
#[test]
fn connections_per_address_and_per_realm_are_limited() {
let s = start(Config { per_ip_connections: 3, ..quick() });
let realm = RealmId::new();
let _h = host(&s, realm, &key(1));
let mut held = Vec::new();
for i in 0..2 {
let mut p = player(&s, realm, &key(20 + i));
assert!(matches!(text(&mut p), Frame::PlayerReady { .. }));
held.push(p);
}
// the host holds one, two players make three: the fourth from this address is refused at the upgrade
let r = tungstenite::connect(format!("ws://{}/coord/v1/player?realm={realm}", s.addr));
assert!(r.is_err(), "the per-address limit applies");
let s = start(Config { per_realm: coa_coordinator::limits::Rate::per_minute(2, 0.0001), ..quick() });
let realm = RealmId::new();
let _h = host(&s, realm, &key(1));
let mut ok = 0;
let mut limited = 0;
let mut keep = Vec::new();
for i in 0..4 {
let mut p = player(&s, realm, &key(30 + i));
match text(&mut p) {
Frame::PlayerReady { .. } => ok += 1,
Frame::Error { code: ErrorCode::RateLimited, .. } => limited += 1,
f => panic!("{f:?}"),
}
keep.push(p);
}
assert_eq!((ok, limited), (2, 2));
}
/// One HTTP response read until its body is complete (keep-alive: a server that closes right after answering can reset the connection on Windows before the client reads).
fn read_response(t: &mut TcpStream) -> String {
use std::io::Read;
let mut data = Vec::new();
let mut buf = [0u8; 1024];
loop {
let n = t.read(&mut buf).expect("a response");
assert!(n > 0, "closed before a full response");
data.extend_from_slice(&buf[..n]);
let text = String::from_utf8_lossy(&data).into_owned();
if let Some(split) = text.find("\r\n\r\n") {
let len = text[..split].lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("content-length:").and_then(|v| v.trim().parse::<usize>().ok())).unwrap_or(0);
if text.len() >= split + 4 + len {
return text;
}
}
}
}
#[test]
fn the_health_endpoint_answers_and_unknown_paths_do_not() {
let s = start(quick());
let mut t = TcpStream::connect(s.addr).unwrap();
use std::io::Write;
t.write_all(b"GET /coord/v1/health HTTP/1.1\r\nHost: x\r\n\r\n").unwrap();
let out = read_response(&mut t);
assert!(out.starts_with("HTTP/1.1 200") && out.contains("\"status\":\"ok\""), "{out}");
let mut t = TcpStream::connect(s.addr).unwrap();
t.write_all(b"GET /admin HTTP/1.1\r\nHost: x\r\n\r\n").unwrap();
let out = read_response(&mut t);
assert!(out.starts_with("HTTP/1.1 404"), "{out}");
}