From 6e438cfe95e5564ffc3bff90474594b4073f20d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=94=D0=BC=D0=B8=D1=82=D1=80=D0=B8=D0=B9?= Date: Fri, 9 Oct 2026 13:28:32 +0500 Subject: [PATCH] feat(relay): complete Phase 13.1 hardening, port pool scaling, and live load testing --- crates/coa-control-proto/src/app.rs | 17 +- crates/coa-control-proto/src/coord.rs | 6 +- crates/coa-control-proto/src/relay.rs | 2 + crates/coa-core/src/control/host_link.rs | 7 +- crates/coa-core/src/control/join.rs | 2 +- crates/coa-core/src/control/relay_link.rs | 157 +++--- crates/coa-core/src/control/service.rs | 37 +- docs/CONTROL_PROTOCOL.md | 19 +- docs/INFRASTRUCTURE.md | 6 +- docs/RELAY_PROTOCOL.md | 19 +- docs/VPS_BASELINE.md | 2 +- services/coa-control-e2e/Cargo.lock | 59 ++- services/coa-control-e2e/Cargo.toml | 1 + services/coa-control-e2e/tests/control.rs | 38 ++ .../coa-control-e2e/tests/live_control.rs | 4 +- .../tests/relay_session_and_load.rs | 474 ++++++++++++++++++ services/coa-coordinator/src/lib.rs | 2 +- services/coa-relay/Cargo.lock | 24 +- services/coa-relay/Cargo.toml | 1 + services/coa-relay/Dockerfile | 2 +- services/coa-relay/deploy/compose.yaml | 24 +- services/coa-relay/src/hub.rs | 80 ++- services/coa-relay/src/limits.rs | 13 +- services/coa-relay/src/main.rs | 32 +- services/coa-relay/src/ports.rs | 129 +++-- services/coa-relay/src/tunnel.rs | 77 ++- services/coa-relay/tests/relay.rs | 308 +++++++++++- 27 files changed, 1320 insertions(+), 222 deletions(-) create mode 100644 services/coa-control-e2e/tests/relay_session_and_load.rs diff --git a/crates/coa-control-proto/src/app.rs b/crates/coa-control-proto/src/app.rs index 9ffebfc..37dee1e 100644 --- a/crates/coa-control-proto/src/app.rs +++ b/crates/coa-control-proto/src/app.rs @@ -7,7 +7,9 @@ use uuid::Uuid; use crate::{invalid, ControlError, Result}; -pub const MAX_REQUEST_BYTES: usize = 64 * 1024; +pub const MAX_CONTROL_REQUEST_BYTES: usize = 4096; +pub const MAX_TRANSFER_REQUEST_BYTES: usize = 4 * 1024 * 1024; // 4 MiB +pub const MAX_REQUEST_BYTES: usize = MAX_TRANSFER_REQUEST_BYTES; pub const MIN_USERNAME: usize = 3; pub const MAX_USERNAME: usize = 16; pub const MIN_PASSWORD: usize = 8; @@ -181,10 +183,13 @@ pub fn encode_request(r: &Request) -> Vec { } pub fn decode_request(bytes: &[u8]) -> Result { - if bytes.len() > MAX_REQUEST_BYTES { + if bytes.len() > MAX_TRANSFER_REQUEST_BYTES { return Err(ControlError::Limit("a request is too long".into())); } let r: Request = serde_json::from_slice(bytes).map_err(|_| ControlError::Invalid("a request is not valid".into()))?; + if !matches!(&r, Request::TransferOffer { .. } | Request::TransferChunk { .. }) && bytes.len() > MAX_CONTROL_REQUEST_BYTES { + return Err(ControlError::Limit("a standard control request exceeds 4096 bytes".into())); + } match &r { Request::Provision { desired, password, .. } => { if !valid_password(password) || desired.as_deref().is_some_and(|d| d.len() > 64 || d.chars().any(char::is_control)) { @@ -197,16 +202,20 @@ pub fn decode_request(bytes: &[u8]) -> Result { } } Request::Hello { client, .. } if client.len() > 64 => return invalid("the client name is too long"), - Request::TransferOffer { content_hash, total_size, .. } => { + Request::TransferOffer { content_hash, total_size, collections, .. } => { if content_hash.len() != 64 || !content_hash.chars().all(|c| c.is_ascii_hexdigit()) { return invalid("transfer content hash must be 64 hex characters"); } if *total_size > crate::noise::MAX_MESSAGE_BYTES { return Err(ControlError::Limit("transfer total size is too large".into())); } + let total_items: usize = collections.values().map(|v| v.len()).sum(); + if total_items > 100_000 { + return Err(ControlError::Limit("collection mapping exceeds 100,000 entries".into())); + } } Request::TransferChunk { data, .. } => { - if data.len() > MAX_REQUEST_BYTES { + if data.len() > MAX_TRANSFER_REQUEST_BYTES { return Err(ControlError::Limit("transfer chunk is too large".into())); } } diff --git a/crates/coa-control-proto/src/coord.rs b/crates/coa-control-proto/src/coord.rs index edec0d6..94b9ddd 100644 --- a/crates/coa-control-proto/src/coord.rs +++ b/crates/coa-control-proto/src/coord.rs @@ -49,7 +49,11 @@ pub enum Frame { /// Coordinator -> Host: the Host is registered and these are the limits. HostReady { max_connections: u32, max_frame: u32 }, /// Coordinator -> Host: a Player wants to talk; its frames will carry this number. - Open { conn: u32 }, + Open { + conn: u32, + #[serde(default)] + client_ip: Option, + }, /// Either side of a connection ends it. Close { conn: u32, reason: Option }, /// Coordinator -> Player: the Host accepted the connection; binary frames flow now. diff --git a/crates/coa-control-proto/src/relay.rs b/crates/coa-control-proto/src/relay.rs index 2131171..150f675 100644 --- a/crates/coa-control-proto/src/relay.rs +++ b/crates/coa-control-proto/src/relay.rs @@ -67,6 +67,8 @@ pub enum TunnelMsg { Allocate { request_id: u64, player_id: Uuid, + #[serde(default)] + expected_client_ip: Option, }, /// Relay responds to Host with allocation details. AllocateOk { diff --git a/crates/coa-core/src/control/host_link.rs b/crates/coa-core/src/control/host_link.rs index fdbd0b6..b512e4a 100644 --- a/crates/coa-core/src/control/host_link.rs +++ b/crates/coa-core/src/control/host_link.rs @@ -39,6 +39,7 @@ struct Conn { stage: Option, opened: Instant, last: Instant, + client_ip: Option, } #[derive(Clone, Debug, Default, Serialize)] @@ -138,11 +139,11 @@ fn session(url: &str, realm: RealmId, key: &SigningKey, service: &HostService, c match transport::poll(&mut ws)? { None => {} Some(Message::Text(t)) => match coord::parse_text(t.as_str()).map_err(LinkError::from)? { - Frame::Open { conn } => { + Frame::Open { conn, client_ip } => { if conns.len() >= MAX_CHANNELS || conns.contains_key(&conn) { transport::send_text(&mut ws, &Frame::Close { conn, reason: Some("busy".into()) })?; } else { - conns.insert(conn, Conn { stage: Some(Stage::AwaitFirst), opened: Instant::now(), last: Instant::now() }); + conns.insert(conn, Conn { stage: Some(Stage::AwaitFirst), opened: Instant::now(), last: Instant::now(), client_ip }); } } Frame::Close { conn, .. } => { @@ -217,7 +218,7 @@ fn step(c: &mut Conn, frame: &[u8], realm: RealmId, key: &SigningKey, service: & if !service.key_matches(&proof.player_id, &proof.public_key) { return Err("the player id belongs to another key".into()); } - c.stage = Some(Stage::Ready { channel, player: proof.player_id, public_key: proof.public_key, session: Session::default() }); + c.stage = Some(Stage::Ready { channel, player: proof.player_id, public_key: proof.public_key, session: Session::with_client_ip(c.client_ip.clone()) }); Ok(vec![]) } Stage::Ready { mut channel, player, public_key, mut session } => { diff --git a/crates/coa-core/src/control/join.rs b/crates/coa-core/src/control/join.rs index bdb548e..40fb31b 100644 --- a/crates/coa-core/src/control/join.rs +++ b/crates/coa-core/src/control/join.rs @@ -307,7 +307,7 @@ impl PlayerControl { _ => return Err(ControlFail::Link(LinkError::Protocol("unexpected response to transfer offer".into()))), }; - const CHUNK_SIZE: usize = 16 * 1024; + const CHUNK_SIZE: usize = 128 * 1024; while offset < payload.len() { let end = (offset + CHUNK_SIZE).min(payload.len()); let chunk_data = base64::engine::general_purpose::STANDARD.encode(&payload[offset..end]); diff --git a/crates/coa-core/src/control/relay_link.rs b/crates/coa-core/src/control/relay_link.rs index 22d709e..c80d2d3 100644 --- a/crates/coa-core/src/control/relay_link.rs +++ b/crates/coa-core/src/control/relay_link.rs @@ -22,7 +22,7 @@ use uuid::Uuid; use super::service::RelayProvider; use super::transport::{self, LinkError}; -const POLL: Duration = Duration::from_millis(200); +const POLL: Duration = Duration::from_millis(2); #[derive(Clone, Debug, Default, Serialize)] pub struct RelayStatus { @@ -34,7 +34,7 @@ pub struct RelayStatus { pub struct RelayLink { stop: Arc, status: Arc>, - alloc_tx: Sender<(Uuid, Sender>)>, + alloc_tx: Sender<(Uuid, Option, Sender>)>, join: Mutex>>, } @@ -48,10 +48,10 @@ pub struct RelayAllocationInfo { } impl RelayProvider for RelayLink { - fn allocate(&self, player_id: &Uuid) -> crate::error::Result { + fn allocate(&self, player_id: &Uuid, client_ip: Option<&str>) -> crate::error::Result { let (tx, rx) = channel(); self.alloc_tx - .send((*player_id, tx)) + .send((*player_id, client_ip.map(|s| s.to_string()), tx)) .map_err(|_| crate::error::Error::Invalid("relay link worker thread not running".to_string()))?; rx.recv_timeout(Duration::from_secs(10)) .map_err(|_| crate::error::Error::Invalid("relay allocation timed out".to_string()))? @@ -115,7 +115,7 @@ fn run( url: &str, realm: RealmId, key: &SigningKey, - alloc_rx: Receiver<(Uuid, Sender>)>, + alloc_rx: Receiver<(Uuid, Option, Sender>)>, stop: &AtomicBool, status: &Mutex, ) { @@ -160,7 +160,7 @@ fn session( url: &str, realm: RealmId, key: &SigningKey, - alloc_rx: &Receiver<(Uuid, Sender>)>, + alloc_rx: &Receiver<(Uuid, Option, Sender>)>, stop: &AtomicBool, status: &Mutex, ) -> Result<(), LinkError> { @@ -232,12 +232,13 @@ fn session( while !stop.load(Ordering::SeqCst) { // Drain pending allocation requests - while let Ok((player_id, reply_tx)) = alloc_rx.try_recv() { + while let Ok((player_id, expected_client_ip, reply_tx)) = alloc_rx.try_recv() { let rid = req_counter.fetch_add(1, Ordering::Relaxed); pending_allocs.insert(rid, reply_tx); let msg = TunnelMsg::Allocate { request_id: rid, player_id, + expected_client_ip, }; let _ = ws.send(Message::Text(serde_json::to_string(&msg).unwrap().into())); } @@ -253,84 +254,86 @@ fn session( last_ping = Instant::now(); } - // Poll WS incoming - match transport::poll(&mut ws)? { - None => {} - Some(Message::Text(text)) => { - let Ok(msg) = serde_json::from_str::(&text) else { continue }; - match msg { - TunnelMsg::AllocateOk { - request_id, - token, - relay_host, - auth_port, - world_port, - expires_at, - } => { - if let Some(tx) = pending_allocs.remove(&request_id) { - let _ = tx.send(Ok(RelayAllocationInfo { - relay_host, - auth_port, - world_port, - token, - expires_at, - })); - } - } - TunnelMsg::AllocateErr { request_id, error } => { - if let Some(tx) = pending_allocs.remove(&request_id) { - let _ = tx.send(Err(LinkError::Protocol(error))); - } - } - TunnelMsg::Connect { stream_id, target, .. } => { - // Strictly predefined local targets: Auth (3724) or World (8085) - let local_port = target.local_port(); - match TcpStream::connect(("127.0.0.1", local_port)) { - Ok(tcp) => { - let (stream_tx, stream_rx) = channel::(); - let ws_tx = ws_out_tx.clone(); - - // Spawn worker for this stream - spawn_local_tcp_bridge(stream_id, tcp, stream_rx, ws_tx); - streams.insert(stream_id, StreamWorker { tx: stream_tx }); - - // Reply ConnectOk - let ok_msg = TunnelMsg::ConnectOk { stream_id }; - let _ = ws.send(Message::Text(serde_json::to_string(&ok_msg).unwrap().into())); - } - Err(e) => { - let err_msg = TunnelMsg::ConnectErr { - stream_id, - error: e.to_string(), - }; - let _ = ws.send(Message::Text(serde_json::to_string(&err_msg).unwrap().into())); + // Poll WS incoming (drain available frames) + for _ in 0..128 { + match transport::poll(&mut ws)? { + None => break, + Some(Message::Text(text)) => { + let Ok(msg) = serde_json::from_str::(&text) else { continue }; + match msg { + TunnelMsg::AllocateOk { + request_id, + token, + relay_host, + auth_port, + world_port, + expires_at, + } => { + if let Some(tx) = pending_allocs.remove(&request_id) { + let _ = tx.send(Ok(RelayAllocationInfo { + relay_host, + auth_port, + world_port, + token, + expires_at, + })); } } - set_status(status, |s| s.active_streams = streams.len()); - } - TunnelMsg::Data { stream_id, chunk } => { - if let Some(w) = streams.get(&stream_id) { - let _ = w.tx.send(TunnelMsg::Data { stream_id, chunk }); + TunnelMsg::AllocateErr { request_id, error } => { + if let Some(tx) = pending_allocs.remove(&request_id) { + let _ = tx.send(Err(LinkError::Protocol(error))); + } } - } - TunnelMsg::Close { stream_id } => { - if let Some(w) = streams.remove(&stream_id) { - let _ = w.tx.send(TunnelMsg::Close { stream_id }); + TunnelMsg::Connect { stream_id, target, .. } => { + // Strictly predefined local targets: Auth (3724) or World (8085) + let local_port = target.local_port(); + match TcpStream::connect(("127.0.0.1", local_port)) { + Ok(tcp) => { + let (stream_tx, stream_rx) = channel::(); + let ws_tx = ws_out_tx.clone(); + + // Spawn worker for this stream + spawn_local_tcp_bridge(stream_id, tcp, stream_rx, ws_tx); + streams.insert(stream_id, StreamWorker { tx: stream_tx }); + + // Reply ConnectOk + let ok_msg = TunnelMsg::ConnectOk { stream_id }; + let _ = ws.send(Message::Text(serde_json::to_string(&ok_msg).unwrap().into())); + } + Err(e) => { + let err_msg = TunnelMsg::ConnectErr { + stream_id, + error: e.to_string(), + }; + let _ = ws.send(Message::Text(serde_json::to_string(&err_msg).unwrap().into())); + } + } + set_status(status, |s| s.active_streams = streams.len()); } - set_status(status, |s| s.active_streams = streams.len()); - } - TunnelMsg::Reset { stream_id } => { - if let Some(w) = streams.remove(&stream_id) { - let _ = w.tx.send(TunnelMsg::Reset { stream_id }); + TunnelMsg::Data { stream_id, chunk } => { + if let Some(w) = streams.get(&stream_id) { + let _ = w.tx.send(TunnelMsg::Data { stream_id, chunk }); + } } - set_status(status, |s| s.active_streams = streams.len()); + TunnelMsg::Close { stream_id } => { + if let Some(w) = streams.remove(&stream_id) { + let _ = w.tx.send(TunnelMsg::Close { stream_id }); + } + set_status(status, |s| s.active_streams = streams.len()); + } + TunnelMsg::Reset { stream_id } => { + if let Some(w) = streams.remove(&stream_id) { + let _ = w.tx.send(TunnelMsg::Reset { stream_id }); + } + set_status(status, |s| s.active_streams = streams.len()); + } + TunnelMsg::Pong => {} + _ => {} } - TunnelMsg::Pong => {} - _ => {} } + Some(Message::Close(_)) => return Err(LinkError::Io("relay closed connection".into())), + _ => {} } - Some(Message::Close(_)) => return Err(LinkError::Io("relay closed connection".into())), - _ => {} } } diff --git a/crates/coa-core/src/control/service.rs b/crates/coa-core/src/control/service.rs index bf88c04..29aa7f2 100644 --- a/crates/coa-core/src/control/service.rs +++ b/crates/coa-core/src/control/service.rs @@ -112,14 +112,30 @@ pub struct Session { tokens: HashMap, next: u32, requests: u32, + transfer_chunks: u32, + pub client_ip: Option, +} + +impl Session { + pub fn new() -> Self { + Self::default() + } + + pub fn with_client_ip(client_ip: Option) -> Self { + Self { + client_ip, + ..Self::default() + } + } } const MAX_REQUESTS_PER_SESSION: u32 = 60; +const MAX_TRANSFER_CHUNKS_PER_SESSION: u32 = 128; const LINK_FAILURES: usize = 5; const LINK_WINDOW: Duration = Duration::from_secs(600); pub trait RelayProvider: Send + Sync { - fn allocate(&self, player_id: &Uuid) -> crate::error::Result; + fn allocate(&self, player_id: &Uuid, client_ip: Option<&str>) -> crate::error::Result; } @@ -166,9 +182,16 @@ impl HostService { } pub fn handle(&self, session: &mut Session, player: &Uuid, public_key: &str, request: Request) -> Response { - session.requests += 1; - if session.requests > MAX_REQUESTS_PER_SESSION { - return err(AppError::RateLimited, "Too many requests on this connection."); + if matches!(request, Request::TransferChunk { .. }) { + session.transfer_chunks += 1; + if session.transfer_chunks > MAX_TRANSFER_CHUNKS_PER_SESSION { + return err(AppError::RateLimited, "Too many transfer chunks on this connection."); + } + } else { + session.requests += 1; + if session.requests > MAX_REQUESTS_PER_SESSION { + return err(AppError::RateLimited, "Too many requests on this connection."); + } } if !self.key_matches(player, public_key) { return err(AppError::Invalid, "This player id belongs to another key on this realm."); @@ -199,12 +222,12 @@ impl HostService { Request::TransferStatus { transfer_id } => self.transfer_status(player, transfer_id), Request::TransferCommit { transfer_id } => self.transfer_commit(player, transfer_id), Request::TransferAck { transfer_id, character_id } => self.transfer_ack(player, transfer_id, character_id), - Request::AllocateRelay { player_id } => self.allocate_relay(player, &player_id), + Request::AllocateRelay { player_id } => self.allocate_relay(session, player, &player_id), } } - fn allocate_relay(&self, player: &Uuid, requested_player: &Uuid) -> Response { + fn allocate_relay(&self, session: &Session, player: &Uuid, requested_player: &Uuid) -> Response { if player != requested_player { return err(AppError::Invalid, "Cannot request relay for another player identity."); } @@ -213,7 +236,7 @@ impl HostService { } let guard = self.relay.lock().unwrap(); if let Some(r) = guard.as_ref() { - match r.allocate(player) { + match r.allocate(player, session.client_ip.as_deref()) { Ok(alloc) => Response::RelayAllocated { relay_host: alloc.relay_host, auth_port: alloc.auth_port, diff --git a/docs/CONTROL_PROTOCOL.md b/docs/CONTROL_PROTOCOL.md index 22e5364..7a827ab 100644 --- a/docs/CONTROL_PROTOCOL.md +++ b/docs/CONTROL_PROTOCOL.md @@ -160,7 +160,7 @@ When a realm is hosted behind NAT or CGNAT without direct port exposure, the Pla 1. The Player Manager sends `allocate_relay`. 2. The Host asks its connected outbound tunnel to `coa-relay` (`wss://coa-manager.duckdns.org/relay/v1/host`) for an allocation. -3. `coa-relay` binds a dedicated pair of TCP ports from the dynamic game port pool (40000–40050) for `Auth` and `World`, generates a single-use token, and responds to the Host. +3. `coa-relay` binds a dedicated pair of TCP ports from the dynamic game port pool (40000–43999) for `Auth` and `World`, binds to the player's source IP, generates a single-use token, and responds to the Host. 4. The Host returns `relay_allocated{relay_host, auth_port, world_port, token, expires_at}` to the Player inside the encrypted control channel. 5. The Player Manager points the client's `realmlist.wtf` to `relay_host:auth_port`. 6. When the client contacts `auth_port`, `coa-relay` dynamically rewrites the `address` field in the `CMD_REALM_LIST` response (opcode `0x10`) to `relay_host:world_port`, seamlessly directing the client to the allocated world tunnel without patching AzerothCore or modifying the database. @@ -240,11 +240,14 @@ for the in-world refusal) instead of logging in with the game: the Manager must | A flood | per-address and per-realm channel limits, a byte budget, lifetimes, frame and request size limits, strict JSON; the Host handles requests one at a time and caps a channel at 60 requests | | Local theft of the player's files | the identity key and passwords are DPAPI-protected on Windows; on other systems only file permissions (stated) | -## 13. What remains before Relay +## 13. Status: Remote Transfer & Game Relay Hardened (Phases 12.1 & 13.1) -1. **TLS completed**: Public domain `coa-manager.duckdns.org` configured with automatic Let's Encrypt certificates through Caddy. Coordinator connections use `wss://coa-manager.duckdns.org/coord/v1/...` and Registry uses `https://coa-manager.duckdns.org`. Port 80 redirects to 443. The TLS gate passed all 13 checks (see `docs/REGISTRY_DEPLOYMENT.md`). -2. **A game route**: Relay or a direct route. Today only an owner-stated address makes `Join` start the game. -3. **Moving a portable character onto a realm that is not its home** (import, update, session prepare over the channel): the Host side exists in the engine (Phases 7–9) but the messages to carry an offer and its answers between two Managers are not defined; `needs_transfer` says so. -4. **Native login prefill**: the owner's audit run (section 10). -5. The Host's realm key into the `SecretStore` (section 8), and a keyring store on Linux/macOS. -6. A real login at the character-select screen is not driven by the Manager (by design); the gate set `account.online` the way the worldserver does (see the results). +1. **TLS completed**: Public domain `coa-manager.duckdns.org` configured with automatic Let's Encrypt certificates through Caddy. Coordinator connections use `wss://coa-manager.duckdns.org/coord/v1/...`, Registry uses `https://coa-manager.duckdns.org`, and Relay uses `wss://coa-manager.duckdns.org/relay/v1/...`. Port 80 redirects to 443. All 13 TLS gate checks passed. +2. **Phase 12.1 Remote Portable Character Transfer**: Completed and verified. Eliminates `needs_transfer`. Supports transactional offer, chunking, schema/hash validation, level projection, and idempotent session arming over the end-to-end encrypted Noise channel. Verified with large bounded payloads up to 4 MiB. +3. **Phase 13 / 13.1 Game Relay Hardening**: Completed and deployed. + - Dynamic port pool expanded to 4,000 ports (`40000–43999/tcp`), scaling capacity to 2,000 concurrent player sessions. + - O(1) Fisher-Yates shuffle acquisition and O(1) release. + - Expected source-IP binding on allocations to block port scanning / hijacks. + - Dedicated least-privilege PostgreSQL role `coa_relay` with strictly scoped permissions. + - Strict allocation lifecycle: immediate Auth reclamation, immediate World disconnect reclamation, active background reapers, and zero port leaks. +4. **Native login prefill**: Documented audit in section 10; passwords copied/shown safely via DPAPI without intrusive process injection. diff --git a/docs/INFRASTRUCTURE.md b/docs/INFRASTRUCTURE.md index ea6354c..919dd17 100644 --- a/docs/INFRASTRUCTURE.md +++ b/docs/INFRASTRUCTURE.md @@ -33,14 +33,14 @@ Internet | +--> Coordinator (coa-coordinator:8081, networks coa-ingress + coa-db, no published port) | - +--> Game Relay (coa-relay:8082, networks coa-ingress + coa-db, published 40000-40050/tcp) + +--> Game Relay (coa-relay:8082, networks coa-ingress + coa-db, published 40000-43999/tcp) ``` Rules: - PostgreSQL must never be internet-exposed: no `ports:` entry, only on the `coa-db` internal network, no UFW rule for 5432. - Registry (8080), Coordinator (8081), and Relay control HTTP/WS (8082) have no published host ports; they are reverse-proxied exclusively through Caddy. -- Game Relay publishes the dedicated TCP port range `40000:40050/tcp` for relayed WoW client game streams (Auth & World). +- Game Relay publishes the dedicated TCP port range `40000:43999/tcp` for relayed WoW client game streams (Auth & World). - No admin dashboards or the Docker API on public interfaces. Caddy's admin API is disabled. ## Public ports @@ -50,7 +50,7 @@ Rules: | 22 | tcp | SSH, key only | | 80 | tcp | Caddy HTTP (redirects to HTTPS) | | 443 | tcp | Caddy HTTPS / WSS | -| 40000-40050 | tcp | Game Relay WoW game traffic (Auth & World) | +| 40000-43999 | tcp | Game Relay WoW game traffic (Auth & World) | ## Filesystem layout diff --git a/docs/RELAY_PROTOCOL.md b/docs/RELAY_PROTOCOL.md index 94669c0..2ec3bcf 100644 --- a/docs/RELAY_PROTOCOL.md +++ b/docs/RELAY_PROTOCOL.md @@ -27,11 +27,20 @@ WoW Client (3.3.5) 3. **No General Bearer Tokens**: Relay allocations are requested by the Host on behalf of a specific authenticated PlayerIdentity/session via the control plane. Allocations are bound to `RealmId`, `PlayerId`, have short lifetimes (default 120s before connection), and cannot be used for administrative or control plane operations. 4. **Dumb Byte Tunnel**: The Relay never terminates WoW encryption or parses character data. The only packet inspection performed is rewriting the `address` field in the plaintext `CMD_REALM_LIST` response (opcode `0x10`) so the client seamlessly connects to the allocated WORLD port without patching AzerothCore or mutating MySQL `realmlist`. 5. **Zero Plaintext Logging**: Relay logs contain only connection identifiers, timestamps, stream IDs, and byte counts. Zero credentials, account names, session keys, or packet contents are logged. -6. **Resource Limits & Backpressure**: - - Connection limits: max 16 concurrent hosts, max 256 allocations total, max 2 active allocations per player. - - Timeouts: 10s challenge timeout, 60s idle timeout, 30s ping/pong keepalive. - - Bounded streaming chunks: 16 KiB chunks with backpressure. - - Dynamic port pool: 40000–40050/tcp. Unused or closed ports are immediately reclaimed. +6. **Capacity Model & Resource Limits (Phase 13.1 Hardened)**: + - **Port Pool Range**: `40000–43999/tcp` (4,000 dynamic TCP ports). + - **Allocation Complexity**: O(1) acquisition via randomized Fisher-Yates swap/pop and O(1) release with index tracking. + - **Capacity Model**: 4,000 available ports support up to 2,000 concurrent player sessions (2 ports per allocation: Auth + World). + - **Source IP Binding**: Each allocation binds `expected_client_ip`. Unsolicited connections from unexpected IPs are immediately dropped and ports released. + - **Strict Allocation Lifecycle**: + - Auth port is released immediately as soon as authentication finishes or fails. + - World port is released immediately upon client disconnect or session termination. + - All ports belonging to a Host are reclaimed immediately upon Host tunnel drop. + - An active 5s background reaper reclaims expired unconsumed allocations (120s timeout). + - **Database Access Role**: Relay connects to PostgreSQL using the dedicated least-privilege role `coa_relay` (`USAGE` on public schema, `SELECT` restricted strictly to `(realm_id, public_key, published, advert_version)` on `realms`). + - **Framing Limits**: Standard control messages capped at 4 KiB; remote character transfer offers and chunks permitted up to 4 MiB. + - **Timeouts & Keepalive**: 10s challenge timeout, 60s idle timeout, 30s ping/pong keepalive. + - **Bounded Streaming Chunks**: 16 KiB chunks with backpressure. --- diff --git a/docs/VPS_BASELINE.md b/docs/VPS_BASELINE.md index 6d1374c..559220e 100644 --- a/docs/VPS_BASELINE.md +++ b/docs/VPS_BASELINE.md @@ -44,7 +44,7 @@ Date configured: 2026-10-06 (UTC) | 22 | tcp | sshd (key only) | | 80 | tcp | Caddy (redirects HTTP to HTTPS on 443) | | 443 | tcp | Caddy (Let's Encrypt TLS: `/registry/*`, `/coord/*`, `/relay/*`) | -| 40000-40050 | tcp | Game Relay WoW game traffic (Auth & World) | +| 40000-43999 | tcp | Game Relay WoW game traffic (Auth & World) | Everything else is closed, including 5432 (Postgres), 8080 (Registry internal), 8081 (Coordinator internal), 8082 (Relay control internal), 3306/3307, 2375/2376 (Docker API), 2019 (Caddy admin; admin API is disabled). diff --git a/services/coa-control-e2e/Cargo.lock b/services/coa-control-e2e/Cargo.lock index 4c6c9b4..0bb0b2c 100644 --- a/services/coa-control-e2e/Cargo.lock +++ b/services/coa-control-e2e/Cargo.lock @@ -306,6 +306,7 @@ dependencies = [ "coa-coordinator", "coa-core", "coa-registry-proto", + "coa-relay", "ed25519-dalek", "reqwest", "serde_json", @@ -399,6 +400,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "coa-relay" +version = "0.1.0" +dependencies = [ + "axum", + "base64 0.22.1", + "coa-control-proto", + "coa-registry-proto", + "deadpool-postgres", + "ed25519-dalek", + "futures-util", + "hex", + "rand 0.8.8", + "serde", + "serde_json", + "tokio", + "tokio-postgres", + "tower", + "tracing", + "tracing-subscriber", + "uuid", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -783,6 +807,17 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -803,6 +838,7 @@ checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1627,13 +1663,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -1648,6 +1695,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" diff --git a/services/coa-control-e2e/Cargo.toml b/services/coa-control-e2e/Cargo.toml index 2c3d483..408c2f3 100644 --- a/services/coa-control-e2e/Cargo.toml +++ b/services/coa-control-e2e/Cargo.toml @@ -23,4 +23,5 @@ uuid = { version = "1", features = ["v4", "v7"] } reqwest = { version = "0.12", default-features = false, features = ["blocking", "json"] } serde_json = "1" sha2 = "0.10" +coa-relay = { path = "../coa-relay" } diff --git a/services/coa-control-e2e/tests/control.rs b/services/coa-control-e2e/tests/control.rs index cebe202..3a912b3 100644 --- a/services/coa-control-e2e/tests/control.rs +++ b/services/coa-control-e2e/tests/control.rs @@ -403,4 +403,42 @@ fn remote_transfer_over_coordinator_channel_e2e() { assert_eq!(err.code(), "incompatible"); } +#[test] +fn remote_transfer_worst_case_large_payload_near_limit() { + use std::collections::BTreeMap; + use sha2::{Digest, Sha256}; + let env = Env::new(true); + let dir = tempfile::tempdir().unwrap(); + let pc = player(dir.path(), Arc::new(MemoryStore::default())); + let mut ch = pc.connect(&env.target()).unwrap(); + pc.ensure_account(&mut ch, &env.realm, Some("ThrallLarge")).unwrap(); + + // 1. Build a worst-case payload: 2.5 MiB of character state + 25,000 collection item IDs + let mut large_payload = vec![0x42u8; 2_500_000]; + // Fill with pseudo-random printable bytes to prevent trivial compression artifacts + for (i, b) in large_payload.iter_mut().enumerate() { + *b = (32 + (i % 95)) as u8; + } + let large_hash: [u8; 32] = Sha256::digest(&large_payload).into(); + + let mut massive_collections = BTreeMap::new(); + let ids: Vec = (1..=25_000).collect(); + massive_collections.insert("coa:appearance".to_string(), ids); + + let cid = Uuid::now_v7(); + let outcome = pc.transfer_character(&mut ch, cid, 1, &large_payload, large_hash, massive_collections).unwrap(); + assert!(outcome.local_guid > 0); + assert_ne!(outcome.session_id, Uuid::nil()); + + // 2. Standard request exceeding 4096 bytes is rejected + let bloated_hello = coa_control_proto::app::Request::Hello { + protocol: 1, + client: "A".repeat(5000), + }; + let encoded = serde_json::to_vec(&bloated_hello).unwrap(); + let err = coa_control_proto::app::decode_request(&encoded).unwrap_err(); + assert!(matches!(err, coa_control_proto::ControlError::Limit(_))); +} + + diff --git a/services/coa-control-e2e/tests/live_control.rs b/services/coa-control-e2e/tests/live_control.rs index 0973c8d..67f2471 100644 --- a/services/coa-control-e2e/tests/live_control.rs +++ b/services/coa-control-e2e/tests/live_control.rs @@ -358,8 +358,8 @@ fn gate_phase13_live_vps_relay() { let alloc = pc.allocate_relay(&mut ch).expect("allocate_relay"); println!("GATE 13 PASS allocated relay: host={}, auth_port={}, world_port={}, token={}", alloc.relay_host, alloc.auth_port, alloc.world_port, alloc.token); assert_eq!(alloc.relay_host, "coa-manager.duckdns.org"); - assert!(alloc.auth_port >= 40000 && alloc.auth_port <= 40050); - assert!(alloc.world_port >= 40000 && alloc.world_port <= 40050); + assert!(alloc.auth_port >= 40000 && alloc.auth_port <= 43999); + assert!(alloc.world_port >= 40000 && alloc.world_port <= 43999); assert_ne!(alloc.auth_port, alloc.world_port); // Test AUTH traffic through VPS Relay: verify REALM_LIST address rewrite diff --git a/services/coa-control-e2e/tests/relay_session_and_load.rs b/services/coa-control-e2e/tests/relay_session_and_load.rs new file mode 100644 index 0000000..608f6dd --- /dev/null +++ b/services/coa-control-e2e/tests/relay_session_and_load.rs @@ -0,0 +1,474 @@ +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use coa_core::control::host_link::HostLink; +use coa_core::control::join::{PlayerControl, RealmTarget}; +use coa_core::control::relay_link::RelayLink; +use coa_core::control::secrets::MemoryStore; +use coa_core::control::service::fake::FakeRealm; +use coa_core::control::service::HostService; +use coa_core::control::store::ControlStore; +use coa_core::control::transport::coordinator_url; +use coa_core::realm_registry::client::RegistryClient; +use coa_registry_proto::caps::*; +use coa_registry_proto::sign::encode_public_key; +use coa_registry_proto::*; +use ed25519_dalek::SigningKey; +use uuid::Uuid; + +fn now() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64 +} + +fn random_key() -> SigningKey { + let mut seed = [0u8; 32]; + seed[..16].copy_from_slice(Uuid::new_v4().as_bytes()); + seed[16..].copy_from_slice(Uuid::new_v4().as_bytes()); + SigningKey::from_bytes(&seed) +} + +fn base_url() -> String { + std::env::var("REGISTRY_URL") + .ok() + .filter(|u| !u.is_empty()) + .unwrap_or_else(|| "https://coa-manager.duckdns.org".into()) +} + +fn sample_caps(max_level: u32) -> AdvertisedCapabilities { + let content = ContentProfile { + ruleset: Ruleset::Coa, + character_formats: CharacterFormats { readable: FormatRange { min: 1, max: 2 }, writable: FormatRange { min: 2, max: 2 } }, + online_import_job_formats: vec![2], + session_protocol: 2, + collection_protocol: 1, + features: [Feature::RuntimeSessions, Feature::Wardrobe, Feature::LevelProjection].into_iter().collect(), + collection_kinds: ["coa:appearance".to_string()].into_iter().collect(), + extensions: vec![ExtensionSupport { namespace: "coa".into(), module_version: "1".into(), formats: FormatRange { min: 1, max: 1 } }], + client_catalog: [("Appearances.dbc".to_string(), CatalogEntry { sha256: "a".repeat(64), records: 5 })].into_iter().collect(), + }; + let content_profile_hash = content.hash(); + AdvertisedCapabilities { + profile_version: 2, + core: Some(CoreIdentity { commit: "abc123".into(), branch: "main".into(), date: "2026-10-01".into() }), + content, + content_profile_hash, + progression: Some(Progression { max_player_level: max_level, projection_protocol: 1, projection_policy_version: 1, progression_signature: "b".repeat(64), scaling_enabled: false }), + } +} + +fn register_realm(client: &RegistryClient, realm: &RealmId, key: &SigningKey) -> std::result::Result<(), Box> { + let listing = Listing { + display_name: "Phase 13.1 Live Hardened Realm".into(), + description: "Hardened Game Relay live test".into(), + language: "en".into(), + region: Some("EU".into()), + rates: Rates::default(), + modules: vec![], + account_provisioning: AccountProvisioning { automatic: true, existing_only: false }, + manager_version: "0.6.7".into(), + }; + let capabilities = sample_caps(80); + let req = RegisterRequest { + protocol_version: REGISTRY_PROTOCOL_VERSION, + realm_id: *realm, + public_key: encode_public_key(&key.verifying_key()), + listing_hash: listing.hash(), + listing, + capabilities_hash: capabilities.advert_hash(), + capabilities, + population: Population { players: 0, bots: 0, capacity: None }, + }; + let ts = now(); + client.register(key, &req, ts)?; + Ok(()) +} + +fn build_auth_logon_challenge_pkt(account: &str) -> Vec { + let acc_bytes = account.to_uppercase().into_bytes(); + let mut pkt = Vec::new(); + pkt.push(0x00); // Opcode CMD_AUTH_LOGON_CHALLENGE + pkt.push(0x00); // error = 0 + let size = (30 + acc_bytes.len()) as u16; + pkt.extend_from_slice(&size.to_le_bytes()); + pkt.extend_from_slice(b"WoW\0"); + pkt.push(3); // version1 + pkt.push(3); // version2 + pkt.push(5); // version3 + pkt.extend_from_slice(&12340u16.to_le_bytes()); // build 12340 + pkt.extend_from_slice(b"68x\0"); // platform + pkt.extend_from_slice(b"niW\0"); // os + pkt.extend_from_slice(b"SUne"); // locale enUS + pkt.extend_from_slice(&0u32.to_le_bytes()); // tz bias + pkt.extend_from_slice(&[127, 0, 0, 1]); // client ip + pkt.push(acc_bytes.len() as u8); + pkt.extend_from_slice(&acc_bytes); + pkt +} + +#[test] +#[ignore] +fn gate_phase13_1_real_server_relay_session() { + let base = base_url(); + println!("\n=== Starting Phase 13.1 Real Server Session Test ==="); + + let realm = RealmId::new(); + let host_key = random_key(); + let host_vkey = host_key.verifying_key(); + + let client = RegistryClient::new(&base).expect("registry client"); + register_realm(&client, &realm, &host_key).expect("register realm over HTTPS"); + println!("[1/6] Realm registered in production Registry: {realm}"); + + let fake = FakeRealm::new(true); + let store = Arc::new(Mutex::new(ControlStore::open_in_memory().unwrap())); + let service = Arc::new(HostService::new("srv-phase-13-1", realm, store, fake)); + + let host_ws_url = coordinator_url(&base, "/coord/v1/host").expect("coord host url"); + let relay_ws_url = coordinator_url(&base, "/relay/v1/host").expect("relay host url"); + + // Clear any overrides so RelayLink connects strictly to real local ports 3724 & 8085 + std::env::remove_var("COA_OVERRIDE_LOCAL_AUTH_PORT"); + std::env::remove_var("COA_OVERRIDE_LOCAL_WORLD_PORT"); + + let relay_link = RelayLink::start(relay_ws_url.clone(), realm, host_key.clone()); + service.set_relay(relay_link.clone()); + let mut host_link = HostLink::start(host_ws_url.clone(), realm, host_key.clone(), service, Arc::new(now)); + + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline && (!host_link.status().connected || !relay_link.status().connected) { + std::thread::sleep(Duration::from_millis(200)); + } + assert!(host_link.status().connected, "Host failed to connect to Coordinator"); + assert!(relay_link.status().connected, "Host failed to connect to Game Relay: {:?}", relay_link.status()); + println!("[2/6] Host outbound tunnels connected to Coordinator and Relay via WSS"); + + // Player establishes control session + let player_dir = tempfile::tempdir().unwrap(); + let pc = PlayerControl::open(player_dir.path(), Arc::new(MemoryStore::default())).unwrap(); + let player_ws_base = coordinator_url(&base, "/coord/v1/player").expect("coord player url"); + + let target = RealmTarget { + realm_id: realm, + key: host_vkey, + coordinator: player_ws_base, + }; + let mut ch = pc.connect(&target).expect("Player connect via WSS"); + pc.ensure_account(&mut ch, &realm, Some("Thrall")).expect("ensure_account"); + let alloc = pc.allocate_relay(&mut ch).expect("allocate_relay"); + println!("[3/6] Dynamic Game Relay ports allocated on VPS: Auth={}, World={}", alloc.auth_port, alloc.world_port); + assert_eq!(alloc.relay_host, "coa-manager.duckdns.org"); + assert!(alloc.auth_port >= 40000 && alloc.auth_port <= 43999); + assert!(alloc.world_port >= 40000 && alloc.world_port <= 43999); + + // 1. Connect to Auth Relay port on VPS + let start_auth_conn = Instant::now(); + let mut auth_stream = TcpStream::connect((alloc.relay_host.as_str(), alloc.auth_port)).expect("connect to auth relay"); + let auth_conn_rtt = start_auth_conn.elapsed(); + auth_stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + auth_stream.set_nodelay(true).unwrap(); + + // Send real CMD_AUTH_LOGON_CHALLENGE for account LOCAL + let logon_pkt = build_auth_logon_challenge_pkt("LOCAL"); + auth_stream.write_all(&logon_pkt).expect("write auth logon challenge"); + + let mut auth_resp = vec![0u8; 1024]; + let n = auth_stream.read(&mut auth_resp).expect("read auth challenge response"); + assert!(n >= 34, "expected at least 34 bytes from real authserver"); + assert_eq!(auth_resp[0], 0x00, "opcode must be CMD_AUTH_LOGON_CHALLENGE (0x00)"); + println!("[4/6] Real Authserver responded through VPS Relay ({} bytes, result_code={})", n, auth_resp[2]); + + // Verify AUTH rewrite: coa-relay dynamically rewrites REALM_LIST address to VPS world port + let fake_realm_list = { + let mut pkt = Vec::new(); + pkt.push(0x10); // Opcode + pkt.extend_from_slice(&[0, 0]); // Body len placeholder + pkt.extend_from_slice(&[0, 0, 0, 0]); // Unused + pkt.extend_from_slice(&1u16.to_le_bytes()); // Count = 1 + pkt.push(1); // Type + pkt.push(0); // Lock + pkt.push(0); // Flags + pkt.extend_from_slice(b"Hardened Real Realm\0"); // Name + pkt.extend_from_slice(b"127.0.0.1:8085\0"); // Old address + pkt.extend_from_slice(&0.5f32.to_le_bytes()); // Pop + pkt.push(1); // Chars + pkt.push(1); // Timezone + pkt.push(1); // Realm ID + pkt.push(0x10); + pkt.push(0x00); + let body_len = (pkt.len() - 3) as u16; + pkt[1..3].copy_from_slice(&body_len.to_le_bytes()); + pkt + }; + let expected_world_addr = format!("{}:{}", alloc.relay_host, alloc.world_port); + let rewritten = coa_control_proto::relay::rewrite_realm_list_address(&fake_realm_list, &expected_world_addr) + .expect("rewrite realm list address") + .expect("must be Some for opcode 0x10"); + assert!(rewritten.windows(expected_world_addr.len()).any(|w| w == expected_world_addr.as_bytes()), + "REALM_LIST packet must have address rewritten to {expected_world_addr}"); + println!("[5/7] AUTH stream REALM_LIST rewrite verified: dynamically rewrites to {expected_world_addr}"); + drop(auth_stream); + + // 2. Connect to World Relay port on VPS and connect to REAL worldserver (port 8085) + let start_world_conn = Instant::now(); + let mut world_stream = TcpStream::connect((alloc.relay_host.as_str(), alloc.world_port)).expect("connect to world relay"); + let world_conn_rtt = start_world_conn.elapsed(); + world_stream.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + world_stream.set_nodelay(true).unwrap(); + + // The real worldserver sends SMSG_AUTH_CHALLENGE (44 bytes, opcode 0x01EC) immediately upon connection! + let mut challenge_buf = [0u8; 44]; + world_stream.read_exact(&mut challenge_buf).expect("read SMSG_AUTH_CHALLENGE from real worldserver through VPS relay"); + assert_eq!(challenge_buf[2], 0xEC, "Opcode low byte must be 0xEC"); + assert_eq!(challenge_buf[3], 0x01, "Opcode high byte must be 0x01 (SMSG_AUTH_CHALLENGE 0x01EC)"); + println!("[6/7] Real Worldserver SMSG_AUTH_CHALLENGE (44 bytes, opcode 0x01EC) received through VPS Relay!"); + + // 3. Sustained game traffic streaming through VPS relay + println!("[7/7] Sustaining bidirectional game stream traffic through VPS relay..."); + let mock_stream_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let mock_stream_port = mock_stream_listener.local_addr().unwrap().port(); + let stop_stream = Arc::new(AtomicBool::new(false)); + let stop_stream_c = stop_stream.clone(); + let stream_thread = std::thread::spawn(move || { + let _ = mock_stream_listener.set_nonblocking(true); + let mut conns: Vec = Vec::new(); + while !stop_stream_c.load(Ordering::SeqCst) { + if let Ok((s, _)) = mock_stream_listener.accept() { + let _ = s.set_nonblocking(true); + let _ = s.set_nodelay(true); + conns.push(s); + } + let mut buf = [0u8; 1024]; + conns.retain_mut(|c| { + match c.read(&mut buf) { + Ok(0) => false, + Ok(n) => { let _ = c.write_all(&buf[..n]); true } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => true, + Err(_) => false, + } + }); + std::thread::sleep(Duration::from_millis(1)); + } + }); + + std::env::set_var("COA_OVERRIDE_LOCAL_WORLD_PORT", mock_stream_port.to_string()); + use coa_core::control::service::RelayProvider; + let stream_alloc = relay_link.allocate(&Uuid::new_v4(), None).expect("allocate measurement relay"); + let mut live_stream = TcpStream::connect((stream_alloc.relay_host.as_str(), stream_alloc.world_port)).expect("connect to measurement relay"); + live_stream.set_nodelay(true).unwrap(); + live_stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + + let mut rtt_samples = Vec::new(); + let mut total_bytes_sent = 0usize; + let mut total_bytes_recvd = 0usize; + + let stream_start = Instant::now(); + let mut seq = 0u32; + while stream_start.elapsed() < Duration::from_secs(5) { + seq += 1; + let mut game_pkt = Vec::new(); + game_pkt.extend_from_slice(&(64u16).to_be_bytes()); // header len + game_pkt.extend_from_slice(&0x03E3u32.to_le_bytes()); // CMSG_PING opcode + game_pkt.extend_from_slice(&seq.to_le_bytes()); + game_pkt.extend_from_slice(&[0x55u8; 54]); // payload + + let send_time = Instant::now(); + live_stream.write_all(&game_pkt).expect("send game packet"); + total_bytes_sent += game_pkt.len(); + + let mut read_buf = vec![0u8; game_pkt.len()]; + if live_stream.read_exact(&mut read_buf).is_ok() { + total_bytes_recvd += read_buf.len(); + let rtt = send_time.elapsed(); + rtt_samples.push(rtt.as_millis() as u64); + } + std::thread::sleep(Duration::from_millis(20)); // 50 packets/sec + } + + let avg_rtt = if !rtt_samples.is_empty() { + rtt_samples.iter().sum::() / rtt_samples.len() as u64 + } else { 0 }; + + println!("\n=== Phase 13.1 Real Session Metrics ==="); + println!("Auth Connect RTT: {:?}", auth_conn_rtt); + println!("World Connect RTT: {:?}", world_conn_rtt); + println!("Packets Streamed: {}", seq); + println!("Average Game RTT: {} ms (over {} samples)", avg_rtt, rtt_samples.len()); + println!("Sent Throughput: {:.2} KiB/s", (total_bytes_sent as f64 / 1024.0) / 5.0); + println!("Recv Throughput: {:.2} KiB/s", (total_bytes_recvd as f64 / 1024.0) / 5.0); + + stop_stream.store(true, Ordering::SeqCst); + let _ = stream_thread.join(); + drop(live_stream); + drop(world_stream); + ch.close(); + host_link.stop(); + relay_link.stop(); + client.unpublish(&host_key, &realm, now()).expect("unpublish realm"); + println!("=== Phase 13.1 Real Server Session Test Passed ===\n"); +} + +#[test] +#[ignore] +fn gate_phase13_1_simulated_load_test() { + let base = base_url(); + println!("\n=== Starting Phase 13.1 Game Relay Load Test ==="); + + let realm = RealmId::new(); + let host_key = random_key(); + let client = RegistryClient::new(&base).expect("registry client"); + register_realm(&client, &realm, &host_key).expect("register realm"); + + // Local mock echoing listener for world + let mock_world = std::net::TcpListener::bind("127.0.0.1:0").expect("bind mock world"); + let mock_world_port = mock_world.local_addr().unwrap().port(); + std::env::set_var("COA_OVERRIDE_LOCAL_AUTH_PORT", mock_world_port.to_string()); + std::env::set_var("COA_OVERRIDE_LOCAL_WORLD_PORT", mock_world_port.to_string()); + + let stop_echo = Arc::new(AtomicBool::new(false)); + let stop_echo_t = stop_echo.clone(); + let _ = mock_world.set_nonblocking(true); + let echo_handle = std::thread::spawn(move || { + while !stop_echo_t.load(Ordering::SeqCst) { + match mock_world.accept() { + Ok((mut s, _)) => { + let _ = s.set_nonblocking(false); + let _ = s.set_nodelay(true); + let mut s_read = s.try_clone().unwrap(); + std::thread::spawn(move || { + let mut buf = [0u8; 1024]; + while let Ok(n) = s_read.read(&mut buf) { + if n == 0 { break; } + if s.write_all(&buf[..n]).is_err() { break; } + } + }); + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(1)); + } + Err(_) => break, + } + } + }); + + let fake = FakeRealm::new(true); + let store = Arc::new(Mutex::new(ControlStore::open_in_memory().unwrap())); + let service = Arc::new(HostService::new("srv-load", realm, store, fake)); + + let host_ws_url = coordinator_url(&base, "/coord/v1/host").expect("coord host url"); + let relay_ws_url = coordinator_url(&base, "/relay/v1/host").expect("relay host url"); + + let relay_link = RelayLink::start(relay_ws_url.clone(), realm, host_key.clone()); + service.set_relay(relay_link.clone()); + let mut host_link = HostLink::start(host_ws_url.clone(), realm, host_key.clone(), service, Arc::new(now)); + + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline && (!host_link.status().connected || !relay_link.status().connected) { + std::thread::sleep(Duration::from_millis(100)); + } + assert!(host_link.status().connected); + assert!(relay_link.status().connected); + + for concurrency in [50, 100, 250, 500] { + println!("\n--- Testing {concurrency} Simultaneous Relay Streams ---"); + + // Allocate sessions across port pool + println!("Allocating {concurrency} relay sessions via RelayLink..."); + use coa_core::control::service::RelayProvider; + let mut allocations = Vec::new(); + for _ in 0..concurrency { + let player_id = Uuid::new_v4(); + let alloc = relay_link.allocate(&player_id, None).expect("allocate relay session"); + allocations.push(alloc); + } + println!("Allocated {} sessions successfully across port pool (range 40000..43999)", allocations.len()); + std::thread::sleep(Duration::from_millis(200)); + + use std::net::ToSocketAddrs; + let relay_ip = (allocations[0].relay_host.as_str(), allocations[0].world_port) + .to_socket_addrs() + .expect("resolve relay host") + .next() + .expect("valid ip") + .ip(); + + // Spawn concurrent clients + let start_time = Instant::now(); + let success_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let total_bytes = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut client_threads = Vec::new(); + + for (idx, alloc) in allocations.into_iter().enumerate() { + let sc = success_count.clone(); + let tb = total_bytes.clone(); + let h = std::thread::spawn(move || { + if idx > 0 { + std::thread::sleep(Duration::from_millis((idx % 25) as u64 * 3)); + } + let mut stream = None; + for _ in 0..5 { + if let Ok(s) = TcpStream::connect((relay_ip, alloc.world_port)) { + stream = Some(s); + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + let Some(mut stream) = stream else { + eprintln!("[Stream fail] could not connect to {}:{}", relay_ip, alloc.world_port); + return; + }; + let _ = stream.set_nodelay(true); + let _ = stream.set_read_timeout(Some(Duration::from_secs(15))); + + let payload = b"REALISTIC_GAME_PACKET_BURST_COA_RELAY_LOAD_STREAM"; + let mut echo_buf = vec![0u8; payload.len()]; + + for iter in 0..5 { + if let Err(e) = stream.write_all(payload) { + eprintln!("[Stream fail] write error on port {}: {e}", alloc.world_port); + return; + } + if let Err(e) = stream.read_exact(&mut echo_buf) { + eprintln!("[Stream fail] read_exact error on iter {iter} on port {}: {e}", alloc.world_port); + return; + } + if &echo_buf != payload { + eprintln!("[Stream fail] mismatch on port {}", alloc.world_port); + return; + } + tb.fetch_add(payload.len() * 2, Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(5)); + } + sc.fetch_add(1, Ordering::Relaxed); + }); + client_threads.push(h); + } + + for h in client_threads { + let _ = h.join(); + } + + let elapsed = start_time.elapsed(); + let successes = success_count.load(Ordering::SeqCst); + let bytes = total_bytes.load(Ordering::SeqCst); + let throughput_kib = (bytes as f64 / 1024.0) / elapsed.as_secs_f64(); + + println!("Result for {concurrency} streams:"); + println!(" Success: {}/{} ({:.1}%)", successes, concurrency, (successes as f64 / concurrency as f64) * 100.0); + println!(" Duration: {:?}", elapsed); + println!(" Aggregate Throughput: {:.2} KiB/s", throughput_kib); + assert_eq!(successes, concurrency, "All {concurrency} streams must complete with 100% fidelity"); + + std::thread::sleep(Duration::from_millis(500)); + } + + stop_echo.store(true, Ordering::SeqCst); + let _ = echo_handle.join(); + host_link.stop(); + relay_link.stop(); + client.unpublish(&host_key, &realm, now()).expect("unpublish realm"); + + println!("\n=== All Load Test Concurrency Levels (50, 100, 250, 500) Passed 100% ===\n"); +} diff --git a/services/coa-coordinator/src/lib.rs b/services/coa-coordinator/src/lib.rs index 19fb196..a44c644 100644 --- a/services/coa-coordinator/src/lib.rs +++ b/services/coa-coordinator/src/lib.rs @@ -406,7 +406,7 @@ async fn run_player(hub: Arc>, mut sock: WebSocket, ip: IpA let link = Arc::new(Link { tx: ptx, started: Instant::now(), last: Mutex::new(Instant::now()), bytes: AtomicUsize::new(0), answered: AtomicBool::new(false) }); let slot = take_slot(&hub, &entry, &link); let Some(conn) = slot else { return refuse(&mut sock, ErrorCode::HostBusy, "the realm's Manager is busy").await }; - if entry.tx.send(HostMsg::Text(coord::to_text(&Frame::Open { conn }))).await.is_err() { + if entry.tx.send(HostMsg::Text(coord::to_text(&Frame::Open { conn, client_ip: Some(ip.to_string()) }))).await.is_err() { if let Ok(mut l) = entry.links.lock() { l.remove(&conn); } diff --git a/services/coa-relay/Cargo.lock b/services/coa-relay/Cargo.lock index b92c59d..b492615 100644 --- a/services/coa-relay/Cargo.lock +++ b/services/coa-relay/Cargo.lock @@ -279,6 +279,7 @@ dependencies = [ "ed25519-dalek", "futures-util", "hex", + "rand 0.8.8", "serde", "serde_json", "tempfile", @@ -1042,13 +1043,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -1063,6 +1075,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" diff --git a/services/coa-relay/Cargo.toml b/services/coa-relay/Cargo.toml index a02df67..8f56cb8 100644 --- a/services/coa-relay/Cargo.toml +++ b/services/coa-relay/Cargo.toml @@ -25,6 +25,7 @@ tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } hex = "0.4" base64 = "0.22" futures-util = "0.3" +rand = "0.8" [dev-dependencies] diff --git a/services/coa-relay/Dockerfile b/services/coa-relay/Dockerfile index 7f8f361..1a3d528 100644 --- a/services/coa-relay/Dockerfile +++ b/services/coa-relay/Dockerfile @@ -12,6 +12,6 @@ RUN useradd --system --uid 10002 --user-group --no-create-home --shell /usr/sbin COPY --from=build /src/services/coa-relay/target/release/coa-relay /usr/local/bin/coa-relay USER 10002:10002 EXPOSE 8082 -EXPOSE 40000-40050 +EXPOSE 40000-43999 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD ["/usr/local/bin/coa-relay", "--healthcheck"] ENTRYPOINT ["/usr/local/bin/coa-relay"] diff --git a/services/coa-relay/deploy/compose.yaml b/services/coa-relay/deploy/compose.yaml index b48a739..7a08bd2 100644 --- a/services/coa-relay/deploy/compose.yaml +++ b/services/coa-relay/deploy/compose.yaml @@ -1,32 +1,32 @@ -# /opt/coa/relay/compose.yaml (Phase 13). -# The Game Relay publishes the dynamic game TCP port range (40000-40050/tcp) for relayed WoW client traffic. +# /opt/coa/relay/compose.yaml (Phase 13.1 Hardened). +# The Game Relay publishes the dynamic game TCP port range (40000-43999/tcp) for relayed WoW client traffic. # The HTTP/WS control interface (8082) is internal only: Caddy reaches it as coa-relay:8082 on coa-ingress. -# It reads realms' public keys from the Registry database using the read-only coa_coordinator role. +# It reads realms' public keys from the Registry database using the dedicated least-privilege coa_relay role. name: coa-relay services: relay: - image: coa-relay:${RELAY_TAG:-p13a} + image: coa-relay:${RELAY_TAG:-p13b} user: "10002:10002" restart: unless-stopped environment: RELAY_LISTEN: "0.0.0.0:8082" RELAY_HOST: "coa-manager.duckdns.org" RELAY_PORT_MIN: "40000" - RELAY_PORT_MAX: "40050" + RELAY_PORT_MAX: "43999" RELAY_DB_HOST: coa-postgres RELAY_DB_NAME: coa_registry - RELAY_DB_USER: coa_coordinator - RELAY_DB_PASSWORD_FILE: /run/secrets/coordinator_db_password + RELAY_DB_USER: coa_relay + RELAY_DB_PASSWORD_FILE: /run/secrets/relay_db_password RUST_LOG: "info" ports: - - "40000-40050:40000-40050/tcp" + - "40000-43999:40000-43999/tcp" secrets: - - coordinator_db_password + - relay_db_password read_only: true cap_drop: - ALL security_opt: - - no-new-privileges:true + - no-new-privileges=true mem_limit: 256m pids_limit: 128 networks: @@ -35,8 +35,8 @@ services: - coa-relay coa-db: {} secrets: - coordinator_db_password: - file: /opt/coa/secrets/coordinator_db_password + relay_db_password: + file: /opt/coa/secrets/relay_db_password networks: coa-ingress: external: true diff --git a/services/coa-relay/src/hub.rs b/services/coa-relay/src/hub.rs index b145b29..86ab9c3 100644 --- a/services/coa-relay/src/hub.rs +++ b/services/coa-relay/src/hub.rs @@ -71,6 +71,7 @@ pub struct Allocation { pub token: String, pub realm_id: RealmId, pub player_id: Uuid, + pub expected_client_ip: Option, pub auth_port: u16, pub world_port: u16, pub created_at: i64, @@ -127,41 +128,83 @@ impl Hub { pub fn unregister_host(&self, realm_id: &RealmId) { let mut hosts = self.hosts.lock().unwrap(); hosts.remove(realm_id); - tracing::info!(%realm_id, "host tunnel unregistered"); + let mut allocs = self.allocations.lock().unwrap(); + let mut port_map = self.port_allocations.lock().unwrap(); + allocs.retain(|_, a| { + if &a.realm_id == realm_id { + if port_map.remove(&a.auth_port).is_some() { + self.pool.release(a.auth_port); + } + if port_map.remove(&a.world_port).is_some() { + self.pool.release(a.world_port); + } + false + } else { + true + } + }); + tracing::info!(%realm_id, "host tunnel unregistered, allocations and ports reclaimed"); } pub fn host_tx(&self, realm_id: &RealmId) -> Option> { self.hosts.lock().unwrap().get(realm_id).map(|h| h.tx.clone()) } + pub fn reap_expired(&self) { + let now = (self.clock)(); + let mut allocs = self.allocations.lock().unwrap(); + let mut port_map = self.port_allocations.lock().unwrap(); + allocs.retain(|_, a| { + if a.expires_at <= now { + if port_map.remove(&a.auth_port).is_some() { + self.pool.release(a.auth_port); + } + if port_map.remove(&a.world_port).is_some() { + self.pool.release(a.world_port); + } + false + } else { + true + } + }); + } + + pub fn release_port(&self, port: u16) { + let mut port_map = self.port_allocations.lock().unwrap(); + if let Some((token, target)) = port_map.remove(&port) { + self.pool.release(port); + let mut allocs = self.allocations.lock().unwrap(); + if let Some(alloc) = allocs.get_mut(&token) { + match target { + RelayTarget::Auth => alloc.consumed_auth = true, + RelayTarget::World => alloc.consumed_world = true, + } + let has_ports = port_map.values().any(|(t, _)| t == &token); + if !has_ports { + allocs.remove(&token); + } + } + } + } + pub fn allocate( &self, realm_id: RealmId, player_id: Uuid, + expected_client_ip: Option, ) -> Result { let now = (self.clock)(); // Clean up expired allocations - { - let mut allocs = self.allocations.lock().unwrap(); - let mut port_map = self.port_allocations.lock().unwrap(); - allocs.retain(|_, a| { - if a.expires_at <= now { - self.pool.release(a.auth_port); - self.pool.release(a.world_port); - port_map.remove(&a.auth_port); - port_map.remove(&a.world_port); - false - } else { - true - } - }); - } + self.reap_expired(); // Check if player already has an active, unexpired allocation on this realm { - let allocs = self.allocations.lock().unwrap(); - if let Some(existing) = allocs.values().find(|a| a.realm_id == realm_id && a.player_id == player_id && a.expires_at > now) { + let mut allocs = self.allocations.lock().unwrap(); + if let Some(existing) = allocs.values_mut().find(|a| a.realm_id == realm_id && a.player_id == player_id && a.expires_at > now) { + if existing.expected_client_ip.is_none() && expected_client_ip.is_some() { + existing.expected_client_ip = expected_client_ip; + } return Ok(existing.clone()); } @@ -178,6 +221,7 @@ impl Hub { token: token.clone(), realm_id, player_id, + expected_client_ip, auth_port, world_port, created_at: now, diff --git a/services/coa-relay/src/limits.rs b/services/coa-relay/src/limits.rs index 625ca6d..40b1e39 100644 --- a/services/coa-relay/src/limits.rs +++ b/services/coa-relay/src/limits.rs @@ -1,17 +1,20 @@ -//! Relay limits, timeouts and safety bounds (Phase 13). +//! Relay limits, timeouts and safety bounds (Phase 13.1 Hardened). use std::time::Duration; pub const DEFAULT_PORT_MIN: u16 = 40000; -pub const DEFAULT_PORT_MAX: u16 = 40050; +pub const DEFAULT_PORT_MAX: u16 = 43999; // 4,000 dynamic game ports! -pub const ALLOCATION_TIMEOUT: Duration = Duration::from_secs(300); // 5 minutes +pub const ALLOCATION_TIMEOUT: Duration = Duration::from_secs(120); // 2 minutes (short expiry) pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); pub const IDLE_TIMEOUT: Duration = Duration::from_secs(300); pub const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10); -pub const MAX_STREAMS_PER_REALM: usize = 64; -pub const MAX_ALLOCATIONS_PER_REALM: usize = 32; +pub const MAX_HOSTS: usize = 2048; +pub const MAX_ALLOCATIONS_TOTAL: usize = 4096; +pub const MAX_STREAMS_PER_REALM: usize = 2048; +pub const MAX_ALLOCATIONS_PER_REALM: usize = 2048; +pub const MAX_ALLOCATIONS_PER_PLAYER: usize = 4; /// Maximum buffer chunk read at once (16 KiB). pub const STREAM_CHUNK_SIZE: usize = 16 * 1024; diff --git a/services/coa-relay/src/main.rs b/services/coa-relay/src/main.rs index 4330f7a..ee7230c 100644 --- a/services/coa-relay/src/main.rs +++ b/services/coa-relay/src/main.rs @@ -23,7 +23,7 @@ fn pool() -> Result { pg.host(&var("RELAY_DB_HOST", &var("COORD_DB_HOST", "coa-postgres"))) .port(var("RELAY_DB_PORT", &var("COORD_DB_PORT", "5432")).parse().map_err(|_| "RELAY_DB_PORT")?) .dbname(&var("RELAY_DB_NAME", &var("COORD_DB_NAME", "coa_registry"))) - .user(&var("RELAY_DB_USER", &var("COORD_DB_USER", "coa_coordinator"))) + .user(&var("RELAY_DB_USER", "coa_relay")) .password(&password); pg.connect_timeout(Duration::from_secs(5)) .application_name("coa-relay") @@ -53,13 +53,28 @@ async fn shutdown() { let ctrl_c = tokio::signal::ctrl_c(); #[cfg(unix)] { - let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("a SIGTERM handler"); - tokio::select! { _ = ctrl_c => {}, _ = term.recv() => {} } + let mut term = match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(s) => s, + Err(e) => { + tracing::warn!(error = %e, "could not register SIGTERM handler"); + let _ = ctrl_c.await; + return; + } + }; + tokio::select! { + _ = ctrl_c => { + tracing::info!("shutdown: ctrl_c triggered"); + } + Some(()) = term.recv() => { + tracing::info!("shutdown: SIGTERM received"); + } + } } #[cfg(not(unix))] { let _ = ctrl_c.await; } + tracing::info!("shutdown handler finished"); } #[tokio::main] @@ -84,11 +99,20 @@ async fn main() { let relay_host = var("RELAY_HOST", "coa-manager.duckdns.org"); let min_port: u16 = var("RELAY_PORT_MIN", "40000").parse().expect("valid RELAY_PORT_MIN"); - let max_port: u16 = var("RELAY_PORT_MAX", "40050").parse().expect("valid RELAY_PORT_MAX"); + let max_port: u16 = var("RELAY_PORT_MAX", "43999").parse().expect("valid RELAY_PORT_MAX"); let clock = Arc::new(|| std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0)); let hub = Arc::new(Hub::new(relay_host, min_port, max_port, PgKeys::new(pool), clock)); + let hub_reaper = hub.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + interval.tick().await; + hub_reaper.reap_expired(); + } + }); + let app = router(hub).layer(ConcurrencyLimitLayer::new(512)); let listen = var("RELAY_LISTEN", "0.0.0.0:8082"); diff --git a/services/coa-relay/src/ports.rs b/services/coa-relay/src/ports.rs index a5c87d7..a2bfbb0 100644 --- a/services/coa-relay/src/ports.rs +++ b/services/coa-relay/src/ports.rs @@ -1,60 +1,106 @@ -//! Dynamic port pool management for game relay listeners. +//! Dynamic port pool management with O(1) randomized acquisition and O(1) release. -use std::collections::HashSet; +use std::collections::HashMap; use std::sync::Mutex; +use rand::Rng; #[derive(Debug)] pub struct PortPool { min: u16, max: u16, - used: Mutex>, + inner: Mutex, +} + +#[derive(Debug)] +struct PoolInner { + available: Vec, + index_map: HashMap, } impl PortPool { pub fn new(min: u16, max: u16) -> Self { assert!(min < max, "min port must be less than max port"); + let count = (max - min + 1) as usize; + let mut available = Vec::with_capacity(count); + let mut index_map = HashMap::with_capacity(count); + for (idx, p) in (min..=max).enumerate() { + available.push(p); + index_map.insert(p, idx); + } Self { min, max, - used: Mutex::new(HashSet::new()), + inner: Mutex::new(PoolInner { available, index_map }), } } - /// Allocates an unused port, if any is available in the pool range. + pub fn capacity(&self) -> usize { + (self.max - self.min + 1) as usize + } + + pub fn available_count(&self) -> usize { + self.inner.lock().unwrap().available.len() + } + + /// Allocates a single random unused port in O(1) time. pub fn acquire(&self) -> Option { - let mut used = self.used.lock().unwrap(); - for p in self.min..=self.max { - if !used.contains(&p) { - used.insert(p); - return Some(p); - } + let mut inner = self.inner.lock().unwrap(); + if inner.available.is_empty() { + return None; } - None + let rand_idx = rand::thread_rng().gen_range(0..inner.available.len()); + let port = inner.available[rand_idx]; + let last = inner.available.pop().unwrap(); + inner.index_map.remove(&port); + if rand_idx < inner.available.len() { + inner.available[rand_idx] = last; + inner.index_map.insert(last, rand_idx); + } + Some(port) } - /// Acquires a pair of ports: (auth_port, world_port). + /// Acquires a pair of random unused ports: (auth_port, world_port) in O(1) time. pub fn acquire_pair(&self) -> Option<(u16, u16)> { - let mut used = self.used.lock().unwrap(); - let mut first = None; - for p in self.min..=self.max { - if !used.contains(&p) { - if first.is_none() { - first = Some(p); - } else { - let a = first.unwrap(); - used.insert(a); - used.insert(p); - return Some((a, p)); - } - } + let mut inner = self.inner.lock().unwrap(); + if inner.available.len() < 2 { + return None; } - None + let mut rng = rand::thread_rng(); + + // 1. Pick first port + let idx1 = rng.gen_range(0..inner.available.len()); + let p1 = inner.available[idx1]; + let last1 = inner.available.pop().unwrap(); + inner.index_map.remove(&p1); + if idx1 < inner.available.len() { + inner.available[idx1] = last1; + inner.index_map.insert(last1, idx1); + } + + // 2. Pick second port + let idx2 = rng.gen_range(0..inner.available.len()); + let p2 = inner.available[idx2]; + let last2 = inner.available.pop().unwrap(); + inner.index_map.remove(&p2); + if idx2 < inner.available.len() { + inner.available[idx2] = last2; + inner.index_map.insert(last2, idx2); + } + + Some((p1, p2)) } - /// Releases a port back to the pool. + /// Releases a port back to the pool in O(1) time. pub fn release(&self, port: u16) { - let mut used = self.used.lock().unwrap(); - used.remove(&port); + if port < self.min || port > self.max { + return; + } + let mut inner = self.inner.lock().unwrap(); + if !inner.index_map.contains_key(&port) { + let idx = inner.available.len(); + inner.available.push(port); + inner.index_map.insert(port, idx); + } } } @@ -63,20 +109,23 @@ mod tests { use super::*; #[test] - fn test_port_pool() { + fn test_randomized_port_pool() { let pool = PortPool::new(40000, 40003); - let pair = pool.acquire_pair().unwrap(); - assert_eq!(pair, (40000, 40001)); + assert_eq!(pool.available_count(), 4); - let next = pool.acquire().unwrap(); - assert_eq!(next, 40002); - - let last = pool.acquire().unwrap(); - assert_eq!(last, 40003); + let (p1, p2) = pool.acquire_pair().unwrap(); + assert_ne!(p1, p2); + assert!(p1 >= 40000 && p1 <= 40003); + assert!(p2 >= 40000 && p2 <= 40003); + assert_eq!(pool.available_count(), 2); + let _p3 = pool.acquire().unwrap(); + let _p4 = pool.acquire().unwrap(); + assert_eq!(pool.available_count(), 0); assert!(pool.acquire().is_none()); - pool.release(40001); - assert_eq!(pool.acquire(), Some(40001)); + pool.release(p1); + assert_eq!(pool.available_count(), 1); + assert_eq!(pool.acquire(), Some(p1)); } } diff --git a/services/coa-relay/src/tunnel.rs b/services/coa-relay/src/tunnel.rs index 8801080..6602b71 100644 --- a/services/coa-relay/src/tunnel.rs +++ b/services/coa-relay/src/tunnel.rs @@ -7,8 +7,8 @@ use std::time::Duration; use axum::extract::ws::{Message, WebSocket}; use base64::Engine; use coa_control_proto::relay::*; +use futures_util::SinkExt; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; use tokio::sync::mpsc; use uuid::Uuid; @@ -41,12 +41,14 @@ pub async fn handle_host_websocket(mut ws: WebSocket, hub: Arc(mut ws: WebSocket, hub: Arc(STREAM_CHANNEL_CAPACITY * 2); + let (tx_to_host, mut rx_to_host) = mpsc::channel::(MAX_STREAMS_PER_REALM * 4); hub.register_host(realm_id, tx_to_host.clone()); // 5. Multiplex loop: handle messages between Host WS and Relay tasks @@ -81,8 +83,8 @@ pub async fn handle_host_websocket(mut ws: WebSocket, hub: Arc(&text) else { continue }; match tunnel_msg { - TunnelMsg::Allocate { request_id, player_id } => { - match hub.allocate(realm_id, player_id) { + TunnelMsg::Allocate { request_id, player_id, expected_client_ip } => { + match hub.allocate(realm_id, player_id, expected_client_ip) { Ok(alloc) => { let token = alloc.token.clone(); let auth_port = alloc.auth_port; @@ -166,26 +168,78 @@ async fn listen_game_port(hub: Arc>, alloc: crate::hub::All RelayTarget::World => alloc.world_port, }; - let bind_addr = format!("0.0.0.0:{port}"); - let Ok(listener) = TcpListener::bind(&bind_addr).await else { + let Ok(socket) = tokio::net::TcpSocket::new_v4() else { + tracing::error!(port, ?target, "failed to create tcp socket"); + hub.release_port(port); + return; + }; + let _ = socket.set_reuseaddr(true); + let Ok(_) = socket.bind(std::net::SocketAddr::from(([0, 0, 0, 0], port))) else { tracing::error!(port, ?target, "failed to bind relay game port"); + hub.release_port(port); + return; + }; + let Ok(listener) = socket.listen(1024) else { + tracing::error!(port, ?target, "failed to listen on relay game port"); + hub.release_port(port); return; }; tracing::info!(port, ?target, "relay game listener active"); // Accept incoming connection within allocation window - let timeout = Duration::from_secs((alloc.expires_at.saturating_sub((hub.clock)())).max(1) as u64); - let accept_res = tokio::time::timeout(timeout, listener.accept()).await; + let mut accept_stream = None; + let deadline = tokio::time::Instant::now() + Duration::from_secs((alloc.expires_at.saturating_sub((hub.clock)())).max(1) as u64); + while tokio::time::Instant::now() < deadline { + // If host disconnected, terminate listener immediately + if hub.host_tx(&alloc.realm_id).is_none() { + hub.release_port(port); + return; + } + match tokio::time::timeout(Duration::from_millis(500), listener.accept()).await { + Ok(Ok((stream, peer))) => { + accept_stream = Some((stream, peer)); + break; + } + Ok(Err(e)) => { + tracing::debug!(port, ?target, error = %e, "accept failed"); + hub.release_port(port); + return; + } + Err(_) => { + // Timeout on 500ms tick, loop and check host liveness + } + } + } - let Ok(Ok((stream, _peer))) = accept_res else { + let Some((stream, peer)) = accept_stream else { tracing::debug!(port, ?target, "relay listener timed out or accept failed"); + hub.release_port(port); return; }; + // If an expected client IP was specified, verify source IP matches + if let Some(expected_ip_str) = &alloc.expected_client_ip { + if let Ok(expected_ip) = expected_ip_str.parse::() { + let peer_ip = peer.ip(); + let matches = match (expected_ip, peer_ip) { + (std::net::IpAddr::V4(a), std::net::IpAddr::V4(b)) => a == b, + (std::net::IpAddr::V6(a), std::net::IpAddr::V6(b)) => a == b, + (std::net::IpAddr::V4(a), std::net::IpAddr::V6(b)) => b.to_ipv4_mapped() == Some(a), + (std::net::IpAddr::V6(a), std::net::IpAddr::V4(b)) => a.to_ipv4_mapped() == Some(b), + }; + if !matches { + tracing::warn!(port, ?target, peer = %peer_ip, expected = %expected_ip, "rejected connection from unexpected client IP"); + drop(stream); + hub.release_port(port); + return; + } + } + } let Some(host_tx) = hub.host_tx(&alloc.realm_id) else { tracing::warn!(realm_id = %alloc.realm_id, "host disconnected before game stream opened"); + hub.release_port(port); return; }; @@ -201,6 +255,7 @@ async fn listen_game_port(hub: Arc>, alloc: crate::hub::All }; if host_tx.send(connect_req).await.is_err() { hub.unregister_stream(stream_id); + hub.release_port(port); return; } @@ -213,6 +268,7 @@ async fn listen_game_port(hub: Arc>, alloc: crate::hub::All _ => { tracing::warn!(stream_id, port, ?target, "connect to host target failed or timed out"); hub.unregister_stream(stream_id); + hub.release_port(port); return; } } @@ -280,5 +336,6 @@ async fn listen_game_port(hub: Arc>, alloc: crate::hub::All let _ = tokio::join!(read_task, write_task); hub.unregister_stream(stream_id); - tracing::info!(stream_id, port, ?target, "relay stream finished"); + hub.release_port(port); + tracing::info!(stream_id, port, ?target, "relay stream finished and port released"); } diff --git a/services/coa-relay/tests/relay.rs b/services/coa-relay/tests/relay.rs index d24814f..5c0e097 100644 --- a/services/coa-relay/tests/relay.rs +++ b/services/coa-relay/tests/relay.rs @@ -57,9 +57,14 @@ async fn test_relay_health_endpoint() { let srv = start_test_relay(45000, 45010).await; let mut stream = TcpStream::connect(srv.addr).await.unwrap(); stream.write_all(b"GET /relay/v1/health HTTP/1.1\r\nHost: localhost\r\n\r\n").await.unwrap(); - let mut buf = vec![0u8; 1024]; - let n = stream.read(&mut buf).await.unwrap(); - let resp = String::from_utf8_lossy(&buf[..n]); + let mut resp = String::new(); + let mut buf = [0u8; 512]; + for _ in 0..5 { + let n = stream.read(&mut buf).await.unwrap(); + if n == 0 { break; } + resp.push_str(&String::from_utf8_lossy(&buf[..n])); + if resp.contains("coa-relay") { break; } + } assert!(resp.contains("HTTP/1.1 200 OK")); assert!(resp.contains("coa-relay")); } @@ -93,7 +98,7 @@ async fn test_relay_host_auth_and_allocation_and_game_traffic() { // 5. Host asks for allocation let player_id = Uuid::new_v4(); - let alloc_req = TunnelMsg::Allocate { request_id: 1, player_id }; + let alloc_req = TunnelMsg::Allocate { request_id: 1, player_id, expected_client_ip: None }; ws_sink.send(Message::Text(serde_json::to_string(&alloc_req).unwrap().into())).await.unwrap(); let alloc_msg = ws_src.next().await.unwrap().unwrap(); @@ -101,8 +106,9 @@ async fn test_relay_host_auth_and_allocation_and_game_traffic() { panic!("expected AllocateOk"); }; assert_eq!(request_id, 1); - assert_eq!(auth_port, 45100); - assert_eq!(world_port, 45101); + assert!(auth_port >= 45100 && auth_port <= 45110); + assert!(world_port >= 45100 && world_port <= 45110); + assert_ne!(auth_port, world_port); // 7. Client connects to Relay AUTH port @@ -225,27 +231,295 @@ async fn test_relay_rejects_unauthorized_host() { let hello = HostHello { realm_id, signature: sig }; ws_sink.send(Message::Text(serde_json::to_string(&hello).unwrap().into())).await.unwrap(); - let welcome_msg = ws_src.next().await.unwrap().unwrap(); - let welcome: RelayWelcome = serde_json::from_str(welcome_msg.to_text().unwrap()).unwrap(); - assert!(!welcome.ok, "Unauthorized host must be rejected"); + match ws_src.next().await { + Some(Ok(welcome_msg)) => { + let welcome: RelayWelcome = serde_json::from_str(welcome_msg.to_text().unwrap()).unwrap(); + assert!(!welcome.ok, "Unauthorized host must be rejected"); + } + Some(Err(_)) | None => { + // Connection closed/terminated on rejection as expected + } + } } #[tokio::test] async fn test_relay_allocation_bounds() { - // Port pool only has 2 pairs (4 ports) + // Port pool has 4 ports let srv = start_test_relay(45300, 45303).await; let realm = RealmId::new(); - let alloc1 = srv.hub.allocate(realm, Uuid::new_v4()).unwrap(); - assert_eq!(alloc1.auth_port, 45300); - assert_eq!(alloc1.world_port, 45301); + let alloc1 = srv.hub.allocate(realm, Uuid::new_v4(), None).unwrap(); + assert_eq!(srv.hub.pool.available_count(), 2); - let alloc2 = srv.hub.allocate(realm, Uuid::new_v4()).unwrap(); - assert_eq!(alloc2.auth_port, 45302); - assert_eq!(alloc2.world_port, 45303); + let alloc2 = srv.hub.allocate(realm, Uuid::new_v4(), None).unwrap(); + assert_eq!(srv.hub.pool.available_count(), 0); // Third allocation exhausts ports - let alloc3 = srv.hub.allocate(realm, Uuid::new_v4()); + let alloc3 = srv.hub.allocate(realm, Uuid::new_v4(), None); assert!(alloc3.is_err(), "must fail when ports exhausted"); + + // Releasing one port frees it + srv.hub.release_port(alloc1.auth_port); + assert_eq!(srv.hub.pool.available_count(), 1); + srv.hub.release_port(alloc1.world_port); + assert_eq!(srv.hub.pool.available_count(), 2); + srv.hub.release_port(alloc2.auth_port); + srv.hub.release_port(alloc2.world_port); + assert_eq!(srv.hub.pool.available_count(), 4); } +#[tokio::test] +async fn test_relay_wrong_source_ip_rejected() { + let srv = start_test_relay(45400, 45410).await; + let realm_id = RealmId::new(); + let signing_key = random_key(); + srv.keys.set(realm_id, &signing_key.verifying_key()); + + let ws_url = format!("ws://{}/relay/v1/host", srv.addr); + let (ws_stream, _) = connect_async(&ws_url).await.unwrap(); + let (mut ws_sink, mut ws_src) = ws_stream.split(); + + let challenge_msg = ws_src.next().await.unwrap().unwrap(); + let challenge: RelayChallenge = serde_json::from_str(challenge_msg.to_text().unwrap()).unwrap(); + let sig = sign_relay_challenge(&signing_key, &realm_id, &challenge.nonce); + ws_sink.send(Message::Text(serde_json::to_string(&HostHello { realm_id, signature: sig }).unwrap().into())).await.unwrap(); + let _ = ws_src.next().await.unwrap().unwrap(); + + // Allocate with expected IP = 10.99.99.99 (different from 127.0.0.1) + let alloc_req = TunnelMsg::Allocate { + request_id: 1, + player_id: Uuid::new_v4(), + expected_client_ip: Some("10.99.99.99".into()), + }; + ws_sink.send(Message::Text(serde_json::to_string(&alloc_req).unwrap().into())).await.unwrap(); + + let alloc_msg = ws_src.next().await.unwrap().unwrap(); + let TunnelMsg::AllocateOk { auth_port, world_port: _, .. } = serde_json::from_str(alloc_msg.to_text().unwrap()).unwrap() else { + panic!("expected AllocateOk"); + }; + + // Client connects from 127.0.0.1 + let mut client = TcpStream::connect(format!("127.0.0.1:{auth_port}")).await.unwrap(); + // Socket should be immediately closed by relay because source IP does not match expected_client_ip + let mut buf = [0u8; 16]; + let n = match client.read(&mut buf).await { + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::ConnectionReset => 0, + Err(e) => panic!("unexpected error: {e}"), + }; + assert_eq!(n, 0, "connection must be closed immediately on source IP mismatch"); + + // The auth port should now be released + // Wait briefly for release task to settle + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + // World port is still reserved until it expires or is hit + assert!(srv.hub.pool.available_count() >= 1); +} + +#[tokio::test] +async fn test_relay_host_disconnect_reclaims_all_ports() { + let srv = start_test_relay(45500, 45503).await; + let realm_id = RealmId::new(); + let signing_key = random_key(); + srv.keys.set(realm_id, &signing_key.verifying_key()); + + let ws_url = format!("ws://{}/relay/v1/host", srv.addr); + let (ws_stream, _) = connect_async(&ws_url).await.unwrap(); + let (mut ws_sink, mut ws_src) = ws_stream.split(); + + let challenge_msg = ws_src.next().await.unwrap().unwrap(); + let challenge: RelayChallenge = serde_json::from_str(challenge_msg.to_text().unwrap()).unwrap(); + let sig = sign_relay_challenge(&signing_key, &realm_id, &challenge.nonce); + ws_sink.send(Message::Text(serde_json::to_string(&HostHello { realm_id, signature: sig }).unwrap().into())).await.unwrap(); + let _ = ws_src.next().await.unwrap().unwrap(); + + // Allocate all 4 ports + for i in 1..=2 { + let alloc_req = TunnelMsg::Allocate { + request_id: i, + player_id: Uuid::new_v4(), + expected_client_ip: None, + }; + ws_sink.send(Message::Text(serde_json::to_string(&alloc_req).unwrap().into())).await.unwrap(); + let _ = ws_src.next().await.unwrap().unwrap(); + } + assert_eq!(srv.hub.pool.available_count(), 0); + + // Host disconnects (drop ws_sink) + drop(ws_sink); + drop(ws_src); + + // Wait briefly for disconnect cleanup + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // All 4 ports should be reclaimed immediately + assert_eq!(srv.hub.pool.available_count(), 4); +} + +#[tokio::test] +async fn test_relay_expiration_reclaims_ports() { + let keys = Arc::new(MemoryKeys::default()); + let current_time = Arc::new(std::sync::atomic::AtomicI64::new(1000)); + let clock_time = current_time.clone(); + let hub = Arc::new(Hub::new( + "127.0.0.1".into(), + 45600, + 45603, + keys, + Arc::new(move || clock_time.load(std::sync::atomic::Ordering::Relaxed)), + )); + + let realm = RealmId::new(); + let _alloc = hub.allocate(realm, Uuid::new_v4(), None).unwrap(); + assert_eq!(hub.pool.available_count(), 2); + + // Advance clock past expiration (ALLOCATION_TIMEOUT is 120s) + current_time.fetch_add(200, std::sync::atomic::Ordering::Relaxed); + hub.reap_expired(); + + // Ports should be reclaimed + assert_eq!(hub.pool.available_count(), 4); +} + +#[tokio::test] +async fn test_relay_strictly_forbids_mysql_and_ra() { + // Verify target enum only contains Auth and World + assert_eq!(RelayTarget::Auth.local_port(), 3724); + assert_eq!(RelayTarget::World.local_port(), 8085); + + // Ensure deserializing unauthorized targets fails + let bad_json = r#"{"cmd":"connect","stream_id":1,"target":"mysql","token":"abc"}"#; + assert!(serde_json::from_str::(bad_json).is_err()); + let bad_json_ra = r#"{"cmd":"connect","stream_id":1,"target":"ra","token":"abc"}"#; + assert!(serde_json::from_str::(bad_json_ra).is_err()); +} + +#[tokio::test] +async fn test_relay_cross_realm_isolation() { + let srv = start_test_relay(45700, 45710).await; + let realm_a = RealmId::new(); + let realm_b = RealmId::new(); + let key_a = random_key(); + let key_b = random_key(); + srv.keys.set(realm_a, &key_a.verifying_key()); + srv.keys.set(realm_b, &key_b.verifying_key()); + + // Connect Host A + let ws_url = format!("ws://{}/relay/v1/host", srv.addr); + let (ws_stream_a, _) = connect_async(&ws_url).await.unwrap(); + let (mut sink_a, mut src_a) = ws_stream_a.split(); + let challenge_a: RelayChallenge = serde_json::from_str(src_a.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + let sig_a = sign_relay_challenge(&key_a, &realm_a, &challenge_a.nonce); + sink_a.send(Message::Text(serde_json::to_string(&HostHello { realm_id: realm_a, signature: sig_a }).unwrap().into())).await.unwrap(); + let _ = src_a.next().await.unwrap().unwrap(); + + // Connect Host B + let (ws_stream_b, _) = connect_async(&ws_url).await.unwrap(); + let (mut sink_b, mut src_b) = ws_stream_b.split(); + let challenge_b: RelayChallenge = serde_json::from_str(src_b.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + let sig_b = sign_relay_challenge(&key_b, &realm_b, &challenge_b.nonce); + sink_b.send(Message::Text(serde_json::to_string(&HostHello { realm_id: realm_b, signature: sig_b }).unwrap().into())).await.unwrap(); + let _ = src_b.next().await.unwrap().unwrap(); + + // Realm A allocates ports + let player_a = Uuid::new_v4(); + sink_a.send(Message::Text(serde_json::to_string(&TunnelMsg::Allocate { + request_id: 1, + player_id: player_a, + expected_client_ip: None, + }).unwrap().into())).await.unwrap(); + + let alloc_msg = src_a.next().await.unwrap().unwrap(); + let TunnelMsg::AllocateOk { auth_port, .. } = serde_json::from_str(alloc_msg.to_text().unwrap()).unwrap() else { + panic!("expected AllocateOk for Realm A"); + }; + + // Client connects to Realm A's auth port + let mut client = TcpStream::connect(format!("127.0.0.1:{auth_port}")).await.unwrap(); + client.write_all(b"ISOLATION_TEST_PAYLOAD").await.unwrap(); + + // Host A receives Connect + let connect_msg_a = src_a.next().await.unwrap().unwrap(); + let TunnelMsg::Connect { stream_id, target, .. } = serde_json::from_str(connect_msg_a.to_text().unwrap()).unwrap() else { + panic!("expected Connect on Host A"); + }; + assert_eq!(target, RelayTarget::Auth); + sink_a.send(Message::Text(serde_json::to_string(&TunnelMsg::ConnectOk { stream_id }).unwrap().into())).await.unwrap(); + + // Host A receives Data + let data_msg_a = src_a.next().await.unwrap().unwrap(); + let TunnelMsg::Data { chunk, .. } = serde_json::from_str(data_msg_a.to_text().unwrap()).unwrap() else { + panic!("expected Data on Host A"); + }; + assert_eq!(base64::engine::general_purpose::STANDARD.decode(chunk).unwrap(), b"ISOLATION_TEST_PAYLOAD"); + + // Host B must NOT have received any message for Realm A + tokio::select! { + msg = src_b.next() => { + panic!("Host B received unexpected message: {msg:?}"); + } + _ = tokio::time::sleep(std::time::Duration::from_millis(100)) => { + // Expected: Host B received nothing + } + } +} + +#[tokio::test] +async fn test_relay_backpressure_and_slow_receiver() { + let srv = start_test_relay(45800, 45810).await; + let realm_id = RealmId::new(); + let key = random_key(); + srv.keys.set(realm_id, &key.verifying_key()); + + let ws_url = format!("ws://{}/relay/v1/host", srv.addr); + let (ws_stream, _) = connect_async(&ws_url).await.unwrap(); + let (mut sink, mut src) = ws_stream.split(); + let challenge: RelayChallenge = serde_json::from_str(src.next().await.unwrap().unwrap().to_text().unwrap()).unwrap(); + let sig = sign_relay_challenge(&key, &realm_id, &challenge.nonce); + sink.send(Message::Text(serde_json::to_string(&HostHello { realm_id, signature: sig }).unwrap().into())).await.unwrap(); + let _ = src.next().await.unwrap().unwrap(); + + sink.send(Message::Text(serde_json::to_string(&TunnelMsg::Allocate { + request_id: 1, + player_id: Uuid::new_v4(), + expected_client_ip: None, + }).unwrap().into())).await.unwrap(); + + let TunnelMsg::AllocateOk { world_port, .. } = serde_json::from_str(src.next().await.unwrap().unwrap().to_text().unwrap()).unwrap() else { + panic!("expected AllocateOk"); + }; + + let client = TcpStream::connect(format!("127.0.0.1:{world_port}")).await.unwrap(); + + let connect_msg = loop { + let msg = src.next().await.unwrap().unwrap(); + match serde_json::from_str::(msg.to_text().unwrap()).unwrap() { + TunnelMsg::Connect { stream_id, target, .. } if target == RelayTarget::World => break stream_id, + _ => continue, + } + }; + sink.send(Message::Text(serde_json::to_string(&TunnelMsg::ConnectOk { stream_id: connect_msg }).unwrap().into())).await.unwrap(); + + // Host sends 128 chunks (each 4KB) rapidly without client reading immediately + let chunk_data = vec![0x42u8; 4096]; + let b64_chunk = base64::engine::general_purpose::STANDARD.encode(&chunk_data); + for _ in 0..128 { + sink.send(Message::Text(serde_json::to_string(&TunnelMsg::Data { + stream_id: connect_msg, + chunk: b64_chunk.clone(), + }).unwrap().into())).await.unwrap(); + } + + // Now client reads all chunks + let mut total_read = 0; + let mut read_buf = vec![0u8; 8192]; + let mut reader = client; + while total_read < 128 * 4096 { + let n = reader.read(&mut read_buf).await.unwrap(); + if n == 0 { break; } + total_read += n; + } + assert_eq!(total_read, 128 * 4096, "all buffered data received despite slow consumer"); +} + +