mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-22 14:26:00 +00:00
Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation (#1531)
Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation Stream Xbox games with Xbox Cloud Gaming (Game Pass) from DroidDeck, outside the Linux session. - Backend: `xodus-cli cloud-token` (tools/xbox/droiddeck-cloud-token.patch) gets the streaming tokens; they stay in memory. - Web player (default): the Greenlight player page (tools/xcloud-player, MIT) in a WebView, plus Microsoft's own xbox.com/play player. In-stream menu with screen effects (sharpen, colour, deband, FXAA, CRT, NTSC, Toon, fps cap), stream quality, performance overlay (level, layout, top right / top left) and rumble. - Native player (opt-in, Android 10+, Player engine: Native): Google's WebRTC (io.github.webrtc-sdk 125.6422.07, BSD) with the Greenlight protocol ported to Kotlin, in its own process (:xcloudnative) with its own instance of DroidDeck's compositor. Decoded frames reach the compositor as AHardwareBuffers (new stream mode, VK_ANDROID_external_memory_android_ hardware_buffer import), so DDFG 2x and LSFG 2x/3x/4x run on them. Live frame generation, frame-rate target (presets and a 10-165 fps slider, even pacing), zero-copy picture, small network buffer, fast decoder, instant controller input, overlay with engine and on-screen fps, background end, restart and web fallback. It takes the main process's tokens through XboxCloudTokenService; they are fetched ahead while a cloud game is shown. - The compositor requests the dma-buf import extensions only where the driver has them (the system driver has none). - Catalog: cloud availability, Game Pass plan CFQ7TTC0K6L8 counted, library filter All / Owned / Game Pass / Cloud, Stream and Add to Games on the store page, stream-only Games-tab entries (Stream + Download / Play + Stream). - Xbox games get VC++ 2015-2022 and the ARM64 runtime registry entries (vcredist2022_arm64, built in until the catalog has it) before each launch. - Controller focus reaches pickers opened from a game page (install location, uninstall); Add / Remove from Games keep the focus. Credits: Greenlight by UnknownSKL (Jim Kroon), MIT, player and protocol; Better xCloud for the resolution / OS mapping; MGamer404's Bannerlator #627 for the ARM64 runtime approach. Ported from the412banner's Bannerlator.
This commit is contained in:
1 parent
5ca1f610ab
commit
c5d7cdc058
78 files changed
+7835
-45
No files matched your search
@@ -66,6 +66,8 @@ static void send_toplevel_configure(struct surface *s) { (void)s; resizes++; }
|
||||
#define pointer_button(...) ((void)0)
|
||||
#define scroll_event(...) ((void)0)
|
||||
#define axis_event(...) ((void)0)
|
||||
#define stream_take(...) ((void)0)
|
||||
#define stream_forget_all(...) ((void)0)
|
||||
#define pointer_delta(...) ((void)0)
|
||||
#define deliver_touch(m) ((void)0)
|
||||
#define deliver_pointer(m) (pointers++)
|
||||
|
||||
@@ -7,6 +7,12 @@ checksums. `runtime.lock.json` pins the Nexus Windows runtime release and each
|
||||
extracted PE DLL. `prepare.py` verifies both source revisions and all Nexus patch
|
||||
checksums before applying the local transport, launch and library patches.
|
||||
|
||||
`droiddeck-cloud-token.patch` adds `xodus-cli cloud-token`: the stored account's
|
||||
Xbox Cloud Gaming sign-in (XSTS for `http://gssv.xboxlive.com/`, then the streaming
|
||||
service's `/v2/login/user`) and the login proof token a session's `/connect` step
|
||||
takes. It prints one JSON object and never opens a login or provisions a device.
|
||||
Ported from Bannerlator.
|
||||
|
||||
`droiddeck-library-subscriptions.patch` queries fresh subscription membership
|
||||
alongside product entitlements using the same Store session, account and market.
|
||||
Android can validate a Game Pass child's parent plan when that plan is absent
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
diff --git a/crates/xodus-cli/src/commands.rs b/crates/xodus-cli/src/commands.rs
|
||||
index 5a66715..5b310e4 100644
|
||||
--- a/crates/xodus-cli/src/commands.rs
|
||||
+++ b/crates/xodus-cli/src/commands.rs
|
||||
@@ -6,6 +6,7 @@ pub mod license;
|
||||
pub mod login;
|
||||
pub mod logout;
|
||||
pub mod profile;
|
||||
+pub mod cloud_token;
|
||||
pub mod game_activity;
|
||||
pub mod achievements;
|
||||
pub mod subscription;
|
||||
diff --git a/crates/xodus-cli/src/commands/cloud_token.rs b/crates/xodus-cli/src/commands/cloud_token.rs
|
||||
new file mode 100644
|
||||
index 0000000..fbd0ed0
|
||||
--- /dev/null
|
||||
+++ b/crates/xodus-cli/src/commands/cloud_token.rs
|
||||
@@ -0,0 +1,124 @@
|
||||
+//! Xbox Cloud Gaming streaming token for the stored account (DroidDeck). Same bearer
|
||||
+//! RPS -> Xbox user -> XSTS exchange as the profile read, for the cloud gaming relying party,
|
||||
+//! then the streaming service's user login (as xbox.com and Greenlight's xal-node do).
|
||||
+//! Never provisions a device or opens login.
|
||||
+use std::{process::ExitCode, time::Duration};
|
||||
+use serde_json::{Value, json};
|
||||
+use xodus::{models::{live::ExchangeUserTokenOutcome, secrets::Token, soap}, tokens::TokenManager};
|
||||
+
|
||||
+const GSSV_RELYING_PARTY: &str = "http://gssv.xboxlive.com/";
|
||||
+const OFFERINGS: [&str; 2] = ["xgpuweb", "xgpuwebf2p"];
|
||||
+
|
||||
+/// The stored account's compact Microsoft account ticket for one service target and policy.
|
||||
+async fn ticket(tokens: &TokenManager, client: &reqwest::Client, target: &str, policy: soap::PolicyReference) -> Result<(String, String), &'static str> {
|
||||
+ let (account, device, user) = super::session::prepare(tokens, client).await.map_err(|stage| {
|
||||
+ super::session::diagnostic(stage); stage
|
||||
+ })?;
|
||||
+ let outcome = super::session::exchange(|| async { xodus::api::live::exchange_user_token(client, user.clone(), account.clone(), device.clone(),
|
||||
+ None, Some("Silent".into()), "{d6d5a677-0872-4ab0-9442-bb792fce85c5}".into(),
|
||||
+ &[(target.into(), Some(policy.clone()))])
|
||||
+ .await.map_err(|error| super::session::classify(&error)) }).await?;
|
||||
+ let response = match outcome {
|
||||
+ ExchangeUserTokenOutcome::Issued(soap::BodyContent::RequestSecurityTokenResponseCollection(mut collection)) if !collection.security_tokens.is_empty() => collection.security_tokens.remove(0),
|
||||
+ ExchangeUserTokenOutcome::Issued(soap::BodyContent::RequestSecurityTokenResponse(response)) => *response,
|
||||
+ _ => return Err("msa-response"),
|
||||
+ };
|
||||
+ // Token::from panics on token types it does not know; only take the compact forms here. The cloud
|
||||
+ // transfer purpose answers with a login proof token, which is what /connect takes (Greenlight
|
||||
+ // sends the MSAL response's "lpt").
|
||||
+ let kind = response.token_type.clone();
|
||||
+ if kind == "urn:passport:loginprooftoken" {
|
||||
+ let shape = match (&response.requested_security_token.binary_security_token, &response.requested_security_token.encrypted_data) {
|
||||
+ (Some(token), _) if !token.value.is_empty() => "binary",
|
||||
+ (_, Some(_)) => "encrypted",
|
||||
+ _ => "empty",
|
||||
+ };
|
||||
+ eprintln!("NEXUS_SESSION_DIAGNOSTIC_V1 login-proof-{shape}");
|
||||
+ return match response.requested_security_token.binary_security_token {
|
||||
+ Some(token) if !token.value.is_empty() => Ok((account, token.value)),
|
||||
+ _ => Err("msa-ticket"),
|
||||
+ };
|
||||
+ }
|
||||
+ let compact = matches!(kind.as_str(), "urn:passport:compact" | "urn:passport:delegationcompact")
|
||||
+ && response.requested_security_token.binary_security_token.is_some();
|
||||
+ if !compact {
|
||||
+ let slug: String = kind.chars().map(|c| if c.is_ascii_alphabetic() { c.to_ascii_lowercase() } else { '-' }).take(40).collect();
|
||||
+ eprintln!("NEXUS_SESSION_DIAGNOSTIC_V1 ticket-type-{slug}");
|
||||
+ return Err("msa-ticket");
|
||||
+ }
|
||||
+ let Token::Compact(ticket) = Token::from(response) else { return Err("msa-ticket"); };
|
||||
+ Ok((account, ticket))
|
||||
+}
|
||||
+
|
||||
+async fn xsts(tokens: &TokenManager, client: &reqwest::Client) -> Result<(String, String), &'static str> {
|
||||
+ let (account, ticket) = ticket(tokens, client, "user.auth.xboxlive.com", soap::PolicyReference::mbi_ssl()).await?;
|
||||
+ let user = xodus::api::xbox::authenticate_xbox_user(client, ticket).await.map_err(|_| "xbox-user")?;
|
||||
+ let xsts = xodus::api::xbox::request_xsts_token(client, user.token, GSSV_RELYING_PARTY).await.map_err(|_| "xsts")?;
|
||||
+ Ok((account, xsts.token))
|
||||
+}
|
||||
+
|
||||
+/// The Microsoft account token a cloud session's /connect step takes (xbox.com asks MSAL for
|
||||
+/// "service::http://Passport.NET/purpose::PURPOSE_XBOX_CLOUD_CONSOLE_TRANSFER_TOKEN").
|
||||
+async fn transfer_token(tokens: &TokenManager, client: &reqwest::Client) -> Result<String, &'static str> {
|
||||
+ let policy = soap::PolicyReference { uri: "PURPOSE_XBOX_CLOUD_CONSOLE_TRANSFER_TOKEN".into(), val: String::default() };
|
||||
+ Ok(ticket(tokens, client, "http://Passport.NET/purpose", policy).await?.1)
|
||||
+}
|
||||
+
|
||||
+fn regions(value: &Value) -> Vec<Value> {
|
||||
+ value["offeringSettings"]["regions"].as_array().map(|list| list.iter().filter_map(|region| {
|
||||
+ let name = region["name"].as_str()?;
|
||||
+ let base = reqwest::Url::parse(region["baseUri"].as_str()?).ok()?;
|
||||
+ let host = base.host_str()?;
|
||||
+ if base.scheme() != "https" || !host.ends_with(".xboxlive.com") || !base.username().is_empty() { return None; }
|
||||
+ Some(json!({ "name": name, "baseUri": base.as_str().trim_end_matches('/'), "isDefault": region["isDefault"].as_bool().unwrap_or(false) }))
|
||||
+ }).take(32).collect()).unwrap_or_default()
|
||||
+}
|
||||
+
|
||||
+async fn login(tokens: &TokenManager) -> Result<Value, String> {
|
||||
+ let client = reqwest::Client::builder().redirect(reqwest::redirect::Policy::none())
|
||||
+ .timeout(super::session::REQUEST_TIMEOUT).user_agent("xodus-nexus-cloud/1").build().map_err(|_| "http-client")?;
|
||||
+ let (account, xsts) = xsts(tokens, &client).await?;
|
||||
+ let mut failures = Vec::new();
|
||||
+ for offering in OFFERINGS {
|
||||
+ let response = client.post(format!("https://{offering}.gssv-play-prod.xboxlive.com/v2/login/user"))
|
||||
+ .header("x-gssv-client", "XboxComBrowser").header("Cache-Control", "no-store, must-revalidate, no-cache")
|
||||
+ .json(&json!({ "token": xsts, "offeringId": offering })).send().await.map_err(|_| "gssv-request")?;
|
||||
+ let status = response.status().as_u16();
|
||||
+ let body = super::profile::bounded(response, 256 * 1024).await;
|
||||
+ let Ok(body) = body else { failures.push(format!("{offering}:{status}")); continue; };
|
||||
+ let value: Value = serde_json::from_slice(&body).map_err(|_| "gssv-json")?;
|
||||
+ let token = value["gsToken"].as_str().filter(|token| !token.is_empty() && token.len() <= 16384).ok_or("gs-token")?;
|
||||
+ // Diagnostic lines are letters and dashes only (the app's allow-list).
|
||||
+ eprintln!("NEXUS_SESSION_DIAGNOSTIC_V1 gssv-login-ok-{}", if offering == "xgpuweb" { "gamepass" } else { "free" });
|
||||
+ let transfer = transfer_token(tokens, &client).await;
|
||||
+ eprintln!("NEXUS_SESSION_DIAGNOSTIC_V1 transfer-token-{}", if transfer.is_ok() { "ok" } else { "missing" });
|
||||
+ if tokens.get_user().map_err(|_| "account-changed")?.username != account { return Err("account-changed".into()); }
|
||||
+ return Ok(json!({ "schema": 1, "account": account, "offering": offering, "gsToken": token, "userToken": transfer.ok(),
|
||||
+ "durationInSeconds": value["durationInSeconds"].as_u64().unwrap_or(0), "market": value["market"].as_str().unwrap_or_default(),
|
||||
+ "regions": regions(&value) }));
|
||||
+ }
|
||||
+ Err(format!("gssv-login {}", failures.join(" ")))
|
||||
+}
|
||||
+
|
||||
+pub async fn run(tokens: &TokenManager) -> ExitCode {
|
||||
+ match tokio::time::timeout(Duration::from_secs(45), login(tokens)).await {
|
||||
+ Ok(Ok(value)) => { println!("{value}"); ExitCode::SUCCESS }
|
||||
+ Ok(Err(stage)) => { eprintln!("Xbox Cloud Gaming sign-in failed ({stage})."); ExitCode::FAILURE }
|
||||
+ Err(_) => { eprintln!("Xbox Cloud Gaming sign-in timed out."); ExitCode::FAILURE }
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+#[cfg(test)]
|
||||
+mod tests {
|
||||
+ use super::*;
|
||||
+ #[test]
|
||||
+ fn only_https_xboxlive_regions_are_kept() {
|
||||
+ let value = json!({"offeringSettings":{"regions":[
|
||||
+ {"name":"WestEurope","baseUri":"https://weu.core.gssv-play-prod.xboxlive.com/","isDefault":true},
|
||||
+ {"name":"Evil","baseUri":"https://evil.test/"},
|
||||
+ {"name":"Plain","baseUri":"http://eus.core.gssv-play-prod.xboxlive.com"}]}});
|
||||
+ let list = regions(&value);
|
||||
+ assert_eq!(list.len(), 1);
|
||||
+ assert_eq!(list[0]["baseUri"], "https://weu.core.gssv-play-prod.xboxlive.com");
|
||||
+ }
|
||||
+}
|
||||
diff --git a/crates/xodus-cli/src/main.rs b/crates/xodus-cli/src/main.rs
|
||||
index 1ae4167..4b0ac26 100644
|
||||
--- a/crates/xodus-cli/src/main.rs
|
||||
+++ b/crates/xodus-cli/src/main.rs
|
||||
@@ -68,6 +68,8 @@ enum SubCommand {
|
||||
#[arg(long, default_value_t = false)]
|
||||
json: bool,
|
||||
},
|
||||
+ #[command(about = "Get an Xbox Cloud Gaming streaming token for the stored account")]
|
||||
+ CloudToken,
|
||||
#[command(about = "Read Xbox gamertag and gamerpic for the stored account")]
|
||||
Profile {
|
||||
#[arg(long, default_value_t = false)]
|
||||
@@ -239,6 +241,7 @@ async fn main() -> ExitCode {
|
||||
args.command,
|
||||
SubCommand::Account { .. }
|
||||
| SubCommand::Profile { .. }
|
||||
+ | SubCommand::CloudToken
|
||||
| SubCommand::GameActivity { .. }
|
||||
| SubCommand::Achievements { .. }
|
||||
| SubCommand::Library { .. }
|
||||
@@ -304,6 +307,7 @@ async fn main() -> ExitCode {
|
||||
}
|
||||
},
|
||||
SubCommand::Profile { json } => commands::profile::run(&tokens, json).await,
|
||||
+ SubCommand::CloudToken => commands::cloud_token::run(&tokens).await,
|
||||
SubCommand::GameActivity { title_id, json } => commands::game_activity::run(&tokens, title_id, json).await,
|
||||
SubCommand::Achievements { title_id, json } => commands::achievements::run(&tokens, title_id, json).await,
|
||||
SubCommand::Login => commands::login::run(&client, &tokens).await,
|
||||
@@ -10,7 +10,7 @@ recipe = Path(__file__).resolve().parent
|
||||
cache = Path(sys.argv[1]).resolve()
|
||||
lock = json.loads((recipe / 'backend.lock.json').read_text())
|
||||
local_patches = ['droiddeck-credentials.patch', 'droiddeck-launch-view.patch', 'droiddeck-library-subscriptions.patch',
|
||||
'droiddeck-credential-fallback.patch', 'droiddeck-credential-diagnostics.patch']
|
||||
'droiddeck-credential-fallback.patch', 'droiddeck-credential-diagnostics.patch', 'droiddeck-cloud-token.patch']
|
||||
identity = hashlib.sha256((recipe / 'backend.lock.json').read_bytes() + b''.join((recipe / name).read_bytes() for name in local_patches)).hexdigest()
|
||||
source = cache / ('source-' + identity[:16])
|
||||
if not (source / '.prepared').is_file():
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Xbox Cloud Gaming player
|
||||
|
||||
`greenlight/lib` is the WebRTC client of **Greenlight** by **UnknownSKL (Jim Kroon)**
|
||||
(https://github.com/unknownskl/greenlight, `packages/player/src/client/lib`, commit ef852c3, MIT —
|
||||
see `greenlight/LICENSE`). DroidDeck changes: `player.ts` renders video with the `<video>`
|
||||
component instead of WebGPU, `render/video.ts` sets a transparent poster, and `input/gamepad.ts` treats missing buttons/axes as unpressed (Android reports no guide button); `uuid` is replaced by `uuid.ts` (crypto.randomUUID).
|
||||
|
||||
`sdp-tweaks.ts` (H.264 profile preference, bitrate cap), the CAS / unsharp-mask sharpening in
|
||||
`renderer.ts` and the "stop quality drops" metadata change follow **Better xCloud** by **redphx**
|
||||
(https://github.com/redphx/better-xcloud, MIT). The colour, deband, FXAA, Toon, CRT and NTSC shaders are
|
||||
ported from Bannerlator's screen effects (The412Banner). `streamstats.ts` is DroidDeck's.
|
||||
|
||||
`entry.ts` is DroidDeck's: it drives a cloud session through the app (`DDCloud` bridge; the
|
||||
streaming and Microsoft account tokens never reach the page) and attaches the first controller.
|
||||
|
||||
Build: `bash tools/xcloud-player/build.sh` (esbuild) writes
|
||||
`app/src/main/assets/xcloud/player.js`; the built file is committed.
|
||||
|
||||
The page, its sources and the Android side (`stores/xbox/XboxCloud*.kt`) are ported from
|
||||
Bannerlator's Xbox Cloud Gaming player (https://github.com/The412Banner/Bannerlator).
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
# Bundles the cloud player page script with esbuild 0.24.0 (ESBUILD=/path/to/esbuild, default: esbuild on PATH).
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
repo=$(cd "$here/../.." && pwd)
|
||||
"${ESBUILD:-esbuild}" "$here/entry.ts" --bundle --format=iife --target=chrome110 --minify \
|
||||
--alias:uuid="$here/uuid.ts" --legal-comments=inline \
|
||||
--banner:js="/* DroidDeck Xbox Cloud Gaming player. Includes Greenlight's player (c) UnknownSKL (Jim Kroon), MIT. */" \
|
||||
--outfile="$repo/app/src/main/assets/xcloud/player.js"
|
||||
@@ -0,0 +1,212 @@
|
||||
// DroidDeck cloud player page: runs one Xbox Cloud Gaming session. Every request to Microsoft goes
|
||||
// through the app (DDCloud), which holds the tokens; this page only does WebRTC, video, sound and input.
|
||||
import xCloudPlayer from './greenlight/lib/player'
|
||||
import Gamepad from './greenlight/lib/input/gamepad'
|
||||
import { EffectRenderer, Effects, needsRenderer } from './renderer'
|
||||
import { StreamStats } from './streamstats'
|
||||
import { capVideoBitrate, preferH264Profile } from './sdp-tweaks'
|
||||
|
||||
interface Reply { status: number, body: string }
|
||||
declare const DDCloud: { call(id: number, op: string, arg: string): void, status(text: string): void, failed(stage: string, detail: string): void, playing(): void, log(text: string): void, settings(): string }
|
||||
|
||||
/** The player settings the app hands over (XboxCloudSettings.toJson); stream-start ones are applied by the app. */
|
||||
interface Settings extends Effects {
|
||||
bitrateMbps: number // 0 = Microsoft's own limit
|
||||
codec: string // 'default' | 'high' | 'normal' | 'low'
|
||||
preventDrops: boolean
|
||||
statsLevel: number // 0 off, 1 fps, 2 basic, 3 detailed, 4 full
|
||||
statsLayout: string // 'horizontal' | 'vertical'
|
||||
statsCorner: string // 'right' | 'left'
|
||||
server: string // the region's host, for the full overlay
|
||||
volume: number // 100..300 %
|
||||
rumble: number // 0..100 %
|
||||
}
|
||||
let settings: Settings = JSON.parse(DDCloud.settings())
|
||||
|
||||
// Codec profile and bitrate cap go into our offer (Better xCloud's approach).
|
||||
const nativeSetLocalDescription = RTCPeerConnection.prototype.setLocalDescription
|
||||
RTCPeerConnection.prototype.setLocalDescription = function (this: RTCPeerConnection, description?: RTCLocalSessionDescriptionInit) {
|
||||
if (description?.sdp) {
|
||||
if (settings.codec === 'high' || settings.codec === 'normal' || settings.codec === 'low') description.sdp = preferH264Profile(description.sdp, settings.codec)
|
||||
if (settings.bitrateMbps > 0) description.sdp = capVideoBitrate(description.sdp, settings.bitrateMbps * 1000)
|
||||
}
|
||||
return nativeSetLocalDescription.call(this, description as RTCLocalSessionDescriptionInit)
|
||||
} as typeof RTCPeerConnection.prototype.setLocalDescription
|
||||
;(window as any).__ddRumble = settings.rumble / 100
|
||||
|
||||
// Page errors go to the app's Xbox log (no tokens ever reach this page).
|
||||
window.addEventListener('error', event => DDCloud.log(`error: ${event.message} @${event.lineno}`))
|
||||
window.addEventListener('unhandledrejection', event => DDCloud.log(`rejection: ${String(event.reason).slice(0, 200)}`))
|
||||
window.addEventListener('gamepadconnected', event => DDCloud.log(`controller: ${event.gamepad.id.slice(0, 60)} mapping=${event.gamepad.mapping || 'none'} buttons=${event.gamepad.buttons.length} axes=${event.gamepad.axes.length}`))
|
||||
|
||||
const pending = new Map<number, (reply: Reply) => void>()
|
||||
let nextId = 1
|
||||
;(window as any).__ddResolve = (id: number, status: number, body: string) => {
|
||||
const done = pending.get(id)
|
||||
pending.delete(id)
|
||||
if (done) done({ status, body })
|
||||
}
|
||||
|
||||
function call(op: string, arg: unknown = ''): Promise<Reply> {
|
||||
return new Promise(resolve => {
|
||||
const id = nextId++
|
||||
pending.set(id, resolve)
|
||||
DDCloud.call(id, op, typeof arg === 'string' ? arg : JSON.stringify(arg))
|
||||
})
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
|
||||
const ok = (reply: Reply) => reply.status >= 200 && reply.status < 300
|
||||
|
||||
function show(text: string) {
|
||||
const element = document.getElementById('status')
|
||||
if (element) element.textContent = text
|
||||
DDCloud.status(text)
|
||||
}
|
||||
|
||||
class Failure extends Error {
|
||||
constructor(public stage: string, public detail: string) { super(stage) }
|
||||
}
|
||||
|
||||
function expect(stage: string, reply: Reply): Reply {
|
||||
if (!ok(reply)) throw new Failure(stage, `${reply.status} ${reply.body.slice(0, 300)}`)
|
||||
return reply
|
||||
}
|
||||
|
||||
function waitForCandidates(pc: RTCPeerConnection, timeout: number): Promise<void> {
|
||||
return new Promise(resolve => {
|
||||
if (pc.iceGatheringState === 'complete') return resolve()
|
||||
const timer = setTimeout(resolve, timeout)
|
||||
pc.addEventListener('icegatheringstatechange', () => {
|
||||
if (pc.iceGatheringState === 'complete') { clearTimeout(timer); resolve() }
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function run() {
|
||||
show('Starting the game on Xbox Cloud Gaming…')
|
||||
expect('start', await call('start'))
|
||||
|
||||
let connected = false
|
||||
let waited = 0
|
||||
for (;;) {
|
||||
const reply = expect('state', await call('state'))
|
||||
const state = JSON.parse(reply.body || '{}')
|
||||
if (state.state === 'Provisioned') break
|
||||
if (state.state === 'Failed') throw new Failure('provision', JSON.stringify(state.errorDetails ?? {}).slice(0, 300))
|
||||
if (state.state === 'ReadyToConnect' && !connected) {
|
||||
connected = true
|
||||
show('Signing in to the cloud console…')
|
||||
expect('connect', await call('connect'))
|
||||
} else if (state.state === 'WaitingForResources') {
|
||||
show(`In the queue for a cloud console… ${Math.round(waited / 1000)} s`)
|
||||
} else {
|
||||
show(`Getting a cloud console ready… ${Math.round(waited / 1000)} s`)
|
||||
}
|
||||
await sleep(1000)
|
||||
waited += 1000
|
||||
}
|
||||
|
||||
show('Connecting…')
|
||||
const player = new xCloudPlayer('player')
|
||||
player.init()
|
||||
const pc = player._peerConnection
|
||||
|
||||
// "Stop quality drops": report a short decode time so the server does not lower the resolution
|
||||
// for a slow decoder (Better xCloud's patchStreamMetadata).
|
||||
const input = player._channels.input as any
|
||||
const queueMetadata = input.queueMetadataFrame.bind(input)
|
||||
input.queueMetadataFrame = (frame: any) => {
|
||||
if (settings.preventDrops) {
|
||||
const render = frame.frameRenderedTimeMs - frame.frameDecodedTimeMs
|
||||
frame.frameDecodedTimeMs = frame.frameSubmittedTimeMs + 10
|
||||
frame.frameRenderedTimeMs = frame.frameDecodedTimeMs + render
|
||||
}
|
||||
queueMetadata(frame)
|
||||
}
|
||||
|
||||
// Volume above 100 %: the stream's sound goes through a gain node instead of the <audio> element.
|
||||
let gain: GainNode | undefined
|
||||
let audioStream: MediaStream | undefined
|
||||
const applyVolume = () => {
|
||||
if (audioStream === undefined || (gain === undefined && settings.volume === 100)) return
|
||||
if (gain === undefined) {
|
||||
const context = new AudioContext()
|
||||
gain = context.createGain()
|
||||
context.createMediaStreamSource(audioStream).connect(gain).connect(context.destination)
|
||||
const element = document.querySelector('#player audio') as HTMLAudioElement | null
|
||||
if (element) element.muted = true
|
||||
}
|
||||
gain.gain.value = settings.volume / 100
|
||||
}
|
||||
pc.addEventListener('track', event => {
|
||||
if (event.track.kind === 'audio') { audioStream = event.streams[0]; applyVolume() }
|
||||
})
|
||||
|
||||
const stats = new StreamStats(pc)
|
||||
let renderer: EffectRenderer | undefined
|
||||
const applyEffects = () => {
|
||||
const video = player.getVideoElement() as HTMLVideoElement | undefined
|
||||
if (video === undefined) return
|
||||
if (!needsRenderer(settings)) { renderer?.pause(); return }
|
||||
if (renderer === undefined) {
|
||||
try { renderer = new EffectRenderer(video, settings) }
|
||||
catch (error) { DDCloud.log(`effects unavailable: ${String(error).slice(0, 200)}`) }
|
||||
} else renderer.resume(settings)
|
||||
}
|
||||
// Live changes from the in-player menu.
|
||||
;(window as any).__ddApply = (json: string) => {
|
||||
settings = JSON.parse(json)
|
||||
;(window as any).__ddRumble = settings.rumble / 100
|
||||
stats.show(settings.statsLevel, settings.statsLayout === 'vertical', settings.server, settings.statsCorner === 'left')
|
||||
applyVolume()
|
||||
applyEffects()
|
||||
}
|
||||
pc.addEventListener('connectionstatechange', () => {
|
||||
if (pc.connectionState === 'connected') {
|
||||
show(''); DDCloud.playing()
|
||||
stats.show(settings.statsLevel, settings.statsLayout === 'vertical', settings.server, settings.statsCorner === 'left')
|
||||
// The video element exists once its track arrived; give it a moment for its first frame.
|
||||
setTimeout(applyEffects, 500)
|
||||
}
|
||||
else if (pc.connectionState === 'failed') DDCloud.failed('webrtc', 'connection failed')
|
||||
else if (pc.connectionState === 'disconnected') show('Connection lost…')
|
||||
})
|
||||
|
||||
const offer = await player.createOffer()
|
||||
const sdp = expect('sdp', await call('sdp', offer.sdp ?? ''))
|
||||
const answer = JSON.parse(JSON.parse(sdp.body).exchangeResponse)
|
||||
if (answer.status && answer.status !== 'success') throw new Failure('sdp-answer', JSON.stringify(answer).slice(0, 300))
|
||||
await player.setRemoteSDP(answer.sdp)
|
||||
|
||||
await waitForCandidates(pc, 3000)
|
||||
const local = (await player.getICECandidates()).map(c => JSON.stringify({
|
||||
candidate: c.candidate, sdpMid: c.sdpMid, sdpMLineIndex: c.sdpMLineIndex, usernameFragment: c.usernameFragment,
|
||||
}))
|
||||
const ice = expect('ice', await call('ice', local))
|
||||
player.setRemoteIceCandidates(JSON.parse(JSON.parse(ice.body).exchangeResponse))
|
||||
|
||||
// The control channel's authorization (sent on the message handshake) announces controller 0 and
|
||||
// then removes it, clearing any handler attached before; attach the controller after that.
|
||||
const handshake = Date.now()
|
||||
while (!player._channels.message._handshakeCompleted && Date.now() - handshake < 20_000) await sleep(100)
|
||||
DDCloud.log(`handshake ${player._channels.message._handshakeCompleted ? 'done' : 'missing'} after ${Date.now() - handshake} ms`)
|
||||
await sleep(300)
|
||||
const pad = new Gamepad(0, { enable_keyboard: false })
|
||||
pad.attach(player)
|
||||
DDCloud.log(`controllers at attach: ${Array.from(navigator.getGamepads()).filter(Boolean).length}`)
|
||||
// The app's "Xbox button" (in the leave dialog) presses Nexus for a moment.
|
||||
;(window as any).__ddNexus = () => { pad.sendButtonState('Nexus', 1); setTimeout(() => pad.sendButtonState('Nexus', 0), 200) }
|
||||
// The app's menu button / a controller's Guide button: Nexus follows the physical button.
|
||||
;(window as any).__ddButton = (name: string, value: number) => pad.sendButtonState(name, value)
|
||||
// The in-player menu pauses controller input to the game while it is open.
|
||||
;(window as any).__ddInput = (enabled: boolean) => { if (enabled) pad.onWindowFocus(); else pad.onWindowBlur() }
|
||||
|
||||
setInterval(() => { call('keepalive') }, 30_000)
|
||||
;(window as any).__ddStop = () => { try { renderer?.destroy(); stats.destroy(); pad.detach(); player.destroy() } catch (e) { /* closing anyway */ } }
|
||||
}
|
||||
|
||||
run().catch((error: unknown) => {
|
||||
if (error instanceof Failure) DDCloud.failed(error.stage, error.detail)
|
||||
else DDCloud.failed('page', String(error).slice(0, 300))
|
||||
})
|
||||
@@ -0,0 +1,21 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2019 UnknownSKL (Jim Kroon)
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,69 @@
|
||||
import xCloudPlayer from '../player'
|
||||
|
||||
export default class Channel {
|
||||
private _player:xCloudPlayer
|
||||
private _dataChannel:RTCDataChannel
|
||||
|
||||
constructor(player:xCloudPlayer) {
|
||||
this._player = player
|
||||
// console.log('creating data channel: ', this.constructor.name, this, this.getChannelConfig())
|
||||
|
||||
this._dataChannel = this._player._peerConnection.createDataChannel(this.getChannelName(), this.getChannelConfig())
|
||||
|
||||
this._dataChannel.onopen = this.onOpen.bind(this)
|
||||
this._dataChannel.onmessage = this.onMessage.bind(this)
|
||||
this._dataChannel.onclosing = this.onClosing.bind(this)
|
||||
this._dataChannel.onclose = this.onClose.bind(this)
|
||||
this._dataChannel.onerror = this.onError.bind(this)
|
||||
}
|
||||
|
||||
// Channel config functions
|
||||
getChannelName() {
|
||||
return 'channel'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return { }
|
||||
}
|
||||
|
||||
// Default channel functions
|
||||
onOpen(event:Event) {
|
||||
console.log(this.getChannelName(), 'Opening channel:', event)
|
||||
}
|
||||
|
||||
onMessage(event:MessageEvent) {
|
||||
console.log(this.getChannelName(), 'Message channel:', event)
|
||||
}
|
||||
|
||||
onClosing(event:Event) {
|
||||
console.log(this.getChannelName(), 'Closing channel:', event)
|
||||
}
|
||||
|
||||
onClose(event:Event) {
|
||||
console.log(this.getChannelName(), 'Closed channel:', event)
|
||||
}
|
||||
|
||||
onError(event:Event) {
|
||||
console.log(this.getChannelName(), 'Error channel:', event)
|
||||
}
|
||||
|
||||
send(data:any) {
|
||||
if(this._dataChannel.readyState !== 'open'){
|
||||
console.log('['+this.getChannelName()+'] Tried to send data while channel is not open. readyState is:', this._dataChannel.readyState)
|
||||
return
|
||||
}
|
||||
|
||||
if(typeof data === 'string'){
|
||||
data = (new TextEncoder).encode(data)
|
||||
}
|
||||
|
||||
return this._dataChannel.send(data)
|
||||
}
|
||||
|
||||
getPlayer(){
|
||||
return this._player
|
||||
}
|
||||
|
||||
// Channel destroy function
|
||||
destroy() {}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import Channel from './channel'
|
||||
|
||||
export default class ChatChannel extends Channel {
|
||||
|
||||
_micPermissions:'granted'|'denied'|'prompt' = 'prompt'
|
||||
_micStream:MediaStream | undefined
|
||||
|
||||
constructor(player:any){
|
||||
super(player)
|
||||
|
||||
navigator.permissions.query({ name: 'microphone' }).then((permissionStatus) => {
|
||||
this._micPermissions = permissionStatus.state
|
||||
|
||||
permissionStatus.onchange = () => {
|
||||
this._micPermissions = permissionStatus.state
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
getChannelName() {
|
||||
return 'chat'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return {
|
||||
ordered: true,
|
||||
protocol: 'chatV1',
|
||||
}
|
||||
}
|
||||
|
||||
// startMicrophone() {
|
||||
// if(this._micPermissions === 'denied'){
|
||||
// alert('Microphone permissions are denied. Please enable them in your browser settings.')
|
||||
// return
|
||||
// }
|
||||
// if(this._micStream !== undefined){
|
||||
// console.log('Microphone already started. You need to stop it first by calling stopMicrophone()')
|
||||
// return
|
||||
// }
|
||||
|
||||
// navigator.mediaDevices.getUserMedia({ audio: {
|
||||
// channelCount: 1,
|
||||
// sampleRate: 24e3,
|
||||
// } }).then((stream) => {
|
||||
// if(this._micStream !== undefined){
|
||||
// return
|
||||
// }
|
||||
|
||||
// this._micStream = stream
|
||||
|
||||
// // Add to SDP
|
||||
// this.getPlayer()._peerConnection.addTrack(stream.getAudioTracks()[0], stream)
|
||||
// this.getPlayer().createOffer().then((offer) => {
|
||||
// if(this.getPlayer()._sdpHandler){
|
||||
// this.getPlayer()._sdpHandler(offer)
|
||||
// } else {
|
||||
// console.log('No SDP handler set. Set an SDP Handler via player.setChatSdpHandler()')
|
||||
// }
|
||||
// })
|
||||
// }).catch((err) => {
|
||||
// alert(`Error connecting to microphone: ${err}`)
|
||||
// })
|
||||
// }
|
||||
|
||||
// stopMicrophone() {
|
||||
// if(this._micStream === undefined) {return}
|
||||
|
||||
// this._micStream.getTracks().forEach((track) => {
|
||||
// track.stop()
|
||||
// })
|
||||
|
||||
// this.getPlayer()._peerConnection.getSenders().forEach((sender) => {
|
||||
// if(sender.track && sender.track.kind === 'audio'){
|
||||
// this.getPlayer()._peerConnection.removeTrack(sender)
|
||||
// }
|
||||
// })
|
||||
|
||||
// this._micStream = undefined
|
||||
// }
|
||||
|
||||
destroy() {
|
||||
// console.log('DebugChannel destroy() called')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import Gamepad from '../input/gamepad'
|
||||
import MouseKeyboard from '../input/mousekeyboard'
|
||||
import Touch from '../input/touch'
|
||||
import Channel from './channel'
|
||||
|
||||
interface GamepadHandlers {
|
||||
0: undefined|Gamepad|MouseKeyboard|Touch;
|
||||
1: undefined|Gamepad|MouseKeyboard|Touch;
|
||||
2: undefined|Gamepad|MouseKeyboard|Touch;
|
||||
3: undefined|Gamepad|MouseKeyboard|Touch;
|
||||
}
|
||||
|
||||
export default class ControlChannel extends Channel {
|
||||
_keyframeInterval
|
||||
_gamepadHandlers:GamepadHandlers = {
|
||||
0: undefined,
|
||||
1: undefined,
|
||||
2: undefined,
|
||||
3: undefined,
|
||||
}
|
||||
|
||||
getChannelName() {
|
||||
return 'control'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return {
|
||||
protocol: 'controlV1',
|
||||
ordered: true,
|
||||
}
|
||||
}
|
||||
|
||||
destroy() {
|
||||
// console.log('DebugChannel destroy() called')
|
||||
clearInterval(this._keyframeInterval)
|
||||
|
||||
this._gamepadHandlers[0]?.detach()
|
||||
this._gamepadHandlers[1]?.detach()
|
||||
this._gamepadHandlers[2]?.detach()
|
||||
this._gamepadHandlers[3]?.detach()
|
||||
}
|
||||
|
||||
sendAuthorization(){
|
||||
const authRequest = JSON.stringify({
|
||||
'message':'authorizationRequest',
|
||||
'accessKey':'4BDB3609-C1F1-4195-9B37-FEFF45DA8B8E',
|
||||
})
|
||||
|
||||
this.send(authRequest)
|
||||
|
||||
this.sendGamepadState(0, true)
|
||||
this.sendGamepadState(0, false)
|
||||
|
||||
// if(this.getPlayer()._config.keyframe_interval > 0) {
|
||||
// this._keyframeInterval = setInterval(() => {
|
||||
// this.requestKeyframeRequest(true)
|
||||
// }, this.getPlayer()._config.keyframe_interval*1000)
|
||||
// }
|
||||
}
|
||||
|
||||
sendGamepadState(gamepadIndex, wasAdded = true, handler:undefined|Gamepad|MouseKeyboard|Touch = undefined) {
|
||||
|
||||
if(wasAdded === true){
|
||||
if(handler !== undefined){
|
||||
if(this._gamepadHandlers[gamepadIndex] !== undefined){
|
||||
this._gamepadHandlers[gamepadIndex]?.detach()
|
||||
}
|
||||
this._gamepadHandlers[gamepadIndex] = handler
|
||||
}
|
||||
} else {
|
||||
this._gamepadHandlers[gamepadIndex] = undefined
|
||||
}
|
||||
|
||||
const gamepadRequest = JSON.stringify({
|
||||
'message': 'gamepadChanged',
|
||||
'gamepadIndex': gamepadIndex,
|
||||
'wasAdded': wasAdded,
|
||||
})
|
||||
this.send(gamepadRequest)
|
||||
}
|
||||
|
||||
requestKeyframeRequest(ifrRequested = false) {
|
||||
// console.log('xCloudPlayer Channel/Control.ts - ['+this._channelName+'] User requested Video KeyFrame')
|
||||
const keyframeRequest = JSON.stringify({
|
||||
message: 'videoKeyframeRequested',
|
||||
ifrRequested: ifrRequested,
|
||||
})
|
||||
|
||||
this.send(keyframeRequest)
|
||||
}
|
||||
|
||||
getGamepadHandlers(){
|
||||
return this._gamepadHandlers
|
||||
}
|
||||
|
||||
getGamepadHandler(index):Gamepad|undefined{
|
||||
for(const gamepad in this.getPlayer()._channels.control.getGamepadHandlers()){
|
||||
if(
|
||||
this.getPlayer()._channels.control.getGamepadHandlers()[gamepad] instanceof Gamepad &&
|
||||
(this.getPlayer()._channels.control.getGamepadHandlers()[gamepad] as Gamepad).getPhysicalGamepadId() === index
|
||||
) {
|
||||
return this.getPlayer()._channels.control.getGamepadHandlers()[gamepad]
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import Channel from './channel'
|
||||
|
||||
export default class DebugChannel extends Channel {
|
||||
getChannelName() {
|
||||
return 'debug'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return { }
|
||||
}
|
||||
|
||||
destroy() {
|
||||
// console.log('DebugChannel destroy() called')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import Channel from './channel'
|
||||
import InputPacket, { MetadataFrame, ReportTypes, InputFrame as GamepadFrame, MouseFrame, KeyboardFrame, PointerFrame } from './input/packet'
|
||||
import InputQueue from './input/queue'
|
||||
|
||||
import Gamepad from '../input/gamepad'
|
||||
|
||||
export interface VibrationFrame {
|
||||
gamepadIndex: number;
|
||||
leftMotorPercent: number;
|
||||
rightMotorPercent: number;
|
||||
leftTriggerMotorPercent: number;
|
||||
rightTriggerMotorPercent: number;
|
||||
durationMs: number;
|
||||
delayMs: number;
|
||||
repeat: number;
|
||||
}
|
||||
|
||||
export default class InputChannel extends Channel {
|
||||
private _serverVideoWidth = 0
|
||||
private _serverVideoHeight = 0
|
||||
|
||||
private _inputQueue = new InputQueue(this.getPlayer())
|
||||
|
||||
getChannelName() {
|
||||
return 'input'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return {
|
||||
ordered: true,
|
||||
protocol: '1.0',
|
||||
}
|
||||
}
|
||||
|
||||
getServerVideoWidth(){
|
||||
return this._serverVideoWidth
|
||||
}
|
||||
|
||||
getServerVideoHeight(){
|
||||
return this._serverVideoHeight
|
||||
}
|
||||
|
||||
start(){
|
||||
const Packet = new InputPacket(0)
|
||||
Packet.setMetadata(navigator.maxTouchPoints > 1 ? navigator.maxTouchPoints : 1)
|
||||
this.send(Packet.toBuffer())
|
||||
|
||||
console.log('[InputChannel] Sent metadata:', Packet, Packet.toBuffer())
|
||||
|
||||
setTimeout(this.gamepadStateLoop.bind(this), 16)
|
||||
}
|
||||
|
||||
onMessage(event: MessageEvent<any>) {
|
||||
const dataView = new DataView(event.data)
|
||||
const reportType = dataView.getUint8(0)
|
||||
|
||||
console.log('[InputChannel] received:', dataView)
|
||||
|
||||
if(reportType === ReportTypes.Vibration){
|
||||
console.log('Received a vibration report')
|
||||
dataView.getUint8(2) // rumbleType: 0 = FourMotorRumble
|
||||
const gamepadIndex = dataView.getUint8(3)
|
||||
|
||||
const gamepad = this.getPlayer()._channels.control.getGamepadHandler(gamepadIndex)
|
||||
|
||||
if(gamepad !== undefined){
|
||||
const leftMotorPercent = dataView.getUint8(4) / 100
|
||||
const rightMotorPercent = dataView.getUint8(5) / 100
|
||||
const leftTriggerMotorPercent = dataView.getUint8(6) / 100
|
||||
const rightTriggerMotorPercent = dataView.getUint8(7) / 100
|
||||
const durationMs = dataView.getUint16(8, true)
|
||||
const delayMs = dataView.getUint16(10, true)
|
||||
const repeat = dataView.getUint8(12)
|
||||
|
||||
const rumble:VibrationFrame = {
|
||||
gamepadIndex,
|
||||
leftMotorPercent,
|
||||
rightMotorPercent,
|
||||
leftTriggerMotorPercent,
|
||||
rightTriggerMotorPercent,
|
||||
durationMs,
|
||||
delayMs,
|
||||
repeat,
|
||||
}
|
||||
|
||||
gamepad.handleVibration(rumble)
|
||||
} else {
|
||||
console.log('[InputChannel] Received a vibration report but no gamepad handler is available')
|
||||
}
|
||||
|
||||
} else if(reportType === ReportTypes.ServerMetadata){
|
||||
console.log('[InputChannel] Received server video dimensions:', dataView.getUint32(2, true), dataView.getUint32(6, true))
|
||||
this._serverVideoHeight = dataView.getUint32(2, true)
|
||||
this._serverVideoWidth = dataView.getUint32(6, true)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
gamepadStateLoop(){
|
||||
const gamepadHandlers = this.getPlayer()._channels.control.getGamepadHandlers()
|
||||
// console.log('[InputChannel] gamepadStateLoop() called', gamepadHandlers)
|
||||
const gamepadFrames:Array<GamepadFrame> = []
|
||||
for(const gamepad in gamepadHandlers){
|
||||
if(gamepadHandlers[gamepad] instanceof Gamepad){
|
||||
const frame = (gamepadHandlers[gamepad] as Gamepad).getGamepadState()
|
||||
if(frame !== undefined){
|
||||
gamepadFrames.push(frame)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(gamepadFrames.length > 0){
|
||||
this.queueGamepadFrames(gamepadFrames)
|
||||
}
|
||||
|
||||
setTimeout(this.gamepadStateLoop.bind(this), 16)
|
||||
}
|
||||
|
||||
queueMetadataFrame(data:MetadataFrame){
|
||||
return this._inputQueue.queueMetadataFrame(data)
|
||||
}
|
||||
|
||||
queueGamepadFrames(frames:Array<GamepadFrame>, forceSend = false){
|
||||
this._inputQueue.queueGamepadFrames(frames, forceSend)
|
||||
}
|
||||
|
||||
queueMouseFrame(data:MouseFrame){
|
||||
this._inputQueue.queueMouseFrame(data)
|
||||
}
|
||||
|
||||
queueKeyboardFrame(data:KeyboardFrame){
|
||||
this._inputQueue.queueKeyboardFrame(data)
|
||||
}
|
||||
|
||||
queuePointerFrame(data:PointerFrame){
|
||||
this._inputQueue.queuePointerFrame(data)
|
||||
}
|
||||
|
||||
destroy() {
|
||||
// console.log('DebugChannel destroy() called')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
// import { InputFrame, PointerFrame, MouseFrame, KeyboardFrame } from '../Input'
|
||||
|
||||
export interface InputFrame {
|
||||
GamepadIndex: number;
|
||||
Nexus: number;
|
||||
Menu: number;
|
||||
View: number;
|
||||
A: number;
|
||||
B: number;
|
||||
X: number;
|
||||
Y: number;
|
||||
DPadUp: number;
|
||||
DPadDown: number;
|
||||
DPadLeft: number;
|
||||
DPadRight: number;
|
||||
LeftShoulder: number;
|
||||
RightShoulder: number;
|
||||
LeftThumb: number;
|
||||
RightThumb: number;
|
||||
|
||||
LeftThumbXAxis: number;
|
||||
LeftThumbYAxis: number;
|
||||
RightThumbXAxis: number;
|
||||
RightThumbYAxis: number;
|
||||
LeftTrigger: number;
|
||||
RightTrigger: number;
|
||||
}
|
||||
|
||||
export interface PointerFrame {
|
||||
events: Array<any>;
|
||||
}
|
||||
|
||||
export interface MouseFrame {
|
||||
X: number;
|
||||
Y: number;
|
||||
WheelX: number;
|
||||
WheelY: number;
|
||||
Buttons: number;
|
||||
Relative: number;
|
||||
}
|
||||
|
||||
export interface KeyboardFrame {
|
||||
pressed: boolean;
|
||||
keyCode: number;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export enum ReportTypes {
|
||||
None = 0,
|
||||
Metadata = 1,
|
||||
Gamepad = 2,
|
||||
Pointer = 4,
|
||||
ClientMetadata = 8,
|
||||
ServerMetadata = 16,
|
||||
Mouse = 32,
|
||||
Keyboard = 64,
|
||||
Vibration = 128,
|
||||
Sensor = 256,
|
||||
UnreliableInputReport = 512,
|
||||
UnreliableInputAck = 1024,
|
||||
}
|
||||
|
||||
export interface MetadataFrame {
|
||||
serverDataKey: number;
|
||||
firstFramePacketArrivalTimeMs: number;
|
||||
frameSubmittedTimeMs: number;
|
||||
frameDecodedTimeMs: number;
|
||||
frameRenderedTimeMs: number;
|
||||
}
|
||||
|
||||
export interface StreamConfig {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export default class InputPacket {
|
||||
|
||||
_reportType = ReportTypes.None
|
||||
_totalSize = -1
|
||||
_sequence = -1
|
||||
|
||||
_metadataFrames:Array<MetadataFrame> = []
|
||||
_gamepadFrames:Array<InputFrame> = []
|
||||
_pointerFrames:Array<PointerFrame> = []
|
||||
_mouseFrames:Array<MouseFrame> = []
|
||||
_keyboardFrames:Array<KeyboardFrame> = []
|
||||
|
||||
_maxTouchpoints = 0
|
||||
_serverVideoWidth = 0
|
||||
_serverVideoHeight = 0
|
||||
|
||||
constructor(sequence:number){
|
||||
this._sequence = sequence
|
||||
}
|
||||
|
||||
setStreamConfig(options:StreamConfig){
|
||||
this._serverVideoWidth = options.width
|
||||
this._serverVideoHeight = options.height
|
||||
}
|
||||
|
||||
setMetadata(maxTouchpoints = 1){
|
||||
this._reportType = ReportTypes.ClientMetadata
|
||||
this._totalSize = 15
|
||||
this._maxTouchpoints = maxTouchpoints
|
||||
}
|
||||
|
||||
setData(metadataQueue:Array<any>, gamepadQueue:Array<InputFrame>, pointerQueue:Array<PointerFrame>, mouseQueue:Array<MouseFrame>, keyboardQueue:Array<KeyboardFrame>){
|
||||
let size = 14
|
||||
|
||||
if(metadataQueue.length > 0){
|
||||
this._reportType |= ReportTypes.Metadata
|
||||
size = size+this._calculateMetadataSize(metadataQueue)
|
||||
this._metadataFrames = metadataQueue
|
||||
}
|
||||
|
||||
if(gamepadQueue.length > 0){
|
||||
this._reportType |= ReportTypes.Gamepad
|
||||
size = size+this._calculateGamepadSize(gamepadQueue)
|
||||
this._gamepadFrames = gamepadQueue
|
||||
}
|
||||
|
||||
if(pointerQueue.length > 0){
|
||||
this._reportType |= ReportTypes.Pointer
|
||||
size = size+this._calculatePointerSize(pointerQueue)
|
||||
this._pointerFrames = pointerQueue
|
||||
}
|
||||
|
||||
if(mouseQueue.length > 0){
|
||||
this._reportType |= ReportTypes.Mouse
|
||||
size = size+this._calculateMouseSize(mouseQueue)
|
||||
this._mouseFrames = mouseQueue
|
||||
}
|
||||
|
||||
if(keyboardQueue.length > 0){
|
||||
this._reportType |= ReportTypes.Keyboard
|
||||
size = size+this._calculateKeyboardSize(keyboardQueue)
|
||||
this._keyboardFrames = keyboardQueue
|
||||
}
|
||||
|
||||
this._totalSize = size
|
||||
}
|
||||
|
||||
_calculateMetadataSize(frames:Array<MetadataFrame>){
|
||||
return 1 + ((7 * 4) * frames.length)
|
||||
}
|
||||
|
||||
_calculateGamepadSize(frames:Array<InputFrame>){
|
||||
return 1 + (23 * frames.length)
|
||||
}
|
||||
|
||||
_calculatePointerSize(frames:Array<PointerFrame>){
|
||||
let pointerSize = 1
|
||||
for(const frame in frames){
|
||||
pointerSize = pointerSize + 1 + (frames[frame].events.length * 20)
|
||||
}
|
||||
|
||||
return pointerSize
|
||||
}
|
||||
|
||||
_calculateMouseSize(frames:Array<MouseFrame>){
|
||||
return 1 + (18 * frames.length)
|
||||
}
|
||||
|
||||
_calculateKeyboardSize(frames:Array<KeyboardFrame>){
|
||||
return 1 + (3 * frames.length)
|
||||
}
|
||||
|
||||
|
||||
|
||||
_writeMetadataData(packet:DataView, offset:number, frames:Array<MetadataFrame>){
|
||||
packet.setUint8(offset, frames.length)
|
||||
offset++
|
||||
|
||||
if(frames.length >= 30){
|
||||
console.warn('metadataQueue is bigger then 30. This might impact reliability!')
|
||||
}
|
||||
|
||||
for(let i = 0; i < frames.length; i++) {
|
||||
const frame = frames[i]
|
||||
|
||||
const firstFramePacketArrivalTimeMs = frame.firstFramePacketArrivalTimeMs
|
||||
const frameSubmittedTimeMs = frame.frameSubmittedTimeMs
|
||||
const frameDecodedTimeMs = frame.frameDecodedTimeMs
|
||||
const frameRenderedTimeMs = frame.frameRenderedTimeMs
|
||||
const framePacketTime = performance.now()
|
||||
const frameDateNow = performance.now()
|
||||
|
||||
packet.setUint32(offset, frame.serverDataKey, true)
|
||||
packet.setUint32(offset+4, firstFramePacketArrivalTimeMs, true)
|
||||
packet.setUint32(offset+8, frameSubmittedTimeMs, true)
|
||||
packet.setUint32(offset+12, frameDecodedTimeMs, true)
|
||||
packet.setUint32(offset+16, frameRenderedTimeMs, true)
|
||||
packet.setUint32(offset+20, framePacketTime, true)
|
||||
packet.setUint32(offset+24, frameDateNow, true)
|
||||
|
||||
offset += 28
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
_writeGamepadData(packet:DataView, offset:number, frames:Array<InputFrame>){
|
||||
packet.setUint8(offset, frames.length)
|
||||
offset++
|
||||
|
||||
if(frames.length >= 30){
|
||||
console.warn('gamepadQueue is bigger then 30. This might impact reliability!')
|
||||
}
|
||||
|
||||
for(let i = 0; i < frames.length; i++) {
|
||||
const input:InputFrame = frames[i]
|
||||
|
||||
packet.setUint8(offset, input.GamepadIndex)
|
||||
offset++
|
||||
|
||||
let buttonMask = 0
|
||||
if(input.Nexus > 0){ buttonMask |= 2 }
|
||||
if(input.Menu > 0){ buttonMask |= 4 }
|
||||
if(input.View > 0){ buttonMask |= 8 }
|
||||
if(input.A > 0){ buttonMask |= 16 }
|
||||
if(input.B > 0){ buttonMask |= 32 }
|
||||
if(input.X > 0){ buttonMask |= 64 }
|
||||
if(input.Y > 0){ buttonMask |= 128 }
|
||||
if(input.DPadUp > 0){ buttonMask |= 256 }
|
||||
if(input.DPadDown > 0){ buttonMask |= 512 }
|
||||
if(input.DPadLeft > 0){ buttonMask |= 1024 }
|
||||
if(input.DPadRight > 0){ buttonMask |= 2048 }
|
||||
if(input.LeftShoulder > 0){ buttonMask |= 4096 }
|
||||
if(input.RightShoulder > 0){ buttonMask |= 8192 }
|
||||
if(input.LeftThumb > 0){ buttonMask |= 16384 }
|
||||
if(input.RightThumb > 0){ buttonMask |= 32768 }
|
||||
|
||||
packet.setUint16(offset, buttonMask, true)
|
||||
packet.setInt16(offset+2, this._normalizeAxisValue(input.LeftThumbXAxis), true) // LeftThumbXAxis
|
||||
packet.setInt16(offset+4, this._normalizeAxisValue(-input.LeftThumbYAxis), true) // LeftThumbYAxis
|
||||
packet.setInt16(offset+6, this._normalizeAxisValue(input.RightThumbXAxis), true) // RightThumbXAxis
|
||||
packet.setInt16(offset+8, this._normalizeAxisValue(-input.RightThumbYAxis), true) // RightThumbYAxis
|
||||
packet.setUint16(offset+10, this._normalizeTriggerValue(input.LeftTrigger), true) // LeftTrigger
|
||||
packet.setUint16(offset+12, this._normalizeTriggerValue(input.RightTrigger), true) // RightTrigger
|
||||
|
||||
packet.setUint32(offset+14, 1, true) // PhysicalPhysicality
|
||||
packet.setUint32(offset+18, 1, false) // VirtualPhysicality
|
||||
offset += 22
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
_writePointerData(packet:DataView, offset:number, frames:Array<PointerFrame>){
|
||||
packet.setUint8(offset, 1)
|
||||
// packet.setUint8(offset, frames.length)
|
||||
offset++
|
||||
|
||||
if(frames.length >= 2){
|
||||
console.warn('pointerQueue is bigger then 1. Only one event will be sent.')
|
||||
}
|
||||
|
||||
// for (; frames.length > 0;) {
|
||||
// this._inputFps.count()
|
||||
const frame = frames[0]
|
||||
if(frame !== undefined){
|
||||
packet.setUint8(offset, frame.events.length)
|
||||
offset++
|
||||
|
||||
const screenWidth = this._serverVideoWidth
|
||||
const screenHeight = this._serverVideoHeight
|
||||
|
||||
for(let i = 0; i < frame.events.length; i++){
|
||||
const event = frame.events[i]
|
||||
const rect = event.target.getBoundingClientRect()
|
||||
|
||||
let e = 0.06575749909301447 * (screenHeight / 1)
|
||||
let n = 0.06575749909301447 * (screenWidth / 1)
|
||||
|
||||
// @TODO: Check if this is correct
|
||||
e =1
|
||||
n =1
|
||||
|
||||
if(event.type === 'pointerup'){
|
||||
e = 0
|
||||
n = 0
|
||||
}
|
||||
|
||||
packet.setUint16(offset, e, true)
|
||||
packet.setUint16(offset+2, n, true)
|
||||
packet.setUint8(offset+4, 255*event.pressure)
|
||||
packet.setUint16(offset+5, event.twist, true)
|
||||
packet.setUint32(offset+7, 0, true)
|
||||
let o = (event.x - rect.left) * (screenWidth / rect.width)
|
||||
, l = (event.y - rect.top) * (screenHeight / rect.height)
|
||||
|
||||
if(event.type === 'pointerup'){
|
||||
// Reset x and y to 0 on pointerup
|
||||
o = 0
|
||||
l = 0
|
||||
}
|
||||
|
||||
packet.setUint32(offset+11, o, true)
|
||||
packet.setUint32(offset+15, l, true)
|
||||
packet.setUint8(offset+19, (event.type === 'pointerdown') ? 1 :
|
||||
(event.type === 'pointerup') ? 2 : (event.type === 'pointermove') ? 3 : 0)
|
||||
|
||||
offset = offset+20
|
||||
}
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
_writeMouseData(packet:DataView, offset:number, frames:Array<MouseFrame>){
|
||||
packet.setUint8(offset, frames.length)
|
||||
offset++
|
||||
|
||||
if(frames.length >= 30){
|
||||
console.warn('mouseQueue is bigger then 30. This might impact reliability!')
|
||||
}
|
||||
|
||||
for(let i = 0; i < frames.length; i++) {
|
||||
const input:MouseFrame = frames[i]
|
||||
|
||||
packet.setUint32(offset, input.X, true)
|
||||
packet.setUint32(offset+4, input.Y, true)
|
||||
packet.setUint32(offset+8, input.WheelX, true)
|
||||
packet.setUint32(offset+12, input.WheelY, true)
|
||||
|
||||
packet.setUint8(offset+16, input.Buttons)
|
||||
packet.setUint8(offset+17, input.Relative)
|
||||
offset += 18
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
_writeKeyboardData(packet:DataView, offset:number, frames:Array<KeyboardFrame>){
|
||||
packet.setUint8(offset, frames.length)
|
||||
offset++
|
||||
|
||||
if(frames.length >= 30){
|
||||
console.warn('keyboardQueue is bigger then 30. This might impact reliability!')
|
||||
}
|
||||
|
||||
for(let i = 0; i < frames.length; i++) {
|
||||
const input:KeyboardFrame = frames[i]
|
||||
|
||||
packet.setUint8(offset, 2) // 1 = Known, 2 = VKey, 3 = AppCommand, 0 = Unknown
|
||||
packet.setUint8(offset+1, input.pressed ? 1 : 0)
|
||||
packet.setUint8(offset+2, input.keyCode)
|
||||
// packet.setUint8(offset+3, 1)
|
||||
// packet.setUint8(offset+4, input.keyCode)
|
||||
|
||||
offset += 3
|
||||
}
|
||||
|
||||
return offset
|
||||
}
|
||||
|
||||
toBuffer(){
|
||||
const metadataAlloc = new Uint8Array(this._totalSize)
|
||||
const packet = new DataView(metadataAlloc.buffer)
|
||||
|
||||
packet.setUint16(0, this._reportType, true)
|
||||
packet.setUint32(2, this._sequence, true)
|
||||
packet.setFloat64(6, performance.now(), true)
|
||||
|
||||
let offset = 14
|
||||
|
||||
if(this._metadataFrames.length > 0) {offset = this._writeMetadataData(packet, offset, this._metadataFrames)}
|
||||
if(this._gamepadFrames.length > 0) {offset = this._writeGamepadData(packet, offset, this._gamepadFrames)}
|
||||
if(this._pointerFrames.length > 0) {offset = this._writePointerData(packet, offset, this._pointerFrames)}
|
||||
if(this._mouseFrames.length > 0) {offset = this._writeMouseData(packet, offset, this._mouseFrames)}
|
||||
if(this._keyboardFrames.length > 0) {offset = this._writeKeyboardData(packet, offset, this._keyboardFrames)}
|
||||
|
||||
if(this._reportType === ReportTypes.ClientMetadata){
|
||||
packet.setUint8(offset, this._maxTouchpoints)
|
||||
offset++
|
||||
}
|
||||
|
||||
if(offset !== offset){
|
||||
throw new Error('Packet length mismatch. Something is wrong!')
|
||||
}
|
||||
|
||||
return packet
|
||||
}
|
||||
|
||||
_normalizeTriggerValue(value: number): number {
|
||||
if (value <= 0 || Number.isNaN(value)) {
|
||||
return 0
|
||||
}
|
||||
|
||||
if (value >= 1) {
|
||||
return 65535
|
||||
}
|
||||
|
||||
// Fast float -> int conversion in range [0, 65535]
|
||||
return (value * 65535) | 0
|
||||
}
|
||||
|
||||
_normalizeAxisValue(value: number) {
|
||||
const max = 32767
|
||||
const min = -32767
|
||||
const scaled = Math.round(value * max)
|
||||
return scaled > max ? max : scaled < min ? min : scaled
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import xCloudPlayer from '../../player'
|
||||
import InputPacket, { InputFrame as GamepadFrame, KeyboardFrame, MetadataFrame, MouseFrame, PointerFrame } from './packet'
|
||||
|
||||
export default class InputQueue {
|
||||
private _player:xCloudPlayer
|
||||
|
||||
private _inputSequenceNum = 0
|
||||
|
||||
private _metadataQueue:Array<MetadataFrame> = []
|
||||
private _gamepadQueue:Array<GamepadFrame> = []
|
||||
private _pointerQueue:Array<PointerFrame> = []
|
||||
private _mouseQueue:Array<MouseFrame> = []
|
||||
private _keyboardQueue:Array<KeyboardFrame> = []
|
||||
|
||||
// Axis change detection and heartbeat tracking
|
||||
private _gamepadHeartbeatTimers = new Map<number, number>()
|
||||
private _lastGamepadState = new Map<number, GamepadFrame>()
|
||||
private _axisChangeThreshold = 0.08
|
||||
|
||||
constructor(player:xCloudPlayer) {
|
||||
this._player = player
|
||||
}
|
||||
|
||||
private _hasQueuedData() {
|
||||
return this._metadataQueue.length > 0 ||
|
||||
this._gamepadQueue.length > 0 ||
|
||||
this._pointerQueue.length > 0 ||
|
||||
this._mouseQueue.length > 0 ||
|
||||
this._keyboardQueue.length > 0
|
||||
}
|
||||
|
||||
private _upsertGamepadFrame(data:GamepadFrame) {
|
||||
for(let i = 0; i < this._gamepadQueue.length; i++){
|
||||
if(this._gamepadQueue[i].GamepadIndex === data.GamepadIndex){
|
||||
this._gamepadQueue[i] = data
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
this._gamepadQueue.push(data)
|
||||
}
|
||||
|
||||
private _hasSignificantAxisChange(current:GamepadFrame, last:GamepadFrame | undefined):boolean {
|
||||
if(!last) return true // First frame always send
|
||||
|
||||
const stickNames = ['LeftThumbXAxis', 'LeftThumbYAxis', 'RightThumbXAxis', 'RightThumbYAxis'] as const
|
||||
const triggerNames = ['LeftTrigger', 'RightTrigger'] as const
|
||||
|
||||
// For analog sticks: only skip if BOTH change is small AND value is near center
|
||||
for(const stick of stickNames){
|
||||
const delta = Math.abs(current[stick] - last[stick])
|
||||
const value = Math.abs(current[stick])
|
||||
if(delta > this._axisChangeThreshold || value > 0.15){
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// For triggers: always send if they changed (higher sensitivity needed)
|
||||
for(const trigger of triggerNames){
|
||||
if(Math.abs(current[trigger] - last[trigger]) > 0.01){
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
private _isHeartbeatDue(gamepadIndex:number):boolean {
|
||||
const lastHeartbeat = this._gamepadHeartbeatTimers.get(gamepadIndex) ?? 0
|
||||
return performance.now() - lastHeartbeat > 33 // Send at least every 33ms
|
||||
}
|
||||
|
||||
queueMetadataFrame(data:MetadataFrame) {
|
||||
this._metadataQueue.push(data)
|
||||
this.checkQueueAndSend()
|
||||
}
|
||||
|
||||
queueGamepadFrame(data:GamepadFrame) {
|
||||
const lastFrame = this._lastGamepadState.get(data.GamepadIndex)
|
||||
const hasChange = this._hasSignificantAxisChange(data, lastFrame)
|
||||
const isDue = this._isHeartbeatDue(data.GamepadIndex)
|
||||
|
||||
if(hasChange || isDue){
|
||||
this._upsertGamepadFrame(data)
|
||||
this._lastGamepadState.set(data.GamepadIndex, data)
|
||||
this._gamepadHeartbeatTimers.set(data.GamepadIndex, performance.now())
|
||||
this.checkQueueAndSend()
|
||||
}
|
||||
}
|
||||
|
||||
queueGamepadFrames(frames:Array<GamepadFrame>, forceSend = false) {
|
||||
for(const frame of frames){
|
||||
this._upsertGamepadFrame(frame)
|
||||
}
|
||||
this.checkQueueAndSend(forceSend)
|
||||
}
|
||||
|
||||
queueMouseFrame(data:MouseFrame) {
|
||||
this._mouseQueue.push(data)
|
||||
this.checkQueueAndSend()
|
||||
}
|
||||
|
||||
queueKeyboardFrame(data:KeyboardFrame) {
|
||||
this._keyboardQueue.push(data)
|
||||
this.checkQueueAndSend()
|
||||
}
|
||||
|
||||
queuePointerFrame(data:PointerFrame) {
|
||||
if(this._pointerQueue.length > 0){
|
||||
this._pointerQueue[0].events = this._pointerQueue[0].events.concat(data.events)
|
||||
} else {
|
||||
this._pointerQueue.push(data)
|
||||
}
|
||||
this.checkQueueAndSend()
|
||||
}
|
||||
|
||||
checkQueueAndSend(forceSend = false) {
|
||||
if(forceSend === true){
|
||||
if(this._hasQueuedData()){
|
||||
this.sendQueue()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Prioritize low-latency gamepad delivery.
|
||||
if(this._gamepadQueue.length > 0){
|
||||
this.sendQueue()
|
||||
return
|
||||
}
|
||||
|
||||
// Non-metadata input should also not wait.
|
||||
if(this._mouseQueue.length > 0 || this._keyboardQueue.length > 0 || this._pointerQueue.length > 0){
|
||||
this.sendQueue()
|
||||
return
|
||||
}
|
||||
|
||||
// Metadata-only can batch slightly.
|
||||
if(this._metadataQueue.length > 5){
|
||||
this.sendQueue()
|
||||
}
|
||||
}
|
||||
|
||||
sendQueue() {
|
||||
// console.log('Sending queues:', this._metadataQueue.length, this._gamepadQueue.length, this._pointerQueue.length, this._mouseQueue.length, this._keyboardQueue.length)
|
||||
|
||||
const packet = new InputPacket(this.getSequenceNum())
|
||||
if(this._pointerQueue.length > 0){
|
||||
packet.setStreamConfig({
|
||||
width: this._player._channels.input.getServerVideoWidth(),
|
||||
height: this._player._channels.input.getServerVideoHeight(),
|
||||
})
|
||||
}
|
||||
packet.setData(this._metadataQueue, this._gamepadQueue, this._pointerQueue, this._mouseQueue, this._keyboardQueue)
|
||||
this._player._channels.input.send(packet.toBuffer())
|
||||
|
||||
this.clearQueues()
|
||||
}
|
||||
|
||||
clearQueues() {
|
||||
this._metadataQueue = []
|
||||
this._gamepadQueue = []
|
||||
this._pointerQueue = []
|
||||
this._mouseQueue = []
|
||||
this._keyboardQueue = []
|
||||
}
|
||||
|
||||
getSequenceNum() {
|
||||
this._inputSequenceNum++
|
||||
return this._inputSequenceNum
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
import Channel from './channel'
|
||||
import MessageHandler from './message/handler'
|
||||
|
||||
export default class MessageChannel extends Channel {
|
||||
_handshakeCompleted = false
|
||||
|
||||
getChannelName() {
|
||||
return 'message'
|
||||
}
|
||||
|
||||
getChannelConfig() {
|
||||
return {
|
||||
protocol: 'messageV1',
|
||||
ordered: true,
|
||||
}
|
||||
}
|
||||
|
||||
onOpen() {
|
||||
console.log('[MessageChannel] Sending handshake...')
|
||||
const handshake = JSON.stringify({
|
||||
'type':'Handshake',
|
||||
'version':'messageV1',
|
||||
'id':'be0bfc6d-1e83-4c8a-90ed-fa8601c5a179',
|
||||
'cv':'0',
|
||||
})
|
||||
this.send(handshake)
|
||||
}
|
||||
|
||||
onMessage(event: MessageEvent<any>) {
|
||||
const data = JSON.parse(event.data)
|
||||
// console.log('[MessageChannel] received:', data)
|
||||
|
||||
switch(data.type){
|
||||
case 'HandshakeAck':
|
||||
console.log('[MessageChannel] Handshake completed')
|
||||
this._handshakeCompleted = true
|
||||
// Start control & input channels
|
||||
this.getPlayer()._channels.control.sendAuthorization()
|
||||
this.getPlayer()._channels.input.start()
|
||||
this.sendConfig()
|
||||
break
|
||||
|
||||
case 'TransactionStart':
|
||||
case 'Message':
|
||||
new MessageHandler(this, data)
|
||||
break
|
||||
|
||||
default:
|
||||
console.log('[MessageChannel] Unhandled message type:', data.type, data)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sendConfig(){
|
||||
const systemUis = []
|
||||
const systemVersion = [0, 2, 0]
|
||||
const uiConfig = JSON.stringify(this.generateMessage('/streaming/systemUi/configuration', {
|
||||
'version': systemVersion,
|
||||
'systemUis': systemUis, // Xbox Windows app has [33], xCloud has [10,19,31,27,32,-41]
|
||||
|
||||
// 8 = unknown
|
||||
// 10 = ShowVirtualKeyboard
|
||||
// 13 = unknown
|
||||
// 15 = unknown
|
||||
// 19 = ShowMessageDialog
|
||||
// 31 = ShowApplication
|
||||
// 27 = ShowPurchase
|
||||
// 32 = ShowTimerExtensions
|
||||
// 33 = Xbox windows app, disables the nexus menu on xCloud (Alt nexus menu?)
|
||||
// -44 = unknown
|
||||
// 40 = unknown
|
||||
// 41 = unknown
|
||||
// -43 = unknown
|
||||
|
||||
// Possible options: Keyboard, PurchaseModal
|
||||
}))
|
||||
this.send(uiConfig)
|
||||
|
||||
const clientConfig = JSON.stringify(this.generateMessage('/streaming/properties/clientappinstallidchanged', { 'clientAppInstallId': 'c97d7ee0-73b2-4239-bf1d-9d805a338429' }))
|
||||
this.send(clientConfig)
|
||||
|
||||
const orientationConfig = JSON.stringify(this.generateMessage('/streaming/characteristics/orientationchanged', { 'orientation': 0 }))
|
||||
this.send(orientationConfig)
|
||||
|
||||
const touchConfig = JSON.stringify(this.generateMessage('/streaming/characteristics/touchinputenabledchanged', { 'touchInputEnabled': false }))
|
||||
this.send(touchConfig)
|
||||
|
||||
const deviceConfig = JSON.stringify(this.generateMessage('/streaming/characteristics/clientdevicecapabilities', {}))
|
||||
this.send(deviceConfig)
|
||||
|
||||
const dimensionsConfig = JSON.stringify(this.generateMessage('/streaming/characteristics/dimensionschanged', {
|
||||
'horizontal': 1920,
|
||||
'vertical': 1080,
|
||||
'preferredWidth': 1920,
|
||||
'preferredHeight': 1080,
|
||||
'safeAreaLeft': 0,
|
||||
'safeAreaTop': 0,
|
||||
'safeAreaRight': 1920,
|
||||
'safeAreaBottom': 1080,
|
||||
'supportsCustomResolution':true,
|
||||
}))
|
||||
this.send(dimensionsConfig)
|
||||
}
|
||||
|
||||
generateMessage(path, data) {
|
||||
return {
|
||||
'type': 'Message',
|
||||
'content': JSON.stringify(data),
|
||||
'id': uuidv4(),
|
||||
'target': path,
|
||||
'cv': '',
|
||||
}
|
||||
}
|
||||
|
||||
destroy() {
|
||||
// console.log('DebugChannel destroy() called')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import Channel from '../../lib/channel'
|
||||
|
||||
export default class MessageHandler {
|
||||
private _channel:Channel
|
||||
private _data:object
|
||||
|
||||
constructor(channel:Channel, data:object) {
|
||||
this._channel = channel
|
||||
this._data = data
|
||||
|
||||
this.handle(this._data)
|
||||
}
|
||||
|
||||
handle(data) {
|
||||
console.log('[MessageHandler] Handling:', data)
|
||||
switch(data.target){
|
||||
case '/streaming/sessionLifetimeManagement/serverInitiatedDisconnect':
|
||||
this.completeTransaction(data.id, '')
|
||||
this._channel.getPlayer().destroy()
|
||||
break
|
||||
|
||||
case '/streaming/properties/titleinfo':
|
||||
console.log('Title update:', data)
|
||||
break
|
||||
|
||||
case '/streaming/touchcontrols/showlayoutv2':
|
||||
// console.log('Showing touch layout:', data)
|
||||
break
|
||||
|
||||
case '/streaming/systemUi/messages/ShowMessageDialog':
|
||||
this.handleMessage(data)
|
||||
break
|
||||
|
||||
default:
|
||||
console.log('[MessageChannel] Unhandled transaction:', data)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
completeTransaction(id, data) {
|
||||
const transaction = JSON.stringify({
|
||||
'type': 'TransactionComplete',
|
||||
'content': JSON.stringify(data),
|
||||
// 'content':'{\'Result\':0}',
|
||||
'id': id,
|
||||
'cv': '',
|
||||
})
|
||||
|
||||
this._channel.send(transaction)
|
||||
}
|
||||
|
||||
cancelTransaction(id, data) {
|
||||
const transaction = JSON.stringify({
|
||||
'type': 'ReceiverCancel',
|
||||
'content': JSON.stringify(data),
|
||||
'id': id,
|
||||
'cv': '',
|
||||
})
|
||||
|
||||
this._channel.send(transaction)
|
||||
}
|
||||
|
||||
handleMessage(data) {
|
||||
const jsonData = JSON.parse(data.content)
|
||||
console.log(jsonData)
|
||||
if(confirm(jsonData.TitleText+'\n\n'+jsonData.ContentText)){
|
||||
this.completeTransaction(data.id, { Result: 0 })
|
||||
} else {
|
||||
this.completeTransaction(data.id, { Result: 1 })
|
||||
// this.cancelTransaction(data.id, '')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import xCloudPlayer from './player'
|
||||
import Teredo from './teredo'
|
||||
|
||||
export default class Ice {
|
||||
private _peerConnection:RTCPeerConnection
|
||||
private _iceCandidates: RTCIceCandidate[] = []
|
||||
|
||||
constructor(player:xCloudPlayer) {
|
||||
this._peerConnection = player._peerConnection
|
||||
|
||||
this._peerConnection.addEventListener('icecandidate', event => {
|
||||
if(event.candidate) {
|
||||
this._iceCandidates.push(event.candidate)
|
||||
|
||||
// @TODO: Implement Teredo port selection for remote streaming
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
getCandidates(){
|
||||
return this._iceCandidates
|
||||
}
|
||||
|
||||
setRemoteCandidates(candidates:RTCIceCandidate[]){
|
||||
if(candidates.length === 0){
|
||||
console.warn('Error: No candidates received!')
|
||||
}
|
||||
|
||||
for(const candidate in candidates){
|
||||
if(candidates[candidate].candidate === 'a=end-of-candidates') {continue}
|
||||
|
||||
const candidateAddress = candidates[candidate].candidate.split(' ')
|
||||
|
||||
if(candidateAddress.length > 4 && candidateAddress[4].substr(0, 4) === '2001'){
|
||||
const teredo = new Teredo(candidateAddress[4])
|
||||
|
||||
this._peerConnection.addIceCandidate({
|
||||
candidate: 'a=candidate:10 1 UDP 1 '+teredo.getIpv4Address()+' 9002 typ host ',
|
||||
sdpMLineIndex: candidates[candidate].sdpMLineIndex,
|
||||
sdpMid: candidates[candidate].sdpMid,
|
||||
})
|
||||
this._peerConnection.addIceCandidate({
|
||||
candidate: 'a=candidate:11 1 UDP 1 '+teredo.getIpv4Address()+' '+teredo.getIpv4Port()+' typ host ',
|
||||
sdpMLineIndex: candidates[candidate].sdpMLineIndex,
|
||||
sdpMid: candidates[candidate].sdpMid,
|
||||
})
|
||||
}
|
||||
|
||||
this._peerConnection.addIceCandidate({
|
||||
candidate: candidates[candidate].candidate,
|
||||
sdpMid: candidates[candidate].sdpMid,
|
||||
sdpMLineIndex: candidates[candidate].sdpMLineIndex,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
import xCloudPlayer from '../player'
|
||||
import { VibrationFrame } from '../channel/input'
|
||||
|
||||
export interface GamepadOptions {
|
||||
enable_keyboard?: boolean;
|
||||
enable_gamepad?: boolean;
|
||||
enable_vibration?: boolean;
|
||||
keyboard_mapping?: {
|
||||
[key:string]: string;
|
||||
};
|
||||
gamepad_mapping?: {
|
||||
[key:string]: string;
|
||||
};
|
||||
gamepad_axes_mapping?: {
|
||||
[key:string]: string;
|
||||
};
|
||||
gamepad_deadzone?: number;
|
||||
gamepad_force_capture?: boolean;
|
||||
}
|
||||
|
||||
export interface GamepadOptionsDefaults extends Required<GamepadOptions> {}
|
||||
|
||||
export default class Gamepad {
|
||||
private _player: xCloudPlayer | undefined
|
||||
private _index: number
|
||||
private _physicalGamepadId = -1
|
||||
private _rumbleInterval
|
||||
private _isFocused = true
|
||||
private _gamepadOverride = {
|
||||
buttons: {},
|
||||
}
|
||||
|
||||
private _options:GamepadOptionsDefaults = {
|
||||
enable_keyboard: false,
|
||||
enable_gamepad: true,
|
||||
enable_vibration: true,
|
||||
keyboard_mapping: {
|
||||
'A': 'Enter',
|
||||
'B': 'Backspace',
|
||||
'X': 'x',
|
||||
'Y': 'y',
|
||||
'DPadUp': 'ArrowUp',
|
||||
'DPadDown': 'ArrowDown',
|
||||
'DPadLeft': 'ArrowLeft',
|
||||
'DPadRight': 'ArrowRight',
|
||||
'LeftShoulder': '[',
|
||||
'RightShoulder': ']',
|
||||
'LeftThumb': 'l',
|
||||
'RightThumb': 'r',
|
||||
'LeftTrigger': '-',
|
||||
'RightTrigger': '=',
|
||||
'Menu': 'm',
|
||||
'View': 'v',
|
||||
'Nexus': 'n',
|
||||
},
|
||||
gamepad_mapping: {
|
||||
'A': '0',
|
||||
'B': '1',
|
||||
'X': '2',
|
||||
'Y': '3',
|
||||
'DPadUp': '12',
|
||||
'DPadDown': '13',
|
||||
'DPadLeft': '14',
|
||||
'DPadRight': '15',
|
||||
'LeftShoulder': '4',
|
||||
'RightShoulder': '5',
|
||||
'LeftThumb': '10',
|
||||
'RightThumb': '11',
|
||||
'LeftTrigger': '6',
|
||||
'RightTrigger': '7',
|
||||
'Menu': '9',
|
||||
'View': '8',
|
||||
'Nexus': '16',
|
||||
},
|
||||
gamepad_axes_mapping: {
|
||||
'LeftThumbXAxis': '0',
|
||||
'LeftThumbYAxis': '1',
|
||||
'RightThumbXAxis': '2',
|
||||
'RightThumbYAxis': '3',
|
||||
},
|
||||
gamepad_deadzone: 0.08,
|
||||
gamepad_force_capture: false,
|
||||
}
|
||||
|
||||
private _listener = {
|
||||
keyDown: this.onKeyDown.bind(this),
|
||||
keyUp: this.onKeyUp.bind(this),
|
||||
gamepadConnected: this.onGamepadConnected.bind(this),
|
||||
gamepadDisconnected: this.onGamepadDisconnected.bind(this),
|
||||
windowFocus: this.onWindowFocus.bind(this),
|
||||
windowBlur: this.onWindowBlur.bind(this),
|
||||
}
|
||||
|
||||
constructor(index:number, options:GamepadOptions = {}){
|
||||
this._index = index
|
||||
|
||||
this._options = {
|
||||
...this._options,
|
||||
...options,
|
||||
}
|
||||
}
|
||||
|
||||
attach(xCloudPlayer:xCloudPlayer){
|
||||
this._player = xCloudPlayer
|
||||
|
||||
this._player._channels.control.sendGamepadState(this._index, true, this)
|
||||
|
||||
if(this._options.enable_keyboard === true){
|
||||
window.addEventListener('keydown', this._listener.keyDown)
|
||||
window.addEventListener('keyup', this._listener.keyUp)
|
||||
}
|
||||
|
||||
if(this._options.enable_gamepad === true){
|
||||
window.addEventListener('gamepadconnected', this._listener.gamepadConnected)
|
||||
window.addEventListener('gamepaddisconnected', this._listener.gamepadDisconnected)
|
||||
}
|
||||
|
||||
if(this._options.enable_gamepad === true && this._options.gamepad_force_capture === false){
|
||||
window.addEventListener('blur', this._listener.windowBlur)
|
||||
window.addEventListener('focus', this._listener.windowFocus)
|
||||
}
|
||||
|
||||
this.detectActiveGamepad()
|
||||
}
|
||||
|
||||
detach(){
|
||||
if(this._player === undefined){
|
||||
console.log('[Gamepad] Player is not attached. this._player is:', this._player)
|
||||
return
|
||||
}
|
||||
this._player._channels.control.sendGamepadState(this._index, false)
|
||||
this._physicalGamepadId = -1
|
||||
|
||||
if(this._rumbleInterval !== undefined){
|
||||
clearInterval(this._rumbleInterval)
|
||||
}
|
||||
|
||||
if(this._options.enable_keyboard === true){
|
||||
window.removeEventListener('keydown', this._listener.keyDown)
|
||||
window.removeEventListener('keyup', this._listener.keyUp)
|
||||
}
|
||||
|
||||
if(this._options.enable_gamepad === true){
|
||||
window.removeEventListener('gamepadconnected', this._listener.gamepadConnected)
|
||||
window.removeEventListener('gamepaddisconnected', this._listener.gamepadDisconnected)
|
||||
}
|
||||
|
||||
if(this._options.enable_gamepad === true && this._options.gamepad_force_capture === false){
|
||||
window.removeEventListener('blur', this._listener.windowBlur)
|
||||
window.removeEventListener('focus', this._listener.windowFocus)
|
||||
}
|
||||
}
|
||||
|
||||
sendButtonState(button, value){
|
||||
if(this._player === undefined){
|
||||
console.log('[Gamepad] Player is not attached. this._player is:', this._player)
|
||||
return
|
||||
}
|
||||
|
||||
if(! (button in this.getDefaultFamepadFrame())){
|
||||
console.log('[Gamepad] Invalid button:', button)
|
||||
return
|
||||
}
|
||||
|
||||
const buttonPreset = this.getDefaultFamepadFrame()
|
||||
buttonPreset.GamepadIndex = this._index
|
||||
this._gamepadOverride.buttons[button] = value
|
||||
}
|
||||
|
||||
onKeyDown(event:KeyboardEvent){
|
||||
if(this._isFocused === false){
|
||||
return
|
||||
}
|
||||
|
||||
for(const button in this._options.keyboard_mapping){
|
||||
if(this._options.keyboard_mapping[button] === event.key){
|
||||
this._gamepadOverride.buttons[button] = 1
|
||||
event.preventDefault()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onKeyUp(event:KeyboardEvent){
|
||||
if(this._isFocused === false){
|
||||
return
|
||||
}
|
||||
|
||||
for(const button in this._options.keyboard_mapping){
|
||||
if(this._options.keyboard_mapping[button] === event.key){
|
||||
this._gamepadOverride.buttons[button] = 0
|
||||
event.preventDefault()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
onWindowFocus(){
|
||||
this._isFocused = true
|
||||
}
|
||||
|
||||
onWindowBlur(){
|
||||
// this._isFocused = false
|
||||
}
|
||||
|
||||
getDefaultFamepadFrame(){
|
||||
return {
|
||||
GamepadIndex: 0,
|
||||
Nexus: 0,
|
||||
Menu: 0,
|
||||
View: 0,
|
||||
A: 0,
|
||||
B: 0,
|
||||
X: 0,
|
||||
Y: 0,
|
||||
DPadUp: 0,
|
||||
DPadDown: 0,
|
||||
DPadLeft: 0,
|
||||
DPadRight: 0,
|
||||
LeftShoulder: 0,
|
||||
RightShoulder: 0,
|
||||
LeftThumb: 0,
|
||||
RightThumb: 0,
|
||||
|
||||
LeftThumbXAxis: 0,
|
||||
LeftThumbYAxis: 0,
|
||||
RightThumbXAxis: 0,
|
||||
RightThumbYAxis: 0,
|
||||
LeftTrigger: 0,
|
||||
RightTrigger: 0,
|
||||
}
|
||||
}
|
||||
|
||||
detectActiveGamepad(){
|
||||
if(this._physicalGamepadId < 0){
|
||||
for(const gamepadBrowser in navigator.getGamepads()){
|
||||
const gamepadIndex = navigator.getGamepads()[gamepadBrowser]?.index
|
||||
if(gamepadIndex === undefined){
|
||||
continue
|
||||
}
|
||||
|
||||
let gamepadFoundInHandler = false
|
||||
for(const gamepad in this._player?._channels.control.getGamepadHandlers()){
|
||||
if(
|
||||
this._player?._channels.control.getGamepadHandlers()[gamepad] instanceof Gamepad &&
|
||||
(this._player?._channels.control.getGamepadHandlers()[gamepad] as Gamepad)._physicalGamepadId === navigator.getGamepads()[gamepadBrowser]?.index
|
||||
) {
|
||||
console.log('[Gamepad] Active gamepad already detected:', gamepadIndex, (this._player?._channels.control.getGamepadHandlers()[gamepad] as Gamepad)._physicalGamepadId)
|
||||
gamepadFoundInHandler = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(gamepadFoundInHandler === false){
|
||||
this._physicalGamepadId = gamepadIndex
|
||||
console.log('[Gamepad] Active gamepad detected:', this._physicalGamepadId)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log('[Gamepad] Physical gamepad already detected:', this._physicalGamepadId)
|
||||
}
|
||||
}
|
||||
|
||||
onGamepadConnected(event:GamepadEvent){
|
||||
if(this._physicalGamepadId < 0){
|
||||
for(const gamepad in this._player?._channels.control.getGamepadHandlers()){
|
||||
if(
|
||||
this._player?._channels.control.getGamepadHandlers()[gamepad] instanceof Gamepad &&
|
||||
(this._player?._channels.control.getGamepadHandlers()[gamepad] as Gamepad)._physicalGamepadId === event.gamepad.index
|
||||
) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
this._physicalGamepadId = event.gamepad.index
|
||||
}
|
||||
}
|
||||
|
||||
onGamepadDisconnected(event:GamepadEvent){
|
||||
if(event.gamepad.index === this._physicalGamepadId){
|
||||
this._physicalGamepadId = -1
|
||||
}
|
||||
}
|
||||
|
||||
hasOverrideButtons(){
|
||||
return Object.keys(this._gamepadOverride.buttons).length > 0
|
||||
}
|
||||
|
||||
getGamepadState(){
|
||||
const gamepad = this.getGamepad(this._physicalGamepadId)
|
||||
if((this._physicalGamepadId < 0 || gamepad === undefined || this._isFocused === false) && (this.hasOverrideButtons() === false)){
|
||||
return undefined
|
||||
}
|
||||
|
||||
const frame = this.getDefaultFamepadFrame()
|
||||
frame.GamepadIndex = this._index
|
||||
|
||||
// Set buttons
|
||||
if(gamepad !== undefined){
|
||||
for(const button in this._options.gamepad_mapping){
|
||||
// DroidDeck: Android often reports no guide button (index 16); a missing button is "not pressed".
|
||||
frame[button] = gamepad.buttons[this._options.gamepad_mapping[button]]?.value ?? 0
|
||||
}
|
||||
|
||||
// Set axis
|
||||
for(const axis in this._options.gamepad_axes_mapping){
|
||||
frame[axis] = this.normaliseAxis(gamepad.axes[this._options.gamepad_axes_mapping[axis]] ?? 0)
|
||||
}
|
||||
}
|
||||
|
||||
// Check for button overrides from the keyboard
|
||||
// for(const button in this._gamepadOverride.buttons){
|
||||
// frame[button] = this._gamepadOverride.buttons[button]
|
||||
|
||||
// // Clean up existing keys if they are 0
|
||||
// if(this._gamepadOverride.buttons[button] === 0){
|
||||
// // Remove key from override
|
||||
// const newOverrideButtons = {}
|
||||
// for(const oButton in this._gamepadOverride.buttons){
|
||||
// if(this._gamepadOverride.buttons[oButton] !== 0){
|
||||
// newOverrideButtons[oButton] = this._gamepadOverride.buttons[oButton]
|
||||
// }
|
||||
// }
|
||||
// this._gamepadOverride.buttons = newOverrideButtons
|
||||
// }
|
||||
// }
|
||||
|
||||
// Check for button overrides from the keyboard
|
||||
for(const button in this._gamepadOverride.buttons){
|
||||
frame[button] = this._gamepadOverride.buttons[button]
|
||||
}
|
||||
|
||||
// Clean up released buttons in a separate pass
|
||||
const buttonsToRemove = []
|
||||
for(const button in this._gamepadOverride.buttons){
|
||||
if(this._gamepadOverride.buttons[button] === 0){
|
||||
buttonsToRemove.push(button)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete released buttons instead of recreating the object
|
||||
for(const button of buttonsToRemove){
|
||||
delete this._gamepadOverride.buttons[button]
|
||||
}
|
||||
|
||||
// Start + Select Nexus menu workaround
|
||||
if(frame.View > 0 && frame.Menu > 0){
|
||||
frame.View = 0
|
||||
frame.Menu = 0
|
||||
|
||||
frame.Nexus = 1
|
||||
}
|
||||
|
||||
return frame
|
||||
}
|
||||
|
||||
getGamepad(index = -1){
|
||||
const gamepad = navigator.getGamepads()[index]
|
||||
if(gamepad !== null) {
|
||||
return gamepad
|
||||
} else {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
getPhysicalGamepadId(){
|
||||
return this._physicalGamepadId
|
||||
}
|
||||
|
||||
handleVibration(report:VibrationFrame){
|
||||
if(this._physicalGamepadId < 0){
|
||||
console.log('[Gamepad] Received a vibration report but no physical gamepad is connected')
|
||||
return
|
||||
}
|
||||
|
||||
const gamepad = this.getGamepad(this._physicalGamepadId)
|
||||
if(gamepad === undefined){
|
||||
console.log('[Gamepad] Received a vibration report but no physical gamepad is connected')
|
||||
return
|
||||
}
|
||||
|
||||
if(this._options.enable_vibration === false){
|
||||
return
|
||||
}
|
||||
|
||||
const rumbleData = {
|
||||
startDelay: 0,
|
||||
duration: report.durationMs,
|
||||
// DroidDeck: the player's rumble strength setting (0..1).
|
||||
weakMagnitude: report.rightMotorPercent * ((window as any).__ddRumble ?? 1),
|
||||
strongMagnitude: report.leftMotorPercent * ((window as any).__ddRumble ?? 1),
|
||||
|
||||
leftTrigger: report.leftTriggerMotorPercent,
|
||||
rightTrigger: report.rightTriggerMotorPercent,
|
||||
}
|
||||
|
||||
// @ts-expect-error - dual-rumble is not in the types
|
||||
if(gamepad.vibrationActuator?.type === 'dual-rumble') {
|
||||
const intensityRumble = report.rightMotorPercent < .6 ? (.6 - report.rightMotorPercent) / 2 : 0
|
||||
const intensityRumbleTriggers = (report.leftTriggerMotorPercent + report.rightTriggerMotorPercent) / 4
|
||||
const endIntensity = Math.min(intensityRumble, intensityRumbleTriggers)
|
||||
|
||||
rumbleData.weakMagnitude = Math.min(1, (report.rightMotorPercent + endIntensity) * ((window as any).__ddRumble ?? 1))
|
||||
|
||||
// Set triggers to 0 as dual-rumble does not support Triggers
|
||||
// @TODO: Check if we can improve rumble: https://github.com/MicrosoftEdge/MSEdgeExplainers/blob/main/GamepadHapticsActuatorTriggerRumble/explainer.md
|
||||
rumbleData.leftTrigger = 0
|
||||
rumbleData.rightTrigger = 0
|
||||
|
||||
gamepad.vibrationActuator?.playEffect('dual-rumble', rumbleData)
|
||||
|
||||
clearInterval(this._rumbleInterval)
|
||||
if(report.repeat > 0) {
|
||||
let repeatCount = report.repeat
|
||||
|
||||
this._rumbleInterval = setInterval(() => {
|
||||
if(repeatCount <= 0){
|
||||
clearInterval(this._rumbleInterval)
|
||||
}
|
||||
|
||||
if(gamepad.vibrationActuator !== undefined) {
|
||||
gamepad.vibrationActuator?.playEffect('dual-rumble', rumbleData)
|
||||
}
|
||||
repeatCount--
|
||||
|
||||
}, report.delayMs + report.durationMs)
|
||||
}
|
||||
} else {
|
||||
console.log('[Gamepad] Unknown vibration type:', (gamepad.vibrationActuator as any).type)
|
||||
}
|
||||
}
|
||||
|
||||
normaliseAxis(value:number) {
|
||||
if(Math.abs(value) < this._options.gamepad_deadzone) {
|
||||
return 0
|
||||
}
|
||||
|
||||
value = value - Math.sign(value) * this._options.gamepad_deadzone
|
||||
value /= (1.0 - this._options.gamepad_deadzone)
|
||||
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import xCloudPlayer from '../player'
|
||||
|
||||
export default class MouseKeyboard {
|
||||
private _player: xCloudPlayer | undefined
|
||||
private _index: number
|
||||
|
||||
private _listener = {
|
||||
keyDown: this.onKeyDown.bind(this),
|
||||
keyUp: this.onKeyUp.bind(this),
|
||||
pointerMove: this.onPointerMove.bind(this),
|
||||
pointerClick: this.onPointerClick.bind(this),
|
||||
pointerScroll: this.onPointerScroll.bind(this),
|
||||
lockMouseKeyboard: this.lockMouseKeyboard.bind(this),
|
||||
}
|
||||
|
||||
|
||||
constructor(index:number){
|
||||
this._index = index
|
||||
}
|
||||
|
||||
attach(xCloudPlayer:xCloudPlayer){
|
||||
this._player = xCloudPlayer
|
||||
|
||||
this._player._channels.control.sendGamepadState(this._index, true, this)
|
||||
|
||||
// Mod the video UI to capture MKB input
|
||||
if(this._player.getVideoElement() === undefined){
|
||||
console.log('[MKB] VideoComponent is not attached. this._player._videoComponent is:', this._player.getVideoElement())
|
||||
return
|
||||
} else {
|
||||
this._player.getVideoElement()?.addEventListener('pointermove', this._listener.pointerMove)
|
||||
this._player.getVideoElement()?.addEventListener('pointerdown', this._listener.pointerClick)
|
||||
this._player.getVideoElement()?.addEventListener('pointerup', this._listener.pointerClick)
|
||||
this._player.getVideoElement()?.addEventListener('wheel', this._listener.pointerScroll)
|
||||
this._player.getVideoElement()?.addEventListener('click', this._listener.lockMouseKeyboard)
|
||||
|
||||
window.addEventListener('keydown', this._listener.keyDown)
|
||||
window.addEventListener('keyup', this._listener.keyUp)
|
||||
|
||||
this._player.getVideoElement()?.addEventListener('contextmenu', event => {
|
||||
event.preventDefault()
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
detach(){
|
||||
if(this._player === undefined){
|
||||
console.log('[MKB] Player is not attached. this._player is:', this._player)
|
||||
return
|
||||
}
|
||||
|
||||
this._player._channels.control.sendGamepadState(this._index, false)
|
||||
|
||||
window.removeEventListener('keydown', this._listener.keyDown)
|
||||
window.removeEventListener('keyup', this._listener.keyUp)
|
||||
|
||||
this._player.getVideoElement()?.removeEventListener('pointermove', this._listener.pointerMove)
|
||||
this._player.getVideoElement()?.removeEventListener('pointerdown', this._listener.pointerClick)
|
||||
this._player.getVideoElement()?.removeEventListener('pointerup', this._listener.pointerClick)
|
||||
this._player.getVideoElement()?.removeEventListener('wheel', this._listener.pointerScroll)
|
||||
this._player.getVideoElement()?.removeEventListener('click', this._listener.lockMouseKeyboard)
|
||||
}
|
||||
|
||||
onPointerMove(event:PointerEvent){
|
||||
this._player?._channels.input.queueMouseFrame({
|
||||
X: event.movementX*2,
|
||||
Y: event.movementY*2,
|
||||
WheelX: 0,
|
||||
WheelY: 0,
|
||||
Buttons: event.buttons,
|
||||
Relative: 0, // 0 = Relative, 1 = Absolute
|
||||
})
|
||||
}
|
||||
|
||||
onPointerClick(event:PointerEvent){
|
||||
// console.log('Pointer clicked:', event)
|
||||
this._player?._channels.input.queueMouseFrame({
|
||||
X: event.movementX*2,
|
||||
Y: event.movementY*2,
|
||||
WheelX: 0,
|
||||
WheelY: 0,
|
||||
Buttons: event.buttons,
|
||||
Relative: 0, // 0 = Relative, 1 = Absolute
|
||||
})
|
||||
}
|
||||
|
||||
onPointerScroll(event:WheelEvent){
|
||||
console.log('[MKB] Pointer scrolled:', event)
|
||||
}
|
||||
|
||||
onKeyDown(event:KeyboardEvent){
|
||||
this._player?._channels.input.queueKeyboardFrame({
|
||||
key: event.key,
|
||||
keyCode: event.keyCode,
|
||||
pressed: true,
|
||||
})
|
||||
event.preventDefault()
|
||||
}
|
||||
|
||||
onKeyUp(event:KeyboardEvent){
|
||||
this._player?._channels.input.queueKeyboardFrame({
|
||||
key: event.key,
|
||||
keyCode: event.keyCode,
|
||||
pressed: false,
|
||||
})
|
||||
}
|
||||
|
||||
lockMouseKeyboard(){
|
||||
if(this._player === undefined || this._player.getVideoElement() === undefined){
|
||||
console.log('[MKB] Player or VideoComponent is not attached. Failed to lock Mouse & Keyboard because there is nothing to lock for.')
|
||||
return
|
||||
}
|
||||
|
||||
// @disable ts-expect-error requestPointerLock() is not defined to consume arguments
|
||||
const promise = this._player?.getVideoElement()?.requestPointerLock({
|
||||
unadjustedMovement: true,
|
||||
}) as Promise<undefined> | undefined
|
||||
|
||||
if(promise === undefined){
|
||||
console.log('[MKB] Failed to lock Mouse & Keyboard because the requestPointerLock() is undefined')
|
||||
return
|
||||
}
|
||||
|
||||
if ('keyboard' in navigator && 'lock' in (navigator.keyboard as any)) {
|
||||
document.body.requestFullscreen().then(() => {
|
||||
(navigator as any).keyboard.lock()
|
||||
})
|
||||
} else {
|
||||
console.log('[MKB] Keyboard lock is not supported. Error:', 'NotSupportedError')
|
||||
return
|
||||
}
|
||||
|
||||
promise.then(() => {
|
||||
console.log('[MKB] pointer is locked')
|
||||
}).catch((error) => {
|
||||
if (error.name === 'NotSupportedError') {
|
||||
console.log('[MKB] Mouse pointer lock is not supported. Error:', 'NotSupportedError')
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import xCloudPlayer from '../player'
|
||||
|
||||
export default class Touch {
|
||||
private _player: xCloudPlayer | undefined
|
||||
private _index: number
|
||||
|
||||
private _pointerEvents = {}
|
||||
private _lastPointerId = -1
|
||||
|
||||
private _mouseDown = false
|
||||
|
||||
private _listener = {
|
||||
onPointer: this.onPointer.bind(this),
|
||||
}
|
||||
|
||||
constructor(index:number){
|
||||
this._index = index
|
||||
}
|
||||
|
||||
attach(xCloudPlayer:xCloudPlayer){
|
||||
this._player = xCloudPlayer
|
||||
|
||||
this._player._channels.control.sendGamepadState(this._index, true, this)
|
||||
|
||||
// Mod the video UI to capture MKB input
|
||||
if(this._player.getVideoElement() === undefined){
|
||||
console.log('VideoComponent is not attached. this._player._videoComponent is:', this._player.getVideoElement())
|
||||
return
|
||||
} else {
|
||||
this._player.getVideoElement()?.addEventListener('pointermove', this._listener.onPointer, { passive: false })
|
||||
this._player.getVideoElement()?.addEventListener('pointerdown', this._listener.onPointer, { passive: false })
|
||||
this._player.getVideoElement()?.addEventListener('pointerup', this._listener.onPointer, { passive: false })
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
detach(){
|
||||
if(this._player === undefined){
|
||||
console.log('Player is not attached. this._player is:', this._player)
|
||||
return
|
||||
}
|
||||
this._player._channels.control.sendGamepadState(this._index, false)
|
||||
|
||||
this._player.getVideoElement()?.removeEventListener('pointermove', this._listener.onPointer)
|
||||
this._player.getVideoElement()?.removeEventListener('pointerdown', this._listener.onPointer)
|
||||
this._player.getVideoElement()?.removeEventListener('pointerup', this._listener.onPointer)
|
||||
}
|
||||
|
||||
onPointer(event:PointerEvent) {
|
||||
this._lastPointerId = event.pointerId
|
||||
|
||||
if(event.pointerType === 'mouse' && event.type === 'pointerdown'){
|
||||
this._mouseDown = true
|
||||
}
|
||||
|
||||
if(event.pointerType === 'mouse' && event.type === 'pointerup'){
|
||||
this._mouseDown = false
|
||||
}
|
||||
|
||||
if(event.pointerType === 'mouse' && event.type === 'pointermove' && this._mouseDown === false){
|
||||
// ignore events if the pointer type is mouse and is not pressed.
|
||||
} else {
|
||||
this._player?._channels.input.queuePointerFrame({
|
||||
events: [event],
|
||||
})
|
||||
}
|
||||
event.preventDefault()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import ChatChannel from './channel/chat'
|
||||
import ControlChannel from './channel/control'
|
||||
import InputChannel from './channel/input'
|
||||
import MessageChannel from './channel/message'
|
||||
|
||||
import Ice from './ice'
|
||||
import Sdp from './sdp'
|
||||
import Stats from './stats'
|
||||
|
||||
import VideoComponent from './render/video'
|
||||
import WebGpuComponent from './render/webgpu'
|
||||
import AudioComponent from './render/audio'
|
||||
|
||||
export default class xCloudPlayer {
|
||||
|
||||
_peerConnection = new RTCPeerConnection({})
|
||||
_channels = {
|
||||
chat: new ChatChannel(this),
|
||||
control: new ControlChannel(this),
|
||||
input: new InputChannel(this),
|
||||
message: new MessageChannel(this),
|
||||
}
|
||||
|
||||
private _elementId: string
|
||||
private _isDestoyed: boolean = false
|
||||
private _iceHelper = new Ice(this)
|
||||
private _sdpHelper = new Sdp(this)
|
||||
private _statsHelper = new Stats(this)
|
||||
|
||||
private _videoComponent: VideoComponent | WebGpuComponent | undefined
|
||||
private _audioComponent: AudioComponent | undefined
|
||||
|
||||
constructor(elementId: string) {
|
||||
console.log('xCloudPlayer constructing...');
|
||||
|
||||
this._elementId = elementId;
|
||||
this._peerConnection.addTransceiver('audio', { direction: 'sendrecv' })
|
||||
const videoTransceiver = this._peerConnection.addTransceiver('video', { direction: 'recvonly' })
|
||||
videoTransceiver.setCodecPreferences(this._sdpHelper.getDefaultCodecPreferences())
|
||||
|
||||
this._peerConnection.ontrack = (event) => {
|
||||
|
||||
if(event.track.kind === 'video'){
|
||||
// DroidDeck: a plain <video> element (hardware decode); WebGPU is not dependable in Android WebView.
|
||||
this._videoComponent = new VideoComponent(this)
|
||||
this._videoComponent.create(event.streams[0])
|
||||
|
||||
} else if(event.track.kind === 'audio'){
|
||||
this._audioComponent = new AudioComponent(this)
|
||||
this._audioComponent.create(event.streams[0])
|
||||
|
||||
} else {
|
||||
console.log('[xPlayer] Detected an unknown stream type: ', event.track.kind)
|
||||
}
|
||||
}
|
||||
|
||||
console.log('xCloudPlayer constructed.');
|
||||
}
|
||||
|
||||
init() {
|
||||
console.log('xCloudPlayer init called.');
|
||||
}
|
||||
|
||||
getElementId() {
|
||||
return this._elementId
|
||||
}
|
||||
|
||||
async createOffer() {
|
||||
const offer = await this._peerConnection.createOffer({
|
||||
offerToReceiveAudio: true,
|
||||
offerToReceiveVideo: true,
|
||||
})
|
||||
const finalOffer = this._sdpHelper.setLocalSDP(offer)
|
||||
this._peerConnection.setLocalDescription(finalOffer)
|
||||
|
||||
return finalOffer
|
||||
}
|
||||
|
||||
async setRemoteSDP(sdp: string) {
|
||||
const finalSdp = this._sdpHelper.setRemoteSDP(sdp)
|
||||
return await this._peerConnection.setRemoteDescription({
|
||||
type: 'answer',
|
||||
sdp: finalSdp,
|
||||
})
|
||||
}
|
||||
|
||||
async getICECandidates() {
|
||||
return this._iceHelper.getCandidates()
|
||||
}
|
||||
|
||||
async setRemoteIceCandidates(candidates:RTCIceCandidate[]) {
|
||||
// @TODO: Sort ipv4 and ipv6, and use best route as preference (ipv6 only etc)
|
||||
|
||||
this._iceHelper.setRemoteCandidates(candidates)
|
||||
}
|
||||
|
||||
getVideoElement() {
|
||||
if(this._videoComponent){
|
||||
return this._videoComponent.getElement()
|
||||
} else {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
getAudioElement() {
|
||||
if(this._audioComponent){
|
||||
return this._audioComponent.getElement()
|
||||
} else {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
toggleDebugOverlay() {
|
||||
if(this._videoComponent){
|
||||
this._videoComponent.toggleDebugOverlay()
|
||||
}
|
||||
}
|
||||
|
||||
getStats() {
|
||||
return this._statsHelper
|
||||
}
|
||||
|
||||
destroy() {
|
||||
if(this._isDestoyed === false){
|
||||
this._peerConnection.close()
|
||||
|
||||
if(this._peerConnection.onconnectionstatechange){
|
||||
this._peerConnection.onconnectionstatechange(new Event('connectionstatechanged'))
|
||||
}
|
||||
|
||||
for(const channel in this._channels) {
|
||||
this._channels[channel].destroy()
|
||||
}
|
||||
|
||||
if(this._videoComponent){ this._videoComponent.destroy() }
|
||||
if(this._audioComponent){ this._audioComponent.destroy() }
|
||||
|
||||
this._isDestoyed = true
|
||||
} else {
|
||||
console.log('Cannot destroy because the player is already destroyed.')
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import xCloudPlayer from '../player'
|
||||
|
||||
export default class AudioComponent {
|
||||
private _player:xCloudPlayer
|
||||
|
||||
private _element:HTMLAudioElement | undefined
|
||||
|
||||
constructor(player:any){
|
||||
this._player = player
|
||||
}
|
||||
|
||||
create(stream:MediaStream) {
|
||||
const audioElement = document.createElement('audio')
|
||||
audioElement.srcObject = stream
|
||||
audioElement.autoplay = true
|
||||
audioElement.muted = false
|
||||
|
||||
this._element = audioElement
|
||||
|
||||
const element = document.getElementById(this._player.getElementId())
|
||||
if(element === null) {return}
|
||||
|
||||
element.appendChild(this._element)
|
||||
}
|
||||
|
||||
getElement(){
|
||||
return this._element
|
||||
}
|
||||
|
||||
destroy(){
|
||||
const streamHolder = document.getElementById(this._player.getElementId())
|
||||
const element = streamHolder?.querySelector('audio')
|
||||
|
||||
if(element){
|
||||
element.remove()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
import xCloudPlayer from '../player'
|
||||
import VideoComponent from './video'
|
||||
import WebGPUComponent from './webgpu'
|
||||
|
||||
interface OverlayInterface {
|
||||
debug: undefined | HTMLElement;
|
||||
}
|
||||
|
||||
export default class Overlay {
|
||||
private _player:xCloudPlayer
|
||||
private _videoComponent:VideoComponent | WebGPUComponent
|
||||
|
||||
private _overlays:OverlayInterface = {
|
||||
debug: undefined,
|
||||
}
|
||||
|
||||
constructor(videoComponent:VideoComponent | WebGPUComponent, player:xCloudPlayer){
|
||||
this._videoComponent = videoComponent
|
||||
this._player = player
|
||||
}
|
||||
|
||||
toggleDebug(){
|
||||
if(this._overlays.debug === undefined){
|
||||
this.createDebugOverlay()
|
||||
} else {
|
||||
this.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
createDebugOverlay(){
|
||||
this._overlays.debug = document.createElement('div')
|
||||
this._overlays.debug.id = 'playerDebugOverlay'
|
||||
|
||||
this._overlays.debug.style.position = 'absolute'
|
||||
this._overlays.debug.style.top = '0'
|
||||
this._overlays.debug.style.left = '0'
|
||||
this._overlays.debug.style.width = this._videoComponent.getElement()?.style.width || '100%'
|
||||
this._overlays.debug.style.height = this._videoComponent.getElement()?.style.height || 'auto'
|
||||
this._overlays.debug.style.background = 'linear-gradient(0deg, rgba(0,0,0, 0) 0%, rgba(0,0,0, 0) 75%, rgba(0,0,0, 0.5) 100%)'
|
||||
this._overlays.debug.style.padding = '10px'
|
||||
this._overlays.debug.style.pointerEvents = 'none'
|
||||
|
||||
document.getElementById(this._player.getElementId())?.appendChild(this._overlays.debug)
|
||||
|
||||
setTimeout(() => {
|
||||
this.refreshDebugOverlay()
|
||||
}, 500)
|
||||
this.refreshDebugOverlay()
|
||||
}
|
||||
|
||||
destroy(){
|
||||
this._overlays.debug?.remove()
|
||||
this._overlays.debug = undefined
|
||||
}
|
||||
|
||||
refreshDebugOverlay(){
|
||||
if(this._overlays.debug !== undefined){
|
||||
this._overlays.debug.innerHTML = ''
|
||||
|
||||
this._overlays.debug.appendChild(this.createLabel('Local Play', this._player.getStats()._remoteIsLocal ? 'Local' : 'Remote', 'ok'))
|
||||
this._overlays.debug.appendChild(this.createLabel('Resolution', this._player.getStats()._videoWidth+'x'+this._player.getStats()._videoHeight, 'ok'))
|
||||
this._overlays.debug.appendChild(this.createLabel('Stream FPS', this._player.getStats()._videoFps.toString(), this._player.getStats()._videoFps >= 58 ? 'ok' : 'warning'))
|
||||
|
||||
// Display rendering FPS if available
|
||||
const frameStats = (this._videoComponent as any).getFrameStats?.();
|
||||
if (frameStats) {
|
||||
this._overlays.debug.appendChild(this.createLabel('Render FPS', frameStats.renderingFps.toString(), frameStats.renderingFps >= 58 ? 'ok' : 'warning'))
|
||||
this._overlays.debug.appendChild(this.createLabel('Render Delay', frameStats.renderDelayMs.toString() + ' ms', frameStats.renderDelayMs < 16 ? 'ok' : frameStats.renderDelayMs < 33 ? 'warning' : 'error'))
|
||||
this._overlays.debug.appendChild(this.createLabel('Dropped Frames', frameStats.droppedFrames.toString(), frameStats.droppedFrames === 0 ? 'ok' : frameStats.droppedFrames < 30 ? 'warning' : 'error'))
|
||||
}
|
||||
|
||||
this._overlays.debug.appendChild(this.createLabel('Connection', this._player.getStats()._remoteIsIpv6 ? 'IPv6' : 'IPv4', 'ok'))
|
||||
this._overlays.debug.appendChild(this.createLabel('Video', this._player.getStats()._videoCodec, 'ok'))
|
||||
const rttButtonState = (this._player.getStats()._rtt*1000 > 75) ? 'error' : (this._player.getStats()._rtt*1000 > 40) ? 'warning' : 'ok'
|
||||
this._overlays.debug.appendChild(this.createLabel('RTT', (this._player.getStats()._rtt*1000).toString()+' ms', rttButtonState))
|
||||
|
||||
setTimeout(() => {
|
||||
this.refreshDebugOverlay()
|
||||
}, 500)
|
||||
}
|
||||
}
|
||||
|
||||
createLabel(text:string, value:string = '', style:string = ''){
|
||||
const labelStyle = {
|
||||
borderRadius: '5px',
|
||||
background: 'linear-gradient(0deg, rgba(28,26,26,1) 0%, rgba(47,45,45,1) 100%)',
|
||||
padding: '10px',
|
||||
fontSize: '12px',
|
||||
marginTop: '3px',
|
||||
marginRight: '10px',
|
||||
marginBottom: '2px',
|
||||
textTransform: 'uppercase',
|
||||
color: 'white',
|
||||
display: 'inline-block',
|
||||
}
|
||||
|
||||
const element = document.createElement('span')
|
||||
|
||||
const colors = {
|
||||
'error': 'rgb(220, 53, 69)',
|
||||
'warning': 'rgb(255, 193, 7)',
|
||||
'ok': 'rgb(25, 135, 84)',
|
||||
}
|
||||
|
||||
if(value !== ''){
|
||||
element.innerHTML = `${text}: <span style="background: ${colors[style]}; border-radius: 5px; padding: 4px; padding-left: 8px; padding-right: 8px;">${value}</span>`
|
||||
} else {
|
||||
element.innerText = text
|
||||
}
|
||||
|
||||
|
||||
for(const key in labelStyle){
|
||||
element.style[key] = labelStyle[key]
|
||||
}
|
||||
return element
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import xCloudPlayer from '../player'
|
||||
import Overlay from './overlay'
|
||||
|
||||
export default class VideoComponent {
|
||||
private _player:xCloudPlayer
|
||||
|
||||
private _element:HTMLVideoElement | undefined
|
||||
private _overlay:Overlay
|
||||
|
||||
constructor(player:any){
|
||||
this._player = player
|
||||
this._overlay = new Overlay(this, this._player)
|
||||
}
|
||||
|
||||
create(stream:MediaStream) {
|
||||
const videoElement = document.createElement('video')
|
||||
videoElement.srcObject = stream
|
||||
videoElement.autoplay = true
|
||||
videoElement.muted = true
|
||||
videoElement.playsInline = true
|
||||
videoElement.style.width = '100%'
|
||||
videoElement.style.height = '100%'
|
||||
videoElement.style.objectFit = 'contain'
|
||||
videoElement.style.backgroundColor = 'black'
|
||||
videoElement.style.touchAction = 'none'
|
||||
// DroidDeck: a transparent poster hides Android WebView's grey play icon before the first frame.
|
||||
videoElement.poster = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'
|
||||
|
||||
const element = document.getElementById(this._player.getElementId())
|
||||
if(element === null) {return}
|
||||
|
||||
this._element = videoElement
|
||||
element.appendChild(this._element)
|
||||
const isStatic = getComputedStyle(element).position === 'static' ? true : false
|
||||
if(isStatic === true){
|
||||
element.style.position = 'relative'
|
||||
}
|
||||
|
||||
this._element.requestVideoFrameCallback(this.processVideoMetadata.bind(this))
|
||||
}
|
||||
|
||||
processVideoMetadata(timestamp, data:VideoFrameCallbackMetadata) {
|
||||
if(this._element === undefined) {return}
|
||||
|
||||
this._element.requestVideoFrameCallback(this.processVideoMetadata.bind(this))
|
||||
|
||||
this._player._channels.input.queueMetadataFrame({
|
||||
serverDataKey: data.rtpTimestamp as number,
|
||||
firstFramePacketArrivalTimeMs: data.receiveTime as number,
|
||||
frameSubmittedTimeMs: data.receiveTime as number,
|
||||
frameDecodedTimeMs: data.expectedDisplayTime,
|
||||
frameRenderedTimeMs: data.expectedDisplayTime,
|
||||
})
|
||||
}
|
||||
|
||||
getElement(){
|
||||
return this._element
|
||||
}
|
||||
|
||||
toggleDebugOverlay(){
|
||||
this._overlay.toggleDebug()
|
||||
}
|
||||
|
||||
destroy(){
|
||||
const streamHolder = document.getElementById(this._player.getElementId())
|
||||
const element = streamHolder?.querySelector('video')
|
||||
|
||||
if(this._overlay !== undefined){
|
||||
this._overlay.destroy()
|
||||
}
|
||||
|
||||
if(element){
|
||||
element.remove()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,483 @@
|
||||
import xCloudPlayer from '../player'
|
||||
import Overlay from './overlay'
|
||||
|
||||
export default class WebGpuComponent {
|
||||
private _player:xCloudPlayer
|
||||
|
||||
private _element:HTMLCanvasElement | undefined
|
||||
private _overlay:Overlay
|
||||
private _isActive: boolean = true
|
||||
|
||||
private _gpuReader:GPUCanvasContext | undefined
|
||||
private _gpuVideoTexture:GPUTexture | null = null
|
||||
private _gpuDevice:GPUDevice | null = null
|
||||
private _gpuContext:GPUCanvasContext | null = null
|
||||
private _gpuBindGroup:GPUBindGroup | null = null
|
||||
private _gpuPipeline:GPURenderPipeline | null = null
|
||||
private _gpuSampler:GPUSampler | null = null
|
||||
private _resizeObserver:ResizeObserver | null = null
|
||||
private _containerElement:HTMLElement | null = null
|
||||
private _isProcessingFrame: boolean = false
|
||||
private _frameQueueSize: number = 0
|
||||
private _droppedFrames: number = 0
|
||||
private _lastFrameTime: number = 0
|
||||
private _renderingFps: number = 0
|
||||
private _frameTimeSamples: number[] = []
|
||||
private _renderDelayMs: number = 0
|
||||
private _renderDelaySamples: number[] = []
|
||||
|
||||
constructor(player:any){
|
||||
this._player = player
|
||||
this._overlay = new Overlay(this, this._player)
|
||||
}
|
||||
|
||||
create(stream:MediaStream) {
|
||||
const videoElement = document.createElement('canvas')
|
||||
console.log('Creating WebGPU canvas element for stream:', stream);
|
||||
|
||||
// videoElement.srcObject = stream
|
||||
// videoElement.autoplay = true
|
||||
// videoElement.muted = true
|
||||
// videoElement.playsInline = true
|
||||
videoElement.width = 1920
|
||||
videoElement.height = 1080
|
||||
videoElement.style.willChange = 'contents'
|
||||
// videoElement.style.imageRendering = 'pixelated' // or 'crisp-edges' for faster rendering
|
||||
// videoElement.style.objectFit = 'contain'
|
||||
// videoElement.style.backgroundColor = 'black'
|
||||
// videoElement.style.touchAction = 'none'
|
||||
|
||||
const that:WebGpuComponent = this
|
||||
|
||||
navigator.gpu.requestAdapter().then((adapter:any) => {
|
||||
if (!adapter) {
|
||||
console.error('Failed to get GPU adapter');
|
||||
return;
|
||||
}
|
||||
console.log('WebGPU adapter:', adapter);
|
||||
adapter.requestDevice().then((device:any) => {
|
||||
const context = videoElement.getContext("webgpu");
|
||||
if (!context) {
|
||||
console.error('Failed to get WebGPU context');
|
||||
return;
|
||||
}
|
||||
const format = navigator.gpu.getPreferredCanvasFormat();
|
||||
context.configure({
|
||||
device,
|
||||
format,
|
||||
alphaMode: 'opaque',
|
||||
// Enable desynchronized mode for lower latency and reduced jitter
|
||||
usage: GPUTextureUsage.RENDER_ATTACHMENT
|
||||
});
|
||||
|
||||
this._gpuDevice = device
|
||||
this._gpuContext = context
|
||||
|
||||
// Additional WebGPU setup and rendering logic would go here
|
||||
|
||||
that._gpuVideoTexture = this._gpuDevice.createTexture({
|
||||
size: [videoElement.width, videoElement.height],
|
||||
format: "rgba8unorm",
|
||||
usage: GPUTextureUsage.TEXTURE_BINDING |
|
||||
GPUTextureUsage.COPY_DST |
|
||||
GPUTextureUsage.RENDER_ATTACHMENT
|
||||
});
|
||||
|
||||
const vertexModule = this._gpuDevice.createShaderModule({
|
||||
code: `
|
||||
struct VertexOutput {
|
||||
@builtin(position) position: vec4<f32>,
|
||||
@location(0) uv: vec2<f32>,
|
||||
}
|
||||
|
||||
@vertex
|
||||
fn main(@builtin(vertex_index) VertexIndex: u32) -> VertexOutput {
|
||||
var pos = array<vec2<f32>, 4>(
|
||||
vec2<f32>(-1.0, -1.0),
|
||||
vec2<f32>( 1.0, -1.0),
|
||||
vec2<f32>(-1.0, 1.0),
|
||||
vec2<f32>( 1.0, 1.0)
|
||||
);
|
||||
var uv = array<vec2<f32>, 4>(
|
||||
vec2<f32>(0.0, 1.0),
|
||||
vec2<f32>(1.0, 1.0),
|
||||
vec2<f32>(0.0, 0.0),
|
||||
vec2<f32>(1.0, 0.0)
|
||||
);
|
||||
var output: VertexOutput;
|
||||
output.position = vec4<f32>(pos[VertexIndex], 0.0, 1.0);
|
||||
output.uv = uv[VertexIndex];
|
||||
return output;
|
||||
}
|
||||
`
|
||||
});
|
||||
|
||||
const fragmentModule = this._gpuDevice.createShaderModule({
|
||||
code: `
|
||||
@group(0) @binding(0) var myTexture: texture_2d<f32>;
|
||||
@group(0) @binding(1) var mySampler: sampler;
|
||||
|
||||
@fragment
|
||||
fn main(@location(0) uv: vec2<f32>) -> @location(0) vec4<f32> {
|
||||
// Get texture dimensions
|
||||
let texSize = vec2<f32>(textureDimensions(myTexture));
|
||||
let invTexSize = 1.0 / texSize;
|
||||
|
||||
// Edge-aware upscaling with Catmull-Rom interpolation
|
||||
// Sample 3x3 neighborhood for edge detection
|
||||
let p00 = textureSample(myTexture, mySampler, uv + vec2<f32>(-1.0, -1.0) * invTexSize).rgb;
|
||||
let p10 = textureSample(myTexture, mySampler, uv + vec2<f32>( 0.0, -1.0) * invTexSize).rgb;
|
||||
let p20 = textureSample(myTexture, mySampler, uv + vec2<f32>( 1.0, -1.0) * invTexSize).rgb;
|
||||
|
||||
let p01 = textureSample(myTexture, mySampler, uv + vec2<f32>(-1.0, 0.0) * invTexSize).rgb;
|
||||
let p11 = textureSample(myTexture, mySampler, uv).rgb;
|
||||
let p21 = textureSample(myTexture, mySampler, uv + vec2<f32>( 1.0, 0.0) * invTexSize).rgb;
|
||||
|
||||
let p02 = textureSample(myTexture, mySampler, uv + vec2<f32>(-1.0, 1.0) * invTexSize).rgb;
|
||||
let p12 = textureSample(myTexture, mySampler, uv + vec2<f32>( 0.0, 1.0) * invTexSize).rgb;
|
||||
let p22 = textureSample(myTexture, mySampler, uv + vec2<f32>( 1.0, 1.0) * invTexSize).rgb;
|
||||
|
||||
// Calculate Sobel edge detection
|
||||
let edgeX = -p00 + p20 - 2.0*p01 + 2.0*p21 - p02 + p22;
|
||||
let edgeY = p00 + 2.0*p10 + p20 - p02 - 2.0*p12 - p22;
|
||||
let edge = length(edgeX) + length(edgeY);
|
||||
|
||||
// Sample with adaptive filtering based on edge strength
|
||||
let centerSample = textureSample(myTexture, mySampler, uv);
|
||||
var resultRgb = centerSample.rgb;
|
||||
|
||||
// For smooth areas, apply slight smoothing
|
||||
// For edge areas, keep sharp sampling
|
||||
if (edge < 0.1) {
|
||||
// Smooth region: blend neighboring pixels for smoothness
|
||||
let neighbors = (p10 + p01 + p21 + p12) * 0.25;
|
||||
resultRgb = mix(resultRgb, neighbors, 0.15);
|
||||
} else {
|
||||
// Edge region: apply subtle sharpening to maintain clarity
|
||||
resultRgb = resultRgb * 1.1 - (p10 + p01 + p21 + p12) * 0.025;
|
||||
}
|
||||
|
||||
return vec4<f32>(resultRgb, centerSample.a);
|
||||
}
|
||||
`
|
||||
});
|
||||
|
||||
// Render pipeline (vertex + fragment shader)
|
||||
this._gpuPipeline = this._gpuDevice.createRenderPipeline({
|
||||
vertex: {
|
||||
module: vertexModule,
|
||||
entryPoint: "main"
|
||||
},
|
||||
fragment: {
|
||||
module: fragmentModule,
|
||||
entryPoint: "main",
|
||||
targets: [{ format: navigator.gpu.getPreferredCanvasFormat() }]
|
||||
},
|
||||
primitive: { topology: "triangle-strip" },
|
||||
layout: "auto",
|
||||
});
|
||||
|
||||
// Create sampler once and reuse it
|
||||
this._gpuSampler = this._gpuDevice.createSampler({
|
||||
magFilter: "linear",
|
||||
minFilter: "linear",
|
||||
addressModeU: "clamp-to-edge",
|
||||
addressModeV: "clamp-to-edge",
|
||||
});
|
||||
|
||||
// Process video frames from the MediaStream
|
||||
// console.log('Stream tracks:', stream);
|
||||
const track = stream.getVideoTracks()[0];
|
||||
const processor = new MediaStreamTrackProcessor({ track });
|
||||
this._gpuReader = processor.readable.getReader();
|
||||
|
||||
this._gpuBindGroup = this._gpuDevice.createBindGroup({
|
||||
layout: this._gpuPipeline.getBindGroupLayout(0),
|
||||
entries: [
|
||||
{ binding: 0, resource: this._gpuVideoTexture.createView() },
|
||||
{ binding: 1, resource: this._gpuSampler },
|
||||
]
|
||||
});
|
||||
|
||||
// Start rendering loop
|
||||
that.processVideoData();
|
||||
});
|
||||
});
|
||||
|
||||
// console.log('WebGPU adapter:', adapter, navigator.gpu);
|
||||
// const device = adapter.requestDevice();
|
||||
// console.log('Device:', device);
|
||||
// const context = videoElement.getContext("webgpu");
|
||||
// console.log('Context:', context);
|
||||
|
||||
// const format = navigator.gpu.getPreferredCanvasFormat();
|
||||
// context.configure({ device, format });
|
||||
|
||||
// Rest
|
||||
|
||||
const element = document.getElementById(this._player.getElementId())
|
||||
if(element === null) {return}
|
||||
|
||||
this._element = videoElement
|
||||
this._containerElement = element
|
||||
element.appendChild(this._element)
|
||||
const isStatic = getComputedStyle(element).position === 'static' ? true : false
|
||||
if(isStatic === true){
|
||||
element.style.position = 'relative'
|
||||
}
|
||||
|
||||
// Make canvas fill the container
|
||||
this._element.style.width = '100%'
|
||||
this._element.style.height = '100%'
|
||||
this._element.style.display = 'block'
|
||||
|
||||
// Set up resize observer to handle canvas resizing
|
||||
this._resizeObserver = new ResizeObserver(() => {
|
||||
this.resizeCanvas()
|
||||
})
|
||||
this._resizeObserver.observe(element)
|
||||
|
||||
// Initial resize
|
||||
this.resizeCanvas()
|
||||
|
||||
// this._element.requestVideoFrameCallback(this.processVideoMetadata.bind(this))
|
||||
}
|
||||
|
||||
async processVideoData(timestamp?:number) {
|
||||
if (this._isProcessingFrame || !this._isActive) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._isProcessingFrame = true;
|
||||
|
||||
try {
|
||||
// Non-blocking read - check if frame is available without waiting
|
||||
const { value: frame, done } = await this._gpuReader.read();
|
||||
if (done || !this._isActive || !frame) {
|
||||
this._isProcessingFrame = false;
|
||||
return;
|
||||
}
|
||||
|
||||
// Frame dropping: if multiple frames queued, skip older ones to maintain order and latency
|
||||
// This prevents frames from being processed out of order
|
||||
if (this._frameQueueSize > 0) {
|
||||
frame.close();
|
||||
this._droppedFrames++;
|
||||
this._isProcessingFrame = false;
|
||||
// Continue processing next frame immediately without delay
|
||||
this.videoLoop();
|
||||
return;
|
||||
}
|
||||
|
||||
this._frameQueueSize++;
|
||||
|
||||
// Resize texture if video frame dimensions changed
|
||||
if (this._gpuVideoTexture &&
|
||||
(this._gpuVideoTexture.width !== frame.codedWidth ||
|
||||
this._gpuVideoTexture.height !== frame.codedHeight)) {
|
||||
|
||||
this._gpuVideoTexture.destroy();
|
||||
|
||||
this._gpuVideoTexture = this._gpuDevice.createTexture({
|
||||
size: [frame.codedWidth, frame.codedHeight],
|
||||
format: "rgba8unorm",
|
||||
usage: GPUTextureUsage.TEXTURE_BINDING |
|
||||
GPUTextureUsage.COPY_DST |
|
||||
GPUTextureUsage.RENDER_ATTACHMENT
|
||||
});
|
||||
|
||||
// Recreate bind group with new texture
|
||||
this._gpuBindGroup = this._gpuDevice.createBindGroup({
|
||||
layout: this._gpuPipeline.getBindGroupLayout(0),
|
||||
entries: [
|
||||
{ binding: 0, resource: this._gpuVideoTexture.createView() },
|
||||
{ binding: 1, resource: this._gpuSampler },
|
||||
]
|
||||
});
|
||||
|
||||
console.log(`Video texture resized to ${frame.codedWidth}x${frame.codedHeight}`);
|
||||
}
|
||||
|
||||
const renderStartTime = performance.now();
|
||||
|
||||
// Track frame timing for FPS calculation
|
||||
if (this._lastFrameTime > 0) {
|
||||
const frameTime = renderStartTime - this._lastFrameTime;
|
||||
this._frameTimeSamples.push(frameTime);
|
||||
// Keep only last 60 samples for smooth FPS average
|
||||
if (this._frameTimeSamples.length > 60) {
|
||||
this._frameTimeSamples.shift();
|
||||
}
|
||||
// Update FPS based on average frame time
|
||||
const avgFrameTime = this._frameTimeSamples.reduce((a, b) => a + b, 0) / this._frameTimeSamples.length;
|
||||
this._renderingFps = Math.round(1000 / avgFrameTime);
|
||||
}
|
||||
this._lastFrameTime = renderStartTime;
|
||||
|
||||
// Upload the video frame to GPU
|
||||
this._gpuDevice.queue.copyExternalImageToTexture(
|
||||
{ source: frame },
|
||||
{ texture: this._gpuVideoTexture },
|
||||
[frame.codedWidth, frame.codedHeight]
|
||||
);
|
||||
|
||||
// Render to canvas
|
||||
const commandEncoder = this._gpuDevice.createCommandEncoder();
|
||||
const passEncoder = commandEncoder.beginRenderPass({
|
||||
colorAttachments: [{
|
||||
view: this._gpuContext.getCurrentTexture().createView(),
|
||||
loadOp: 'clear',
|
||||
storeOp: 'store',
|
||||
clearValue: { r: 0, g: 0, b: 0, a: 1 },
|
||||
}],
|
||||
});
|
||||
|
||||
passEncoder.setPipeline(this._gpuPipeline);
|
||||
passEncoder.setBindGroup(0, this._gpuBindGroup);
|
||||
passEncoder.draw(4);
|
||||
passEncoder.end();
|
||||
|
||||
this._gpuDevice.queue.submit([commandEncoder.finish()]);
|
||||
|
||||
const renderEndTime = performance.now();
|
||||
const renderDelay = renderEndTime - renderStartTime;
|
||||
|
||||
// Track render delay (time to process and submit frame to GPU)
|
||||
this._renderDelaySamples.push(renderDelay);
|
||||
// Keep only last 60 samples for smooth average
|
||||
if (this._renderDelaySamples.length > 60) {
|
||||
this._renderDelaySamples.shift();
|
||||
}
|
||||
// Update average render delay
|
||||
this._renderDelayMs = Math.round(
|
||||
this._renderDelaySamples.reduce((a, b) => a + b, 0) / this._renderDelaySamples.length * 100
|
||||
) / 100; // Round to 2 decimal places
|
||||
|
||||
// Queue metadata frames with actual render timing
|
||||
this._player._channels.input.queueMetadataFrame({
|
||||
serverDataKey: frame.timestamp as number,
|
||||
firstFramePacketArrivalTimeMs: frame.timestamp as number,
|
||||
frameSubmittedTimeMs: renderStartTime,
|
||||
frameDecodedTimeMs: renderStartTime,
|
||||
frameRenderedTimeMs: renderEndTime,
|
||||
})
|
||||
|
||||
frame.close();
|
||||
this._frameQueueSize--;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error processing video frame:', error);
|
||||
} finally {
|
||||
this._isProcessingFrame = false;
|
||||
|
||||
// Continue processing next frame immediately without vsync delay for low-latency streaming
|
||||
if (this._isActive) {
|
||||
this.videoLoop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
videoLoop() {
|
||||
this.processVideoData();
|
||||
}
|
||||
|
||||
resizeCanvas() {
|
||||
if (!this._element || !this._containerElement || !this._gpuContext || !this._gpuDevice) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the display size (CSS pixels)
|
||||
const displayWidth = this._containerElement.clientWidth
|
||||
const displayHeight = this._containerElement.clientHeight
|
||||
|
||||
// Get device pixel ratio for sharp rendering on high-DPI displays
|
||||
const devicePixelRatio = window.devicePixelRatio || 1
|
||||
|
||||
// Calculate the actual canvas size in device pixels
|
||||
const width = Math.floor(displayWidth * devicePixelRatio)
|
||||
const height = Math.floor(displayHeight * devicePixelRatio)
|
||||
|
||||
// Check if canvas needs resizing
|
||||
if (this._element.width !== width || this._element.height !== height) {
|
||||
this._element.width = width
|
||||
this._element.height = height
|
||||
|
||||
// Reconfigure the WebGPU context with the new size
|
||||
const format = navigator.gpu.getPreferredCanvasFormat()
|
||||
this._gpuContext.configure({
|
||||
device: this._gpuDevice,
|
||||
format,
|
||||
alphaMode: 'opaque',
|
||||
usage: GPUTextureUsage.RENDER_ATTACHMENT
|
||||
})
|
||||
|
||||
console.log(`Canvas resized to ${width}x${height} (display: ${displayWidth}x${displayHeight}, DPR: ${devicePixelRatio})`)
|
||||
}
|
||||
}
|
||||
|
||||
// processVideoMetadata(timestamp, data:VideoFrameCallbackMetadata) {
|
||||
// if(this._element === undefined) {return}
|
||||
|
||||
// this._element.requestVideoFrameCallback(this.processVideoMetadata.bind(this))
|
||||
|
||||
// this._player._channels.input.queueMetadataFrame({
|
||||
// serverDataKey: data.rtpTimestamp as number,
|
||||
// firstFramePacketArrivalTimeMs: data.receiveTime as number,
|
||||
// frameSubmittedTimeMs: data.receiveTime as number,
|
||||
// frameDecodedTimeMs: data.expectedDisplayTime,
|
||||
// frameRenderedTimeMs: data.expectedDisplayTime,
|
||||
// })
|
||||
// }
|
||||
|
||||
getElement(){
|
||||
return this._element
|
||||
}
|
||||
|
||||
toggleDebugOverlay(){
|
||||
this._overlay.toggleDebug()
|
||||
}
|
||||
|
||||
getFrameStats() {
|
||||
return {
|
||||
droppedFrames: this._droppedFrames,
|
||||
queueSize: this._frameQueueSize,
|
||||
isProcessing: this._isProcessingFrame,
|
||||
renderingFps: this._renderingFps,
|
||||
renderDelayMs: this._renderDelayMs
|
||||
}
|
||||
}
|
||||
|
||||
destroy(){
|
||||
const streamHolder = document.getElementById(this._player.getElementId())
|
||||
this._isActive = false
|
||||
const element = streamHolder?.querySelector('canvas')
|
||||
|
||||
// Clean up GPU reader
|
||||
if(this._gpuReader){
|
||||
this._gpuReader.cancel();
|
||||
this._gpuReader = undefined;
|
||||
}
|
||||
|
||||
if(this._resizeObserver){
|
||||
this._resizeObserver.disconnect()
|
||||
this._resizeObserver = null
|
||||
}
|
||||
|
||||
if(this._gpuVideoTexture){
|
||||
this._gpuVideoTexture.destroy();
|
||||
this._gpuVideoTexture = null;
|
||||
}
|
||||
|
||||
if(this._overlay !== undefined){
|
||||
this._overlay.destroy()
|
||||
}
|
||||
|
||||
if(element){
|
||||
element.remove()
|
||||
}
|
||||
|
||||
if(this._element){
|
||||
this._element.remove()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import xCloudPlayer from './player'
|
||||
|
||||
export default class Sdp {
|
||||
private _player: xCloudPlayer
|
||||
private _peerConnection:RTCPeerConnection
|
||||
|
||||
constructor(player:xCloudPlayer) {
|
||||
this._player = player
|
||||
this._peerConnection = player._peerConnection
|
||||
}
|
||||
|
||||
getAvailableCodecs():Array<any> {
|
||||
const capabilities = RTCRtpReceiver.getCapabilities('video')?.codecs
|
||||
if(capabilities === undefined) {return []}
|
||||
|
||||
const codecs:Array<string> = []
|
||||
for(const codec in capabilities) {
|
||||
if(!codecs.includes(capabilities[codec].mimeType)) {codecs.push(capabilities[codec].mimeType)}
|
||||
}
|
||||
|
||||
return codecs
|
||||
}
|
||||
|
||||
// getServerCodecs() {
|
||||
// for(const sender in this._peerConnection.getSenders()){
|
||||
// console.log('Sender:', this._peerConnection.getSenders()[sender])
|
||||
// }
|
||||
// }
|
||||
|
||||
setLocalSDP(sdp:RTCSessionDescriptionInit) {
|
||||
console.log('[SDP] setLocalSDP() Setting local SDP:', sdp)
|
||||
|
||||
// if(this._player._config.video_bitrate > 0){
|
||||
// console.log('[SDP] Set max video bitrate to:', this._player._config.video_bitrate, 'kbps')
|
||||
// sdp.sdp = this._setBitrate(sdp.sdp, 'video', this._player._config.video_bitrate)
|
||||
// }
|
||||
|
||||
// if(this._player._config.audio_bitrate > 0){
|
||||
// console.log('[SDP] Set max audio bitrate to:', this._player._config.audio_bitrate, 'kbps')
|
||||
// sdp.sdp = this._setBitrate(sdp.sdp, 'audio', this._player._config.audio_bitrate)
|
||||
// }
|
||||
|
||||
// if(this._player._config.audio_mono !== true){
|
||||
// console.log('[SDP] setLocalSDP() Set audio to stereo')
|
||||
// sdp.sdp = sdp.sdp?.replace('useinbandfec=1', 'useinbandfec=1; stereo=1')
|
||||
// }
|
||||
|
||||
sdp.sdp = sdp.sdp?.replace('useinbandfec=1', 'useinbandfec=1; stereo=1')
|
||||
|
||||
return sdp
|
||||
}
|
||||
|
||||
setRemoteSDP(sdp:string) {
|
||||
console.log('[SDP] setRemoteSDP() Setting remote SDP:', sdp)
|
||||
return sdp
|
||||
}
|
||||
|
||||
getDefaultCodecPreferences() {
|
||||
const capabilities = RTCRtpReceiver.getCapabilities('video')?.codecs
|
||||
if(capabilities === undefined) {return []}
|
||||
|
||||
const t1:Array<any> = []
|
||||
const t2:Array<any> = []
|
||||
const t3:Array<any> = []
|
||||
const t4:Array<any> = []
|
||||
|
||||
for(const codec in capabilities) {
|
||||
if(capabilities[codec].mimeType.includes('H264')) {
|
||||
if(capabilities[codec].sdpFmtpLine?.includes('profile-level-id=4d')) {
|
||||
t1.push(capabilities[codec])
|
||||
|
||||
} else if(capabilities[codec].sdpFmtpLine?.includes('profile-level-id=42e')) {
|
||||
t2.push(capabilities[codec])
|
||||
|
||||
} else if(capabilities[codec].sdpFmtpLine?.includes('profile-level-id=420')) {
|
||||
t3.push(capabilities[codec])
|
||||
}
|
||||
} else if(capabilities[codec].mimeType.includes('ulpfec')) {
|
||||
t4.push(capabilities[codec])
|
||||
} else if(capabilities[codec].mimeType.includes('flexfec')) {
|
||||
t4.push(capabilities[codec])
|
||||
} else if(capabilities[codec].mimeType.includes('VP9')) {
|
||||
t4.push(capabilities[codec])
|
||||
} else if(capabilities[codec].mimeType.includes('VP8')) {
|
||||
t4.push(capabilities[codec])
|
||||
}
|
||||
// else {
|
||||
// t4.push(capabilities[codec])
|
||||
// }
|
||||
}
|
||||
|
||||
const codecOrder = [...t1, ...t2, ...t3, ...t4]
|
||||
|
||||
return codecOrder
|
||||
}
|
||||
|
||||
_setBitrate(sdp, media, bitrate) {
|
||||
const lines = sdp.split('\n')
|
||||
let line = -1
|
||||
for(let i=0; i < lines.length; i++) {
|
||||
if(lines[i].indexOf('m='+media) === 0) {
|
||||
line = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if (line === -1) {
|
||||
console.debug('Could not find the m line for', media)
|
||||
return sdp
|
||||
}
|
||||
line++
|
||||
|
||||
while(lines[line].indexOf('i=') === 0 || lines[line].indexOf('c=') === 0) {
|
||||
line++
|
||||
}
|
||||
|
||||
if (lines[line].indexOf('b') === 0) {
|
||||
lines[line] = 'b=AS:'+bitrate
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
let newLines = lines.slice(0, line)
|
||||
newLines.push('b=AS:'+bitrate)
|
||||
newLines = newLines.concat(lines.slice(line, lines.length))
|
||||
|
||||
return newLines.join('\n')
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import xCloudPlayer from '../player'
|
||||
|
||||
export default class Stats {
|
||||
private _player: xCloudPlayer
|
||||
private _peerConnection:RTCPeerConnection
|
||||
|
||||
_videoCodec:string = ''
|
||||
_audioCodec:string = ''
|
||||
|
||||
_videoWidth:number = 0
|
||||
_videoHeight:number = 0
|
||||
_videoFps:number = 0
|
||||
_rtt:number = 0.0
|
||||
|
||||
_remoteHost:string = ''
|
||||
_remotePort:number = 0
|
||||
_remoteIsLocal = false
|
||||
_remoteIsIpv6 = false
|
||||
_activeRemoteCandidate:string = ''
|
||||
|
||||
constructor(player:xCloudPlayer) {
|
||||
this._player = player
|
||||
this._peerConnection = player._peerConnection
|
||||
|
||||
setInterval(this.loop.bind(this), 1000)
|
||||
}
|
||||
|
||||
loop(){
|
||||
this._peerConnection.getStats().then((stats) => {
|
||||
stats.forEach((report) => {
|
||||
// @TODO: Implement chat stats using type outbound-rtp
|
||||
|
||||
if(report.type === 'inbound-rtp'){
|
||||
this.readInboundRtp(report)
|
||||
|
||||
} else if(report.type === 'candidate-pair' && report.packetsReceived !== undefined && report.packetsReceived > 0){
|
||||
this.readCandidatePair(report)
|
||||
|
||||
} else if(report.type === 'remote-candidate'){
|
||||
this.readRemoteCandidate(report)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
readInboundRtp(report:RTCInboundRtpStreamStats){
|
||||
if(report.kind === 'video' && report.codecId && report.packetsReceived !== undefined && report.packetsReceived > 0){
|
||||
|
||||
if(report.codecId.includes('profile-level-id=4d')){
|
||||
this._videoCodec = 'H264 (High)'
|
||||
|
||||
} else if(report.codecId.includes('profile-level-id=42e')) {
|
||||
this._videoCodec = 'H264 (Normal)'
|
||||
|
||||
} else if(report.codecId.includes('profile-level-id=420')) {
|
||||
this._videoCodec = 'H264 (Low)'
|
||||
|
||||
} else {
|
||||
this._videoCodec = 'Other ('+report.codecId+')'
|
||||
}
|
||||
|
||||
if(report.frameWidth){ this._videoWidth = report.frameWidth }
|
||||
if(report.frameHeight){ this._videoHeight = report.frameHeight }
|
||||
if(report.framesPerSecond){ this._videoFps = report.framesPerSecond }
|
||||
// report.jitter
|
||||
|
||||
} else if(report.kind === 'audio' && report.codecId && report.packetsReceived !== undefined && report.packetsReceived > 0){
|
||||
this._audioCodec = report.codecId
|
||||
|
||||
// report.jitter
|
||||
}
|
||||
}
|
||||
|
||||
readCandidatePair(report){
|
||||
this._activeRemoteCandidate = report.remoteCandidateId
|
||||
this._rtt = report.currentRoundTripTime
|
||||
}
|
||||
|
||||
readRemoteCandidate(report){
|
||||
if(report.id !== this._activeRemoteCandidate) {return}
|
||||
|
||||
if(this.isPrivateIP(report.address)){
|
||||
this._remoteIsLocal = true
|
||||
} else {
|
||||
this._remoteIsLocal = false
|
||||
}
|
||||
|
||||
if(this.isIpv6(report.address)){
|
||||
this._remoteIsIpv6 = true
|
||||
} else {
|
||||
this._remoteIsIpv6 = false
|
||||
}
|
||||
|
||||
this._remoteHost = report.address
|
||||
this._remotePort = report.port
|
||||
}
|
||||
|
||||
isPrivateIP(ip) {
|
||||
const parts = ip.split('.')
|
||||
return parts[0] === '10' || (parts[0] === '172' && (parseInt(parts[1], 10) >= 16 && parseInt(parts[1], 10) <= 31)) || (parts[0] === '192' && parts[1] === '168')
|
||||
}
|
||||
|
||||
isIpv6(ip) {
|
||||
const parts = ip.split(':')
|
||||
return parts.length > 3 ? true : false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
export default class Teredo {
|
||||
private _address:string
|
||||
|
||||
private _ipv4Address:string
|
||||
private _ipv4Port:number
|
||||
|
||||
constructor(address:string){
|
||||
this._address = address
|
||||
|
||||
const split_str = this._address.split(':')
|
||||
const ipv4_part = split_str[6] + split_str[7]
|
||||
const ipv4_port_part = split_str[5]
|
||||
|
||||
const ip_1 =~ parseInt(ipv4_part.substring(0, 2), 16) & 0xFF
|
||||
const ip_2 =~ parseInt(ipv4_part.substring(2, 4), 16) & 0xFF
|
||||
const ip_3 =~ parseInt(ipv4_part.substring(4, 6), 16) & 0xFF
|
||||
const ip_4 =~ parseInt(ipv4_part.substring(6, 8), 16) & 0xFF
|
||||
const port_1 =~ parseInt(ipv4_port_part.substring(0, 4), 16) & 0xFFFF
|
||||
|
||||
this._ipv4Address = ip_1+'.'+ip_2+'.'+ip_3+'.'+ip_4
|
||||
this._ipv4Port = port_1
|
||||
}
|
||||
|
||||
getIpv4Address(){
|
||||
return this._ipv4Address
|
||||
}
|
||||
|
||||
getIpv4Port(){
|
||||
return this._ipv4Port
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
// Picture effects for the stream: the video is drawn through WebGL2 shaders onto a canvas over it.
|
||||
// Base pass: sharpening (CAS / unsharp mask, from Better xCloud by redphx, MIT) plus DroidDeck's colour
|
||||
// grading (brightness, contrast, saturation, gamma) and deband dither. Extra passes are DroidDeck's own
|
||||
// screen effects (FXAA, Toon, CRT, NTSC, from Bannerlator). Off (no canvas) when nothing is set.
|
||||
|
||||
export interface Effects {
|
||||
brightness: number // -50..50 (DroidDeck's slider units)
|
||||
contrast: number // -100..100
|
||||
gamma: number // 0.5..3
|
||||
saturation: number // 0..200 %
|
||||
sharpen: number // 0..10
|
||||
sharpenMode: string // 'cas' | 'usm'
|
||||
deband: number // 0..8 (dither amplitude in 8-bit steps)
|
||||
fxaa: boolean
|
||||
toon: boolean
|
||||
crt: boolean
|
||||
ntsc: boolean
|
||||
maxFps: number // 0 = every frame
|
||||
}
|
||||
|
||||
export function needsRenderer(e: Effects): boolean {
|
||||
return e.brightness !== 0 || e.contrast !== 0 || Math.abs(e.gamma - 1) > 0.001 || e.saturation !== 100 ||
|
||||
e.sharpen > 0 || e.deband > 0 || e.fxaa || e.toon || e.crt || e.ntsc || e.maxFps > 0
|
||||
}
|
||||
|
||||
const VERTEX_300 = `#version 300 es
|
||||
in vec2 position;
|
||||
out vec2 vUV;
|
||||
void main() { vUV = position * 0.5 + 0.5; gl_Position = vec4(position, 0.0, 1.0); }`
|
||||
|
||||
const VERTEX_100 = `attribute vec2 position;
|
||||
varying vec2 vUV;
|
||||
void main() { vUV = position * 0.5 + 0.5; gl_Position = vec4(position, 0.0, 1.0); }`
|
||||
|
||||
const BASE = `#version 300 es
|
||||
precision mediump float;
|
||||
uniform sampler2D screenTexture;
|
||||
uniform vec2 resolution;
|
||||
uniform int filterId;
|
||||
uniform float sharpenFactor;
|
||||
uniform float brightness;
|
||||
uniform float contrast;
|
||||
uniform float gamma;
|
||||
uniform float saturation;
|
||||
uniform float deband;
|
||||
out vec4 fragColor;
|
||||
const float CAS_CONTRAST_PEAK = 0.8 * -3.0 + 8.0;
|
||||
|
||||
vec3 sharpen(vec2 coord, vec3 e) {
|
||||
vec2 px = 1.0 / resolution;
|
||||
vec3 b = texture(screenTexture, coord + px * vec2(0, 1)).rgb;
|
||||
vec3 d = texture(screenTexture, coord + px * vec2(-1, 0)).rgb;
|
||||
vec3 f = texture(screenTexture, coord + px * vec2(1, 0)).rgb;
|
||||
vec3 h = texture(screenTexture, coord + px * vec2(0, -1)).rgb;
|
||||
vec3 a = texture(screenTexture, coord + px * vec2(-1, 1)).rgb;
|
||||
vec3 c = texture(screenTexture, coord + px * vec2(1, 1)).rgb;
|
||||
vec3 g = texture(screenTexture, coord + px * vec2(-1, -1)).rgb;
|
||||
vec3 i = texture(screenTexture, coord + px * vec2(1, -1)).rgb;
|
||||
if (filterId == 1) {
|
||||
vec3 blur = ((a + c + g + i) + (b + d + f + h) * 2.0 + e * 4.0) / 16.0;
|
||||
return e + (e - blur) * sharpenFactor / 3.0;
|
||||
}
|
||||
vec3 minRgb = min(min(min(d, e), min(f, b)), h) + min(min(a, c), min(g, i));
|
||||
vec3 maxRgb = max(max(max(d, e), max(f, b)), h) + max(max(a, c), max(g, i));
|
||||
vec3 amplify = inversesqrt(clamp(min(minRgb, 2.0 - maxRgb) / maxRgb, 0.0, 1.0));
|
||||
vec3 weight = -(1.0 / (amplify * CAS_CONTRAST_PEAK));
|
||||
vec3 outColor = clamp(((b + d + f + h) * weight + e) / (4.0 * weight + 1.0), 0.0, 1.0);
|
||||
return mix(e, outColor, sharpenFactor / 2.0);
|
||||
}
|
||||
|
||||
float ign(vec2 p) { return fract(52.9829189 * fract(dot(p, vec2(0.06711056, 0.00583715)))); }
|
||||
|
||||
void main() {
|
||||
vec2 uv = gl_FragCoord.xy / resolution;
|
||||
vec3 color = texture(screenTexture, uv).rgb;
|
||||
if (sharpenFactor > 0.0) color = sharpen(uv, color);
|
||||
color = clamp(color + brightness, 0.0, 1.0);
|
||||
color = (color - 0.5) * clamp(contrast + 1.0, 0.5, 2.0) + 0.5;
|
||||
color = mix(vec3(dot(color, vec3(0.2126, 0.7152, 0.0722))), color, saturation);
|
||||
color = pow(max(color, 0.0), vec3(1.0 / gamma));
|
||||
if (deband > 0.0) color += vec3(((ign(gl_FragCoord.xy) + ign(gl_FragCoord.xy + vec2(11.0, 17.0))) - 1.0) * (deband / 255.0));
|
||||
fragColor = vec4(color, 1.0);
|
||||
}`
|
||||
|
||||
// Screen-effect shaders from Bannerlator (GLSL ES 1.00), unchanged.
|
||||
const FXAA = `precision highp float;
|
||||
#define FXAA_MIN_REDUCE (1.0 / 128.0)
|
||||
#define FXAA_MUL_REDUCE (1.0 / 8.0)
|
||||
#define MAX_SPAN 8.0
|
||||
uniform sampler2D screenTexture;
|
||||
uniform vec2 resolution;
|
||||
const vec3 luma = vec3(0.299, 0.587, 0.114);
|
||||
void main() {
|
||||
vec2 invResolution = 1.0 / resolution;
|
||||
vec3 rgbNW = texture2D(screenTexture, (gl_FragCoord.xy + vec2(-1.0, -1.0)) * invResolution).rgb;
|
||||
vec3 rgbNE = texture2D(screenTexture, (gl_FragCoord.xy + vec2( 1.0, -1.0)) * invResolution).rgb;
|
||||
vec3 rgbSW = texture2D(screenTexture, (gl_FragCoord.xy + vec2(-1.0, 1.0)) * invResolution).rgb;
|
||||
vec3 rgbSE = texture2D(screenTexture, (gl_FragCoord.xy + vec2( 1.0, 1.0)) * invResolution).rgb;
|
||||
vec3 rgbM = texture2D(screenTexture, gl_FragCoord.xy * invResolution).rgb;
|
||||
float lumaNW = dot(rgbNW, luma);
|
||||
float lumaNE = dot(rgbNE, luma);
|
||||
float lumaSW = dot(rgbSW, luma);
|
||||
float lumaSE = dot(rgbSE, luma);
|
||||
float lumaM = dot(rgbM, luma);
|
||||
float lumaMin = min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE)));
|
||||
float lumaMax = max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE)));
|
||||
vec2 dir;
|
||||
dir.x = -((lumaNW + lumaNE) - (lumaSW + lumaSE));
|
||||
dir.y = ((lumaNW + lumaSW) - (lumaNE + lumaSE));
|
||||
float dirReduce = max((lumaNW + lumaNE + lumaSW + lumaSE) * 0.25 * FXAA_MUL_REDUCE, FXAA_MIN_REDUCE);
|
||||
float minDirFactor = 1.0 / (min(abs(dir.x), abs(dir.y)) + dirReduce);
|
||||
dir = clamp(dir * minDirFactor, vec2(-MAX_SPAN), vec2(MAX_SPAN)) * invResolution;
|
||||
vec4 rgbA = 0.5 * (
|
||||
texture2D(screenTexture, gl_FragCoord.xy * invResolution + dir * (1.0 / 3.0 - 0.5)) +
|
||||
texture2D(screenTexture, gl_FragCoord.xy * invResolution + dir * (2.0 / 3.0 - 0.5)));
|
||||
vec4 rgbB = rgbA * 0.5 + 0.25 * (
|
||||
texture2D(screenTexture, gl_FragCoord.xy * invResolution + dir * -0.5) +
|
||||
texture2D(screenTexture, gl_FragCoord.xy * invResolution + dir * 0.5));
|
||||
float lumaB = dot(rgbB, vec4(luma, 0.0));
|
||||
gl_FragColor = lumaB < lumaMin || lumaB > lumaMax ? rgbA : rgbB;
|
||||
}`
|
||||
|
||||
const TOON = `precision highp float;
|
||||
uniform sampler2D screenTexture;
|
||||
uniform vec2 resolution;
|
||||
void main() {
|
||||
vec2 uv = gl_FragCoord.xy / resolution;
|
||||
float edgeThreshold = 0.2;
|
||||
vec2 offset = vec2(1.0) / resolution;
|
||||
vec3 colorCenter = texture2D(screenTexture, uv).rgb;
|
||||
vec3 colorLeft = texture2D(screenTexture, uv - vec2(offset.x, 0.0)).rgb;
|
||||
vec3 colorRight = texture2D(screenTexture, uv + vec2(offset.x, 0.0)).rgb;
|
||||
vec3 colorUp = texture2D(screenTexture, uv - vec2(0.0, offset.y)).rgb;
|
||||
vec3 colorDown = texture2D(screenTexture, uv + vec2(0.0, offset.y)).rgb;
|
||||
float diffHorizontal = length(colorRight - colorLeft);
|
||||
float diffVertical = length(colorUp - colorDown);
|
||||
float edgeFactor = step(edgeThreshold, diffHorizontal + diffVertical);
|
||||
vec3 outlineColor = mix(colorCenter, vec3(0.0), edgeFactor);
|
||||
gl_FragColor = vec4(outlineColor, 1.0);
|
||||
}`
|
||||
|
||||
const CRT = `precision highp float;
|
||||
#define CA_AMOUNT 1.0025
|
||||
#define SCANLINE_INTENSITY_X 0.125
|
||||
#define SCANLINE_INTENSITY_Y 0.375
|
||||
#define SCANLINE_SIZE 1024.0
|
||||
uniform sampler2D screenTexture;
|
||||
varying vec2 vUV;
|
||||
void main() {
|
||||
vec4 finalColor = texture2D(screenTexture, vUV);
|
||||
finalColor.rgb = vec3(
|
||||
texture2D(screenTexture, (vUV - 0.5) * CA_AMOUNT + 0.5).r,
|
||||
finalColor.g,
|
||||
texture2D(screenTexture, (vUV - 0.5) / CA_AMOUNT + 0.5).b
|
||||
);
|
||||
float scanlineX = abs(sin(vUV.x * SCANLINE_SIZE) * 0.5 * SCANLINE_INTENSITY_X);
|
||||
float scanlineY = abs(sin(vUV.y * SCANLINE_SIZE) * 0.5 * SCANLINE_INTENSITY_Y);
|
||||
gl_FragColor = vec4(mix(finalColor.rgb, vec3(0.0), scanlineX + scanlineY), finalColor.a);
|
||||
}`
|
||||
|
||||
const NTSC = `precision highp float;
|
||||
#define PI 3.14159265
|
||||
#define SCANLINE_INTENSITY 0.35
|
||||
#define CHROMA_OFFSET 0.005
|
||||
#define BLUR_RADIUS 0.002
|
||||
#define WARP_AMOUNT 0.01
|
||||
#define SCANLINE_DARKEN 0.5
|
||||
uniform sampler2D screenTexture;
|
||||
uniform int FrameCount;
|
||||
uniform vec2 TextureSize;
|
||||
uniform vec2 resolution;
|
||||
varying vec2 vUV;
|
||||
const mat3 yiq_mat = mat3(
|
||||
0.299, 0.587, 0.114,
|
||||
0.596, -0.275, -0.321,
|
||||
0.212, -0.523, 0.311
|
||||
);
|
||||
const mat3 yiq2rgb_mat = mat3(
|
||||
1.0, 0.956, 0.621,
|
||||
1.0, -0.272, -0.647,
|
||||
1.0, -1.106, 1.705
|
||||
);
|
||||
vec3 applyNTSC(vec2 uv) {
|
||||
vec3 col = texture2D(screenTexture, uv).rgb;
|
||||
vec3 yiq = col * yiq_mat;
|
||||
float chromaPhase = PI * (mod(uv.y * TextureSize.y, 2.0) + float(FrameCount));
|
||||
yiq.y *= cos(chromaPhase * 0.5);
|
||||
yiq.z *= sin(chromaPhase * 0.5);
|
||||
vec3 rgb = yiq * yiq2rgb_mat;
|
||||
vec3 finalColor;
|
||||
finalColor.r = texture2D(screenTexture, uv + vec2(CHROMA_OFFSET, 0.0)).r;
|
||||
finalColor.g = texture2D(screenTexture, uv + vec2(0.0, BLUR_RADIUS)).g;
|
||||
finalColor.b = texture2D(screenTexture, uv - vec2(CHROMA_OFFSET, 0.0)).b;
|
||||
return finalColor;
|
||||
}
|
||||
vec3 applyScanlines(vec2 uv) {
|
||||
vec3 col = texture2D(screenTexture, uv).rgb;
|
||||
float scanline = abs(sin(uv.y * resolution.y * 2.0)) * SCANLINE_INTENSITY;
|
||||
col *= 1.0 - (scanline * SCANLINE_DARKEN);
|
||||
return col;
|
||||
}
|
||||
vec2 applyWarp(vec2 uv) {
|
||||
uv = uv * 2.0 - 1.0;
|
||||
float r = sqrt(uv.x * uv.x + uv.y * uv.y);
|
||||
uv += uv * (r * r) * WARP_AMOUNT;
|
||||
return uv * 0.5 + 0.5;
|
||||
}
|
||||
void main() {
|
||||
vec2 warpedUV = applyWarp(vUV);
|
||||
vec3 ntscColor = applyNTSC(warpedUV);
|
||||
vec3 scanlineColor = applyScanlines(warpedUV);
|
||||
vec3 finalColor = mix(ntscColor, scanlineColor, 0.7);
|
||||
gl_FragColor = vec4(finalColor, 1.0);
|
||||
}`
|
||||
|
||||
interface Pass { program: WebGLProgram, uniforms: Map<string, WebGLUniformLocation | null> }
|
||||
|
||||
export class EffectRenderer {
|
||||
private canvas: HTMLCanvasElement
|
||||
private gl: WebGL2RenderingContext
|
||||
private passes = new Map<string, Pass>()
|
||||
private videoTexture: WebGLTexture
|
||||
private targets: { framebuffer: WebGLFramebuffer, texture: WebGLTexture }[] = []
|
||||
private width = 0
|
||||
private height = 0
|
||||
private frame = 0
|
||||
private lastDraw = 0
|
||||
private stopped = false
|
||||
private idle = false
|
||||
private pending = false
|
||||
|
||||
constructor(private video: HTMLVideoElement, private effects: Effects) {
|
||||
this.canvas = document.createElement('canvas')
|
||||
Object.assign(this.canvas.style, { position: 'absolute', inset: '0', width: '100%', height: '100%', objectFit: 'contain', zIndex: '1' })
|
||||
video.parentElement?.appendChild(this.canvas)
|
||||
const gl = this.canvas.getContext('webgl2', { alpha: false, antialias: false, depth: false, desynchronized: true, powerPreference: 'high-performance' })
|
||||
if (gl === null) throw new Error('WebGL2 unavailable')
|
||||
this.gl = gl
|
||||
const buffer = gl.createBuffer()
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, buffer)
|
||||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-1, -1, 3, -1, -1, 3]), gl.STATIC_DRAW)
|
||||
this.passes.set('base', this.compile(VERTEX_300, BASE, ['screenTexture', 'resolution', 'filterId', 'sharpenFactor', 'brightness', 'contrast', 'gamma', 'saturation', 'deband']))
|
||||
this.videoTexture = this.texture()
|
||||
this.schedule()
|
||||
}
|
||||
|
||||
update(effects: Effects) { this.effects = effects }
|
||||
|
||||
// Turning effects off hides the canvas but keeps the GL context: losing and recreating it can stall the stream.
|
||||
pause() { if (this.idle || this.stopped) return; this.idle = true; this.canvas.style.display = 'none' }
|
||||
|
||||
resume(effects: Effects) {
|
||||
this.effects = effects
|
||||
if (!this.idle || this.stopped) return
|
||||
this.idle = false
|
||||
this.canvas.style.display = ''
|
||||
this.schedule()
|
||||
}
|
||||
|
||||
private schedule() {
|
||||
if (this.pending) return
|
||||
this.pending = true
|
||||
this.video.requestVideoFrameCallback(this.onFrame)
|
||||
}
|
||||
|
||||
private compile(vertex: string, fragment: string, names: string[]): Pass {
|
||||
const gl = this.gl
|
||||
const shader = (type: number, source: string) => {
|
||||
const s = gl.createShader(type)!
|
||||
gl.shaderSource(s, source)
|
||||
gl.compileShader(s)
|
||||
if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(`shader: ${gl.getShaderInfoLog(s)}`)
|
||||
return s
|
||||
}
|
||||
const program = gl.createProgram()!
|
||||
gl.attachShader(program, shader(gl.VERTEX_SHADER, vertex))
|
||||
gl.attachShader(program, shader(gl.FRAGMENT_SHADER, fragment))
|
||||
gl.bindAttribLocation(program, 0, 'position')
|
||||
gl.linkProgram(program)
|
||||
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) throw new Error(`program: ${gl.getProgramInfoLog(program)}`)
|
||||
return { program, uniforms: new Map(names.map(name => [name, gl.getUniformLocation(program, name)])) }
|
||||
}
|
||||
|
||||
private extra(name: string): Pass {
|
||||
let pass = this.passes.get(name)
|
||||
if (pass === undefined) {
|
||||
const source = { fxaa: FXAA, toon: TOON, crt: CRT, ntsc: NTSC }[name]!
|
||||
pass = this.compile(VERTEX_100, source, ['screenTexture', 'resolution', 'TextureSize', 'FrameCount'])
|
||||
this.passes.set(name, pass)
|
||||
}
|
||||
return pass
|
||||
}
|
||||
|
||||
private texture(): WebGLTexture {
|
||||
const gl = this.gl
|
||||
const texture = gl.createTexture()!
|
||||
gl.bindTexture(gl.TEXTURE_2D, texture)
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR)
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR)
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE)
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE)
|
||||
return texture
|
||||
}
|
||||
|
||||
private resize(width: number, height: number) {
|
||||
const gl = this.gl
|
||||
this.width = width; this.height = height
|
||||
this.canvas.width = width; this.canvas.height = height
|
||||
for (const target of this.targets) { gl.deleteFramebuffer(target.framebuffer); gl.deleteTexture(target.texture) }
|
||||
this.targets = [0, 1].map(() => {
|
||||
const texture = this.texture()
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null)
|
||||
const framebuffer = gl.createFramebuffer()!
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer)
|
||||
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, texture, 0)
|
||||
return { framebuffer, texture }
|
||||
})
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, null)
|
||||
}
|
||||
|
||||
private onFrame = (now: number) => {
|
||||
this.pending = false
|
||||
if (this.stopped || this.idle) return
|
||||
this.schedule()
|
||||
const max = this.effects.maxFps
|
||||
if (max > 0 && now - this.lastDraw < 1000 / max - 2) return
|
||||
this.lastDraw = now
|
||||
try { this.draw() } catch (error) { (window as any).DDCloud?.log(`effects: ${String(error).slice(0, 200)}`); this.destroy() }
|
||||
}
|
||||
|
||||
private draw() {
|
||||
const gl = this.gl
|
||||
const e = this.effects
|
||||
const width = this.video.videoWidth, height = this.video.videoHeight
|
||||
if (width === 0 || height === 0) return
|
||||
if (width !== this.width || height !== this.height) this.resize(width, height)
|
||||
this.frame++
|
||||
|
||||
gl.activeTexture(gl.TEXTURE0)
|
||||
gl.bindTexture(gl.TEXTURE_2D, this.videoTexture)
|
||||
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true)
|
||||
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.video)
|
||||
|
||||
const extras = ['fxaa', 'toon', 'crt', 'ntsc'].filter(name => (e as any)[name])
|
||||
gl.viewport(0, 0, width, height)
|
||||
gl.enableVertexAttribArray(0)
|
||||
gl.vertexAttribPointer(0, 2, gl.FLOAT, false, 0, 0)
|
||||
|
||||
const base = this.passes.get('base')!
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, extras.length > 0 ? this.targets[0].framebuffer : null)
|
||||
gl.useProgram(base.program)
|
||||
const u = base.uniforms
|
||||
gl.uniform1i(u.get('screenTexture')!, 0)
|
||||
gl.uniform2f(u.get('resolution')!, width, height)
|
||||
gl.uniform1i(u.get('filterId')!, e.sharpenMode === 'usm' ? 1 : 2)
|
||||
gl.uniform1f(u.get('sharpenFactor')!, e.sharpen)
|
||||
gl.uniform1f(u.get('brightness')!, e.brightness / 100)
|
||||
gl.uniform1f(u.get('contrast')!, e.contrast / 100)
|
||||
gl.uniform1f(u.get('gamma')!, Math.min(5, Math.max(0.1, e.gamma)))
|
||||
gl.uniform1f(u.get('saturation')!, e.saturation / 100)
|
||||
gl.uniform1f(u.get('deband')!, e.deband)
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 3)
|
||||
|
||||
extras.forEach((name, index) => {
|
||||
const pass = this.extra(name)
|
||||
const source = this.targets[index % 2]
|
||||
const last = index === extras.length - 1
|
||||
gl.bindFramebuffer(gl.FRAMEBUFFER, last ? null : this.targets[(index + 1) % 2].framebuffer)
|
||||
gl.useProgram(pass.program)
|
||||
gl.bindTexture(gl.TEXTURE_2D, source.texture)
|
||||
const pu = pass.uniforms
|
||||
gl.uniform1i(pu.get('screenTexture') ?? null, 0)
|
||||
gl.uniform2f(pu.get('resolution') ?? null, width, height)
|
||||
gl.uniform2f(pu.get('TextureSize') ?? null, width, height)
|
||||
gl.uniform1i(pu.get('FrameCount') ?? null, this.frame)
|
||||
gl.drawArrays(gl.TRIANGLES, 0, 3)
|
||||
})
|
||||
}
|
||||
|
||||
destroy() {
|
||||
this.stopped = true
|
||||
this.canvas.remove()
|
||||
this.gl.getExtension('WEBGL_lose_context')?.loseContext()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// Offer tweaks after Better xCloud by redphx (MIT, src/utils/sdp.ts): prefer an H.264 profile, cap bitrate.
|
||||
|
||||
/** Moves the H.264 payloads of the chosen profile to the front of the video m-line. */
|
||||
export function preferH264Profile(sdp: string, profile: 'high' | 'normal' | 'low'): string {
|
||||
const prefix = profile === 'high' ? '4d' : (profile === 'low' ? '420' : '42e')
|
||||
const preferred: string[] = []
|
||||
for (const match of sdp.matchAll(/a=fmtp:(\d+).*profile-level-id=([0-9a-f]{6})/g)) {
|
||||
if (match[2].startsWith(prefix)) preferred.push(match[1])
|
||||
}
|
||||
if (preferred.length === 0) return sdp
|
||||
const lines = sdp.split('\r\n')
|
||||
const index = lines.findIndex(line => line.startsWith('m=video'))
|
||||
if (index < 0) return sdp
|
||||
const parts = lines[index].trim().split(' ')
|
||||
const ids = parts.slice(3).filter(id => !preferred.includes(id))
|
||||
lines[index] = parts.slice(0, 3).concat(preferred, ids).join(' ')
|
||||
return lines.join('\r\n')
|
||||
}
|
||||
|
||||
/** Sets b=AS (kbit/s) on the video m-section. */
|
||||
export function capVideoBitrate(sdp: string, kbps: number): string {
|
||||
const lines = sdp.split('\r\n')
|
||||
const start = lines.findIndex(line => line.startsWith('m=video'))
|
||||
if (start < 0) return sdp
|
||||
const bLine = `b=AS:${kbps}`
|
||||
for (let i = start + 1; i <= lines.length; i++) {
|
||||
const line = lines[i]
|
||||
if (line === undefined || line.startsWith('m=')) { lines.splice(i, 0, bLine); break }
|
||||
if (line.startsWith('i=') || line.startsWith('c=')) continue
|
||||
if (line.startsWith('b=AS:')) { lines[i] = bLine; break }
|
||||
lines.splice(i, 0, bLine); break
|
||||
}
|
||||
return lines.join('\r\n')
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
// Performance overlay in the top-right or top-left corner, from WebRTC getStats. Levels: 1 fps; 2 + ping, resolution;
|
||||
// 3 + bitrate, decode time, packet loss; 4 + codec, jitter, dropped frames, server. Horizontal or vertical.
|
||||
|
||||
export class StreamStats {
|
||||
private element: HTMLDivElement
|
||||
private timer: ReturnType<typeof setInterval> | undefined
|
||||
private level = 0
|
||||
private vertical = false
|
||||
private left = false
|
||||
private server = ''
|
||||
private last = { bytes: 0, time: 0, decode: 0, frames: 0, lost: 0, received: 0 }
|
||||
|
||||
constructor(private pc: RTCPeerConnection) {
|
||||
this.element = document.createElement('div')
|
||||
Object.assign(this.element.style, {
|
||||
position: 'absolute', top: '8px', right: '8px', zIndex: '3', padding: '4px 8px', borderRadius: '6px',
|
||||
background: 'rgba(0,0,0,0.55)', color: '#e8e8e8', font: '12px monospace', lineHeight: '16px',
|
||||
pointerEvents: 'none', display: 'none', whiteSpace: 'pre', textAlign: 'right',
|
||||
})
|
||||
document.body.appendChild(this.element)
|
||||
}
|
||||
|
||||
show(level: number, vertical: boolean, server: string, left = false) {
|
||||
this.level = level; this.vertical = vertical; this.server = server; this.left = left
|
||||
// Top left keeps clear of the clock Xbox draws top right while its guide is open.
|
||||
Object.assign(this.element.style, left ? { left: '8px', right: '', textAlign: 'left' } : { left: '', right: '8px', textAlign: 'right' })
|
||||
this.element.style.display = level > 0 ? 'block' : 'none'
|
||||
if (level > 0 && this.timer === undefined) { this.timer = setInterval(() => this.update(), 1000); this.update() }
|
||||
if (level === 0 && this.timer !== undefined) { clearInterval(this.timer); this.timer = undefined }
|
||||
}
|
||||
|
||||
private async update() {
|
||||
const report = await this.pc.getStats()
|
||||
let video: any, pair: any
|
||||
const codecs = new Map<string, any>()
|
||||
report.forEach((s: any) => {
|
||||
if (s.type === 'inbound-rtp' && s.kind === 'video') video = s
|
||||
else if (s.type === 'candidate-pair' && s.state === 'succeeded' && s.nominated) pair = s
|
||||
else if (s.type === 'codec') codecs.set(s.id, s)
|
||||
})
|
||||
if (video === undefined) return
|
||||
const seconds = this.last.time > 0 ? (video.timestamp - this.last.time) / 1000 : 0
|
||||
const mbps = seconds > 0 ? (video.bytesReceived - this.last.bytes) * 8 / seconds / 1e6 : 0
|
||||
const frames = (video.framesDecoded ?? 0) - this.last.frames
|
||||
const decodeMs = frames > 0 ? ((video.totalDecodeTime ?? 0) - this.last.decode) * 1000 / frames : 0
|
||||
const lost = (video.packetsLost ?? 0) - this.last.lost
|
||||
const received = (video.packetsReceived ?? 0) - this.last.received
|
||||
const loss = lost + received > 0 ? lost * 100 / (lost + received) : 0
|
||||
this.last = { bytes: video.bytesReceived, time: video.timestamp, decode: video.totalDecodeTime ?? 0,
|
||||
frames: video.framesDecoded ?? 0, lost: video.packetsLost ?? 0, received: video.packetsReceived ?? 0 }
|
||||
|
||||
const items: string[] = ['web', `${Math.round(video.framesPerSecond ?? 0)} fps`]
|
||||
if (this.level >= 2) {
|
||||
if (pair?.currentRoundTripTime !== undefined) items.push(`ping ${Math.round(pair.currentRoundTripTime * 1000)} ms`)
|
||||
items.push(`${video.frameWidth ?? '?'}x${video.frameHeight ?? '?'}`)
|
||||
}
|
||||
if (this.level >= 3) items.push(`${mbps.toFixed(1)} Mb/s`, `decode ${decodeMs.toFixed(1)} ms`, `loss ${loss.toFixed(1)}%`)
|
||||
if (this.level >= 4) {
|
||||
const codec = codecs.get(video.codecId)
|
||||
if (codec) items.push(String(codec.mimeType ?? '').replace('video/', ''))
|
||||
items.push(`jitter ${Math.round((video.jitter ?? 0) * 1000)} ms`, `dropped ${video.framesDropped ?? 0}`)
|
||||
if (this.server) items.push(this.server)
|
||||
}
|
||||
this.element.textContent = items.join(this.vertical ? '\n' : ' · ')
|
||||
}
|
||||
|
||||
destroy() { this.show(0, this.vertical, this.server, this.left); this.element.remove() }
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
// Stand-in for the "uuid" package (only v4 is used by the Greenlight player).
|
||||
export function v4(): string { return crypto.randomUUID() }
|
||||
Reference in new issue
Block a user