Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation (#1531)

Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation

Stream Xbox games with Xbox Cloud Gaming (Game Pass) from DroidDeck, outside
the Linux session.

- Backend: `xodus-cli cloud-token` (tools/xbox/droiddeck-cloud-token.patch)
  gets the streaming tokens; they stay in memory.
- Web player (default): the Greenlight player page (tools/xcloud-player, MIT)
  in a WebView, plus Microsoft's own xbox.com/play player. In-stream menu with
  screen effects (sharpen, colour, deband, FXAA, CRT, NTSC, Toon, fps cap),
  stream quality, performance overlay (level, layout, top right / top left)
  and rumble.
- Native player (opt-in, Android 10+, Player engine: Native): Google's WebRTC
  (io.github.webrtc-sdk 125.6422.07, BSD) with the Greenlight protocol ported
  to Kotlin, in its own process (:xcloudnative) with its own instance of
  DroidDeck's compositor. Decoded frames reach the compositor as
  AHardwareBuffers (new stream mode, VK_ANDROID_external_memory_android_
  hardware_buffer import), so DDFG 2x and LSFG 2x/3x/4x run on them. Live
  frame generation, frame-rate target (presets and a 10-165 fps slider, even
  pacing), zero-copy picture, small network buffer, fast decoder, instant
  controller input, overlay with engine and on-screen fps, background end,
  restart and web fallback. It takes the main process's tokens through
  XboxCloudTokenService; they are fetched ahead while a cloud game is shown.
- The compositor requests the dma-buf import extensions only where the driver
  has them (the system driver has none).
- Catalog: cloud availability, Game Pass plan CFQ7TTC0K6L8 counted, library
  filter All / Owned / Game Pass / Cloud, Stream and Add to Games on the store
  page, stream-only Games-tab entries (Stream + Download / Play + Stream).
- Xbox games get VC++ 2015-2022 and the ARM64 runtime registry entries
  (vcredist2022_arm64, built in until the catalog has it) before each launch.
- Controller focus reaches pickers opened from a game page (install location,
  uninstall); Add / Remove from Games keep the focus.

Credits: Greenlight by UnknownSKL (Jim Kroon), MIT, player and protocol;
Better xCloud for the resolution / OS mapping; MGamer404's Bannerlator #627
for the ARM64 runtime approach. Ported from the412banner's Bannerlator.
This commit is contained in:
The412Banner authored and GitHub committed 2026-10-24 23:17:57 -04:00
1 parent 5ca1f610ab
commit c5d7cdc058
78 files changed
+7835 -45

No files matched your search

@@ -0,0 +1,257 @@
package org.webrtc;
import android.graphics.ImageFormat;
import android.media.Image;
import android.media.ImageReader;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.SystemClock;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
/**
* DroidDeck's hardware video decoder factory for the native Xbox Cloud Gaming player. It builds WebRTC's own
* AndroidVideoDecoder (hence this package) around a MediaCodec wrapper that can:
* - ask the decoder for low-latency output (Android's KEY_LOW_LATENCY and Qualcomm's low-latency / picture-order
* vendor keys; unknown keys are ignored, and a decoder that refuses them is configured without them);
* - for zero-copy, decode straight onto the player's SurfaceView instead of WebRTC's texture, rendering every
* output buffer there (no GL copy, no compositor; WebRTC then gets no decoded frames to draw).
* Without either option it behaves like WebRTC's HardwareVideoDecoderFactory.
* The route is switchable while the decoder runs (setOutputSurface): {@link #useScreen} sends frames straight to the
* screen (zero-copy) or, with null, back to WebRTC's texture (for the Vulkan compositor and frame generation);
* {@link #setZeroCopySurface} moves a zero-copy decoder to a stand-in surface while the screen surface is gone.
* The factory also reports the decoded size and the decode time (input queued to output ready), which WebRTC's
* stats lack in zero-copy.
*/
public class DroidDeckLowLatencyDecoderFactory implements VideoDecoderFactory {
/** Counts frames sent to the display in zero-copy mode (WebRTC's own stats see none). */
public interface FrameCounter { void onFrameShown(); }
private final HardwareVideoDecoderFactory hardware;
private final EglBase.Context sharedContext;
private final boolean lowLatency;
/** Frames go to {@link #zeroCopySurface} (the screen or the stand-in), not WebRTC's texture. */
private volatile boolean zeroCopy;
private volatile Surface zeroCopySurface;
/** Back on WebRTC's texture: render the next buffer there, so WebRTC's pending texture frame completes. */
private volatile boolean renderNext;
private final FrameCounter counter;
private volatile Wrapper live;
private ImageReader standIn;
private HandlerThread standInThread;
private volatile int width, height;
private final Object decodeLock = new Object();
private final HashMap<Long, Long> queuedAt = new HashMap<>();
private double decodeSumMs;
private int decodeCount;
public DroidDeckLowLatencyDecoderFactory(EglBase.Context sharedContext, boolean lowLatency, Surface zeroCopySurface, FrameCounter counter) {
this.hardware = new HardwareVideoDecoderFactory(sharedContext);
this.sharedContext = sharedContext;
this.lowLatency = lowLatency;
this.zeroCopy = zeroCopySurface != null;
this.zeroCopySurface = zeroCopySurface;
this.counter = counter;
}
/**
* Zero-copy: the surface to decode onto from now on; null = a stand-in that drops every frame (the screen
* surface is being destroyed). Must be called before a destroyed surface's callback returns.
*/
public synchronized void setZeroCopySurface(Surface surface) {
if (!zeroCopy) return;
moveTo(surface != null ? surface : standInSurface(), surface != null ? "the screen" : "a stand-in");
}
/**
* Switch the route live: a surface = zero-copy onto it; null = WebRTC's texture again. Returns false when the
* decoder refused the new surface (the route is then unchanged).
*/
public synchronized boolean useScreen(Surface screen) {
if (screen != null) {
Wrapper w = live;
if (w != null && !moveTo(screen, "the screen")) return false;
zeroCopySurface = screen;
zeroCopy = true;
return true;
}
if (!zeroCopy) return true;
Wrapper w = live;
if (w != null && w.texture != null && !moveTo(w.texture, "WebRTC's texture")) return false;
zeroCopy = false;
renderNext = true;
return true;
}
private boolean moveTo(Surface target, String what) {
Wrapper w = live;
if (w == null || target == null) { if (zeroCopy) zeroCopySurface = target; return true; }
try {
w.raw.setOutputSurface(target);
if (zeroCopy) zeroCopySurface = target;
Logging.d("DroidDeckDecoder", "decoder output moved to " + what);
return true;
} catch (RuntimeException e) {
Logging.w("DroidDeckDecoder", "decoder output change to " + what + " failed: " + e);
return false;
}
}
private Surface standInSurface() {
if (standIn == null) {
standInThread = new HandlerThread("bl-decoder-standin");
standInThread.start();
standIn = ImageReader.newInstance(Math.max(width, 16), Math.max(height, 16), ImageFormat.PRIVATE, 3);
standIn.setOnImageAvailableListener(reader -> {
Image image = reader.acquireLatestImage();
if (image != null) image.close();
}, new Handler(standInThread.getLooper()));
}
return standIn.getSurface();
}
/** The decoder's picture size, or null before the first decoder is configured. */
public String videoSize() {
return width > 0 ? width + "x" + height : null;
}
/** Average decode time since the last call, in ms (-1 when no frame was decoded in between). */
public double takeDecodeMs() {
synchronized (decodeLock) {
double avg = decodeCount > 0 ? decodeSumMs / decodeCount : -1;
decodeSumMs = 0; decodeCount = 0;
return avg;
}
}
public synchronized void release() {
if (standIn != null) { standIn.close(); standIn = null; }
if (standInThread != null) { standInThread.quitSafely(); standInThread = null; }
}
@Override
public VideoCodecInfo[] getSupportedCodecs() {
return hardware.getSupportedCodecs();
}
@Override
public VideoDecoder createDecoder(VideoCodecInfo info) {
VideoCodecMimeType type;
try { type = VideoCodecMimeType.valueOf(info.getName()); } catch (IllegalArgumentException e) { return hardware.createDecoder(info); }
MediaCodecInfo codec = findHardwareDecoder(type);
if (codec == null) return hardware.createDecoder(info);
Integer color = MediaCodecUtils.selectColorFormat(MediaCodecUtils.DECODER_COLOR_FORMATS, codec.getCapabilitiesForType(type.mimeType()));
if (color == null) return hardware.createDecoder(info);
return new AndroidVideoDecoder(new WrapperFactory(), codec.getName(), type, color, sharedContext);
}
private static MediaCodecInfo findHardwareDecoder(VideoCodecMimeType type) {
for (MediaCodecInfo info : new MediaCodecList(MediaCodecList.REGULAR_CODECS).getCodecInfos()) {
if (info.isEncoder()) continue;
if (MediaCodecUtils.codecSupportsType(info, type) && MediaCodecUtils.isHardwareAccelerated(info)) return info;
}
return null;
}
private class WrapperFactory implements MediaCodecWrapperFactory {
@Override
public MediaCodecWrapper createByCodecName(String name) throws IOException {
return new Wrapper(MediaCodec.createByCodecName(name));
}
}
/** MediaCodec itself (not WebRTC's wrapper) so the output surface can be changed while it runs. */
private class Wrapper implements MediaCodecWrapper {
final MediaCodec raw;
/** WebRTC's texture surface, the route back from zero-copy. */
Surface texture;
Wrapper(MediaCodec raw) { this.raw = raw; }
@Override
public void configure(MediaFormat format, Surface surface, MediaCrypto crypto, int flags) {
width = format.getInteger(MediaFormat.KEY_WIDTH);
height = format.getInteger(MediaFormat.KEY_HEIGHT);
texture = surface;
live = this;
Surface target = surface;
if (zeroCopy) { synchronized (DroidDeckLowLatencyDecoderFactory.this) { target = zeroCopySurface != null ? zeroCopySurface : standInSurface(); } }
if (lowLatency) {
MediaFormat fast = MediaFormat.createVideoFormat(format.getString(MediaFormat.KEY_MIME), width, height);
copyInt(format, fast, MediaFormat.KEY_COLOR_FORMAT);
if (Build.VERSION.SDK_INT >= 30) fast.setInteger(MediaFormat.KEY_LOW_LATENCY, 1);
fast.setInteger(MediaFormat.KEY_PRIORITY, 0);
fast.setInteger("vendor.qti-ext-dec-low-latency.enable", 1);
fast.setInteger("vendor.qti-ext-dec-picture-order.enable", 1);
try {
raw.configure(fast, target, crypto, flags);
Logging.d("DroidDeckDecoder", "low-latency decoder configured" + (zeroCopy ? " (zero-copy)" : ""));
return;
} catch (RuntimeException e) {
Logging.w("DroidDeckDecoder", "decoder refused low-latency keys: " + e);
}
}
raw.configure(format, target, crypto, flags);
}
private void copyInt(MediaFormat from, MediaFormat to, String key) {
if (from.containsKey(key)) to.setInteger(key, from.getInteger(key));
}
// Zero-copy: AndroidVideoDecoder renders one buffer to its texture and drops the rest until the texture
// hands the frame back, which never happens with the display surface; show every buffer instead.
@Override public void releaseOutputBuffer(int index, boolean render) {
if (zeroCopy) { raw.releaseOutputBuffer(index, true); if (counter != null) counter.onFrameShown(); }
else if (renderNext) { renderNext = false; raw.releaseOutputBuffer(index, true); }
else raw.releaseOutputBuffer(index, render);
}
@Override public void queueInputBuffer(int index, int offset, int size, long presentationTimeUs, int flags) {
synchronized (decodeLock) {
if (queuedAt.size() > 64) queuedAt.clear();
queuedAt.put(presentationTimeUs, SystemClock.elapsedRealtimeNanos());
}
raw.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
}
@Override public int dequeueOutputBuffer(MediaCodec.BufferInfo info, long timeoutUs) {
int index = raw.dequeueOutputBuffer(info, timeoutUs);
if (index >= 0) synchronized (decodeLock) {
Long at = queuedAt.remove(info.presentationTimeUs);
if (at != null) { decodeSumMs += (SystemClock.elapsedRealtimeNanos() - at) / 1e6; decodeCount++; }
}
return index;
}
@Override public MediaFormat getOutputFormat() {
MediaFormat f = raw.getOutputFormat();
if (f.containsKey(MediaFormat.KEY_WIDTH) && f.containsKey(MediaFormat.KEY_HEIGHT)) {
width = f.containsKey("crop-right") ? f.getInteger("crop-right") - f.getInteger("crop-left") + 1 : f.getInteger(MediaFormat.KEY_WIDTH);
height = f.containsKey("crop-bottom") ? f.getInteger("crop-bottom") - f.getInteger("crop-top") + 1 : f.getInteger(MediaFormat.KEY_HEIGHT);
}
return f;
}
@Override public void start() { raw.start(); }
@Override public void flush() { synchronized (decodeLock) { queuedAt.clear(); } raw.flush(); }
@Override public void stop() { raw.stop(); }
@Override public void release() { if (live == this) live = null; raw.release(); }
@Override public int dequeueInputBuffer(long timeoutUs) { return raw.dequeueInputBuffer(timeoutUs); }
@Override public MediaFormat getInputFormat() { return raw.getInputFormat(); }
@Override public MediaFormat getOutputFormat(int index) { return raw.getOutputFormat(index); }
@Override public ByteBuffer getInputBuffer(int index) { return raw.getInputBuffer(index); }
@Override public ByteBuffer getOutputBuffer(int index) { return raw.getOutputBuffer(index); }
@Override public Surface createInputSurface() { return raw.createInputSurface(); }
@Override public void setParameters(Bundle params) { raw.setParameters(params); }
@Override public MediaCodecInfo getCodecInfo() { return raw.getCodecInfo(); }
}
}