Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation (#1531)

Xbox Cloud Gaming: stream from the store and Games tab, native player with DroidDeck frame generation

Stream Xbox games with Xbox Cloud Gaming (Game Pass) from DroidDeck, outside
the Linux session.

- Backend: `xodus-cli cloud-token` (tools/xbox/droiddeck-cloud-token.patch)
  gets the streaming tokens; they stay in memory.
- Web player (default): the Greenlight player page (tools/xcloud-player, MIT)
  in a WebView, plus Microsoft's own xbox.com/play player. In-stream menu with
  screen effects (sharpen, colour, deband, FXAA, CRT, NTSC, Toon, fps cap),
  stream quality, performance overlay (level, layout, top right / top left)
  and rumble.
- Native player (opt-in, Android 10+, Player engine: Native): Google's WebRTC
  (io.github.webrtc-sdk 125.6422.07, BSD) with the Greenlight protocol ported
  to Kotlin, in its own process (:xcloudnative) with its own instance of
  DroidDeck's compositor. Decoded frames reach the compositor as
  AHardwareBuffers (new stream mode, VK_ANDROID_external_memory_android_
  hardware_buffer import), so DDFG 2x and LSFG 2x/3x/4x run on them. Live
  frame generation, frame-rate target (presets and a 10-165 fps slider, even
  pacing), zero-copy picture, small network buffer, fast decoder, instant
  controller input, overlay with engine and on-screen fps, background end,
  restart and web fallback. It takes the main process's tokens through
  XboxCloudTokenService; they are fetched ahead while a cloud game is shown.
- The compositor requests the dma-buf import extensions only where the driver
  has them (the system driver has none).
- Catalog: cloud availability, Game Pass plan CFQ7TTC0K6L8 counted, library
  filter All / Owned / Game Pass / Cloud, Stream and Add to Games on the store
  page, stream-only Games-tab entries (Stream + Download / Play + Stream).
- Xbox games get VC++ 2015-2022 and the ARM64 runtime registry entries
  (vcredist2022_arm64, built in until the catalog has it) before each launch.
- Controller focus reaches pickers opened from a game page (install location,
  uninstall); Add / Remove from Games keep the focus.

Credits: Greenlight by UnknownSKL (Jim Kroon), MIT, player and protocol;
Better xCloud for the resolution / OS mapping; MGamer404's Bannerlator #627
for the ARM64 runtime approach. Ported from the412banner's Bannerlator.
This commit is contained in:
The412Banner authored and GitHub committed 2026-10-24 23:17:57 -04:00
1 parent 5ca1f610ab
commit c5d7cdc058
78 files changed
+7835 -45

No files matched your search

@@ -95,4 +95,5 @@ target_link_libraries(droiddeckwayland
dl
log
android
nativewindow
)
@@ -16,6 +16,7 @@
#include <sys/syscall.h>
#include <sys/sysmacros.h>
#include <android/log.h>
#include <android/hardware_buffer.h>
#include <wayland-server.h>
#include "xdg-shell-server-protocol.h"
@@ -1845,7 +1846,121 @@ static void hdr_count_copied(const struct draw_list *dl, const char *reason) {
}
}
/*
* Stream mode (the Xbox Cloud Gaming native player, in its own process): no Wayland clients, the scene is one
* picture handed over as AHardwareBuffers (ImageReader images of the decoded video). Each buffer is imported once
* and kept while the reader cycles through its few buffers; the newest frame is the scene, so the screen effects
* and frame generation apply to it as to a game.
*/
#define STREAM_SLOTS 6
static pthread_mutex_t g_stream_lock = PTHREAD_MUTEX_INITIALIZER;
static atomic_int g_stream_on;
static AHardwareBuffer *g_stream_pending;
static struct { AHardwareBuffer *ahb; struct vkp_image *img; uint64_t used; } g_stream_slots[STREAM_SLOTS];
static struct vkp_image *g_stream_img;
static uint64_t g_stream_seq;
static atomic_uint g_stream_frames;
static void stream_forget_all(void) {
vkp_wait_idle();
for (int i = 0; i < STREAM_SLOTS; i++) {
if (g_stream_slots[i].img) vkp_image_destroy(g_stream_slots[i].img);
if (g_stream_slots[i].ahb) AHardwareBuffer_release(g_stream_slots[i].ahb);
g_stream_slots[i].img = NULL; g_stream_slots[i].ahb = NULL; g_stream_slots[i].used = 0;
}
g_stream_img = NULL;
}
/* Compositor thread: the newest handed-over buffer becomes the scene picture. */
static void stream_take(void) {
pthread_mutex_lock(&g_stream_lock);
AHardwareBuffer *ahb = g_stream_pending;
g_stream_pending = NULL;
pthread_mutex_unlock(&g_stream_lock);
if (!ahb) return;
if (!atomic_load(&g_stream_on)) { AHardwareBuffer_release(ahb); return; }
int slot = -1, oldest = 0;
for (int i = 0; i < STREAM_SLOTS; i++) {
if (g_stream_slots[i].ahb == ahb) { slot = i; break; }
if (g_stream_slots[i].used < g_stream_slots[oldest].used) oldest = i;
}
if (slot >= 0) {
AHardwareBuffer_release(ahb); /* the slot keeps its own reference */
} else {
struct vkp_image *img = vkp_image_from_ahb(ahb);
if (!img) {
static int said;
if (!said) { said = 1; droiddeck_log("stream", "a decoded frame could not be imported: nothing to show"); }
AHardwareBuffer_release(ahb);
return;
}
droiddeck_log("stream", "decoded frame buffer imported (%dx%d)", vkp_image_width(img), vkp_image_height(img));
slot = oldest;
if (g_stream_slots[slot].img) {
vkp_wait_idle(); /* the GPU may still read the picture being replaced */
if (g_stream_img == g_stream_slots[slot].img) g_stream_img = NULL;
vkp_image_destroy(g_stream_slots[slot].img);
}
if (g_stream_slots[slot].ahb) AHardwareBuffer_release(g_stream_slots[slot].ahb);
g_stream_slots[slot].ahb = ahb;
g_stream_slots[slot].img = img;
}
g_stream_slots[slot].used = ++g_stream_seq;
g_stream_img = g_stream_slots[slot].img;
atomic_fetch_add(&g_stream_frames, 1);
schedule_render();
}
static void render_stream(void) {
g_dirty = 0;
struct vkp_image *img = g_stream_img;
if (!img) return;
int w = vkp_image_width(img), h = vkp_image_height(img);
if (w != g_scene_w || h != g_scene_h) {
droiddeck_log("stream", "picture %dx%d", w, h);
g_scene_w = w;
g_scene_h = h;
}
struct vkp_draw d = {.img = img, .sx = 0, .sy = 0, .sw = (float)w, .sh = (float)h, .dx = 0, .dy = 0, .dw = w, .dh = h, .blend = 0};
vkp_apply_window_request();
sc_layer_hide();
int r = vkp_render(w, h, &d, 1);
if (r == 0) {
g_stat_frames++;
static int first;
if (!first) { first = 1; droiddeck_log("stream", "first stream frame presented"); }
} else {
static int failed;
if (failed++ % 600 == 0) droiddeck_log("stream", "presenting a stream frame failed (%d, %d times)", r, failed);
}
}
void droiddeck_stream_set(int on) {
atomic_store(&g_stream_on, on ? 1 : 0);
if (g_input_pipe[1] < 0) return;
struct input_msg m = { 21, on ? 1 : 0, 0, 0, 0 };
ssize_t n = write(g_input_pipe[1], &m, sizeof(m));
(void)n;
}
/* Any thread: hands over one decoded frame (acquires its own reference). */
void droiddeck_stream_frame(AHardwareBuffer *ahb) {
if (!ahb || !atomic_load(&g_stream_on) || g_input_pipe[1] < 0) return;
AHardwareBuffer_acquire(ahb);
pthread_mutex_lock(&g_stream_lock);
AHardwareBuffer *dropped = g_stream_pending;
g_stream_pending = ahb;
pthread_mutex_unlock(&g_stream_lock);
if (dropped) { AHardwareBuffer_release(dropped); return; } /* a wake-up is already queued */
struct input_msg m = { 20, 0, 0, 0, 0 };
ssize_t n = write(g_input_pipe[1], &m, sizeof(m));
(void)n;
}
unsigned droiddeck_stream_frames(void) { return atomic_load(&g_stream_frames); }
static void render_scene(void) {
if (atomic_load(&g_stream_on)) { render_stream(); return; }
struct draw_list dl = {0};
struct surface *s;
int w, h;
@@ -3005,6 +3120,8 @@ static int on_input_readable(int fd, uint32_t mask, void *data) {
break;
case 10: break;
case 11: axis_event(m.p1, m.p2); break;
case 20: stream_take(); break;
case 21: if (!m.p1) stream_forget_all(); schedule_render(); break;
default: deliver_pointer(&m); break;
}
}
+1 -1
View File
@@ -21,7 +21,7 @@
X(BindImageMemory) X(DestroyImage) X(FreeMemory) X(AcquireNextImageKHR) \
X(ResetCommandBuffer) X(BeginCommandBuffer) X(CmdPipelineBarrier) X(CmdBlitImage) \
X(EndCommandBuffer) X(ResetFences) X(QueueSubmit) X(QueuePresentKHR) X(WaitForFences) \
X(QueueWaitIdle) X(GetMemoryFdPropertiesKHR) \
X(QueueWaitIdle) X(GetMemoryFdPropertiesKHR) X(GetAndroidHardwareBufferPropertiesANDROID) \
X(GetImageSubresourceLayout) X(MapMemory) X(UnmapMemory) X(CmdClearColorImage) \
\
X(CreateShaderModule) X(DestroyShaderModule) X(CreatePipelineLayout) X(DestroyPipelineLayout) \
+72 -7
View File
@@ -19,6 +19,7 @@
#include <time.h>
#include <pthread.h>
#include <android/log.h>
#include <android/hardware_buffer.h>
#define TAG "DroidDeckWayland"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
@@ -39,6 +40,7 @@ struct vkp_image {
};
static ANativeWindow *g_window;
static int g_ahb_import;
static char *g_driver_path, *g_library_name, *g_native_lib_dir;
static int g_dev_state;
static VkInstance g_inst;
@@ -459,17 +461,19 @@ static int dev_init(void) {
}
g_vk.GetPhysicalDeviceMemoryProperties(g_pd, &g_memprops);
const char *dev_exts[7] = {VK_KHR_SWAPCHAIN_EXTENSION_NAME, "VK_KHR_external_memory_fd",
"VK_EXT_external_memory_dma_buf", "VK_EXT_image_drm_format_modifier",
"VK_KHR_image_format_list", NULL, NULL};
uint32_t n_dev_exts = 5;
static const char *const import_exts[4] = {"VK_KHR_external_memory_fd", "VK_EXT_external_memory_dma_buf",
"VK_EXT_image_drm_format_modifier", "VK_KHR_image_format_list"};
const char *dev_exts[9] = {VK_KHR_SWAPCHAIN_EXTENSION_NAME};
uint32_t n_dev_exts = 1;
uint32_t ne = 0;
g_vk.EnumerateDeviceExtensionProperties(g_pd, NULL, &ne, NULL);
VkExtensionProperties *exts = calloc(ne ? ne : 1, sizeof(*exts));
g_vk.EnumerateDeviceExtensionProperties(g_pd, NULL, &ne, exts);
for (unsigned i = 0; i < 5; i++)
if (!has_ext(exts, ne, dev_exts[i]))
LOGE("present: driver MISSING %s (dmabuf import will fail)", dev_exts[i]);
/* Only what the driver has: the system driver lacks the dma-buf ones, which the cloud player never needs. */
for (unsigned i = 0; i < 4; i++) {
if (has_ext(exts, ne, import_exts[i])) dev_exts[n_dev_exts++] = import_exts[i];
else LOGE("present: driver MISSING %s (dmabuf import will fail)", import_exts[i]);
}
int want_hdr_md = 0;
if (droiddeck_color_requested()) {
want_hdr_md = has_ext(exts, ne, VK_EXT_HDR_METADATA_EXTENSION_NAME);
@@ -477,6 +481,13 @@ static int dev_init(void) {
}
const int want_sem_fd = has_ext(exts, ne, VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME);
if (want_sem_fd) dev_exts[n_dev_exts++] = VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME;
/* The cloud player's decoded frames arrive as AHardwareBuffers (vkp_image_from_ahb). */
g_ahb_import = has_ext(exts, ne, VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME) &&
has_ext(exts, ne, VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME);
if (g_ahb_import) {
dev_exts[n_dev_exts++] = VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME;
dev_exts[n_dev_exts++] = VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME;
}
free(exts);
float prio = 1.0f;
@@ -987,6 +998,60 @@ struct vkp_image *vkp_image_import_dmabuf(int fd, uint32_t drm_format, uint64_t
return img;
}
void vkp_wait_idle(void) {
if (g_dev && g_vk.DeviceWaitIdle) g_vk.DeviceWaitIdle(g_dev);
}
struct vkp_image *vkp_image_from_ahb(AHardwareBuffer *ahb) {
if (!ahb || dev_init() != 0) return NULL;
if (!g_ahb_import || !g_vk.GetAndroidHardwareBufferPropertiesANDROID) {
static int said;
if (!said) { said = 1; LOGE("ahb: the driver cannot import AHardwareBuffers (VK_ANDROID_external_memory_android_hardware_buffer)"); }
return NULL;
}
AHardwareBuffer_Desc desc;
AHardwareBuffer_describe(ahb, &desc);
VkAndroidHardwareBufferFormatPropertiesANDROID fp = {
.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_FORMAT_PROPERTIES_ANDROID};
VkAndroidHardwareBufferPropertiesANDROID props = {
.sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, .pNext = &fp};
VkResult pr = g_vk.GetAndroidHardwareBufferPropertiesANDROID(g_dev, ahb, &props);
if (pr != VK_SUCCESS || fp.format == VK_FORMAT_UNDEFINED) {
LOGE("ahb: %ux%u format %u: properties %s, Vulkan format %d", desc.width, desc.height, desc.format,
vk_result_name(pr), (int)fp.format);
return NULL;
}
struct vkp_image *img = calloc(1, sizeof(*img));
if (!img) return NULL;
img->w = (int)desc.width; img->h = (int)desc.height; img->dmabuf = 1; img->fmt = fp.format; img->sampled = 1;
VkExternalMemoryImageCreateInfo ext = {
.sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID};
VkImageCreateInfo ici = {
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .pNext = &ext,
.imageType = VK_IMAGE_TYPE_2D, .format = fp.format, .extent = {desc.width, desc.height, 1},
.mipLevels = 1, .arrayLayers = 1, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL,
.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE, .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED};
VkResult cr = g_vk.CreateImage(g_dev, &ici, NULL, &img->image);
if (cr != VK_SUCCESS) { LOGE("ahb: vkCreateImage %ux%u -> %s", desc.width, desc.height, vk_result_name(cr)); free(img); return NULL; }
int idx = -1;
for (int i = 0; i < 32; i++) if (props.memoryTypeBits & (1u << i)) { idx = i; break; }
VkImportAndroidHardwareBufferInfoANDROID imp = {
.sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, .buffer = ahb};
VkMemoryDedicatedAllocateInfo ded = {.sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
.pNext = &imp, .image = img->image};
VkMemoryAllocateInfo mai = {.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, .pNext = &ded,
.allocationSize = props.allocationSize, .memoryTypeIndex = (uint32_t)(idx < 0 ? 0 : idx)};
VkResult mr = idx < 0 ? VK_ERROR_FEATURE_NOT_PRESENT : g_vk.AllocateMemory(g_dev, &mai, NULL, &img->mem);
if (mr != VK_SUCCESS || g_vk.BindImageMemory(g_dev, img->image, img->mem, 0) != VK_SUCCESS) {
LOGE("ahb: importing %ux%u -> %s", desc.width, desc.height, vk_result_name(mr));
if (img->mem) g_vk.FreeMemory(g_dev, img->mem, NULL);
g_vk.DestroyImage(g_dev, img->image, NULL); free(img); return NULL;
}
return img;
}
static int shm_sampled(void) {
static int known;
if (!known) {
@@ -34,6 +34,9 @@ struct vkp_image *vkp_image_import_dmabuf(int fd, uint32_t drm_format, uint64_t
int vkp_image_is_dmabuf(const struct vkp_image *img);
struct vkp_image *vkp_image_create_shm(int w, int h);
struct AHardwareBuffer;
struct vkp_image *vkp_image_from_ahb(struct AHardwareBuffer *ahb);
void vkp_wait_idle(void);
void vkp_image_upload_shm(struct vkp_image *img, const void *data, int stride);
int vkp_image_width(const struct vkp_image *img);
@@ -9,6 +9,7 @@
#include <sys/resource.h>
#include <android/log.h>
#include <android/native_window_jni.h>
#include <android/hardware_buffer_jni.h>
#include "vk_present.h"
#include "droiddeck_ext.h"
#include "ahb_swapchain.h"
@@ -27,6 +28,9 @@ extern int droiddeck_cursor_serial(void);
extern void droiddeck_wayland_send_key(int evdev, int state);
extern void droiddeck_wayland_send_scene_input(int type, int a, int b);
extern void droiddeck_wayland_vsync(int64_t frame_time_ns);
extern void droiddeck_stream_set(int on);
extern void droiddeck_stream_frame(AHardwareBuffer *ahb);
extern unsigned droiddeck_stream_frames(void);
extern void droiddeck_wayland_output_resized(void);
extern void droiddeck_wayland_set_output_refresh_rate(int mhz);
extern volatile int g_fps_limit;
@@ -667,3 +671,21 @@ Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeSetCardboardSpatialP
(*env)->GetFloatArrayRegion(env, pos, 0, 3, p);
vkp_cardboard_set_spatial_pose(m, p, live ? 1 : 0);
}
/* The Xbox Cloud Gaming native player: its decoded frames are the whole scene (no Wayland clients). */
JNIEXPORT void JNICALL
Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeStreamMode(JNIEnv *env, jclass clazz, jboolean on) {
droiddeck_stream_set(on ? 1 : 0);
}
JNIEXPORT void JNICALL
Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeStreamFrame(JNIEnv *env, jclass clazz, jobject hardwareBuffer) {
if (!hardwareBuffer) return;
AHardwareBuffer *ahb = AHardwareBuffer_fromHardwareBuffer(env, hardwareBuffer);
if (ahb) droiddeck_stream_frame(ahb);
}
JNIEXPORT jint JNICALL
Java_com_droiddeck_launcher_wayland_WaylandCompositor_nativeStreamFrames(JNIEnv *env, jclass clazz) {
return (jint)droiddeck_stream_frames();
}