mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-18 20:36:04 +00:00
Decode Remote Play streams on the arm64 Steam client
Steam's arm64 streaming_client only decodes through a V4L2 stateful decoder and libdrm dumb buffers, neither of which exist on Android, so streams from a PC never showed a picture. The preload now offers it a /dev/video-dec0 and a libdrm stand-in backed by dma-heap buffers, and each frame is decoded by the app's MediaCodec (Qualcomm low-latency mode) over the encoder socket, falling back to the FFmpeg in Steam's runtime.
This commit is contained in:
1 parent
9555cb9f65
commit
916a39585f
9 files changed
+1511
-1
No files matched your search
@@ -9,7 +9,7 @@
|
||||
|
||||
#define VENC_MAX_BUFFERS 16
|
||||
|
||||
enum venc_msg { VENC_CONFIG = 1, VENC_FRAME = 2, VENC_DONE = 3, VENC_BITRATE = 4, VENC_TOUCH = 5 };
|
||||
enum venc_msg { VENC_CONFIG = 1, VENC_FRAME = 2, VENC_DONE = 3, VENC_BITRATE = 4, VENC_TOUCH = 5, VDEC_CONFIG = 6, VDEC_CAPTURE = 7, VDEC_PACKET = 8, VDEC_RESULT = 9 };
|
||||
enum venc_touch_action { VENC_TOUCH_DOWN = 0, VENC_TOUCH_MOVE = 1, VENC_TOUCH_UP = 2 };
|
||||
enum venc_codec { VENC_H264 = 0, VENC_HEVC = 1 };
|
||||
|
||||
@@ -52,4 +52,30 @@ struct venc_touch {
|
||||
uint32_t x, y;
|
||||
};
|
||||
|
||||
struct vdec_config {
|
||||
uint32_t type;
|
||||
uint32_t codec;
|
||||
uint32_t width, height;
|
||||
uint32_t stride, rows;
|
||||
uint32_t bitstreams, bitstream_size;
|
||||
};
|
||||
|
||||
struct vdec_capture {
|
||||
uint32_t type;
|
||||
uint32_t index;
|
||||
uint32_t size;
|
||||
};
|
||||
|
||||
struct vdec_packet {
|
||||
uint32_t type;
|
||||
uint32_t input, bytes, capture;
|
||||
int64_t pts_us;
|
||||
};
|
||||
|
||||
struct vdec_result {
|
||||
uint32_t type;
|
||||
int32_t status;
|
||||
uint32_t frame;
|
||||
};
|
||||
|
||||
#endif
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <jni.h>
|
||||
#include <media/NdkMediaCodec.h>
|
||||
#include <media/NdkMediaFormat.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -11,6 +12,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cerrno>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
@@ -45,6 +47,218 @@ int32_t avc_profile(int32_t v4l2) {
|
||||
}
|
||||
}
|
||||
|
||||
struct DmaBufSync {
|
||||
uint64_t flags;
|
||||
};
|
||||
constexpr unsigned long kDmaBufIoctlSync = _IOW('b', 0, DmaBufSync);
|
||||
constexpr uint64_t kDmaBufSyncWrite = 2, kDmaBufSyncStart = 0, kDmaBufSyncEnd = 4;
|
||||
constexpr int32_t kColorFormatYuv420Planar = 19;
|
||||
|
||||
class Decoder {
|
||||
public:
|
||||
~Decoder() { release(); }
|
||||
|
||||
void release() {
|
||||
if (codec_) {
|
||||
AMediaCodec_stop(codec_);
|
||||
AMediaCodec_delete(codec_);
|
||||
codec_ = nullptr;
|
||||
}
|
||||
if (bitstream_) munmap((void *)bitstream_, bitstream_size_);
|
||||
bitstream_ = nullptr;
|
||||
for (auto &c : captures_) drop(c);
|
||||
}
|
||||
|
||||
bool configure(const vdec_config &c, int fd) {
|
||||
release();
|
||||
if (c.width < 16 || c.height < 16 || c.stride < c.width || c.rows < c.height || c.bitstreams == 0 ||
|
||||
c.bitstreams > VENC_MAX_BUFFERS || c.bitstream_size == 0) {
|
||||
LOG("bad decoder config: %ux%u stride %u rows %u, %u bitstreams", c.width, c.height, c.stride, c.rows,
|
||||
c.bitstreams);
|
||||
return false;
|
||||
}
|
||||
bitstream_size_ = (size_t)c.bitstreams * c.bitstream_size;
|
||||
void *p = mmap(nullptr, bitstream_size_, PROT_READ, MAP_SHARED, fd, 0);
|
||||
if (p == MAP_FAILED) {
|
||||
LOG("could not map the bitstream buffers: %s", strerror(errno));
|
||||
return false;
|
||||
}
|
||||
bitstream_ = (const uint8_t *)p;
|
||||
config_ = c;
|
||||
codec_ = create(c, true);
|
||||
if (!codec_) codec_ = create(c, false);
|
||||
if (!codec_) {
|
||||
LOG("no %s decoder for %ux%u", c.codec == VENC_HEVC ? "HEVC" : "H.264", c.width, c.height);
|
||||
release();
|
||||
return false;
|
||||
}
|
||||
src_stride_ = (int32_t)c.width;
|
||||
src_rows_ = (int32_t)c.height;
|
||||
src_width_ = (int32_t)c.width;
|
||||
src_height_ = (int32_t)c.height;
|
||||
src_planar_ = false;
|
||||
shown_ = false;
|
||||
using GetName = media_status_t (*)(AMediaCodec *, char **);
|
||||
using ReleaseName = void (*)(AMediaCodec *, char *);
|
||||
static auto get_name = (GetName)dlsym(RTLD_DEFAULT, "AMediaCodec_getName");
|
||||
static auto release_name = (ReleaseName)dlsym(RTLD_DEFAULT, "AMediaCodec_releaseName");
|
||||
char *name = nullptr;
|
||||
if (get_name && get_name(codec_, &name) != AMEDIA_OK) name = nullptr;
|
||||
LOG("decoding %s %ux%u with %s", c.codec == VENC_HEVC ? "HEVC" : "H.264", c.width, c.height,
|
||||
name ? name : "MediaCodec");
|
||||
if (name && release_name) release_name(codec_, name);
|
||||
return true;
|
||||
}
|
||||
|
||||
void capture(const vdec_capture &c, int fd) {
|
||||
if (c.index >= VENC_MAX_BUFFERS) {
|
||||
close(fd);
|
||||
return;
|
||||
}
|
||||
Capture &cap = captures_[c.index];
|
||||
drop(cap);
|
||||
void *p = mmap(nullptr, c.size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
|
||||
if (p == MAP_FAILED) {
|
||||
LOG("could not map capture buffer %u: %s", c.index, strerror(errno));
|
||||
close(fd);
|
||||
return;
|
||||
}
|
||||
cap = {fd, (uint8_t *)p, c.size};
|
||||
}
|
||||
|
||||
bool decode(const vdec_packet &p, bool *frame) {
|
||||
*frame = false;
|
||||
if (!codec_ || p.capture >= VENC_MAX_BUFFERS || !captures_[p.capture].data || p.input >= config_.bitstreams ||
|
||||
p.bytes == 0 || p.bytes > config_.bitstream_size)
|
||||
return false;
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
ssize_t in = AMediaCodec_dequeueInputBuffer(codec_, 20000);
|
||||
if (in < 0) return false;
|
||||
size_t capacity = 0;
|
||||
uint8_t *dst = AMediaCodec_getInputBuffer(codec_, (size_t)in, &capacity);
|
||||
if (!dst || capacity < p.bytes) {
|
||||
AMediaCodec_queueInputBuffer(codec_, (size_t)in, 0, 0, 0, 0);
|
||||
return false;
|
||||
}
|
||||
memcpy(dst, bitstream_ + (size_t)p.input * config_.bitstream_size, p.bytes);
|
||||
if (AMediaCodec_queueInputBuffer(codec_, (size_t)in, 0, p.bytes, (uint64_t)std::max<int64_t>(p.pts_us, 0), 0) !=
|
||||
AMEDIA_OK)
|
||||
return false;
|
||||
auto wait = std::chrono::milliseconds(shown_ ? 25 : 250);
|
||||
while (std::chrono::steady_clock::now() - start < wait) {
|
||||
AMediaCodecBufferInfo info;
|
||||
ssize_t o = AMediaCodec_dequeueOutputBuffer(codec_, &info, 2000);
|
||||
if (o == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
read_format();
|
||||
continue;
|
||||
}
|
||||
if (o < 0) continue;
|
||||
size_t size = 0;
|
||||
uint8_t *data = AMediaCodec_getOutputBuffer(codec_, (size_t)o, &size);
|
||||
if (data && info.size > 0 && (size_t)info.offset + (size_t)info.size <= size)
|
||||
*frame = copy(data + info.offset, (size_t)info.size, captures_[p.capture]) || *frame;
|
||||
AMediaCodec_releaseOutputBuffer(codec_, (size_t)o, false);
|
||||
if (info.presentationTimeUs >= p.pts_us) break;
|
||||
}
|
||||
shown_ = shown_ || *frame;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
struct Capture {
|
||||
int fd = -1;
|
||||
uint8_t *data = nullptr;
|
||||
size_t size = 0;
|
||||
};
|
||||
|
||||
static void drop(Capture &c) {
|
||||
if (c.data) munmap(c.data, c.size);
|
||||
if (c.fd >= 0) close(c.fd);
|
||||
c = Capture();
|
||||
}
|
||||
|
||||
AMediaCodec *create(const vdec_config &c, bool low_latency) {
|
||||
const char *mime = c.codec == VENC_HEVC ? "video/hevc" : "video/avc";
|
||||
AMediaCodec *codec = AMediaCodec_createDecoderByType(mime);
|
||||
if (!codec) return nullptr;
|
||||
AMediaFormat *f = AMediaFormat_new();
|
||||
AMediaFormat_setString(f, AMEDIAFORMAT_KEY_MIME, mime);
|
||||
AMediaFormat_setInt32(f, AMEDIAFORMAT_KEY_WIDTH, (int32_t)c.width);
|
||||
AMediaFormat_setInt32(f, AMEDIAFORMAT_KEY_HEIGHT, (int32_t)c.height);
|
||||
AMediaFormat_setInt32(f, AMEDIAFORMAT_KEY_COLOR_FORMAT, kColorFormatYuv420SemiPlanar);
|
||||
AMediaFormat_setInt32(f, AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, (int32_t)c.bitstream_size);
|
||||
if (low_latency) {
|
||||
AMediaFormat_setInt32(f, "low-latency", 1);
|
||||
AMediaFormat_setInt32(f, "vendor.qti-ext-dec-low-latency.enable", 1);
|
||||
AMediaFormat_setInt32(f, "vendor.qti-ext-dec-picture-order.enable", 1);
|
||||
AMediaFormat_setInt32(f, "priority", 0);
|
||||
AMediaFormat_setFloat(f, "operating-rate", 120.0f);
|
||||
}
|
||||
media_status_t s = AMediaCodec_configure(codec, f, nullptr, nullptr, 0);
|
||||
AMediaFormat_delete(f);
|
||||
if (s == AMEDIA_OK) s = AMediaCodec_start(codec);
|
||||
if (s != AMEDIA_OK) {
|
||||
AMediaCodec_delete(codec);
|
||||
return nullptr;
|
||||
}
|
||||
return codec;
|
||||
}
|
||||
|
||||
void read_format() {
|
||||
AMediaFormat *f = AMediaCodec_getOutputFormat(codec_);
|
||||
if (!f) return;
|
||||
int32_t v;
|
||||
if (AMediaFormat_getInt32(f, AMEDIAFORMAT_KEY_WIDTH, &v) && v > 0) src_width_ = v;
|
||||
if (AMediaFormat_getInt32(f, AMEDIAFORMAT_KEY_HEIGHT, &v) && v > 0) src_height_ = v;
|
||||
src_stride_ = AMediaFormat_getInt32(f, "stride", &v) && v >= src_width_ ? v : src_width_;
|
||||
src_rows_ = AMediaFormat_getInt32(f, "slice-height", &v) && v >= src_height_ ? v : src_height_;
|
||||
src_planar_ = AMediaFormat_getInt32(f, AMEDIAFORMAT_KEY_COLOR_FORMAT, &v) && v == kColorFormatYuv420Planar;
|
||||
LOG("decoder output %dx%d stride %d rows %d %s", src_width_, src_height_, src_stride_, src_rows_,
|
||||
src_planar_ ? "planar" : "semi-planar");
|
||||
AMediaFormat_delete(f);
|
||||
}
|
||||
|
||||
bool copy(const uint8_t *src, size_t size, const Capture &cap) {
|
||||
size_t luma = (size_t)src_stride_ * src_rows_;
|
||||
size_t chroma = src_planar_ ? 2 * (size_t)(src_stride_ / 2) * (src_rows_ / 2) : (size_t)src_stride_ * (src_rows_ / 2);
|
||||
if (size < luma + chroma) return false;
|
||||
uint32_t w = std::min<uint32_t>((uint32_t)src_width_, config_.stride);
|
||||
uint32_t h = std::min<uint32_t>((uint32_t)src_height_, config_.rows);
|
||||
if ((size_t)config_.stride * config_.rows * 3 / 2 > cap.size) return false;
|
||||
DmaBufSync sync = {kDmaBufSyncStart | kDmaBufSyncWrite};
|
||||
ioctl(cap.fd, kDmaBufIoctlSync, &sync);
|
||||
uint8_t *y = cap.data, *uv = cap.data + (size_t)config_.stride * config_.rows;
|
||||
for (uint32_t row = 0; row < h; row++)
|
||||
memcpy(y + (size_t)row * config_.stride, src + (size_t)row * src_stride_, w);
|
||||
const uint8_t *su = src + luma;
|
||||
for (uint32_t row = 0; row < h / 2; row++) {
|
||||
uint8_t *d = uv + (size_t)row * config_.stride;
|
||||
if (!src_planar_) {
|
||||
memcpy(d, su + (size_t)row * src_stride_, w & ~1u);
|
||||
continue;
|
||||
}
|
||||
const uint8_t *u = su + (size_t)row * (src_stride_ / 2);
|
||||
const uint8_t *vv = su + (size_t)(src_stride_ / 2) * (src_rows_ / 2) + (size_t)row * (src_stride_ / 2);
|
||||
for (uint32_t x = 0; x < w / 2; x++) {
|
||||
d[2 * x] = u[x];
|
||||
d[2 * x + 1] = vv[x];
|
||||
}
|
||||
}
|
||||
sync.flags = kDmaBufSyncEnd | kDmaBufSyncWrite;
|
||||
ioctl(cap.fd, kDmaBufIoctlSync, &sync);
|
||||
return true;
|
||||
}
|
||||
|
||||
AMediaCodec *codec_ = nullptr;
|
||||
vdec_config config_ = {};
|
||||
const uint8_t *bitstream_ = nullptr;
|
||||
size_t bitstream_size_ = 0;
|
||||
Capture captures_[VENC_MAX_BUFFERS];
|
||||
int32_t src_width_ = 0, src_height_ = 0, src_stride_ = 0, src_rows_ = 0;
|
||||
bool src_planar_ = false;
|
||||
bool shown_ = false;
|
||||
};
|
||||
|
||||
class Session {
|
||||
public:
|
||||
explicit Session(int fd) : fd_(fd) {}
|
||||
@@ -85,6 +299,24 @@ public:
|
||||
memcpy(&b, buf, sizeof b);
|
||||
set_bitrate(b.bitrate);
|
||||
for (int i = 0; i < nfds; i++) close(fds[i]);
|
||||
} else if (type == VDEC_CONFIG && n == (ssize_t)sizeof(vdec_config)) {
|
||||
vdec_config c;
|
||||
memcpy(&c, buf, sizeof c);
|
||||
bool ok = nfds == 1 && decoder_.configure(c, fds[0]);
|
||||
for (int i = 0; i < nfds; i++) close(fds[i]);
|
||||
decoded(ok ? 0 : -1, false);
|
||||
} else if (type == VDEC_CAPTURE && n == (ssize_t)sizeof(vdec_capture)) {
|
||||
vdec_capture c;
|
||||
memcpy(&c, buf, sizeof c);
|
||||
if (nfds == 1) decoder_.capture(c, fds[0]);
|
||||
else for (int i = 0; i < nfds; i++) close(fds[i]);
|
||||
} else if (type == VDEC_PACKET && n == (ssize_t)sizeof(vdec_packet)) {
|
||||
vdec_packet p;
|
||||
memcpy(&p, buf, sizeof p);
|
||||
bool frame = false;
|
||||
bool ok = decoder_.decode(p, &frame);
|
||||
for (int i = 0; i < nfds; i++) close(fds[i]);
|
||||
decoded(ok ? 0 : -1, frame);
|
||||
} else {
|
||||
LOG("unexpected message %u of %zd bytes; closing", type, n);
|
||||
for (int i = 0; i < nfds; i++) close(fds[i]);
|
||||
@@ -117,7 +349,13 @@ private:
|
||||
send(fd_, &d, sizeof d, MSG_NOSIGNAL);
|
||||
}
|
||||
|
||||
void decoded(int32_t status, bool frame) {
|
||||
vdec_result r = {VDEC_RESULT, status, frame ? 1u : 0u};
|
||||
send(fd_, &r, sizeof r, MSG_NOSIGNAL);
|
||||
}
|
||||
|
||||
void release() {
|
||||
decoder_.release();
|
||||
if (codec_) {
|
||||
AMediaCodec_stop(codec_);
|
||||
AMediaCodec_delete(codec_);
|
||||
@@ -292,6 +530,7 @@ private:
|
||||
int32_t dst_stride_ = 0, dst_rows_ = 0;
|
||||
std::vector<Mapping> inputs_, outputs_;
|
||||
std::vector<uint8_t> header_;
|
||||
Decoder decoder_;
|
||||
};
|
||||
|
||||
std::atomic<int> g_listener{-1};
|
||||
|
||||
Reference in new issue
Block a user