mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 21:06:09 +00:00
feat: Cardboard VR mode with ARM64 SteamVR (#704)
* feat: add Cardboard mode with side-by-side lens-distorted eyes and 3DoF head tracking * fix: derive the Cardboard field of view from the panel and recentre fully when far from upright * feat: share the Cardboard head pose with the guest and add a SteamVR driver for it * feat: run SteamVR's compositor on Turnip/KGSL with a display-extension layer and a driver-direct HMD * feat: show SteamVR frames in Cardboard mode through a driver-side presenter and a fake direct-mode display * feat: install and start SteamVR for Cardboard mode from the session drawer * docs: describe Cardboard mode and the SteamVR path * feat: drive SteamVR hands from the game pad and keep the Cardboard headset out of standby * docs: record where the SteamVR dashboard stops * fix: map Cardboard head tracking to the screen's axes and keep turns about the real vertical as yaw * fix: give Cardboard its previous picture back when SteamVR stops * fix: report sem_otime from emulated SysV semaphores so SteamVR's web helper can start * fix: bridge SteamVR's shared semaphores across processes and keep Steam alive while SteamVR runs * feat: present SteamVR straight to DroidDeck's compositor at panel resolution * docs: update Cardboard VR notes for the dashboard, semaphore bridge and direct presenter * fix: keep Wine and FEX signal handlers in the VR layer; add semaphore tracing * fix: clear a stale SteamVR active marker on status * fix: clear only an old SteamVR active marker * fix: unblock SteamVR clients on Turnip and keep Steam's web helper alive across SteamVR runs * fix: judge shared waits before a submit's own signals and on Vulkan 1.0 devices * docs: record VR games, SteamVR Home and OpenXR results on the Thor * docs: add Cardboard VR learnings from the Thor bring-up * docs: add Cardboard VR screenshots from the Thor * fix: keep the VR layer's display stand-in to SteamVR runs and clear a leftover marker at session start * fix: report SteamVR start and stop failures from the helper's exit code
This commit is contained in:
1 parent
92d038b428
commit
f4b0ab6867
57 files changed
+17799
-15
No files matched your search
@@ -0,0 +1,202 @@
|
||||
package com.droiddeck.launcher.gpu
|
||||
|
||||
import com.droiddeck.launcher.wayland.WaylandCompositor
|
||||
import org.json.JSONObject
|
||||
import kotlin.math.sqrt
|
||||
|
||||
/**
|
||||
* Cardboard mode: the compositor draws the session as two lens-distorted eye views side by side
|
||||
* (waylandcomp/src/cardboard.c), for a phone or handheld dropped into a Google Cardboard-style
|
||||
* viewer. Units are the menu's own:
|
||||
*
|
||||
* - [ipdMm] distance between the eyes, millimetres
|
||||
* - [screenWidthDm] the virtual screen's width, decimetres (40 = 4 m)
|
||||
* - [screenDistanceDm] how far away it hangs, decimetres (30 = 3 m)
|
||||
* - [viewer] a [Viewer] id: the lens it is drawn for
|
||||
*
|
||||
* [source] SCREEN shows the frame as a flat virtual screen, world-locked by the rotation sensor
|
||||
* when [headTracking] is on (3DoF), with real stereo depth from the IPD. SBS takes a frame that is
|
||||
* already a left/right pair (SteamVR, SBS 3D games and video) and only adds the lens.
|
||||
*/
|
||||
data class Cardboard(
|
||||
val enabled: Boolean = false,
|
||||
val source: Int = SOURCE_SCREEN,
|
||||
val viewer: String = Viewer.CARDBOARD_V2.id,
|
||||
val ipdMm: Int = 64,
|
||||
val screenWidthDm: Int = 40,
|
||||
val screenDistanceDm: Int = 30,
|
||||
val headTracking: Boolean = true,
|
||||
) {
|
||||
val lens: Viewer get() = Viewer.byId(viewer)
|
||||
|
||||
/**
|
||||
* Hands the settings to the compositor. [panelWidthMm] x [panelHeightMm] is the physical size of
|
||||
* the panel the eyes are drawn on (0 = unknown). The width decides how far each lens centre sits
|
||||
* from its viewport's centre, so the two pictures line up with the viewer's lenses; the height,
|
||||
* seen through the lens from its focal distance, is the field of view the distortion works in.
|
||||
*/
|
||||
fun push(panelWidthMm: Float, panelHeightMm: Float) {
|
||||
val l = lens
|
||||
WaylandCompositor.nativeSetCardboard(enabled, source, ipdMm / 1000f, screenDistanceDm / 10f,
|
||||
screenWidthDm / 10f, l.k1, l.k2, fovDeg(l, panelHeightMm), lensInset(panelWidthMm, l.lensSeparationMm), l.chroma)
|
||||
}
|
||||
|
||||
fun encode(): JSONObject = JSONObject()
|
||||
.put("enabled", enabled).put("source", source).put("viewer", viewer).put("ipdMm", ipdMm)
|
||||
.put("screenWidthDm", screenWidthDm).put("screenDistanceDm", screenDistanceDm).put("headTracking", headTracking)
|
||||
|
||||
/**
|
||||
* A lens the eyes can be drawn for: its distortion (Cardboard viewer profile coefficients, in
|
||||
* tangent units at the panel), how far it sits from the panel, and how far apart the two are.
|
||||
* [fallbackFovDeg] is the eye's field of view when the panel's size is unknown, or with no lens.
|
||||
*/
|
||||
enum class Viewer(val id: String, val k1: Float, val k2: Float, val screenToLensMm: Float,
|
||||
val lensSeparationMm: Float, val chroma: Float, val fallbackFovDeg: Float) {
|
||||
/** Google Cardboard (2015): the common plastic and cardboard viewers. */
|
||||
CARDBOARD_V2("cardboard2", 0.34f, 0.55f, 39.3f, 63.9f, 0.006f, 90f),
|
||||
/** The original 2014 Cardboard: narrower, gentler lenses. */
|
||||
CARDBOARD_V1("cardboard1", 0.441f, 0.156f, 42f, 60f, 0.004f, 80f),
|
||||
/** No lens: side-by-side 3D glasses, a 3D TV, or just looking at the stereo pair. */
|
||||
NONE("none", 0f, 0f, 0f, 0f, 0f, 60f);
|
||||
|
||||
companion object {
|
||||
fun byId(id: String): Viewer = entries.firstOrNull { it.id == id } ?: CARDBOARD_V2
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
const val SOURCE_SCREEN = 0
|
||||
const val SOURCE_SBS = 1
|
||||
|
||||
val OFF = Cardboard()
|
||||
|
||||
fun decode(json: JSONObject): Cardboard = Cardboard(
|
||||
enabled = json.optBoolean("enabled", false),
|
||||
source = if (json.optInt("source", SOURCE_SCREEN) == SOURCE_SBS) SOURCE_SBS else SOURCE_SCREEN,
|
||||
viewer = Viewer.byId(json.optString("viewer", Viewer.CARDBOARD_V2.id)).id,
|
||||
ipdMm = json.optInt("ipdMm", 64).coerceIn(50, 80),
|
||||
screenWidthDm = json.optInt("screenWidthDm", 40).coerceIn(5, 200),
|
||||
screenDistanceDm = json.optInt("screenDistanceDm", 30).coerceIn(5, 200),
|
||||
headTracking = json.optBoolean("headTracking", true),
|
||||
)
|
||||
|
||||
/**
|
||||
* One eye's vertical field of view on the panel side of the lens: half the panel's height
|
||||
* seen from the lens's focal distance. The lens then spreads it wider (the distortion).
|
||||
*/
|
||||
fun fovDeg(viewer: Viewer, panelHeightMm: Float): Float {
|
||||
if (panelHeightMm <= 0f || viewer.screenToLensMm <= 0f) return viewer.fallbackFovDeg
|
||||
val half = Math.toDegrees(kotlin.math.atan((panelHeightMm / 2f / viewer.screenToLensMm).toDouble())).toFloat()
|
||||
return (2f * half).coerceIn(30f, 140f)
|
||||
}
|
||||
|
||||
/**
|
||||
* The lens centre's inset towards the nose, as a fraction of one eye viewport's width. Each
|
||||
* eye gets half the panel, so its viewport centre is a quarter of the panel from the middle;
|
||||
* the lens centre is half the lens separation from it. 0 when either is unknown.
|
||||
*/
|
||||
fun lensInset(panelWidthMm: Float, lensSeparationMm: Float): Float {
|
||||
if (panelWidthMm <= 0f || lensSeparationMm <= 0f) return 0f
|
||||
val inset = (panelWidthMm / 4f - lensSeparationMm / 2f) / (panelWidthMm / 2f)
|
||||
return inset.coerceIn(-0.25f, 0.25f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Head pose maths for Cardboard mode, kept free of Android types so it is unit-tested on the JVM.
|
||||
* Matrices are row-major 3x3 FloatArrays. "World" is Android's sensor world (x east, y north,
|
||||
* z up); "screen" axes are x right, y up, z out of the panel towards the viewer's eyes.
|
||||
*/
|
||||
object HeadPose {
|
||||
val IDENTITY = floatArrayOf(1f, 0f, 0f, 0f, 1f, 0f, 0f, 0f, 1f)
|
||||
|
||||
/** Unit quaternion (x, y, z, w) -> rotation matrix: what SensorManager.getRotationMatrixFromVector gives. */
|
||||
fun fromQuaternion(x: Float, y: Float, z: Float, w: Float): FloatArray {
|
||||
val n = sqrt(x * x + y * y + z * z + w * w).takeIf { it > 1e-6f } ?: 1f
|
||||
val qx = x / n; val qy = y / n; val qz = z / n; val qw = w / n
|
||||
return floatArrayOf(
|
||||
1 - 2 * (qy * qy + qz * qz), 2 * (qx * qy - qz * qw), 2 * (qx * qz + qy * qw),
|
||||
2 * (qx * qy + qz * qw), 1 - 2 * (qx * qx + qz * qz), 2 * (qy * qz - qx * qw),
|
||||
2 * (qx * qz - qy * qw), 2 * (qy * qz + qx * qw), 1 - 2 * (qx * qx + qy * qy),
|
||||
)
|
||||
}
|
||||
|
||||
/** The rotation vector sensor's values -> (x, y, z, w), computing w when the sensor leaves it out. */
|
||||
fun quaternionFromRotationVector(v: FloatArray): FloatArray {
|
||||
val x = v[0]; val y = v[1]; val z = v[2]
|
||||
val w = if (v.size >= 4) v[3] else sqrt((1f - x * x - y * y - z * z).coerceAtLeast(0f))
|
||||
return floatArrayOf(x, y, z, w)
|
||||
}
|
||||
|
||||
/**
|
||||
* Device -> screen axes for a Display rotation (Surface.ROTATION_* as 0..3): the returned matrix
|
||||
* maps screen to world, given the device's own device -> world matrix. The same transform as
|
||||
* input/PadMotion's and SensorManager.remapCoordinateSystem's (ROTATION_90 = AXIS_Y, AXIS_MINUS_X:
|
||||
* the device's x is the screen's y, its y the screen's -x).
|
||||
*/
|
||||
fun toScreen(deviceToWorld: FloatArray, rotation: Int): FloatArray {
|
||||
// Columns of A: where each screen axis points in device coordinates.
|
||||
val a = when (rotation and 3) {
|
||||
1 -> floatArrayOf(0f, 1f, 0f, -1f, 0f, 0f, 0f, 0f, 1f) // screen x = -device y, y = device x
|
||||
2 -> floatArrayOf(-1f, 0f, 0f, 0f, -1f, 0f, 0f, 0f, 1f) // screen x = -device x, y = -device y
|
||||
3 -> floatArrayOf(0f, -1f, 0f, 1f, 0f, 0f, 0f, 0f, 1f) // screen x = device y, y = -device x
|
||||
else -> IDENTITY
|
||||
}
|
||||
return multiply(deviceToWorld, a)
|
||||
}
|
||||
|
||||
/** sin(60 degrees): a view further from level than this is "held flat" for recentring. */
|
||||
private const val FLAT_SIN = 0.866f
|
||||
|
||||
/**
|
||||
* What a recentre locks to. The pose the compositor gets is basis^T * screenToWorld * tilt:
|
||||
* - [basis]: a level frame (columns right, up, back in world) facing the recentred heading. Every
|
||||
* rotation is measured in it, so turning about the real vertical always pans the view;
|
||||
* - [tilt]: the device's own tilt at the recentre, taken out so that pose starts straight ahead.
|
||||
* Identity when the device was held upright: the horizon then stays the true one.
|
||||
*/
|
||||
class Reference(val basis: FloatArray, val tilt: FloatArray)
|
||||
|
||||
/**
|
||||
* Recentre on the current orientation. Held upright (the screen within 60 degrees of vertical, as
|
||||
* in a viewer), only the heading resets: forward is where the screen faces. Held flat (lying back,
|
||||
* or the device on a table or in the hands like a tray), forward is where the top of the screen
|
||||
* points and the tilt resets too; turning at the hips still pans, tipping the device still tilts.
|
||||
*/
|
||||
fun recenter(screenToWorld: FloatArray): Reference {
|
||||
val fz = -screenToWorld[8]
|
||||
val flat = kotlin.math.abs(fz) > FLAT_SIN
|
||||
// Upright: the screen's -z. Flat: the screen's +y (its top edge), which is level then.
|
||||
var fx = if (flat) screenToWorld[1] else -screenToWorld[2]
|
||||
var fy = if (flat) screenToWorld[4] else -screenToWorld[5]
|
||||
val len = sqrt(fx * fx + fy * fy).takeIf { it > 1e-4f } ?: run { fx = 0f; fy = 1f; 1f }
|
||||
fx /= len; fy /= len
|
||||
// right = forward x up(0,0,1) = (fy, -fx, 0); up = (0,0,1); back = -forward.
|
||||
val basis = floatArrayOf(
|
||||
fy, 0f, -fx,
|
||||
-fx, 0f, -fy,
|
||||
0f, 1f, 0f,
|
||||
)
|
||||
val tilt = if (flat) multiply(transpose(screenToWorld), basis) else IDENTITY
|
||||
return Reference(basis, tilt)
|
||||
}
|
||||
|
||||
/** head -> recentred world: what the compositor takes. */
|
||||
fun relative(ref: Reference, screenToWorld: FloatArray): FloatArray =
|
||||
multiply(multiply(transpose(ref.basis), screenToWorld), ref.tilt)
|
||||
|
||||
fun multiply(a: FloatArray, b: FloatArray): FloatArray = FloatArray(9) { i ->
|
||||
val r = i / 3; val c = i % 3
|
||||
a[r * 3] * b[c] + a[r * 3 + 1] * b[3 + c] + a[r * 3 + 2] * b[6 + c]
|
||||
}
|
||||
|
||||
fun transpose(m: FloatArray): FloatArray = floatArrayOf(m[0], m[3], m[6], m[1], m[4], m[7], m[2], m[5], m[8])
|
||||
|
||||
/** m * v for a 3-vector. */
|
||||
fun apply(m: FloatArray, v: FloatArray): FloatArray = floatArrayOf(
|
||||
m[0] * v[0] + m[1] * v[1] + m[2] * v[2],
|
||||
m[3] * v[0] + m[4] * v[1] + m[5] * v[2],
|
||||
m[6] * v[0] + m[7] * v[1] + m[8] * v[2],
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package com.droiddeck.launcher.gpu
|
||||
|
||||
import android.content.Context
|
||||
import android.hardware.Sensor
|
||||
import android.hardware.SensorEvent
|
||||
import android.hardware.SensorEventListener
|
||||
import android.hardware.SensorManager
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.util.Log
|
||||
import com.droiddeck.launcher.wayland.WaylandCompositor
|
||||
|
||||
/**
|
||||
* 3DoF head tracking for Cardboard mode: the rotation sensor -> the compositor's head pose
|
||||
* ([HeadPose] for the maths). Game rotation vector first (gyro + accelerometer: no compass jumps
|
||||
* indoors, and only the heading drifts, which a recentre fixes); the plain rotation vector where a
|
||||
* device has no game one. Events arrive on a thread of their own and go straight to native code,
|
||||
* so the session's UI thread never sits between the sensor and the screen.
|
||||
*
|
||||
* [rotation] is the Display rotation (Surface.ROTATION_*) the session is drawn in.
|
||||
*/
|
||||
class HeadTracker(context: Context, private val rotation: () -> Int) : SensorEventListener {
|
||||
private val sensors = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
|
||||
private val sensor: Sensor? = sensors.getDefaultSensor(Sensor.TYPE_GAME_ROTATION_VECTOR)
|
||||
?: sensors.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR)
|
||||
private var thread: HandlerThread? = null
|
||||
@Volatile private var reference: HeadPose.Reference? = null
|
||||
@Volatile private var recenterPending = true
|
||||
@Volatile private var latest: FloatArray? = null
|
||||
|
||||
/** Whether this device has a rotation sensor at all (no sensor = a fixed, head-locked screen). */
|
||||
val available: Boolean get() = sensor != null
|
||||
val sensorName: String get() = sensor?.let { "${it.name} (${if (it.type == Sensor.TYPE_GAME_ROTATION_VECTOR) "game rotation vector" else "rotation vector"})" } ?: "none"
|
||||
|
||||
fun start() {
|
||||
val s = sensor ?: run {
|
||||
WaylandCompositor.nativeLog("cardboard", "no rotation sensor: the virtual screen stays head-locked")
|
||||
WaylandCompositor.nativeSetCardboardPose(HeadPose.IDENTITY, false)
|
||||
return
|
||||
}
|
||||
if (thread != null) return
|
||||
recenterPending = true
|
||||
val t = HandlerThread("droiddeck-headtracker").also { it.start() }
|
||||
thread = t
|
||||
sensors.registerListener(this, s, SensorManager.SENSOR_DELAY_FASTEST, Handler(t.looper))
|
||||
WaylandCompositor.nativeLog("cardboard", "head tracking on: $sensorName")
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
val t = thread ?: return
|
||||
sensors.unregisterListener(this)
|
||||
t.quitSafely()
|
||||
thread = null
|
||||
WaylandCompositor.nativeSetCardboardPose(HeadPose.IDENTITY, false)
|
||||
}
|
||||
|
||||
/** Make where the viewer looks now straight ahead (see HeadPose.recenter). */
|
||||
fun recenter() {
|
||||
recenterPending = true
|
||||
latest?.let { apply(it) }
|
||||
}
|
||||
|
||||
override fun onSensorChanged(event: SensorEvent) {
|
||||
val q = HeadPose.quaternionFromRotationVector(event.values)
|
||||
val m = HeadPose.toScreen(HeadPose.fromQuaternion(q[0], q[1], q[2], q[3]), rotation())
|
||||
latest = m
|
||||
apply(m)
|
||||
}
|
||||
|
||||
private fun apply(screenToWorld: FloatArray) {
|
||||
val ref = reference.takeUnless { recenterPending } ?: HeadPose.recenter(screenToWorld).also {
|
||||
reference = it
|
||||
recenterPending = false
|
||||
}
|
||||
WaylandCompositor.nativeSetCardboardPose(HeadPose.relative(ref, screenToWorld), true)
|
||||
}
|
||||
|
||||
override fun onAccuracyChanged(sensor: Sensor?, accuracy: Int) {
|
||||
Log.d("DroidDeck", "head tracker accuracy $accuracy")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package com.droiddeck.launcher.gpu
|
||||
|
||||
import android.content.Context
|
||||
import com.droiddeck.launcher.agent.AgentGuest
|
||||
import org.json.JSONObject
|
||||
|
||||
/**
|
||||
* Cardboard mode's SteamVR (docs/cardboard-vr.md): Valve's ARM64 SteamVR in the guest, rendering
|
||||
* through the DroidDeck driver as a side-by-side pair that the Cardboard pass puts the lens on. Driven
|
||||
* through the guest agent's fixed "vr" request (droiddeck-steamvr), so it needs no agent commands.
|
||||
* Every call blocks on the guest: run them off the main thread.
|
||||
*/
|
||||
object SteamVr {
|
||||
enum class Status {
|
||||
/** No session, no agent, or a runtime without the helper or the driver. */
|
||||
UNAVAILABLE,
|
||||
NOT_INSTALLED,
|
||||
/** Downloading or unpacking (571 MB). */
|
||||
INSTALLING,
|
||||
READY,
|
||||
RUNNING,
|
||||
}
|
||||
|
||||
fun status(context: Context): Status = runCatching {
|
||||
parse(AgentGuest.call(context, JSONObject().put("kind", "vr").put("action", "status"), STATUS_TIMEOUT_MS))
|
||||
}.getOrDefault(Status.UNAVAILABLE)
|
||||
|
||||
/** install, start or stop; true when the guest did it. */
|
||||
fun run(context: Context, action: String): Boolean = runCatching {
|
||||
require(action in setOf("install", "start", "stop"))
|
||||
succeeded(action, AgentGuest.call(context, JSONObject().put("kind", "vr").put("action", action), ACTION_TIMEOUT_MS))
|
||||
}.getOrDefault(false)
|
||||
|
||||
/**
|
||||
* The agent answers ok whenever it ran the request, even when droiddeck-steamvr then failed; the
|
||||
* helper's own exit code says whether start or stop worked. install runs detached and only reports
|
||||
* that it started (its progress shows in status).
|
||||
*/
|
||||
fun succeeded(action: String, answer: JSONObject): Boolean = when {
|
||||
!answer.optBoolean("ok", false) -> false
|
||||
action == "install" -> answer.optBoolean("started", false)
|
||||
else -> answer.optInt("exitCode", -1) == 0
|
||||
}
|
||||
|
||||
/** The agent's status answer -> what the menu shows. */
|
||||
fun parse(answer: JSONObject): Status = when {
|
||||
!answer.optBoolean("ok", false) -> Status.UNAVAILABLE
|
||||
answer.optInt("exitCode", 0) != 0 -> Status.UNAVAILABLE
|
||||
answer.optBoolean("installing") -> Status.INSTALLING
|
||||
!answer.optBoolean("driver") -> Status.UNAVAILABLE
|
||||
!answer.optBoolean("installed") -> Status.NOT_INSTALLED
|
||||
answer.optBoolean("running") -> Status.RUNNING
|
||||
else -> Status.READY
|
||||
}
|
||||
|
||||
private const val STATUS_TIMEOUT_MS = 10_000L
|
||||
private const val ACTION_TIMEOUT_MS = 40_000L
|
||||
}
|
||||
Reference in new issue
Block a user