feat: Rokid Max and Air glasses with 3DoF, SBS modes, 120 Hz and brightness (#1266)

This commit is contained in:
Kurt Himebauch authored and GitHub committed 2026-10-21 14:11:01 -04:00
1 parent e9891d5a29
commit d44c20f2bb
11 files changed
+427 -19

No files matched your search

@@ -50,6 +50,7 @@ data class GlassesState(
val noSignal: Boolean = false,
val sbsSupport: Int = SBS_UNKNOWN,
val sbs90: Boolean = false,
val rates2d: List<Int> = listOf(60, 90, 120),
val experimental: Boolean = false,
val supportsScreen3dof: Boolean = false,
val supports30Hz: Boolean = false,
@@ -613,12 +614,13 @@ object Glasses {
switching3d = modeState.pendingStereo,
brightnessMax = m?.brightnessMax ?: 8, volumeMax = m?.volumeMax ?: 8,
sunblock = if (m === XrealModel) it.sunblock else _settings.value.sunblock ?: it.sunblock,
hardwareControls = device?.vendorId == VitureProtocol.VENDOR_ID || m === XrealModel,
hardwareControls = device?.vendorId == VitureProtocol.VENDOR_ID || m === XrealModel || m is RokidModel,
hasSunblock = m?.hasSunblock == true,
diagonalFovDeg = m?.diagonalFovDeg ?: 46f,
tracking = it.tracking && link != null,
noSignal = noSignal,
sbs90 = m?.sbs90 == true,
rates2d = m?.rates2d ?: listOf(60, 90, 120),
experimental = m === XrealModel,
supportsScreen3dof = link?.hasImu == true,
supports30Hz = display?.let(::supports30Hz) == true,
@@ -634,7 +636,8 @@ object Glasses {
scheduleModeCheck()
}
internal fun isGlassesVendor(vid: Int) = vid == VitureProtocol.VENDOR_ID || RayNeoModel.isVendor(vid) || vid == XrealModel.VENDOR_ID
internal fun isGlassesVendor(vid: Int) = vid == VitureProtocol.VENDOR_ID || RayNeoModel.isVendor(vid) ||
vid == XrealModel.VENDOR_ID || vid == RokidModel.VENDOR_ID
internal fun supports30Hz(display: Display): Boolean {
val current = display.mode
@@ -654,11 +657,13 @@ object Glasses {
?: VitureProtocol.model(device.productId, name)
}
else if (XrealModel.matches(device.vendorId, device.productId)) XrealModel
else RayNeoModel.model(device.vendorId, device.productId, name)
else RokidModel.model(device.vendorId, device.productId, name)
?: RayNeoModel.model(device.vendorId, device.productId, name)
}
internal fun isGlassesDisplayName(name: String) =
name.contains("VITURE", ignoreCase = true) || RayNeoModel.isDisplay(name) || XrealModel.isDisplay(name)
name.contains("VITURE", ignoreCase = true) || RayNeoModel.isDisplay(name) || XrealModel.isDisplay(name) ||
RokidModel.isDisplay(name)
private fun findDisplay(usbPresent: Boolean): Display? {
return selectGlassesDisplay(displays.getDisplays(DisplayManager.DISPLAY_CATEGORY_PRESENTATION).toList(), usbPresent)
@@ -723,6 +728,8 @@ object Glasses {
} else VitureLink.open(usb, device, linkListener(generation))
} else if (model === XrealModel) {
XrealLink.open(app, xrealListener(generation))
} else if (model is RokidModel) {
RokidLink.open(usb, device, rokidListener(generation))
} else {
val baseKey = calibrationKey!!
val gtUsb = RayNeoProtocol.isGt(device.vendorId, device.productId)
@@ -790,6 +797,33 @@ object Glasses {
}
}
private fun rokidListener(generation: Long) = object : RokidLink.Listener {
override fun onReady() = post {
if (generation != linkGeneration) return@post
publish { it.copy(connected = true, controlError = null) }
updateImu()
applyMode(force = false)
}
override fun onValue(message: Int, value: Int) = post {
if (generation != linkGeneration) return@post
when (message) {
VitureProtocol.MSG_GET_DISPLAY -> glassesMode = value
VitureProtocol.MSG_BRIGHTNESS -> publish { it.copy(brightness = value) }
}
}
override fun onPose(raw: ByteArray, pose: FloatArray, timeNs: Long) {
if (generation != linkGeneration) return
imuPackets++
probe?.invoke(raw, pose)
if (simulating.get() == null) deliver(pose, timeNs)
}
override fun onLost() = post {
if (generation != linkGeneration) return@post
closeLink()
refresh()
}
}
private fun sdkListener(generation: Long) = object : VitureSdkLink.Listener {
override fun onSpatial(pose: FloatArray, timeNs: Long, valid: Boolean) {
if (generation == linkGeneration) SpatialTracking.accept(pose, timeNs, generation, valid)
@@ -8,6 +8,7 @@ interface GlassesModel {
val volumeMax: Int
val diagonalFovDeg: Float
val sbs90: Boolean
val rates2d: List<Int> get() = listOf(60, 90, 120)
val hasSunblock: Boolean get() = true
val resolutions: List<GlassesResolution> get() = emptyList()
fun mode(hz: Int, sbs: Boolean, resolution: GlassesResolution?): Int = mode(hz, sbs)
@@ -12,8 +12,8 @@ internal object GlassesRendering {
fun refreshRates(settings: GlassesSettings, state: GlassesState): List<Int> = buildList {
if (native30HzAvailable(settings, state)) add(30)
add(60)
if (!state.sbs || state.sbs90) add(90)
if (!state.sbs) add(120)
if (state.sbs) { if (state.sbs90) add(90) }
else addAll(state.rates2d.filter { it > 60 })
}
fun outputHz(settings: GlassesSettings, state: GlassesState): Int {
@@ -21,6 +21,7 @@ internal object GlassesRendering {
return when {
requested == 30 -> if (native30HzAvailable(settings, state)) 30 else 60
state.sbs && !state.sbs90 -> 60
!state.sbs && requested !in state.rates2d -> state.rates2d.filter { it <= requested }.maxOrNull() ?: 60
else -> requested
}
}
@@ -0,0 +1,148 @@
package com.droiddeck.launcher.glasses
import android.hardware.usb.UsbConstants
import android.hardware.usb.UsbDevice
import android.hardware.usb.UsbDeviceConnection
import android.hardware.usb.UsbEndpoint
import android.hardware.usb.UsbInterface
import android.hardware.usb.UsbManager
import android.os.SystemClock
import android.util.Log
/** Controls are vendor control transfers; the HID interrupt endpoint streams IMU reports unprompted. */
internal class RokidLink private constructor(
private val connection: UsbDeviceConnection,
private val usbInterface: UsbInterface,
private val endpoint: UsbEndpoint,
private val listener: Listener,
) : GlassesLink {
interface Listener {
fun onReady()
fun onValue(message: Int, value: Int)
fun onPose(raw: ByteArray, pose: FloatArray, timeNs: Long)
fun onLost()
}
@Volatile private var running = true
@Volatile override var ready = false
private set
@Volatile private var tracking = false
private val controlLock = Any()
private lateinit var worker: Thread
override val hasImu = true
override fun send(msgId: Int, payload: ByteArray): Boolean {
if (!running) return false
val value = payload.firstOrNull()?.toInt()?.and(0xff) ?: -1
return when (msgId) {
VitureProtocol.MSG_IMU -> (value in 0..1).also { if (it) tracking = value == 1 }
VitureProtocol.MSG_GET_DISPLAY -> display()?.let { listener.onValue(msgId, it) } != null
VitureProtocol.MSG_DISPLAY -> value in RokidModel.MODES &&
control(RokidModel.REQUEST_SET_DISPLAY, value, RokidModel.INDEX_DISPLAY) &&
true.also { listener.onValue(VitureProtocol.MSG_GET_DISPLAY, value) }
VitureProtocol.MSG_GET_BRIGHTNESS -> read(RokidModel.REQUEST_GET_BRIGHTNESS, RokidModel.INDEX_BRIGHTNESS)
?.let { listener.onValue(VitureProtocol.MSG_BRIGHTNESS, RokidModel.brightnessStep(it)) } != null
VitureProtocol.MSG_BRIGHTNESS -> value in 0..RokidModel.BRIGHTNESS_STEPS &&
control(RokidModel.REQUEST_SET_BRIGHTNESS, RokidModel.brightnessRaw(value), RokidModel.INDEX_BRIGHTNESS)
else -> false // Volume is the glasses' USB audio device; there is no film or IMU-rate control.
}
}
private fun display(): Int? = read(RokidModel.REQUEST_GET_DISPLAY, RokidModel.INDEX_DISPLAY)
private fun read(request: Int, index: Int): Int? = synchronized(controlLock) {
if (!running) return null
val buffer = ByteArray(64)
val count = connection.controlTransfer(VENDOR_IN, request, 0, index, buffer, buffer.size, TIMEOUT_MS)
// Display mode is reported in the second byte, brightness in the first.
val offset = if (request == RokidModel.REQUEST_GET_DISPLAY) 1 else 0
if (count > offset) buffer[offset].toInt() and 0xff else null
}
private fun control(request: Int, value: Int, index: Int): Boolean = synchronized(controlLock) {
running && connection.controlTransfer(VENDOR_OUT, request, value, index, byteArrayOf(1), 1, TIMEOUT_MS) >= 0
}
private fun run() {
val buffer = ByteArray(maxOf(endpoint.maxPacketSize, 64))
val parser = RokidProtocol.Parser()
var fusion = ImuFusion()
var wasTracking = false
var brightness = -1
var lastSensor = SystemClock.elapsedRealtime()
try {
ready = true
listener.onReady()
while (running) {
val count = connection.bulkTransfer(endpoint, buffer, buffer.size, 250)
val now = SystemClock.elapsedRealtime()
val sample = if (count > 0) parser.parse(buffer, count) else null
if (sample == null) {
if (now - lastSensor > 3000) error("Rokid IMU stream stalled")
continue
}
lastSensor = now
sample.brightness?.takeIf { it != brightness }?.let {
brightness = it
listener.onValue(VitureProtocol.MSG_BRIGHTNESS, RokidModel.brightnessStep(it))
}
val enabled = tracking
if (enabled && !wasTracking) fusion = ImuFusion()
wasTracking = enabled
if (!enabled) continue
// Max captures: already right/up/back; level gravity +Y, turn left +gyroY, look up +gyroX, roll right -gyroZ.
fusion.update(sample.timeNs, sample.accel, sample.gyro)?.let {
if (running && tracking) listener.onPose(buffer.copyOf(count), it, SystemClock.elapsedRealtimeNanos())
}
}
} catch (e: Exception) {
if (running) Log.w(TAG, "Rokid stream failed", e)
} finally {
ready = false
if (running) listener.onLost()
}
}
override fun close() {
synchronized(controlLock) {
if (!running) return
running = false
}
if (Thread.currentThread() !== worker) worker.join(500)
runCatching { connection.releaseInterface(usbInterface) }
connection.close()
}
companion object {
private const val TAG = "RokidLink"
private const val TIMEOUT_MS = 250
private const val VENDOR_IN = UsbConstants.USB_DIR_IN or UsbConstants.USB_TYPE_VENDOR
private const val VENDOR_OUT = UsbConstants.USB_DIR_OUT or UsbConstants.USB_TYPE_VENDOR
internal fun sensorEndpoint(device: UsbDevice): Pair<UsbInterface, UsbEndpoint>? {
for (i in 0 until device.interfaceCount) {
val iface = device.getInterface(i)
if (iface.interfaceClass != UsbConstants.USB_CLASS_HID) continue
for (e in 0 until iface.endpointCount) {
val endpoint = iface.getEndpoint(e)
if (endpoint.direction == UsbConstants.USB_DIR_IN &&
endpoint.type == UsbConstants.USB_ENDPOINT_XFER_INT) return iface to endpoint
}
}
return null
}
fun open(usb: UsbManager, device: UsbDevice, listener: Listener): RokidLink? {
val (iface, endpoint) = sensorEndpoint(device) ?: return null
val conn = usb.openDevice(device) ?: return null
if (!conn.claimInterface(iface, true)) {
conn.close()
return null
}
return RokidLink(conn, iface, endpoint, listener).also { link ->
link.worker = Thread({ link.run() }, "rokid-imu").apply { isDaemon = true }
link.worker.start()
}
}
}
}
@@ -0,0 +1,104 @@
package com.droiddeck.launcher.glasses
import java.nio.ByteBuffer
import java.nio.ByteOrder
/** Rokid Air/Max share one USB identity; the product string distinguishes them. */
internal class RokidModel private constructor(override val name: String, private val max: Boolean) : GlassesModel {
override val controllable = true
override val brightnessMax = BRIGHTNESS_STEPS
override val volumeMax = 0
override val hasSunblock = false
override val diagonalFovDeg = if (max) 50f else 43f
override val sbs90 = max
override val rates2d = if (max) listOf(60, 120) else listOf(60)
override val resolutions = if (max) listOf(GlassesResolution(1920, 1080), GlassesResolution(1920, 1200))
else listOf(GlassesResolution(1920, 1080))
override fun mode(hz: Int, sbs: Boolean): Int = mode(hz, sbs, null)
override fun mode(hz: Int, sbs: Boolean, resolution: GlassesResolution?): Int = when {
!max -> if (sbs) MODE_SBS_1080_60 else MODE_2D_60
sbs && resolution?.height == 1080 -> MODE_SBS_1080_60
sbs -> if (hz >= 90) MODE_SBS_1200_90 else MODE_SBS_1200_60
hz >= 120 -> MODE_2D_120
else -> MODE_2D_60
}
override fun modeForStep(step: Int): Int? = null
override fun equals(other: Any?) = other is RokidModel && other.name == name && other.max == max
override fun hashCode() = name.hashCode()
companion object {
const val VENDOR_ID = 0x04d2
const val PRODUCT_ID = 0x162f
const val BRIGHTNESS_STEPS = 10
// Vendor control requests, device recipient.
const val REQUEST_SET_DISPLAY = 0x01
const val REQUEST_GET_DISPLAY = 0x81
const val REQUEST_SET_BRIGHTNESS = 0x02
const val REQUEST_GET_BRIGHTNESS = 0x82
const val INDEX_DISPLAY = 0x01
const val INDEX_BRIGHTNESS = 0x02
const val MODE_2D_60 = 0
const val MODE_SBS_1080_60 = 1
const val MODE_2D_120 = 3
const val MODE_SBS_1200_90 = 4
const val MODE_SBS_1200_60 = 5
val MODES = setOf(MODE_2D_60, MODE_SBS_1080_60, MODE_2D_120, MODE_SBS_1200_90, MODE_SBS_1200_60)
// 0x04d2 is a shared vendor ID, so the Rokid product string is required too.
fun model(vid: Int, pid: Int, name: String?): GlassesModel? {
if (vid != VENDOR_ID || pid != PRODUCT_ID) return null
val product = name?.trim()?.takeIf { it.startsWith("Rokid", ignoreCase = true) } ?: return null
return RokidModel(product, product.contains("Max", ignoreCase = true))
}
fun isDisplay(name: String) = name.contains("Rokid", ignoreCase = true)
/** Device brightness is 0..100; the shared slider uses ten steps. */
fun brightnessStep(raw: Int): Int = ((raw.coerceIn(0, 100) + 5) / 10).coerceIn(0, BRIGHTNESS_STEPS)
fun brightnessRaw(step: Int): Int = step.coerceIn(0, BRIGHTNESS_STEPS) * 10
}
}
internal object RokidProtocol {
data class Sample(val timeNs: Long, val accel: Vec3, val gyro: Vec3, val brightness: Int?)
private const val COMBINED = 17
private const val SEPARATE = 4
private const val SENSOR_ACCEL = 1
private const val SENSOR_GYRO = 2
/** Max sends one combined report; Air sends accel and gyro reports that share a timestamp. */
class Parser {
private var accel: Vec3? = null
private var accelTime = -1L
private var gyro: Vec3? = null
private var gyroTime = -2L
fun parse(packet: ByteArray, length: Int = packet.size): Sample? {
if (length < 64) return null
val b = ByteBuffer.wrap(packet, 0, length).order(ByteOrder.LITTLE_ENDIAN)
return when (packet[0].toInt() and 0xff) {
COMBINED -> Sample(b.getLong(1), vec(b, 9), vec(b, 21), packet[59].toInt() and 0xff)
SEPARATE -> {
val time = b.getLong(9) * 1000 // Air reports microseconds; Max reports nanoseconds.
when (packet[1].toInt() and 0xff) {
SENSOR_ACCEL -> { accel = vec(b, 21); accelTime = time }
SENSOR_GYRO -> { gyro = vec(b, 21); gyroTime = time }
else -> return null
}
if (accelTime != gyroTime) return null
Sample(time, accel!!, gyro!!, null).also { accelTime = -1L; gyroTime = -2L }
}
else -> null
}
}
private fun vec(b: ByteBuffer, offset: Int) = Vec3(b.getFloat(offset), b.getFloat(offset + 4), b.getFloat(offset + 8))
}
}
@@ -4,20 +4,27 @@ import kotlin.math.abs
/** 1S radians/sec and device nanoseconds, mapped using directed hardware captures. */
internal class XrealOrientationFilter {
private val fusion = ImuFusion()
// 1S reports right/down/forward. Rotate 180 degrees about X into the
// shared right/up/back basis before either gravity alignment or fusion.
// Captures: level gravity -Y; looking left -gyroY; up +gyroX; right tilt +gyroZ.
fun update(sample: XrealProtocol.Sample): FloatArray? = fusion.update(sample.timeNs,
Vec3(sample.accel.x, -sample.accel.y, -sample.accel.z),
Vec3(sample.gyro.x, -sample.gyro.y, -sample.gyro.z))
}
/** Gyro integration with gravity correction in the shared right/up/back basis, rad/s and device ns. */
internal class ImuFusion {
private var previous = 0L
private var rotation = Quaternion.IDENTITY
private var initialized = false
private var bias = Vec3(0f, 0f, 0f)
fun update(sample: XrealProtocol.Sample): FloatArray? {
if (sample.timeNs <= previous) return null
val dt = if (previous == 0L) 0f else (sample.timeNs - previous) * 1e-9f
previous = sample.timeNs
// 1S reports right/down/forward. Rotate 180 degrees about X into the
// shared right/up/back basis before either gravity alignment or fusion.
// Captures: level gravity -Y; looking left -gyroY; up +gyroX; right tilt +gyroZ.
val accel = Vec3(sample.accel.x, -sample.accel.y, -sample.accel.z)
val gyro = Vec3(sample.gyro.x, -sample.gyro.y, -sample.gyro.z)
fun update(timeNs: Long, accel: Vec3, gyro: Vec3): FloatArray? {
if (timeNs <= previous) return null
val dt = if (previous == 0L) 0f else (timeNs - previous) * 1e-9f
previous = timeNs
val norm = accel.length()
val gravityValid = abs(norm - 9.81f) < 1.5f
if (dt > 0.25f) initialized = false // Never integrate a reconnect/stall as motion.