mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-18 12:28:40 +00:00
Merge remote-tracking branch 'upstream/main' into wallentx/game-debug-logging
This commit is contained in:
commit
cbd3d8c313
264 files changed
+24276
-1133
No files matched your search
@@ -16,10 +16,11 @@ class App : Application() {
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
// The restart screen must not initialize USB, guest processes, or another graphics context.
|
||||
val process = (getSystemService(ACTIVITY_SERVICE) as android.app.ActivityManager)
|
||||
.runningAppProcesses?.firstOrNull { it.pid == android.os.Process.myPid() }?.processName
|
||||
if (process == "$packageName:vr_recovery") return
|
||||
// Helper processes must not initialize the main USB provider or guest graphics context.
|
||||
val process = if (android.os.Build.VERSION.SDK_INT >= 28) Application.getProcessName()
|
||||
else (getSystemService(ACTIVITY_SERVICE) as android.app.ActivityManager)
|
||||
.runningAppProcesses?.firstOrNull { it.pid == android.os.Process.myPid() }?.processName
|
||||
if (process == "$packageName:vr_recovery" || process == "$packageName:rgb_diagnostic") return
|
||||
CrashHandler.install(this)
|
||||
com.droiddeck.launcher.session.QuestExperimental.initialize(this)
|
||||
com.droiddeck.launcher.session.SessionPrefs.settleGpuDriverMode(this)
|
||||
@@ -27,6 +28,7 @@ class App : Application() {
|
||||
com.droiddeck.launcher.core.DownloadSource.init(this)
|
||||
GpuClockPin.clearLeftover(this)
|
||||
com.droiddeck.launcher.glasses.Glasses.init(this)
|
||||
com.droiddeck.launcher.glasses.GlassesMenuMirror.initialize(this)
|
||||
BwrapSpawner.start(this)
|
||||
com.droiddeck.launcher.frontend.GameFileSync.start(this)
|
||||
com.droiddeck.launcher.stores.StoresState.init(this)
|
||||
|
||||
@@ -1209,7 +1209,9 @@ class MainActivity : ComponentActivity() {
|
||||
}
|
||||
|
||||
private fun playSteamVr() {
|
||||
if (com.droiddeck.launcher.handtracking.HandTracking.wanted(this)) placementCheck = true
|
||||
val tracking = com.droiddeck.launcher.glasses.Glasses.settings.value
|
||||
if (!com.droiddeck.launcher.glasses.Glasses.state.value.usesSpatialControllers(tracking) &&
|
||||
com.droiddeck.launcher.handtracking.HandTracking.wanted(this)) placementCheck = true
|
||||
else startSteamSession(steamVr = true)
|
||||
}
|
||||
|
||||
@@ -1929,6 +1931,9 @@ class MainActivity : ComponentActivity() {
|
||||
val scanGeneration = ++libraryScanGeneration
|
||||
val artRefreshesAtStart = HashMap(steamArtRefreshes)
|
||||
if (shortcutPicker) shortcutLibraryScanning = true
|
||||
if (SessionPrefs.steamClientArch(this) == "x86_64" && !OfflineMode.enabled(this)) {
|
||||
Thread({ com.droiddeck.launcher.session.FexCatalog.refresh(this) }, "fex-catalog").start()
|
||||
}
|
||||
Thread({
|
||||
val emus = Library.emulators(this) { id -> DesktopCatalog.installed(this, id) != null }
|
||||
ui.post { if (scanGeneration == libraryScanGeneration) emulatorList = emus }
|
||||
|
||||
@@ -56,6 +56,8 @@ import com.droiddeck.launcher.gpu.HeadTracker
|
||||
import com.droiddeck.launcher.glasses.Glasses
|
||||
import com.droiddeck.launcher.glasses.GlassesScreen
|
||||
import com.droiddeck.launcher.glasses.GlassesRendering
|
||||
import com.droiddeck.launcher.glasses.SpatialTracking
|
||||
import com.droiddeck.launcher.glasses.TrackingRecovery
|
||||
import kotlinx.coroutines.cancel
|
||||
import kotlinx.coroutines.launch
|
||||
import com.droiddeck.launcher.gpu.SteamVr
|
||||
@@ -306,11 +308,13 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
private var glassesSettings by mutableStateOf(Glasses.settings.value)
|
||||
private var glassesScreen: GlassesScreen? = null
|
||||
private var glassesOutputActive by mutableStateOf(false)
|
||||
private var trackingRecovery by mutableStateOf<TrackingRecovery?>(null)
|
||||
private var glasses3dForVr = false
|
||||
private var ended = false
|
||||
private var glassesPresent = false
|
||||
private var steamVrAutoTried = false
|
||||
private var steamVrInstallTried = false
|
||||
@Volatile private var steamVrRequestGeneration = 0L
|
||||
private var headTracking by mutableStateOf(false)
|
||||
private var glassesSlot by mutableStateOf(0)
|
||||
private var glassesKey: List<Any?> = emptyList()
|
||||
@@ -412,6 +416,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
} else if (intent.getBooleanExtra(EXTRA_STEAMVR, false) && SessionState.mode == SessionService.MODE_STEAM) {
|
||||
SessionState.steamVr = true
|
||||
}
|
||||
if (SessionState.steamVr) Glasses.prepareSteamVrStart()
|
||||
if (intent.action == SessionService.ACTION_AGENT_START) {
|
||||
SessionEvents.record("agent.start_requested", mapOf("mode" to SessionState.mode))
|
||||
}
|
||||
@@ -519,9 +524,25 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
sessionOverlay = createSessionOverlay()
|
||||
root.addView(sessionOverlay)
|
||||
setContentView(root)
|
||||
glassesScope.launch { Glasses.state.collect { glasses = it; onGlassesChanged() } }
|
||||
glassesScope.launch { Glasses.settings.collect { glassesSettings = it; onGlassesChanged() } }
|
||||
glassesScope.launch { Glasses.state.collect {
|
||||
glasses = it
|
||||
onGlassesChanged()
|
||||
updateJoyConTracking()
|
||||
updateHandCamera()
|
||||
} }
|
||||
glassesScope.launch { Glasses.settings.collect {
|
||||
glassesSettings = it
|
||||
onGlassesChanged()
|
||||
updateJoyConTracking()
|
||||
updateHandCamera()
|
||||
} }
|
||||
glassesScope.launch { com.droiddeck.launcher.handtracking.HandTracking.settings.collect { updateHandCamera() } }
|
||||
glassesScope.launch {
|
||||
while (true) {
|
||||
updateTrackingRecovery()
|
||||
kotlinx.coroutines.delay(100)
|
||||
}
|
||||
}
|
||||
glassesScope.launch {
|
||||
while (true) {
|
||||
if (SessionState.running && SessionState.mode == SessionService.MODE_STEAM &&
|
||||
@@ -564,6 +585,13 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
isFocusableInTouchMode = true
|
||||
defaultFocusHighlightEnabled = false
|
||||
setContent {
|
||||
androidx.compose.runtime.SideEffect {
|
||||
com.droiddeck.launcher.glasses.GlassesMenuMirror.setSessionMenuVisible(this@SessionActivity,
|
||||
(drawerOpen || achievementsOpen || pcKeyboardOpen) && !pipUi && steamVrStatus != SteamVr.Status.RUNNING)
|
||||
}
|
||||
androidx.compose.runtime.DisposableEffect(Unit) {
|
||||
onDispose { com.droiddeck.launcher.glasses.GlassesMenuMirror.setSessionMenuVisible(this@SessionActivity, false) }
|
||||
}
|
||||
androidx.compose.runtime.LaunchedEffect(drawerOpen, pcKeyboardOpen, achievementsOpen, pipUi) { updatePointerCapture() }
|
||||
androidx.compose.runtime.LaunchedEffect(picturePosition, SessionState.running, pipUi, cardboard.enabled) { applyGameAlignment() }
|
||||
DroidDeckTheme {
|
||||
@@ -581,9 +609,20 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
cursorImage, cursorHotX, cursorHotY, cursorImageScale)
|
||||
if (hud.text.isNotEmpty()) HudText(hud.text)
|
||||
if (glassesOutputActive && !loading.visible) com.droiddeck.launcher.ui.GlassesPlayingHint(glasses.model)
|
||||
if (trackingRecovery != null && !drawerOpen && !pcKeyboardOpen && !loading.visible) {
|
||||
androidx.compose.ui.viewinterop.AndroidView(
|
||||
factory = { com.droiddeck.launcher.ui.TrackingRecoveryView(it) },
|
||||
modifier = Modifier.fillMaxSize(),
|
||||
update = { it.update(trackingRecovery, !glassesOutputActive && glasses.usable && glasses.sbs) },
|
||||
)
|
||||
}
|
||||
if (GlassesRendering.usesGlasses(glasses, glassesOutputActive, steamVrStatus == SteamVr.Status.RUNNING) &&
|
||||
headTracking && !drawerOpen && !pcKeyboardOpen && !loading.visible) com.droiddeck.launcher.ui.GlassesRecenterButton(
|
||||
controllerSettings?.tint ?: android.graphics.Color.rgb(90, 160, 255), controllerSettings?.opacity ?: 100) { headTracker?.recenter() }
|
||||
controllerSettings?.tint ?: android.graphics.Color.rgb(90, 160, 255), controllerSettings?.opacity ?: 100,
|
||||
onRecenterJoyCons = if (steamVrStatus == SteamVr.Status.RUNNING &&
|
||||
glasses.usesSpatialControllers(glassesSettings) && glassesSettings.joyConTracking)
|
||||
joyConVr?.let { { it.recenter() } } else null,
|
||||
) { headTracker?.recenter() }
|
||||
if (glasses.present && !drawerOpen && !pcKeyboardOpen && !loading.visible) com.droiddeck.launcher.ui.GlassesCornerButton(glassesSlot,
|
||||
controllerSettings?.tint ?: android.graphics.Color.rgb(90, 160, 255), controllerSettings?.opacity ?: 100) {
|
||||
drawerPage = com.droiddeck.launcher.ui.SessionDrawerPage.GLASSES
|
||||
@@ -621,10 +660,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
steamVrSwitchedPicture, glassesVr, steamVrRequested) || (drawerOpen && drawerPage == SessionDrawerPage.VR)
|
||||
LaunchedEffect(pollSteamVr) {
|
||||
while (pollSteamVr) {
|
||||
val request = steamVrRequestGeneration
|
||||
val session = SessionState.sessionId
|
||||
val now = kotlinx.coroutines.withContext(kotlinx.coroutines.Dispatchers.IO) {
|
||||
SteamVr.status(this@SessionActivity)
|
||||
}
|
||||
onSteamVrStatus(now)
|
||||
if (request == steamVrRequestGeneration && session == SessionState.sessionId) onSteamVrStatus(now)
|
||||
kotlinx.coroutines.delay(3000)
|
||||
}
|
||||
}
|
||||
@@ -661,7 +702,10 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
onCardboardRecenter = { headTracker?.recenter() },
|
||||
glassesPresent = glasses.present,
|
||||
glassesOutput = glassesOutputActive,
|
||||
onGlassesRecenter = { headTracker?.recenter() },
|
||||
onGlassesRecenter = {
|
||||
headTracker?.recenter()
|
||||
com.droiddeck.launcher.glasses.GlassesMenuMirror.recenter()
|
||||
},
|
||||
steamVr = steamVrStatus,
|
||||
immersiveSteamVr = com.droiddeck.launcher.session.QuestExperimental.isSupported(),
|
||||
onSteamVr = { action -> steamVrAction(action) },
|
||||
@@ -1426,6 +1470,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
activeCardboard = effectiveCardboard(SessionPrefs.cardboard(this))
|
||||
pushCardboard(activeCardboard)
|
||||
WaylandCompositor.nativeSetCardboardPoseFile(File(filesDir, ".wayland-rt/droiddeck-headpose").path)
|
||||
com.droiddeck.launcher.glasses.SpatialTracking.attachPublisher(File(filesDir, ".wayland-rt/droiddeck-spatialpose").path)
|
||||
if (!SessionState.running) {
|
||||
val waylandLog = File(SessionPaths.beginOrCurrent(this), "wayland.log")
|
||||
WaylandCompositor.setSessionLogFile(waylandLog)
|
||||
@@ -1646,6 +1691,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
|
||||
private fun onSessionEnded(status: Int) {
|
||||
ended = true
|
||||
++steamVrRequestGeneration
|
||||
if (isInPictureInPictureMode) {
|
||||
runOnUiThread { finish() }
|
||||
return
|
||||
@@ -2677,19 +2723,27 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
startActivity(android.content.Intent(this, QuestXrActivity::class.java))
|
||||
return
|
||||
}
|
||||
val request = ++steamVrRequestGeneration
|
||||
val session = SessionState.sessionId
|
||||
fun currentRequest() = request == steamVrRequestGeneration && session == SessionState.sessionId && !isDestroyed && !ended
|
||||
if (action == "start") {
|
||||
SessionState.steamVr = true
|
||||
steamVrAutoTried = true
|
||||
}
|
||||
if (action == "stop") SessionState.steamVr = false
|
||||
if (action == "stop") {
|
||||
SessionState.steamVr = false
|
||||
Glasses.cancelSteamVrStart()
|
||||
}
|
||||
if (glasses.present) {
|
||||
// Optional stereo starts only after RUNNING is observed. READY can
|
||||
// arrive repeatedly while the launch worker is still starting VR.
|
||||
if (action == "install") steamVrStatus = SteamVr.Status.INSTALLING
|
||||
steamVrWorker.execute {
|
||||
runSteamVrAction(action, ::currentRequest) {
|
||||
val ok = SteamVr.run(this, action)
|
||||
val now = SteamVr.status(this)
|
||||
uiHandler.post {
|
||||
if (!currentRequest()) return@post
|
||||
if (!ok && now != SteamVr.Status.RUNNING) Glasses.cancelSteamVrStart()
|
||||
onSteamVrStatus(now)
|
||||
if (!ok) Toast.makeText(this, getString(R.string.cardboard_steamvr_failed), Toast.LENGTH_LONG).show()
|
||||
}
|
||||
@@ -2702,10 +2756,12 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
}
|
||||
if (action == "stop") restoreSteamVrPicture()
|
||||
if (action == "install") steamVrStatus = SteamVr.Status.INSTALLING
|
||||
steamVrWorker.execute {
|
||||
runSteamVrAction(action, ::currentRequest) {
|
||||
val ok = SteamVr.run(this, action)
|
||||
val now = SteamVr.status(this)
|
||||
uiHandler.post {
|
||||
if (!currentRequest()) return@post
|
||||
if (!ok && now != SteamVr.Status.RUNNING) Glasses.cancelSteamVrStart()
|
||||
if (!ok && action == "start" && now != SteamVr.Status.RUNNING) restoreSteamVrPicture()
|
||||
onSteamVrStatus(now)
|
||||
if (!ok) Toast.makeText(this, getString(R.string.cardboard_steamvr_failed), Toast.LENGTH_LONG).show()
|
||||
@@ -2713,6 +2769,20 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
}
|
||||
}
|
||||
|
||||
private fun runSteamVrAction(action: String, currentRequest: () -> Boolean, run: () -> Unit) {
|
||||
if (action != "start") {
|
||||
steamVrWorker.execute(run)
|
||||
return
|
||||
}
|
||||
Glasses.prepareSteamVrStart {
|
||||
uiHandler.post {
|
||||
if (SessionState.steamVr && currentRequest()) steamVrWorker.execute {
|
||||
if (SessionState.steamVr && currentRequest()) run()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun switchPictureForSteamVr() {
|
||||
if (cardboard.source != Cardboard.SOURCE_SBS) {
|
||||
SessionPrefs.setCardboardSourceBeforeSteamVr(this, cardboard.source)
|
||||
@@ -2723,10 +2793,21 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
steamVrSwitchedPicture = SessionPrefs.cardboardSourceBeforeSteamVr(this) >= 0
|
||||
}
|
||||
|
||||
private fun usesSpatialControllers() =
|
||||
GlassesRendering.usesGlasses(glasses, glassesScreen != null, steamVrStatus == SteamVr.Status.RUNNING) &&
|
||||
glasses.usesSpatialControllers(glassesSettings)
|
||||
|
||||
private fun updateJoyConTracking() {
|
||||
joyConVr?.setActive(steamVrStatus == SteamVr.Status.RUNNING &&
|
||||
(ControllerPrefs.joyConVr(this) ||
|
||||
(usesSpatialControllers() && glassesSettings.joyConTracking)))
|
||||
}
|
||||
|
||||
private fun updateHandCamera() {
|
||||
val joyCons = joyConVr?.holding == true
|
||||
val bare = !joyCons && !controllerConnected()
|
||||
val want = started && steamVrStatus == SteamVr.Status.RUNNING && (joyCons || bare) &&
|
||||
!usesSpatialControllers() &&
|
||||
com.droiddeck.launcher.handtracking.HandTracking.wanted(this)
|
||||
handCamera?.bareHands = bare
|
||||
if (want == (handCamera != null)) return
|
||||
@@ -2748,12 +2829,14 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
}
|
||||
|
||||
private fun onSteamVrStatus(now: SteamVr.Status) {
|
||||
if (ended || isDestroyed) return
|
||||
val was = steamVrStatus
|
||||
steamVrStatus = now
|
||||
joyConVr?.setActive(now == SteamVr.Status.RUNNING && ControllerPrefs.joyConVr(this))
|
||||
updateJoyConTracking()
|
||||
updateHandCamera()
|
||||
autoStartSteamVr(now)
|
||||
if ((was == SteamVr.Status.RUNNING) != (now == SteamVr.Status.RUNNING)) applyCardboard(cardboard)
|
||||
if (now != SteamVr.Status.UNAVAILABLE) Glasses.setVrActive(now == SteamVr.Status.RUNNING)
|
||||
if (glasses.present) onGlassesSteamVr(now)
|
||||
if (GlassesRendering.usesGlasses(glasses, glassesScreen != null, now == SteamVr.Status.RUNNING)) {
|
||||
return
|
||||
@@ -2809,15 +2892,17 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
private fun updateHeadTracker() {
|
||||
val onGlasses = GlassesRendering.usesGlasses(glasses, glassesScreen != null,
|
||||
steamVrStatus == SteamVr.Status.RUNNING)
|
||||
SpatialTracking.setOutputUsesGlasses(onGlasses && !ended)
|
||||
val c = activeCardboard
|
||||
val centerView = onGlasses && c.source == Cardboard.SOURCE_SCREEN
|
||||
val want = !ended && (resumed || (glassesScreen != null && started)) && c.enabled && c.headTracking
|
||||
val current = headTracker
|
||||
if (current != null && (!want || current.usesGlasses != onGlasses)) {
|
||||
if (current != null && (!want || current.usesGlasses != onGlasses || current.centerView != centerView)) {
|
||||
current.stop()
|
||||
headTracker = null
|
||||
}
|
||||
if (want) {
|
||||
val t = headTracker ?: HeadTracker(this, glasses = onGlasses) {
|
||||
val t = headTracker ?: HeadTracker(this, glasses = onGlasses, centerView = centerView) {
|
||||
@Suppress("DEPRECATION")
|
||||
(if (Build.VERSION.SDK_INT >= 30) display else windowManager.defaultDisplay)?.rotation ?: android.view.Surface.ROTATION_0
|
||||
}.also { headTracker = it }
|
||||
@@ -2826,6 +2911,18 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
headTracking = headTracker != null
|
||||
}
|
||||
|
||||
private fun updateTrackingRecovery() {
|
||||
val onGlasses = GlassesRendering.usesGlasses(glasses, glassesScreen != null,
|
||||
steamVrStatus == SteamVr.Status.RUNNING)
|
||||
val spatialView = onGlasses && SpatialTracking.selected && glassesSettings.positionalTracking &&
|
||||
(SessionState.steamVr || steamVrStatus == SteamVr.Status.RUNNING ||
|
||||
(glassesOutputActive && glassesSettings.headTracking && glassesSettings.picture3d == com.droiddeck.launcher.glasses.GlassesSettings.PICTURE_SCREEN))
|
||||
val active = started && !ended && SessionState.running && !loading.visible && !pipUi && spatialView
|
||||
if (active) SpatialTracking.refreshStatus()
|
||||
trackingRecovery = TrackingRecovery.forSession(active, glasses.canControl, SpatialTracking.status.value)
|
||||
glassesScreen?.trackingRecovery?.update(trackingRecovery, glasses.sbs)
|
||||
}
|
||||
|
||||
private fun outputHolder(): SurfaceHolder = glassesScreen?.holder ?: surfaceView.holder
|
||||
|
||||
private fun onOutputAttached(holder: SurfaceHolder) {
|
||||
@@ -2864,7 +2961,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
private fun onGlassesChanged() {
|
||||
val g = glasses
|
||||
val s = glassesSettings
|
||||
val key = listOf(g.present, g.usable, g.displayId, g.sbs, g.sbs90, g.switching3d, g.canControl, g.width, g.height, g.hz, s)
|
||||
val key = listOf(g.present, g.usable, g.displayId, g.sbs, g.sbs90, g.supports30Hz, g.switching3d, g.canControl, g.width, g.height, g.hz, g.resolutions, g.vrActive, s)
|
||||
if (key == glassesKey) return
|
||||
glassesKey = key
|
||||
updateGlassesScreen()
|
||||
@@ -2898,7 +2995,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
val target = if (want) Glasses.glassesDisplay()?.takeIf { it.isValid } else null
|
||||
val current = glassesScreen
|
||||
if (current != null && current.display.displayId == target?.displayId) {
|
||||
current.prefer(Glasses.outputHz())
|
||||
current.prefer(Glasses.outputHz(), Glasses.outputResolution())
|
||||
applyOutputPacing()
|
||||
return
|
||||
}
|
||||
@@ -2907,7 +3004,7 @@ class SessionActivity : ComponentActivity(), SurfaceHolder.Callback {
|
||||
runCatching { current.dismiss() }
|
||||
}
|
||||
if (target != null) {
|
||||
val screen = GlassesScreen(this, target, Glasses.outputHz(), this)
|
||||
val screen = GlassesScreen(this, target, Glasses.outputHz(), this, Glasses.outputResolution())
|
||||
// show() can create the surface synchronously. Select its holder
|
||||
// first, otherwise surfaceCreated() rejects the glasses callback.
|
||||
glassesScreen = screen
|
||||
|
||||
@@ -108,15 +108,19 @@ class AgentBridgeProvider : ContentProvider() {
|
||||
val value = request.optString("value")
|
||||
fun on() = value == "1" || value.equals("true", true) || value.equals("on", true)
|
||||
val number = value.toIntOrNull()
|
||||
val numeric = setOf("refreshHz", "sbsHz", "brightness", "volume", "picture3d", "screenWidthDm", "screenDistanceDm", "vrFovDeg", "sbsSupport")
|
||||
val numeric = setOf("refreshHz", "sbsHz", "brightness", "volume", "picture3d", "screenWidthDm", "screenDistanceDm", "vrFovDeg", "sbsSupport", "eyeHeightCm")
|
||||
if (key in numeric && number == null) return error("INVALID_VALUE", "'$key' takes a number")
|
||||
when (key) {
|
||||
"refreshHz" -> g.setRefreshHz(number!!)
|
||||
"sbsHz" -> g.setSbsHz(number!!)
|
||||
"resolution2d" -> g.setResolution(com.droiddeck.launcher.glasses.GlassesDisplayProfile.TWO_D, value)
|
||||
"resolution3d" -> g.setResolution(com.droiddeck.launcher.glasses.GlassesDisplayProfile.THREE_D, value)
|
||||
"resolutionVr" -> g.setResolution(com.droiddeck.launcher.glasses.GlassesDisplayProfile.VR, value)
|
||||
"brightness" -> g.setBrightness(number!!)
|
||||
"volume" -> g.setVolume(number!!)
|
||||
"sunblock" -> g.setSunblock(on())
|
||||
"3d" -> g.set3d(on())
|
||||
"3dof" -> g.set3dof(on())
|
||||
"gameOnGlasses" -> g.setGameOnGlasses(on())
|
||||
"headTracking" -> g.setHeadTracking(on())
|
||||
"auto3d" -> g.setAuto3d(on())
|
||||
@@ -125,6 +129,10 @@ class AgentBridgeProvider : ContentProvider() {
|
||||
"screenDistanceDm" -> g.setScreenDistanceDm(number!!)
|
||||
"vrFovDeg" -> g.setVrFovDeg(number!!)
|
||||
"sbsSupport" -> g.setSbsSupport(number!!)
|
||||
"positionalTracking" -> g.setPositionalTracking(on())
|
||||
"handControllers" -> g.setHandControllers(on())
|
||||
"joyConTracking" -> g.setJoyConTracking(on())
|
||||
"eyeHeightCm" -> g.setEyeHeight(number!!)
|
||||
else -> return error("INVALID_KEY", "Unknown glasses setting '$key'")
|
||||
}
|
||||
}
|
||||
@@ -136,6 +144,34 @@ class AgentBridgeProvider : ContentProvider() {
|
||||
"calibrate" -> g.startCalibration()
|
||||
"cancel-calibration" -> g.cancelCalibration()
|
||||
"reset-calibration" -> g.resetCalibration()
|
||||
"spatial-diagnostics" -> com.droiddeck.launcher.glasses.SpatialTracking.diagnostics(request.optBoolean("enabled",true))
|
||||
"spatial-recenter" -> com.droiddeck.launcher.glasses.SpatialTracking.recenter(request.optBoolean("screen",false))
|
||||
"spatial-floor" -> g.setSpatialFloor()
|
||||
"spatial-restart" -> g.restartSpatialTracking()
|
||||
"spatial-hand-options" -> {
|
||||
AgentAccess.requireCommands(context)
|
||||
if(request.has("lite")) com.droiddeck.launcher.glasses.SpatialTracking.setHandModel(request.getBoolean("lite"))
|
||||
if(request.has("parallel")) com.droiddeck.launcher.glasses.SpatialTracking.setHandCpuParallel(request.getBoolean("parallel"))
|
||||
if(request.has("halfRate")) com.droiddeck.launcher.glasses.SpatialTracking.setHandHalfRate(request.getBoolean("halfRate"))
|
||||
if(request.has("adaptive")) com.droiddeck.launcher.glasses.SpatialTracking.setHandAdaptiveInput(request.getBoolean("adaptive"))
|
||||
if(request.has("scheduled")) com.droiddeck.launcher.glasses.SpatialTracking.setHandScheduledTracking(request.getBoolean("scheduled"))
|
||||
}
|
||||
"spatial-hand-benchmark" -> {
|
||||
AgentAccess.requireCommands(context)
|
||||
com.droiddeck.launcher.glasses.SpatialTracking.benchmarkSavedCapture(request.getString("capture"))
|
||||
}
|
||||
"spatial-hand-capture" -> {
|
||||
AgentAccess.requireCommands(context)
|
||||
com.droiddeck.launcher.glasses.SpatialTracking.captureDiagnostic()
|
||||
}
|
||||
"spatial-rgb-capture", "spatial-rgb-status" -> {
|
||||
AgentAccess.requireCommands(context)
|
||||
return (if (request.getString("op") == "spatial-rgb-capture")
|
||||
com.droiddeck.launcher.glasses.LumaRgbDiagnostic.capture(context!!,request.optString("transport","sdk"),
|
||||
request.optInt("width",640),request.optInt("height",480),request.optInt("fps",30),
|
||||
request.optInt("durationMs",5000))
|
||||
else com.droiddeck.launcher.glasses.LumaRgbDiagnostic.status(context!!)).put("ok", true)
|
||||
}
|
||||
"query" -> { g.query(); Thread.sleep(600) }
|
||||
"mode" -> {
|
||||
val mode = hex(request.optString("mode"), 0xFF) ?: return error("INVALID_VALUE", "mode must be one hex byte, e.g. 0x31")
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.content.Context
|
||||
import com.droiddeck.launcher.R
|
||||
import kotlin.math.abs
|
||||
|
||||
internal data class CalibrationDistanceResult(val accepted: Boolean, val message: String)
|
||||
|
||||
internal object CalibrationDistanceCheck {
|
||||
fun evaluate(context: Context, expected: Float, corners: Int, points: List<SpatialVector>?): CalibrationDistanceResult {
|
||||
require(expected.isFinite() && expected in 0.15f..1.5f)
|
||||
if (corners != 24) return CalibrationDistanceResult(false,
|
||||
context.getString(R.string.spatial_distance_corners, corners))
|
||||
if (points?.size != 24 || points.any { !it.finite() }) return CalibrationDistanceResult(false,
|
||||
context.getString(R.string.spatial_distance_depth))
|
||||
val measured=(points.reduce { a,b -> a+b }*(1f/points.size)).length()
|
||||
if (!measured.isFinite()) return CalibrationDistanceResult(false,context.getString(R.string.spatial_distance_invalid))
|
||||
val accepted=abs(measured-expected)<0.05f
|
||||
return CalibrationDistanceResult(accepted,
|
||||
context.getString(R.string.spatial_distance_result,
|
||||
SpatialMeasurements.validation(measured), SpatialMeasurements.validation(expected),
|
||||
context.getString(if (accepted) R.string.spatial_distance_passed else R.string.spatial_distance_retry)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import org.json.JSONObject
|
||||
|
||||
internal data class CalibrationObservationData(
|
||||
val objects: List<DoubleArray>, val left: List<DoubleArray>, val right: List<DoubleArray>, val head: SpatialPose,
|
||||
) {
|
||||
companion object {
|
||||
fun parse(view: JSONObject): CalibrationObservationData {
|
||||
fun points(name: String, dimensions: Int): List<DoubleArray> {
|
||||
val rows=view.getJSONArray(name)
|
||||
require(rows.length() in 12..24) { "Invalid saved board corner count" }
|
||||
return (0 until rows.length()).map { row ->
|
||||
val values=rows.getJSONArray(row)
|
||||
require(values.length()==dimensions)
|
||||
DoubleArray(dimensions) { values.getDouble(it).also { value -> require(value.isFinite()) } }
|
||||
}
|
||||
}
|
||||
val objects=points("objects",3); val left=points("left",2); val right=points("right",2)
|
||||
require(objects.size==left.size && objects.size==right.size)
|
||||
val pose=view.getJSONArray("head")
|
||||
require(pose.length()==7)
|
||||
val head=SpatialPose.fromSdk(FloatArray(7) { pose.getDouble(it).toFloat() }) ?: error("Invalid saved head pose")
|
||||
return CalibrationObservationData(objects,left,right,head)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -51,10 +51,18 @@ data class GlassesState(
|
||||
val sbsSupport: Int = SBS_UNKNOWN,
|
||||
val sbs90: Boolean = false,
|
||||
val experimental: Boolean = false,
|
||||
val supportsScreen3dof: Boolean = false,
|
||||
val supports30Hz: Boolean = false,
|
||||
val supportsSpatial: Boolean = false,
|
||||
val resolutions: List<GlassesResolution> = emptyList(),
|
||||
val vrActive: Boolean = false,
|
||||
) {
|
||||
val hasDisplay: Boolean get() = displayId != Display.INVALID_DISPLAY
|
||||
val usable: Boolean get() = hasDisplay && !noSignal
|
||||
val canControl: Boolean get() = connected && controllable
|
||||
val spatialConnected: Boolean get() = canControl && supportsSpatial
|
||||
fun usesSpatialControllers(settings: GlassesSettings) = spatialConnected && settings.positionalTracking &&
|
||||
(settings.handControllers || settings.joyConTracking)
|
||||
val can3d: Boolean get() = canControl && !noSignal && sbsSupport != SBS_FAILS
|
||||
val verticalFovDeg: Float get() {
|
||||
val half = Math.toRadians(diagonalFovDeg / 2.0)
|
||||
@@ -74,6 +82,9 @@ data class GlassesState(
|
||||
data class GlassesSettings(
|
||||
val refreshHz: Int = 120,
|
||||
val sbsHz: Int = 60,
|
||||
val resolution2d: String = "",
|
||||
val resolution3d: String = "",
|
||||
val resolutionVr: String = "",
|
||||
val sunblock: Boolean? = null,
|
||||
val gameOnGlasses: Boolean = true,
|
||||
val headTracking: Boolean = true,
|
||||
@@ -82,7 +93,14 @@ data class GlassesSettings(
|
||||
val screenWidthDm: Int = 20,
|
||||
val screenDistanceDm: Int = 30,
|
||||
val vrFovDeg: Int = VR_FOV_DEFAULT,
|
||||
val positionalTracking: Boolean = false,
|
||||
val handControllers: Boolean = false,
|
||||
val joyConTracking: Boolean = false,
|
||||
val steadyHandAim: Boolean = false,
|
||||
val eyeHeightCm: Int = 160,
|
||||
) {
|
||||
val screen3dof: Boolean get() = picture3d == PICTURE_SCREEN && headTracking
|
||||
|
||||
companion object {
|
||||
const val VR_FOV_MIN = 24
|
||||
const val VR_FOV_MAX = 90
|
||||
@@ -129,10 +147,13 @@ object Glasses {
|
||||
private var lastModeSentAt = 0L
|
||||
private val modeState = GlassesModeState(SBS_CONFIRM_MS)
|
||||
private val confirmMode = Runnable { refresh() }
|
||||
private val steamVrSpatialGate = SteamVrSpatialGate()
|
||||
private val resumeSteamVrSpatial = Runnable { updateSpatialMode() }
|
||||
private var glassesMode = 0
|
||||
private var autoModeSent = false
|
||||
|
||||
@Volatile private var sink: PoseSink? = null
|
||||
private val poseObservers = java.util.concurrent.CopyOnWriteArraySet<PoseSink>()
|
||||
@Volatile private var probe: ((ByteArray?, FloatArray?) -> Unit)? = null
|
||||
@Volatile private var latest: FloatArray? = null
|
||||
@Volatile private var imuPackets = 0L
|
||||
@@ -142,6 +163,7 @@ object Glasses {
|
||||
fun init(context: Context) {
|
||||
if (::handler.isInitialized) return
|
||||
app = context.applicationContext
|
||||
SpatialTracking.initialize(app)
|
||||
usb = app.getSystemService(Context.USB_SERVICE) as UsbManager
|
||||
displays = app.getSystemService(Context.DISPLAY_SERVICE) as DisplayManager
|
||||
_settings.value = load()
|
||||
@@ -200,13 +222,84 @@ object Glasses {
|
||||
}
|
||||
|
||||
fun setRefreshHz(hz: Int) = post {
|
||||
update { it.copy(refreshHz = hz.coerceIn(60, 120)) }
|
||||
if (hz == 30 && (_state.value.sbs || !GlassesRendering.native30HzAvailable(_settings.value, _state.value))) return@post
|
||||
update { it.copy(refreshHz = if (hz == 30) 30 else hz.coerceIn(60, 120)) }
|
||||
if (!_state.value.sbs) applyMode(force = true)
|
||||
}
|
||||
|
||||
fun setSbsHz(hz: Int) = post {
|
||||
update { it.copy(sbsHz = if (hz >= 90) 90 else 60) }
|
||||
if (_state.value.sbs && _state.value.can3d) sendMode(true)
|
||||
if (hz == 30 && (!_state.value.sbs || !GlassesRendering.native30HzAvailable(_settings.value, _state.value))) return@post
|
||||
update { it.copy(sbsHz = when { hz == 30 -> 30; hz >= 90 -> 90; else -> 60 }) }
|
||||
if (hz != 30 && _state.value.sbs && _state.value.can3d) sendMode(true)
|
||||
}
|
||||
|
||||
fun setResolution(profile: GlassesDisplayProfile, value: String) = post {
|
||||
val s = _state.value
|
||||
val choice = if (value == "auto") "" else value
|
||||
if (choice.isNotEmpty() && s.resolutions.none { it.key == choice }) return@post
|
||||
if (GlassesDisplayModes.choice(_settings.value, profile) == choice) return@post
|
||||
update { when (profile) {
|
||||
GlassesDisplayProfile.TWO_D -> it.copy(resolution2d = choice)
|
||||
GlassesDisplayProfile.THREE_D -> it.copy(resolution3d = choice)
|
||||
GlassesDisplayProfile.VR -> it.copy(resolutionVr = choice)
|
||||
} }
|
||||
val active = GlassesDisplayModes.profile(s.sbs, s.vrActive)
|
||||
val inheritedByVr = s.vrActive && _settings.value.resolutionVr.isEmpty() &&
|
||||
profile == GlassesDisplayModes.profile(s.sbs, false)
|
||||
if ((profile == active || inheritedByVr) && s.canControl && !s.switching3d &&
|
||||
model?.resolutions?.isNotEmpty() == true) sendMode(s.sbs)
|
||||
}
|
||||
|
||||
fun prepareSteamVrStart(onPrepared: () -> Unit = {}) {
|
||||
if (!::handler.isInitialized) {
|
||||
steamVrSpatialGate.prepare()
|
||||
onPrepared()
|
||||
return
|
||||
}
|
||||
post {
|
||||
if (steamVrSpatialGate.prepare()) {
|
||||
handler.removeCallbacks(resumeSteamVrSpatial)
|
||||
updateSpatialMode()
|
||||
}
|
||||
onPrepared()
|
||||
}
|
||||
}
|
||||
|
||||
fun cancelSteamVrStart() {
|
||||
if (!::handler.isInitialized) {
|
||||
steamVrSpatialGate.cancel()
|
||||
return
|
||||
}
|
||||
post {
|
||||
steamVrSpatialGate.cancel()
|
||||
handler.removeCallbacks(resumeSteamVrSpatial)
|
||||
updateSpatialMode()
|
||||
}
|
||||
}
|
||||
|
||||
fun setVrActive(on: Boolean) = post {
|
||||
val s = _state.value
|
||||
val trackingChanged = steamVrSpatialGate.setRunning(on, SystemClock.uptimeMillis())
|
||||
if (trackingChanged) {
|
||||
handler.removeCallbacks(resumeSteamVrSpatial)
|
||||
steamVrSpatialGate.deadline?.let {
|
||||
Log.i(TAG, "SteamVR RUNNING; spatial resume scheduled at uptime ${it}ms")
|
||||
handler.postAtTime(resumeSteamVrSpatial, it)
|
||||
}
|
||||
}
|
||||
if (s.vrActive == on) {
|
||||
if (trackingChanged) updateSpatialMode()
|
||||
return@post
|
||||
}
|
||||
val previous = GlassesDisplayModes.selected(_settings.value, s)
|
||||
publish { it.copy(vrActive = on) }
|
||||
val restore2d = !on && modeState.stopVr()
|
||||
val stereo = (s.sbs || modeState.pendingStereo) && !restore2d
|
||||
val selected = GlassesDisplayModes.selected(_settings.value, _state.value, stereo)
|
||||
if (s.canControl && s.usable && (restore2d ||
|
||||
(previous != selected && model?.resolutions?.isNotEmpty() == true)) &&
|
||||
!(on && _settings.value.auto3d && !s.sbs)) sendMode(stereo)
|
||||
updateSpatialMode()
|
||||
}
|
||||
|
||||
fun set3d(on: Boolean) = post {
|
||||
@@ -255,7 +348,59 @@ object Glasses {
|
||||
markSbsFailed(linkDevice?.productId, s == GlassesState.SBS_FAILS)
|
||||
rememberSbs(s)
|
||||
}
|
||||
fun setGameOnGlasses(on: Boolean) = post { update { it.copy(gameOnGlasses = on) } }
|
||||
fun setGameOnGlasses(on: Boolean) = post {
|
||||
val s = _state.value
|
||||
if (!on && s.canControl && (s.sbs || modeState.pendingStereo)) {
|
||||
modeState.manual()
|
||||
sendMode(false)
|
||||
}
|
||||
update {
|
||||
it.copy(
|
||||
gameOnGlasses = on,
|
||||
picture3d = if (!on && it.picture3d == GlassesSettings.PICTURE_SCREEN)
|
||||
GlassesSettings.PICTURE_AUTO else it.picture3d,
|
||||
)
|
||||
}
|
||||
}
|
||||
fun set3dof(on: Boolean) = post {
|
||||
if (!_state.value.supportsScreen3dof) return@post
|
||||
update {
|
||||
it.copy(
|
||||
gameOnGlasses = on || it.gameOnGlasses,
|
||||
headTracking = on || it.headTracking,
|
||||
picture3d = if (on) GlassesSettings.PICTURE_SCREEN
|
||||
else if (it.picture3d == GlassesSettings.PICTURE_SCREEN) GlassesSettings.PICTURE_AUTO else it.picture3d,
|
||||
)
|
||||
}
|
||||
}
|
||||
fun setPositionalTracking(on: Boolean) = post {
|
||||
if (!_state.value.spatialConnected || _settings.value.positionalTracking == on) return@post
|
||||
update { it.copy(positionalTracking = on, handControllers = it.handControllers && on,
|
||||
joyConTracking = it.joyConTracking && on) }
|
||||
if (!on) SpatialTracking.setControllers(false, false)
|
||||
closeLink()
|
||||
refresh()
|
||||
}
|
||||
fun setHandControllers(on: Boolean) = post {
|
||||
val wanted = on && _settings.value.positionalTracking && _state.value.spatialConnected
|
||||
update { it.copy(handControllers = wanted, joyConTracking = it.joyConTracking && !wanted) }
|
||||
SpatialTracking.setControllers(wanted, _settings.value.joyConTracking)
|
||||
}
|
||||
fun setSteadyHandAim(on: Boolean) = post {
|
||||
update { it.copy(steadyHandAim = on) }
|
||||
SpatialTracking.setSteadyHandAim(on)
|
||||
}
|
||||
fun setJoyConTracking(on: Boolean) = post {
|
||||
val wanted = on && _settings.value.positionalTracking && _state.value.spatialConnected
|
||||
update { it.copy(joyConTracking = wanted, handControllers = it.handControllers && !wanted) }
|
||||
SpatialTracking.setControllers(_settings.value.handControllers, wanted)
|
||||
}
|
||||
fun setEyeHeight(cm: Int) = post { update { it.copy(eyeHeightCm = cm.coerceIn(50, 250)) } }
|
||||
fun setSpatialFloor() = SpatialTracking.setFloor(_settings.value.eyeHeightCm / 100f)
|
||||
fun restartSpatialTracking() = post {
|
||||
if (_state.value.spatialConnected && _settings.value.positionalTracking) { closeLink(); refresh() }
|
||||
}
|
||||
|
||||
fun setHeadTracking(on: Boolean) = post { update { it.copy(headTracking = on) } }
|
||||
fun setAuto3d(on: Boolean) = post { update { it.copy(auto3d = on) } }
|
||||
fun setPicture3d(picture: Int) = post { update { it.copy(picture3d = picture.coerceIn(0, 2)) } }
|
||||
@@ -264,6 +409,7 @@ object Glasses {
|
||||
fun setVrFovDeg(deg: Int) = post { update { it.copy(vrFovDeg = deg.coerceIn(GlassesSettings.VR_FOV_MIN, GlassesSettings.VR_FOV_MAX)) } }
|
||||
|
||||
fun outputHz(): Int = GlassesRendering.outputHz(_settings.value, _state.value)
|
||||
fun outputResolution(): GlassesResolution? = GlassesDisplayModes.selected(_settings.value, _state.value)
|
||||
|
||||
fun glassesDisplay(): Display? = _state.value.displayId.takeIf { it != Display.INVALID_DISPLAY }?.let { displays.getDisplay(it) }
|
||||
|
||||
@@ -277,6 +423,16 @@ object Glasses {
|
||||
post { updateImu() }
|
||||
}
|
||||
|
||||
internal fun addPoseObserver(observer: PoseSink) {
|
||||
poseObservers.add(observer)
|
||||
post { updateImu() }
|
||||
}
|
||||
|
||||
internal fun removePoseObserver(observer: PoseSink) {
|
||||
poseObservers.remove(observer)
|
||||
post { updateImu() }
|
||||
}
|
||||
|
||||
fun simulate(seconds: Int, holdYaw: Double? = null, holdPitch: Double? = null): Boolean {
|
||||
val hold = holdYaw != null || holdPitch != null
|
||||
val t = Thread({
|
||||
@@ -312,6 +468,15 @@ object Glasses {
|
||||
.put("permission", s.permission.name).put("connected", s.connected)
|
||||
.put("controlError", s.controlError ?: JSONObject.NULL)
|
||||
.put("experimental", s.experimental)
|
||||
.put("supportsScreen3dof", s.supportsScreen3dof)
|
||||
.put("supportsSpatial", s.supportsSpatial)
|
||||
.put("firmware", (link as? VitureSdkLink)?.firmware ?: JSONObject.NULL)
|
||||
.put("spatialSdk", (link as? VitureSdkLink)?.spatialDiagnostics()?.let {
|
||||
JSONObject().put("status",it[0]).put("callbackAgeMs",it[1]).put("cameraPoseDeltaMs",it[2])
|
||||
} ?: JSONObject.NULL)
|
||||
.put("spatial", SpatialTracking.diagnosticStatus())
|
||||
.put("supports30Hz", s.supports30Hz).put("vrActive", s.vrActive)
|
||||
.put("resolutions", JSONArray(s.resolutions.map { it.key }))
|
||||
.put("display", if (s.hasDisplay) JSONObject().put("id", s.displayId).put("width", s.width).put("height", s.height).put("hz", s.hz.toDouble()) else JSONObject.NULL)
|
||||
.put("sbs", s.sbs).put("deviceMode", glassesMode).put("switching3d", s.switching3d)
|
||||
.put("brightness", s.brightness).put("volume", s.volume)
|
||||
@@ -325,6 +490,11 @@ object Glasses {
|
||||
.put("fovDeg", s.diagonalFovDeg.toDouble()).put("noSignal", s.noSignal).put("sbsSupport", s.sbsSupport)
|
||||
.put("settings", JSONObject().put("refreshHz", g.refreshHz).put("sbsHz", g.sbsHz).put("sunblock", g.sunblock ?: JSONObject.NULL)
|
||||
.put("gameOnGlasses", g.gameOnGlasses).put("headTracking", g.headTracking).put("auto3d", g.auto3d)
|
||||
.put("screen3dof", g.screen3dof)
|
||||
.put("resolution2d", g.resolution2d).put("resolution3d", g.resolution3d).put("resolutionVr", g.resolutionVr)
|
||||
.put("positionalTracking", g.positionalTracking).put("handControllers", g.handControllers)
|
||||
.put("joyConTracking", g.joyConTracking).put("eyeHeightCm", g.eyeHeightCm)
|
||||
.put("steadyHandAim", g.steadyHandAim)
|
||||
.put("picture3d", g.picture3d).put("screenWidthDm", g.screenWidthDm).put("screenDistanceDm", g.screenDistanceDm)
|
||||
.put("vrFovDeg", g.vrFovDeg))
|
||||
}
|
||||
@@ -450,6 +620,14 @@ object Glasses {
|
||||
noSignal = noSignal,
|
||||
sbs90 = m?.sbs90 == true,
|
||||
experimental = m === XrealModel,
|
||||
supportsScreen3dof = link?.hasImu == true,
|
||||
supports30Hz = display?.let(::supports30Hz) == true,
|
||||
supportsSpatial = (link as? VitureSdkLink)?.isCarina == true,
|
||||
resolutions = m?.resolutions?.takeIf { modes -> modes.isNotEmpty() }
|
||||
?: display?.supportedModes.orEmpty().filter { mode ->
|
||||
mode.physicalWidth >= 1280 && (mode.physicalWidth >= 3000) == sbsByDisplay
|
||||
}.map { mode -> GlassesResolution(mode.physicalWidth / (if (sbsByDisplay) 2 else 1), mode.physicalHeight) }
|
||||
.distinct().sortedWith(compareBy({ size -> size.height }, { size -> size.width })),
|
||||
)
|
||||
}
|
||||
if (link != null) applyMode(force = false)
|
||||
@@ -458,6 +636,14 @@ object Glasses {
|
||||
|
||||
internal fun isGlassesVendor(vid: Int) = vid == VitureProtocol.VENDOR_ID || RayNeoModel.isVendor(vid) || vid == XrealModel.VENDOR_ID
|
||||
|
||||
internal fun supports30Hz(display: Display): Boolean {
|
||||
val current = display.mode
|
||||
return display.supportedModes.any {
|
||||
it.physicalWidth == current.physicalWidth && it.physicalHeight == current.physicalHeight &&
|
||||
abs(it.refreshRate - 30f) < 0.5f
|
||||
}
|
||||
}
|
||||
|
||||
private fun canOpenModel(m: GlassesModel): Boolean = m.controllable &&
|
||||
(m !is VitureProtocol.Model || !m.needsSdk || VitureSdk.available)
|
||||
|
||||
@@ -511,14 +697,29 @@ object Glasses {
|
||||
runCatching { usb.requestPermission(device, pi) }.onFailure { Log.w(TAG, "requestPermission", it) }
|
||||
}
|
||||
|
||||
private fun updateSpatialMode() {
|
||||
val current = link as? VitureSdkLink ?: return
|
||||
if (!current.ready || !current.isCarina || current.spatialEnabled ==
|
||||
steamVrSpatialGate.spatialEnabled(_settings.value.positionalTracking, SystemClock.uptimeMillis())) return
|
||||
val device = linkDevice
|
||||
closeLink()
|
||||
refresh()
|
||||
val resumed = link as? VitureSdkLink
|
||||
if (!current.spatialEnabled && resumed?.isCarina == true && resumed.spatialEnabled &&
|
||||
device != null && linkDevice?.deviceName == device.deviceName) SpatialTracking.recenter(false)
|
||||
}
|
||||
|
||||
private fun openLink(device: UsbDevice) {
|
||||
val generation = ++linkGeneration
|
||||
val spatial = steamVrSpatialGate.spatialEnabled(_settings.value.positionalTracking, SystemClock.uptimeMillis())
|
||||
calibrationKey = calibrationStore.key(device.vendorId, device.productId,
|
||||
runCatching { device.serialNumber }.getOrNull(), model?.name.orEmpty())
|
||||
val l = if (device.vendorId == VitureProtocol.VENDOR_ID) {
|
||||
if ((model as? VitureProtocol.Model)?.needsSdk == true) {
|
||||
VitureSdkLink.open(usb, device, File(app.cacheDir, "viture"), sdkListener(generation),
|
||||
(model as? VitureProtocol.Model)?.family)
|
||||
// Android can evict cacheDir while the SDK is running.
|
||||
Log.i(TAG, "opening VITURE SDK pid=0x%04X spatial=%s".format(device.productId, spatial))
|
||||
VitureSdkLink.open(usb, device, File(app.noBackupFilesDir, "viture"), sdkListener(generation),
|
||||
(model as? VitureProtocol.Model)?.family, spatial)
|
||||
} else VitureLink.open(usb, device, linkListener(generation))
|
||||
} else if (model === XrealModel) {
|
||||
XrealLink.open(app, xrealListener(generation))
|
||||
@@ -536,6 +737,12 @@ object Glasses {
|
||||
return
|
||||
}
|
||||
link = opened
|
||||
if (opened is VitureSdkLink && opened.isCarina) SpatialTracking.connect(opened.sdkToken, generation,
|
||||
opened.spatialIdentity ?: runCatching { device.serialNumber }.getOrNull()?.takeIf { it.isNotBlank() }
|
||||
?.let { "${device.vendorId}:${device.productId}:$it" },
|
||||
spatial, _settings.value.eyeHeightCm / 100f, _settings.value.handControllers,
|
||||
_settings.value.joyConTracking, _settings.value.steadyHandAim)
|
||||
else SpatialTracking.useLegacyTracking()
|
||||
_calibration.value = GlassesCalibrationState(supported = opened is RayNeoLink,
|
||||
saved = opened is RayNeoLink && calibrationKey?.let(calibrationStore::load) != null)
|
||||
linkDevice = device
|
||||
@@ -584,6 +791,9 @@ object Glasses {
|
||||
}
|
||||
|
||||
private fun sdkListener(generation: Long) = object : VitureSdkLink.Listener {
|
||||
override fun onSpatial(pose: FloatArray, timeNs: Long, valid: Boolean) {
|
||||
if (generation == linkGeneration) SpatialTracking.accept(pose, timeNs, generation, valid)
|
||||
}
|
||||
override fun onValue(msgId: Int, value: Int) = post {
|
||||
if (generation != linkGeneration) return@post
|
||||
when (msgId) {
|
||||
@@ -652,13 +862,14 @@ object Glasses {
|
||||
private fun closeLink() {
|
||||
val l = link ?: return
|
||||
linkGeneration++
|
||||
SpatialTracking.disconnect()
|
||||
link = null
|
||||
linkDevice = null
|
||||
calibrationKey = null
|
||||
_calibration.value = GlassesCalibrationState(phase = if (_calibration.value.running) CalibrationPhase.FAILED else CalibrationPhase.IDLE)
|
||||
glassesMode = 0
|
||||
l.close()
|
||||
publish { it.copy(connected = false, tracking = false) }
|
||||
publish { it.copy(connected = false, tracking = false, supportsScreen3dof = false, supportsSpatial = false) }
|
||||
Log.i(TAG, "link closed")
|
||||
}
|
||||
|
||||
@@ -688,10 +899,14 @@ object Glasses {
|
||||
}
|
||||
if (!s.usable || s.sbs || modeState.pendingStereo) return
|
||||
val hz = _settings.value.refreshHz
|
||||
if (glassesMode == m.mode(hz, false)) return
|
||||
// Native 30 Hz is selected by the Android presentation, without a vendor mode command.
|
||||
if (hz == 30 && GlassesRendering.native30HzAvailable(_settings.value, s)) return
|
||||
if (glassesMode == m.mode(hz, false, outputResolution())) return
|
||||
if (!force) {
|
||||
if (autoModeSent || SystemClock.uptimeMillis() - lastModeSentAt < 8_000) return
|
||||
if (glassesMode == 0 && abs(s.hz - hz) < 1.5f) return
|
||||
val resolution = outputResolution()
|
||||
if (glassesMode == 0 && abs(s.hz - hz) < 1.5f &&
|
||||
(resolution == null || (s.width == resolution.width && s.height == resolution.height))) return
|
||||
autoModeSent = true
|
||||
}
|
||||
sendMode(false)
|
||||
@@ -699,8 +914,10 @@ object Glasses {
|
||||
|
||||
private fun sendMode(sbs: Boolean): Boolean {
|
||||
val m = model ?: return false
|
||||
if (sbs && !_settings.value.gameOnGlasses) update { it.copy(gameOnGlasses = true) }
|
||||
val g = _settings.value
|
||||
val mode = m.mode(if (sbs) g.sbsHz else g.refreshHz, sbs)
|
||||
val resolution = GlassesDisplayModes.selected(g, _state.value, sbs)
|
||||
val mode = m.mode(if (sbs) g.sbsHz else g.refreshHz, sbs, resolution)
|
||||
if (link?.send(VitureProtocol.MSG_DISPLAY, byteArrayOf(mode.toByte())) == true) {
|
||||
if (link is RayNeoLink) glassesMode = mode
|
||||
lastModeSentAt = SystemClock.uptimeMillis()
|
||||
@@ -728,7 +945,7 @@ object Glasses {
|
||||
private fun updateImu() {
|
||||
val l = link ?: return
|
||||
if (!l.ready) return
|
||||
val want = (sink != null || probe != null || _calibration.value.running) && l.hasImu
|
||||
val want = (sink != null || poseObservers.isNotEmpty() || probe != null || _calibration.value.running || SpatialTracking.enabled) && l.hasImu
|
||||
if (want == _state.value.tracking) return
|
||||
val accepted = l.send(VitureProtocol.MSG_IMU, byteArrayOf(if (want) 1 else 0))
|
||||
if (want && !accepted) return
|
||||
@@ -769,6 +986,7 @@ object Glasses {
|
||||
private fun deliver(q: FloatArray, timeNs: Long) {
|
||||
latest = q
|
||||
sink?.onPose(q[0], q[1], q[2], q[3], timeNs)
|
||||
poseObservers.forEach { it.onPose(q[0], q[1], q[2], q[3], timeNs) }
|
||||
}
|
||||
|
||||
private fun onReport(frame: VitureProtocol.Frame) {
|
||||
@@ -832,8 +1050,16 @@ object Glasses {
|
||||
private fun load(): GlassesSettings {
|
||||
val p = app.getSharedPreferences(PREFS, Context.MODE_PRIVATE)
|
||||
return GlassesSettings(
|
||||
positionalTracking = p.getBoolean("positionalTracking", false),
|
||||
handControllers = p.getBoolean("handControllers", false),
|
||||
joyConTracking = p.getBoolean("joyConTracking", false),
|
||||
steadyHandAim = p.getBoolean("steadyHandAim", false),
|
||||
eyeHeightCm = p.getInt("eyeHeightCm", 160).coerceIn(50, 250),
|
||||
refreshHz = p.getInt("refreshHz", 120),
|
||||
sbsHz = p.getInt("sbsHz", 60),
|
||||
resolution2d = p.getString("resolution2d", "").orEmpty(),
|
||||
resolution3d = p.getString("resolution3d", "").orEmpty(),
|
||||
resolutionVr = p.getString("resolutionVr", "").orEmpty(),
|
||||
sunblock = if (p.contains("sunblock")) p.getBoolean("sunblock", false) else null,
|
||||
gameOnGlasses = p.getBoolean("gameOnGlasses", true),
|
||||
headTracking = p.getBoolean("headTracking", true),
|
||||
@@ -847,8 +1073,16 @@ object Glasses {
|
||||
|
||||
private fun save(s: GlassesSettings) {
|
||||
app.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().apply {
|
||||
putBoolean("positionalTracking", s.positionalTracking)
|
||||
putBoolean("handControllers", s.handControllers)
|
||||
putBoolean("joyConTracking", s.joyConTracking)
|
||||
putBoolean("steadyHandAim", s.steadyHandAim)
|
||||
putInt("eyeHeightCm", s.eyeHeightCm)
|
||||
putInt("refreshHz", s.refreshHz)
|
||||
putInt("sbsHz", s.sbsHz)
|
||||
putString("resolution2d", s.resolution2d)
|
||||
putString("resolution3d", s.resolution3d)
|
||||
putString("resolutionVr", s.resolutionVr)
|
||||
s.sunblock?.let { putBoolean("sunblock", it) } ?: remove("sunblock")
|
||||
putBoolean("gameOnGlasses", s.gameOnGlasses)
|
||||
putBoolean("headTracking", s.headTracking)
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
data class GlassesResolution(val width: Int, val height: Int) {
|
||||
val key: String get() = "${width}x$height"
|
||||
}
|
||||
|
||||
enum class GlassesDisplayProfile { TWO_D, THREE_D, VR }
|
||||
|
||||
internal object GlassesDisplayModes {
|
||||
fun profile(stereo: Boolean, vr: Boolean): GlassesDisplayProfile = when {
|
||||
vr -> GlassesDisplayProfile.VR
|
||||
stereo -> GlassesDisplayProfile.THREE_D
|
||||
else -> GlassesDisplayProfile.TWO_D
|
||||
}
|
||||
|
||||
fun choice(settings: GlassesSettings, profile: GlassesDisplayProfile): String = when (profile) {
|
||||
GlassesDisplayProfile.TWO_D -> settings.resolution2d
|
||||
GlassesDisplayProfile.THREE_D -> settings.resolution3d
|
||||
GlassesDisplayProfile.VR -> settings.resolutionVr
|
||||
}
|
||||
|
||||
fun selected(settings: GlassesSettings, state: GlassesState,
|
||||
stereo: Boolean = state.sbs, vr: Boolean = state.vrActive): GlassesResolution? {
|
||||
val selected = choice(settings, profile(stereo, vr)).ifEmpty {
|
||||
if (vr) choice(settings, profile(stereo, false)) else ""
|
||||
}
|
||||
return state.resolutions.firstOrNull { it.key == selected }
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ interface GlassesModel {
|
||||
val diagonalFovDeg: Float
|
||||
val sbs90: Boolean
|
||||
val hasSunblock: Boolean get() = true
|
||||
val resolutions: List<GlassesResolution> get() = emptyList()
|
||||
fun mode(hz: Int, sbs: Boolean, resolution: GlassesResolution?): Int = mode(hz, sbs)
|
||||
fun mode(hz: Int, sbs: Boolean): Int
|
||||
fun modeForStep(step: Int): Int?
|
||||
}
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.content.ContextWrapper
|
||||
import android.graphics.Color
|
||||
import android.graphics.PorterDuff
|
||||
import android.graphics.SurfaceTexture
|
||||
import android.os.Build
|
||||
import android.os.Looper
|
||||
import android.view.Choreographer
|
||||
import android.view.Surface
|
||||
import android.view.View
|
||||
import android.view.WindowManager
|
||||
import android.view.inspector.WindowInspector
|
||||
import kotlin.math.min
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
class GlassesMenuCapture(private val activity: Activity, private val onSize: (Int, Int) -> Unit,
|
||||
private val onFailure: (Throwable) -> Unit = {}) : Choreographer.FrameCallback {
|
||||
private var texture: SurfaceTexture? = null
|
||||
private var surface: Surface? = null
|
||||
private var running = false
|
||||
private var framePending = false
|
||||
private var lastFrameNs = Long.MIN_VALUE
|
||||
private var failures = 0
|
||||
private var bufferWidth = 0
|
||||
private var bufferHeight = 0
|
||||
private val basePosition = IntArray(2)
|
||||
private val windowPosition = IntArray(2)
|
||||
|
||||
fun attach(texture: SurfaceTexture, surface: Surface) {
|
||||
checkMainThread()
|
||||
this.texture = texture
|
||||
this.surface = surface
|
||||
bufferWidth = 0
|
||||
bufferHeight = 0
|
||||
failures = 0
|
||||
lastFrameNs = Long.MIN_VALUE
|
||||
}
|
||||
|
||||
fun start() {
|
||||
checkMainThread()
|
||||
if (running || activity.isDestroyed || activity.isFinishing) return
|
||||
running = true
|
||||
failures = 0
|
||||
lastFrameNs = Long.MIN_VALUE
|
||||
schedule()
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
checkMainThread()
|
||||
running = false
|
||||
if (framePending) Choreographer.getInstance().removeFrameCallback(this)
|
||||
framePending = false
|
||||
}
|
||||
|
||||
override fun doFrame(frameTimeNanos: Long) {
|
||||
framePending = false
|
||||
if (!running) return
|
||||
if (activity.isDestroyed || activity.isFinishing) {
|
||||
stop()
|
||||
return
|
||||
}
|
||||
if (lastFrameNs == Long.MIN_VALUE || frameTimeNanos - lastFrameNs >= FRAME_INTERVAL_NS) {
|
||||
lastFrameNs = frameTimeNanos
|
||||
try {
|
||||
capture()
|
||||
failures = 0
|
||||
} catch (error: Exception) {
|
||||
if (++failures >= MAX_FAILURES) {
|
||||
stop()
|
||||
onFailure(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (running) schedule()
|
||||
}
|
||||
|
||||
private fun schedule() {
|
||||
if (framePending) return
|
||||
framePending = true
|
||||
Choreographer.getInstance().postFrameCallback(this)
|
||||
}
|
||||
|
||||
private fun capture() {
|
||||
val target = surface ?: return
|
||||
val input = texture ?: return
|
||||
val decor = activity.window.decorView
|
||||
if (!decor.isAttachedToWindow || !decor.isShown) return
|
||||
val size = captureSize(decor.width, decor.height) ?: return
|
||||
check(target.isValid)
|
||||
if (bufferWidth != size.first || bufferHeight != size.second) {
|
||||
input.setDefaultBufferSize(size.first, size.second)
|
||||
onSize(size.first, size.second)
|
||||
bufferWidth = size.first
|
||||
bufferHeight = size.second
|
||||
}
|
||||
val candidates = if (Build.VERSION.SDK_INT >= 29) WindowInspector.getGlobalWindowViews() else emptyList()
|
||||
val roots = captureRoots(activity, candidates)
|
||||
val canvas = target.lockHardwareCanvas()
|
||||
val initialSave = canvas.save()
|
||||
try {
|
||||
canvas.drawColor(Color.BLACK, PorterDuff.Mode.SRC)
|
||||
val scale = min(canvas.width.toFloat() / decor.width, canvas.height.toFloat() / decor.height)
|
||||
canvas.translate((canvas.width - decor.width * scale) / 2f, (canvas.height - decor.height * scale) / 2f)
|
||||
canvas.scale(scale, scale)
|
||||
canvas.clipRect(0, 0, decor.width, decor.height)
|
||||
decor.getLocationOnScreen(basePosition)
|
||||
for (root in roots) {
|
||||
val params = root.layoutParams as? WindowManager.LayoutParams
|
||||
if (root !== decor && params != null && params.flags and WindowManager.LayoutParams.FLAG_DIM_BEHIND != 0) {
|
||||
canvas.drawColor(Color.argb((params.dimAmount.coerceIn(0f, 1f) * 255).roundToInt(), 0, 0, 0))
|
||||
}
|
||||
root.getLocationOnScreen(windowPosition)
|
||||
val saved = canvas.save()
|
||||
try {
|
||||
canvas.translate((windowPosition[0] - basePosition[0]).toFloat(), (windowPosition[1] - basePosition[1]).toFloat())
|
||||
root.draw(canvas)
|
||||
} finally {
|
||||
canvas.restoreToCount(saved)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
canvas.restoreToCount(initialSave)
|
||||
} finally {
|
||||
target.unlockCanvasAndPost(canvas)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun checkMainThread() {
|
||||
check(Looper.myLooper() == Looper.getMainLooper())
|
||||
}
|
||||
|
||||
internal companion object {
|
||||
private const val FRAME_INTERVAL_NS = 33_333_334L
|
||||
private const val MAX_FAILURES = 3
|
||||
private const val MAX_EDGE = 1920
|
||||
|
||||
internal fun captureSize(width: Int, height: Int): Pair<Int, Int>? {
|
||||
if (width <= 0 || height <= 0) return null
|
||||
val scale = min(1.0, MAX_EDGE.toDouble() / maxOf(width, height))
|
||||
return (width * scale).roundToInt().coerceAtLeast(1) to (height * scale).roundToInt().coerceAtLeast(1)
|
||||
}
|
||||
|
||||
internal fun captureRoots(activity: Activity, candidates: List<View>): List<View> {
|
||||
val decor = activity.window.decorView
|
||||
if (activity.window.attributes.flags and WindowManager.LayoutParams.FLAG_SECURE != 0 ||
|
||||
!decor.isAttachedToWindow || !decor.isShown) return emptyList()
|
||||
if (Build.VERSION.SDK_INT < 29) return listOf(decor)
|
||||
val display = decor.display?.displayId ?: return listOf(decor)
|
||||
val token = decor.applicationWindowToken
|
||||
return listOf(decor) + candidates.filter { root ->
|
||||
if (root === decor || !root.isAttachedToWindow || !root.isShown || root.display?.displayId != display) return@filter false
|
||||
val params = root.layoutParams as? WindowManager.LayoutParams ?: return@filter false
|
||||
if (params.flags and WindowManager.LayoutParams.FLAG_SECURE != 0 ||
|
||||
params.type !in WindowManager.LayoutParams.FIRST_APPLICATION_WINDOW..WindowManager.LayoutParams.LAST_SUB_WINDOW) return@filter false
|
||||
ownsContext(root.context, activity) || (token != null && root.applicationWindowToken == token)
|
||||
}.distinct()
|
||||
}
|
||||
|
||||
private fun ownsContext(context: Context, activity: Activity): Boolean {
|
||||
var current = context
|
||||
repeat(16) {
|
||||
if (current === activity) return true
|
||||
val next = (current as? ContextWrapper)?.baseContext ?: return false
|
||||
if (next === current) return false
|
||||
current = next
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.app.Activity
|
||||
import android.app.Application
|
||||
import android.app.Presentation
|
||||
import android.os.Bundle
|
||||
import android.util.Log
|
||||
import android.view.Display
|
||||
import android.view.WindowManager
|
||||
import com.droiddeck.launcher.SessionActivity
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.combine
|
||||
import kotlinx.coroutines.launch
|
||||
|
||||
object GlassesMenuMirror {
|
||||
private var controller: GlassesMenuController? = null
|
||||
|
||||
fun initialize(app: Application) {
|
||||
if (controller != null) return
|
||||
val pose = GlassesMenuPose()
|
||||
val menus = GlassesMenuController(pose,
|
||||
display = Glasses::glassesDisplay,
|
||||
observe = { active -> if (active) Glasses.addPoseObserver(pose) else Glasses.removePoseObserver(pose) },
|
||||
create = { activity, display, failure -> MenuPresentation(activity, display, pose, failure) })
|
||||
controller = menus
|
||||
app.registerActivityLifecycleCallbacks(object : Application.ActivityLifecycleCallbacks {
|
||||
override fun onActivityResumed(activity: Activity) = menus.resume(activity)
|
||||
override fun onActivityPaused(activity: Activity) = menus.pause(activity)
|
||||
override fun onActivityDestroyed(activity: Activity) = menus.pause(activity)
|
||||
override fun onActivityCreated(activity: Activity, state: Bundle?) = Unit
|
||||
override fun onActivityStarted(activity: Activity) = Unit
|
||||
override fun onActivityStopped(activity: Activity) = Unit
|
||||
override fun onActivitySaveInstanceState(activity: Activity, state: Bundle) = Unit
|
||||
})
|
||||
CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate).launch {
|
||||
combine(Glasses.state, Glasses.settings) { state, settings -> state to settings }
|
||||
.collect { (state, settings) -> menus.update(state, settings) }
|
||||
}
|
||||
}
|
||||
|
||||
fun setSessionMenuVisible(activity: Activity, visible: Boolean) {
|
||||
controller?.setSessionMenuVisible(activity, visible)
|
||||
}
|
||||
|
||||
fun recenter() { controller?.recenter() }
|
||||
}
|
||||
|
||||
internal class GlassesMenuController(
|
||||
private val pose: GlassesMenuPose,
|
||||
private val display: () -> Display?,
|
||||
private val observe: (Boolean) -> Unit,
|
||||
private val create: (Activity, Display, (Throwable) -> Unit) -> Output,
|
||||
private val schedule: (() -> Unit) -> Unit = { android.os.Handler(android.os.Looper.getMainLooper()).post(it) },
|
||||
) {
|
||||
interface Output {
|
||||
fun configure(state: GlassesState, settings: GlassesSettings)
|
||||
fun show()
|
||||
fun close()
|
||||
}
|
||||
|
||||
private var activity: Activity? = null
|
||||
private val sessionMenus = java.util.WeakHashMap<Activity, Boolean>()
|
||||
private var state = GlassesState()
|
||||
private var settings = GlassesSettings()
|
||||
private var output: Output? = null
|
||||
private var outputDisplay = Display.INVALID_DISPLAY
|
||||
private var failed = false
|
||||
private var dismissalRetried = false
|
||||
private var generation = 0
|
||||
|
||||
fun resume(owner: Activity) {
|
||||
if (activity !== owner) close()
|
||||
activity = owner
|
||||
failed = false
|
||||
dismissalRetried = false
|
||||
reconcile()
|
||||
}
|
||||
|
||||
fun pause(owner: Activity) {
|
||||
if (activity !== owner) return
|
||||
close()
|
||||
activity = null
|
||||
}
|
||||
|
||||
fun setSessionMenuVisible(owner: Activity, visible: Boolean) {
|
||||
if (visible) sessionMenus[owner] = true else sessionMenus.remove(owner)
|
||||
if (activity === owner) reconcile()
|
||||
}
|
||||
|
||||
fun update(next: GlassesState, preferences: GlassesSettings) {
|
||||
val deviceChanged = state.canControl != next.canControl || state.model != next.model
|
||||
if (deviceChanged) pose.reset()
|
||||
if (deviceChanged || state.displayId != next.displayId || state.width != next.width ||
|
||||
state.height != next.height || state.sbs != next.sbs || settings != preferences) {
|
||||
failed = false
|
||||
dismissalRetried = false
|
||||
}
|
||||
state = next
|
||||
settings = preferences
|
||||
pose.configure(preferences)
|
||||
reconcile()
|
||||
}
|
||||
|
||||
fun recenter() = pose.recenter()
|
||||
|
||||
private fun reconcile() {
|
||||
val owner = activity
|
||||
val eligible = owner != null && !owner.isFinishing && !owner.isDestroyed && !owner.isInPictureInPictureMode &&
|
||||
owner.window.attributes.flags and WindowManager.LayoutParams.FLAG_SECURE == 0 &&
|
||||
owner !is com.droiddeck.launcher.QuestXrActivity &&
|
||||
(owner !is SessionActivity || sessionMenus[owner] == true) &&
|
||||
settings.gameOnGlasses && settings.screen3dof && state.usable && state.canControl && state.supportsScreen3dof
|
||||
val target = if (eligible && !failed) display() else null
|
||||
if (target == null || owner == null || owner.windowManager.defaultDisplay.displayId == target.displayId) {
|
||||
close()
|
||||
return
|
||||
}
|
||||
if (outputDisplay != target.displayId) close()
|
||||
try {
|
||||
output?.let { it.configure(state, settings); return }
|
||||
val token = ++generation
|
||||
val next = create(owner, target) { error ->
|
||||
if (generation == token) {
|
||||
close()
|
||||
if (error is GlassesMenuDisplayStopped && !dismissalRetried) {
|
||||
dismissalRetried = true
|
||||
val retry = generation
|
||||
schedule { if (generation == retry && activity === owner) reconcile() }
|
||||
} else {
|
||||
Log.w("GlassesMenu", "Tracked menus are unavailable for this display", error)
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if (generation != token) {
|
||||
next.close()
|
||||
return
|
||||
}
|
||||
output = next
|
||||
outputDisplay = target.displayId
|
||||
next.configure(state, settings)
|
||||
if (generation != token) return
|
||||
observe(true)
|
||||
next.show()
|
||||
} catch (error: RuntimeException) {
|
||||
Log.w("GlassesMenu", "Could not show tracked menus", error)
|
||||
failed = true
|
||||
close()
|
||||
}
|
||||
}
|
||||
|
||||
private fun close() {
|
||||
++generation
|
||||
val previous = output
|
||||
output = null
|
||||
outputDisplay = Display.INVALID_DISPLAY
|
||||
if (previous != null) {
|
||||
observe(false)
|
||||
try { previous.close() }
|
||||
catch (error: RuntimeException) { Log.w("GlassesMenu", "Could not close tracked menus", error) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal class GlassesMenuDisplayStopped : IllegalStateException("The glasses menu display stopped")
|
||||
|
||||
private class MenuPresentation(
|
||||
private val owner: Activity,
|
||||
display: Display,
|
||||
pose: GlassesMenuPose,
|
||||
private val onFailure: (Throwable) -> Unit,
|
||||
) : Presentation(owner, display), GlassesMenuController.Output {
|
||||
private var closed = false
|
||||
private val capture: GlassesMenuCapture = GlassesMenuCapture(owner,
|
||||
onSize = { width, height -> surface.setSourceSize(width, height) }, onFailure = onFailure)
|
||||
private val surface: GlassesMenuSurface = GlassesMenuSurface(context, pose, onFailure = onFailure) { texture, input ->
|
||||
if (!closed) {
|
||||
capture.attach(texture, input)
|
||||
capture.start()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
window?.addFlags(WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE or
|
||||
WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE or WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
setContentView(surface)
|
||||
}
|
||||
|
||||
override fun configure(state: GlassesState, settings: GlassesSettings) {
|
||||
surface.configure(settings, state)
|
||||
val mode = display.mode
|
||||
val hz = if (state.sbs) settings.sbsHz else settings.refreshHz
|
||||
val best = display.supportedModes.filter {
|
||||
it.physicalWidth == mode.physicalWidth && it.physicalHeight == mode.physicalHeight
|
||||
}.minByOrNull { kotlin.math.abs(it.refreshRate - hz) }
|
||||
if (best != null) window?.let { it.attributes = it.attributes.apply { preferredDisplayModeId = best.modeId } }
|
||||
}
|
||||
|
||||
override fun close() {
|
||||
if (closed) return
|
||||
closed = true
|
||||
capture.stop()
|
||||
surface.close()
|
||||
dismiss()
|
||||
}
|
||||
|
||||
override fun onStop() {
|
||||
capture.stop()
|
||||
surface.close()
|
||||
super.onStop()
|
||||
if (!closed) {
|
||||
closed = true
|
||||
onFailure(GlassesMenuDisplayStopped())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import com.droiddeck.launcher.gpu.HeadPose
|
||||
|
||||
class GlassesMenuPose internal constructor(
|
||||
private val spatialEnabled: () -> Boolean = { SpatialTracking.enabled },
|
||||
private val spatialView: (Long) -> SpatialPose? = { now ->
|
||||
synchronized(SpatialTracking.session) {
|
||||
if (SpatialTracking.session.valid(now)) SpatialTracking.session.view(true) else null
|
||||
}
|
||||
},
|
||||
private val recenterSpatial: () -> Unit = { SpatialTracking.recenter(true) },
|
||||
) : Glasses.PoseSink {
|
||||
data class Frame(
|
||||
val rotation: FloatArray,
|
||||
val position: FloatArray,
|
||||
val valid: Boolean,
|
||||
val spatial: Boolean,
|
||||
)
|
||||
|
||||
private var tracking = true
|
||||
private var positional = false
|
||||
private var reference: HeadPose.Reference? = null
|
||||
private var latest: FloatArray? = null
|
||||
private var latestTime = 0L
|
||||
private var safe = identity()
|
||||
|
||||
@Synchronized fun configure(settings: GlassesSettings) {
|
||||
if (tracking != settings.headTracking) {
|
||||
reference = null
|
||||
safe = identity()
|
||||
}
|
||||
tracking = settings.headTracking
|
||||
positional = settings.positionalTracking
|
||||
}
|
||||
|
||||
@Synchronized override fun onPose(x: Float, y: Float, z: Float, w: Float, timeNs: Long) {
|
||||
val norm = x*x + y*y + z*z + w*w
|
||||
if (!norm.isFinite() || norm !in 0.25f..2.25f || timeNs <= latestTime) return
|
||||
val matrix = HeadPose.fromGlasses(x, y, z, w)
|
||||
if (matrix.any { !it.isFinite() }) return
|
||||
latest = matrix
|
||||
latestTime = timeNs
|
||||
}
|
||||
|
||||
@Synchronized fun snapshot(now: Long = System.nanoTime()): Frame {
|
||||
if (!tracking) return identity()
|
||||
val spatial = positional && spatialEnabled()
|
||||
if (spatial) {
|
||||
val pose = spatialView(now)
|
||||
val normalized = pose?.rotation?.normalized()
|
||||
if (pose != null && pose.position.finite() && normalized != null) {
|
||||
safe = Frame(normalized.matrix(), pose.position.array(), true, true)
|
||||
return copy(safe)
|
||||
}
|
||||
} else {
|
||||
val matrix = latest
|
||||
if (matrix != null && latestTime > 0 && now-latestTime in 0..MAX_AGE_NS) {
|
||||
val anchor = reference ?: HeadPose.recenter(matrix, centerView = true).also { reference = it }
|
||||
safe = Frame(HeadPose.relative(anchor, matrix), FloatArray(3), true, false)
|
||||
return copy(safe)
|
||||
}
|
||||
}
|
||||
return copy(safe).copy(valid = false, spatial = spatial)
|
||||
}
|
||||
|
||||
@Synchronized fun recenter() {
|
||||
if (positional && spatialEnabled()) recenterSpatial()
|
||||
else reference = null
|
||||
}
|
||||
|
||||
@Synchronized fun reset() {
|
||||
reference = null
|
||||
latest = null
|
||||
latestTime = 0L
|
||||
safe = identity()
|
||||
}
|
||||
|
||||
private fun copy(frame: Frame) = frame.copy(rotation = frame.rotation.copyOf(), position = frame.position.copyOf())
|
||||
|
||||
private fun identity() = Frame(HeadPose.IDENTITY.copyOf(), FloatArray(3), false, false)
|
||||
|
||||
companion object {
|
||||
internal const val MAX_AGE_NS = 100_000_000L
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.content.Context
|
||||
import android.graphics.SurfaceTexture
|
||||
import android.opengl.GLES11Ext
|
||||
import android.opengl.GLES20
|
||||
import android.opengl.GLSurfaceView
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import android.view.Surface
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import javax.microedition.khronos.egl.EGLConfig
|
||||
import javax.microedition.khronos.opengles.GL10
|
||||
import kotlin.math.tan
|
||||
|
||||
class GlassesMenuSurface(
|
||||
context: Context,
|
||||
private val pose: GlassesMenuPose,
|
||||
private val onFailure: (Throwable) -> Unit = {},
|
||||
private val onInput: (SurfaceTexture, Surface) -> Unit,
|
||||
) : GLSurfaceView(context) {
|
||||
private data class Scene(val settings: GlassesSettings, val state: GlassesState)
|
||||
|
||||
private val main = Handler(Looper.getMainLooper())
|
||||
private val closed = AtomicBoolean()
|
||||
private val failed = AtomicBoolean()
|
||||
private val generation = AtomicInteger()
|
||||
private val pendingFrame = AtomicBoolean()
|
||||
private val inputLock = Any()
|
||||
private var inputTexture: SurfaceTexture? = null
|
||||
private var inputSurface: Surface? = null
|
||||
@Volatile private var scene = Scene(GlassesSettings(), GlassesState())
|
||||
@Volatile private var sourceAspect = 16f / 9f
|
||||
private val renderer = MenuRenderer()
|
||||
|
||||
init {
|
||||
setEGLContextClientVersion(2)
|
||||
setEGLConfigChooser(8, 8, 8, 8, 0, 0)
|
||||
preserveEGLContextOnPause = true
|
||||
setRenderer(renderer)
|
||||
renderMode = RENDERMODE_CONTINUOUSLY
|
||||
}
|
||||
|
||||
fun configure(settings: GlassesSettings, state: GlassesState) {
|
||||
pose.configure(settings)
|
||||
scene = Scene(settings, state)
|
||||
}
|
||||
|
||||
fun setSourceSize(width: Int, height: Int) {
|
||||
if (width > 0 && height > 0) sourceAspect = width.toFloat() / height
|
||||
}
|
||||
|
||||
fun close() {
|
||||
if (!closed.compareAndSet(false, true)) return
|
||||
generation.incrementAndGet()
|
||||
queueEvent { renderer.release() }
|
||||
}
|
||||
|
||||
override fun onDetachedFromWindow() {
|
||||
close()
|
||||
super.onDetachedFromWindow()
|
||||
// GLSurfaceView may exit its GL thread before queued cleanup is consumed.
|
||||
releaseInput()
|
||||
}
|
||||
|
||||
private fun releaseInput() = synchronized(inputLock) {
|
||||
inputTexture?.setOnFrameAvailableListener(null)
|
||||
inputSurface?.release()
|
||||
inputTexture?.release()
|
||||
inputSurface = null
|
||||
inputTexture = null
|
||||
pendingFrame.set(false)
|
||||
}
|
||||
|
||||
private fun fail(error: Throwable) {
|
||||
if (!failed.compareAndSet(false, true)) return
|
||||
generation.incrementAndGet()
|
||||
Log.e("GlassesMenu", "Menu rendering is unavailable", error)
|
||||
main.post { if (!closed.get()) onFailure(error) }
|
||||
}
|
||||
|
||||
private inner class MenuRenderer : Renderer {
|
||||
private val vertices = ByteBuffer.allocateDirect(8 * 4).order(ByteOrder.nativeOrder()).asFloatBuffer().apply {
|
||||
put(floatArrayOf(-1f, -1f, 1f, -1f, -1f, 1f, 1f, 1f)); position(0)
|
||||
}
|
||||
private val textureMatrix = FloatArray(16)
|
||||
private var texture = 0
|
||||
private var program = 0
|
||||
private var width = 1
|
||||
private var height = 1
|
||||
private var hasFrame = false
|
||||
private var positionLocation = 0
|
||||
private var textureLocation = 0
|
||||
private var transformLocation = 0
|
||||
private var rotationLocation = IntArray(3)
|
||||
private var headLocation = 0
|
||||
private var screenLocation = 0
|
||||
private var opticsLocation = 0
|
||||
|
||||
override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
|
||||
releaseInput()
|
||||
texture = 0
|
||||
program = 0
|
||||
hasFrame = false
|
||||
if (closed.get() || failed.get()) return
|
||||
val token = generation.incrementAndGet()
|
||||
try {
|
||||
program = linkProgram()
|
||||
positionLocation = GLES20.glGetAttribLocation(program, "aPosition")
|
||||
textureLocation = GLES20.glGetUniformLocation(program, "uTexture")
|
||||
transformLocation = GLES20.glGetUniformLocation(program, "uTextureMatrix")
|
||||
rotationLocation = IntArray(3) { GLES20.glGetUniformLocation(program, "uRotation$it") }
|
||||
headLocation = GLES20.glGetUniformLocation(program, "uHead")
|
||||
screenLocation = GLES20.glGetUniformLocation(program, "uScreen")
|
||||
opticsLocation = GLES20.glGetUniformLocation(program, "uOptics")
|
||||
val names = IntArray(1)
|
||||
GLES20.glGenTextures(1, names, 0)
|
||||
texture = names[0]
|
||||
check(texture != 0) { "Could not allocate the menu texture" }
|
||||
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture)
|
||||
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR)
|
||||
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)
|
||||
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE)
|
||||
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE)
|
||||
checkGl("initialize menu texture")
|
||||
val incoming = SurfaceTexture(texture)
|
||||
val surface = synchronized(inputLock) {
|
||||
inputTexture = incoming
|
||||
Surface(incoming).also { inputSurface = it }
|
||||
}
|
||||
incoming.setOnFrameAvailableListener({
|
||||
if (!closed.get() && generation.get() == token) pendingFrame.set(true)
|
||||
}, main)
|
||||
main.post {
|
||||
if (!closed.get() && !failed.get() && generation.get() == token) onInput(incoming, surface)
|
||||
}
|
||||
} catch (e: RuntimeException) {
|
||||
fail(e)
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
|
||||
this.width = width.coerceAtLeast(1)
|
||||
this.height = height.coerceAtLeast(1)
|
||||
GLES20.glViewport(0, 0, this.width, this.height)
|
||||
}
|
||||
|
||||
override fun onDrawFrame(gl: GL10?) {
|
||||
if (closed.get() || failed.get()) return
|
||||
GLES20.glClearColor(0.015f, 0.015f, 0.02f, 1f)
|
||||
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT)
|
||||
if (program == 0) return
|
||||
if (pendingFrame.getAndSet(false)) synchronized(inputLock) {
|
||||
inputTexture?.let {
|
||||
try {
|
||||
it.updateTexImage()
|
||||
it.getTransformMatrix(textureMatrix)
|
||||
checkGl("update menu texture")
|
||||
hasFrame = true
|
||||
} catch (e: RuntimeException) {
|
||||
fail(e)
|
||||
release()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!hasFrame) return
|
||||
val current = scene
|
||||
val frame = pose.snapshot()
|
||||
val halfWidth = current.settings.screenWidthDm.coerceIn(5, 200) / 20f
|
||||
val distance = current.settings.screenDistanceDm.coerceIn(5, 200) / 10f
|
||||
val mono = !current.state.sbs
|
||||
val fov = current.state.verticalFovDeg.takeIf { it.isFinite() }?.coerceIn(20f, 140f) ?: 23.3f
|
||||
val tangent = tan(Math.toRadians(fov.toDouble()) / 2).toFloat()
|
||||
GLES20.glUseProgram(program)
|
||||
GLES20.glActiveTexture(GLES20.GL_TEXTURE0)
|
||||
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture)
|
||||
GLES20.glUniform1i(textureLocation, 0)
|
||||
GLES20.glUniformMatrix4fv(transformLocation, 1, false, textureMatrix, 0)
|
||||
for (row in 0..2) GLES20.glUniform3fv(rotationLocation[row], 1, frame.rotation, row * 3)
|
||||
GLES20.glUniform3fv(headLocation, 1, frame.position, 0)
|
||||
GLES20.glUniform3f(screenLocation, halfWidth, distance, sourceAspect)
|
||||
GLES20.glUniform4f(opticsLocation, tangent, width.toFloat() / height / (if (mono) 1f else 2f),
|
||||
if (mono) 0f else IPD_METERS / 2f, if (mono) 1f else 0f)
|
||||
vertices.position(0)
|
||||
GLES20.glEnableVertexAttribArray(positionLocation)
|
||||
GLES20.glVertexAttribPointer(positionLocation, 2, GLES20.GL_FLOAT, false, 0, vertices)
|
||||
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
|
||||
GLES20.glDisableVertexAttribArray(positionLocation)
|
||||
}
|
||||
|
||||
fun release() {
|
||||
releaseInput()
|
||||
if (texture != 0) GLES20.glDeleteTextures(1, intArrayOf(texture), 0)
|
||||
if (program != 0) GLES20.glDeleteProgram(program)
|
||||
texture = 0
|
||||
program = 0
|
||||
hasFrame = false
|
||||
}
|
||||
|
||||
private fun checkGl(operation: String) {
|
||||
val error = GLES20.glGetError()
|
||||
check(error == GLES20.GL_NO_ERROR) { "Could not $operation: GL error 0x${error.toString(16)}" }
|
||||
}
|
||||
|
||||
private fun shader(type: Int, source: String): Int {
|
||||
val shader = GLES20.glCreateShader(type)
|
||||
GLES20.glShaderSource(shader, source)
|
||||
GLES20.glCompileShader(shader)
|
||||
val status = IntArray(1)
|
||||
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, status, 0)
|
||||
if (status[0] == 0) {
|
||||
val error = GLES20.glGetShaderInfoLog(shader)
|
||||
GLES20.glDeleteShader(shader)
|
||||
throw IllegalStateException(error)
|
||||
}
|
||||
return shader
|
||||
}
|
||||
|
||||
private fun linkProgram(): Int {
|
||||
val vertex = shader(GLES20.GL_VERTEX_SHADER, VERTEX)
|
||||
var fragment = 0
|
||||
var linked = 0
|
||||
try {
|
||||
fragment = shader(GLES20.GL_FRAGMENT_SHADER, FRAGMENT)
|
||||
linked = GLES20.glCreateProgram()
|
||||
GLES20.glAttachShader(linked, vertex)
|
||||
GLES20.glAttachShader(linked, fragment)
|
||||
GLES20.glLinkProgram(linked)
|
||||
val status = IntArray(1)
|
||||
GLES20.glGetProgramiv(linked, GLES20.GL_LINK_STATUS, status, 0)
|
||||
check(status[0] != 0) { GLES20.glGetProgramInfoLog(linked) }
|
||||
return linked
|
||||
} catch (e: RuntimeException) {
|
||||
if (linked != 0) GLES20.glDeleteProgram(linked)
|
||||
throw e
|
||||
} finally {
|
||||
GLES20.glDeleteShader(vertex)
|
||||
if (fragment != 0) GLES20.glDeleteShader(fragment)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
internal const val IPD_METERS = 0.064f
|
||||
private const val VERTEX = """
|
||||
attribute vec2 aPosition;
|
||||
varying vec2 vUv;
|
||||
void main() {
|
||||
gl_Position = vec4(aPosition, 0.0, 1.0);
|
||||
vUv = aPosition * 0.5 + 0.5;
|
||||
}
|
||||
"""
|
||||
private const val FRAGMENT = """
|
||||
#extension GL_OES_EGL_image_external : require
|
||||
precision highp float;
|
||||
uniform samplerExternalOES uTexture;
|
||||
uniform mat4 uTextureMatrix;
|
||||
uniform vec3 uRotation0, uRotation1, uRotation2, uHead, uScreen;
|
||||
uniform vec4 uOptics;
|
||||
varying vec2 vUv;
|
||||
vec3 rotate(vec3 v) {
|
||||
return vec3(dot(uRotation0, v), dot(uRotation1, v), dot(uRotation2, v));
|
||||
}
|
||||
void main() {
|
||||
bool mono = uOptics.w > 0.5;
|
||||
float eye = mono || vUv.x < 0.5 ? 0.0 : 1.0;
|
||||
vec2 local = vec2(mono ? vUv.x : vUv.x * 2.0 - eye, 1.0 - vUv.y);
|
||||
vec2 ray = (local - 0.5) * vec2(uOptics.y, 1.0) * (2.0 * uOptics.x);
|
||||
vec3 direction = rotate(vec3(ray.x, -ray.y, -1.0));
|
||||
vec3 origin = uHead + rotate(vec3((eye < 0.5 ? -1.0 : 1.0) * uOptics.z, 0.0, 0.0));
|
||||
vec3 color = vec3(0.015, 0.015, 0.02);
|
||||
if (direction.z < -0.0001) {
|
||||
float distance = (-uScreen.y - origin.z) / direction.z;
|
||||
vec3 hit = origin + direction * distance;
|
||||
vec2 source = vec2(hit.x / uScreen.x, -hit.y * uScreen.z / uScreen.x) * 0.5 + 0.5;
|
||||
if (distance > 0.0 && all(greaterThanEqual(source, vec2(0.0))) && all(lessThanEqual(source, vec2(1.0)))) {
|
||||
vec2 uv = (uTextureMatrix * vec4(source.x, 1.0 - source.y, 0.0, 1.0)).xy;
|
||||
color = texture2D(uTexture, uv).rgb;
|
||||
}
|
||||
}
|
||||
gl_FragColor = vec4(color, 1.0);
|
||||
}
|
||||
"""
|
||||
}
|
||||
}
|
||||
@@ -4,13 +4,32 @@ import com.droiddeck.launcher.gpu.Cardboard
|
||||
|
||||
/** Share the actual picture mode with settings so controls match the rendered view. */
|
||||
internal object GlassesRendering {
|
||||
enum class Picture { FLAT, VIRTUAL_SCREEN, STEREO }
|
||||
enum class Picture { FLAT, DUPLICATED, VIRTUAL_SCREEN, STEREO }
|
||||
|
||||
fun outputHz(settings: GlassesSettings, state: GlassesState): Int =
|
||||
if (state.sbs) if (state.sbs90) settings.sbsHz else 60 else settings.refreshHz
|
||||
fun native30HzAvailable(settings: GlassesSettings, state: GlassesState): Boolean =
|
||||
state.usable && state.supports30Hz && settings.gameOnGlasses
|
||||
|
||||
fun mirrorRefreshLimit(settings: GlassesSettings, state: GlassesState, outputActive: Boolean): Int =
|
||||
if (state.usable && !outputActive) outputHz(settings, state) else 0
|
||||
fun refreshRates(settings: GlassesSettings, state: GlassesState): List<Int> = buildList {
|
||||
if (native30HzAvailable(settings, state)) add(30)
|
||||
add(60)
|
||||
if (!state.sbs || state.sbs90) add(90)
|
||||
if (!state.sbs) add(120)
|
||||
}
|
||||
|
||||
fun outputHz(settings: GlassesSettings, state: GlassesState): Int {
|
||||
val requested = if (state.sbs) settings.sbsHz else settings.refreshHz
|
||||
return when {
|
||||
requested == 30 -> if (native30HzAvailable(settings, state)) 30 else 60
|
||||
state.sbs && !state.sbs90 -> 60
|
||||
else -> requested
|
||||
}
|
||||
}
|
||||
|
||||
fun mirrorRefreshLimit(settings: GlassesSettings, state: GlassesState, outputActive: Boolean): Int {
|
||||
if (!state.usable || outputActive) return 0
|
||||
val hz = outputHz(settings, state)
|
||||
return if (hz == 30 && kotlin.math.abs(state.hz - 30f) >= 0.5f) 60 else hz
|
||||
}
|
||||
|
||||
fun fpsLimit(requested: Int, mirrorRefreshLimit: Int): Int = when {
|
||||
mirrorRefreshLimit <= 0 -> requested
|
||||
@@ -22,14 +41,12 @@ internal object GlassesRendering {
|
||||
outputActive || (steamVrRunning && state.usable)
|
||||
|
||||
fun picture(settings: GlassesSettings, stereoDisplay: Boolean, steamVrRunning: Boolean): Picture =
|
||||
when (settings.picture3d) {
|
||||
GlassesSettings.PICTURE_SCREEN -> Picture.VIRTUAL_SCREEN
|
||||
GlassesSettings.PICTURE_SBS -> Picture.STEREO
|
||||
else -> when {
|
||||
steamVrRunning -> Picture.STEREO
|
||||
stereoDisplay -> Picture.VIRTUAL_SCREEN
|
||||
else -> Picture.FLAT
|
||||
}
|
||||
when {
|
||||
steamVrRunning -> Picture.STEREO
|
||||
settings.picture3d == GlassesSettings.PICTURE_SCREEN -> Picture.VIRTUAL_SCREEN
|
||||
settings.picture3d == GlassesSettings.PICTURE_SBS -> Picture.STEREO
|
||||
stereoDisplay -> Picture.DUPLICATED
|
||||
else -> Picture.FLAT
|
||||
}
|
||||
|
||||
fun cardboard(
|
||||
@@ -55,12 +72,17 @@ internal object GlassesRendering {
|
||||
if (picture == Picture.FLAT) return Cardboard.OFF
|
||||
return Cardboard(
|
||||
enabled = true,
|
||||
source = if (picture == Picture.STEREO) Cardboard.SOURCE_SBS else Cardboard.SOURCE_SCREEN,
|
||||
source = when (picture) {
|
||||
Picture.STEREO -> Cardboard.SOURCE_SBS
|
||||
Picture.DUPLICATED -> Cardboard.SOURCE_FLAT
|
||||
else -> Cardboard.SOURCE_SCREEN
|
||||
},
|
||||
viewer = Cardboard.Viewer.VITURE.id,
|
||||
ipdMm = base.ipdMm,
|
||||
screenWidthDm = settings.screenWidthDm,
|
||||
screenDistanceDm = settings.screenDistanceDm,
|
||||
headTracking = settings.headTracking && state.canControl,
|
||||
headTracking = settings.headTracking && state.canControl &&
|
||||
(picture == Picture.VIRTUAL_SCREEN || steamVrRunning),
|
||||
mono = !state.sbs,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -13,10 +13,12 @@ import android.view.WindowManager
|
||||
import android.widget.FrameLayout
|
||||
import kotlin.math.abs
|
||||
|
||||
class GlassesScreen(context: Context, display: Display, refreshHz: Int, private val callback: SurfaceHolder.Callback) : Presentation(context, display) {
|
||||
class GlassesScreen(context: Context, display: Display, refreshHz: Int, private val callback: SurfaceHolder.Callback, resolution: GlassesResolution? = null) : Presentation(context, display) {
|
||||
val surfaceView = SurfaceView(getContext())
|
||||
internal val trackingRecovery = com.droiddeck.launcher.ui.TrackingRecoveryView(getContext())
|
||||
val holder: SurfaceHolder get() = surfaceView.holder
|
||||
private var wantHz = refreshHz
|
||||
private var wantResolution = resolution
|
||||
|
||||
val hz: Float get() = display.mode.refreshRate
|
||||
|
||||
@@ -24,14 +26,16 @@ class GlassesScreen(context: Context, display: Display, refreshHz: Int, private
|
||||
super.onCreate(savedInstanceState)
|
||||
val root = FrameLayout(context).apply { setBackgroundColor(Color.BLACK) }
|
||||
root.addView(surfaceView, FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT))
|
||||
root.addView(trackingRecovery, FrameLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT))
|
||||
setContentView(root)
|
||||
window?.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
|
||||
applyMode()
|
||||
surfaceView.holder.addCallback(callback)
|
||||
}
|
||||
|
||||
fun prefer(hz: Int) {
|
||||
fun prefer(hz: Int, resolution: GlassesResolution? = null) {
|
||||
wantHz = hz
|
||||
wantResolution = resolution
|
||||
// A 2D/SBS transition can replace the active mode without changing Hz.
|
||||
applyMode()
|
||||
}
|
||||
@@ -39,9 +43,14 @@ class GlassesScreen(context: Context, display: Display, refreshHz: Int, private
|
||||
private fun applyMode() {
|
||||
val w = window ?: return
|
||||
val current = display.mode
|
||||
val best = display.supportedModes.filter {
|
||||
val target = wantResolution
|
||||
val width = target?.width?.times(if (current.physicalWidth >= 3000) 2 else 1) ?: current.physicalWidth
|
||||
val height = target?.height ?: current.physicalHeight
|
||||
val requested = display.supportedModes.filter { it.physicalWidth == width && it.physicalHeight == height }
|
||||
val candidates = requested.ifEmpty { display.supportedModes.filter {
|
||||
it.physicalWidth == current.physicalWidth && it.physicalHeight == current.physicalHeight
|
||||
}.minByOrNull { abs(it.refreshRate - wantHz) } ?: return
|
||||
} }
|
||||
val best = candidates.minByOrNull { abs(it.refreshRate - wantHz) } ?: return
|
||||
if (w.attributes.preferredDisplayModeId != best.modeId) {
|
||||
Log.i("GlassesScreen", "display ${display.displayId}: prefer mode ${best.modeId} " +
|
||||
"${best.physicalWidth}x${best.physicalHeight}@${best.refreshRate} (requested $wantHz Hz)")
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import kotlin.math.*
|
||||
|
||||
internal class HandGestures(private val rightHand: Boolean) {
|
||||
private var grip = false
|
||||
private var last: SpatialHand? = null
|
||||
var lastOutcome = HandObservationOutcome.LANDMARK_COUNT_INVALID; private set
|
||||
fun reset() { grip = false; last = null }
|
||||
fun resetAfterRejection(outcome: HandObservationOutcome) {
|
||||
if (outcome != HandObservationOutcome.PALM_MOTION_REJECTED) reset()
|
||||
}
|
||||
private fun reject(outcome: HandObservationOutcome): SpatialHand? { lastOutcome = outcome; return null }
|
||||
fun estimate(points: List<SpatialVector?>, worldFromCamera: SpatialPose, time: Long,
|
||||
leftImage: List<SpatialVector>? = null, rightImage: List<SpatialVector>? = null): SpatialHand? {
|
||||
if (points.size != 21) return reject(HandObservationOutcome.LANDMARK_COUNT_INVALID)
|
||||
val p = points.map { it?.takeIf(SpatialVector::finite) }
|
||||
val wrist = p[0] ?: return reject(HandObservationOutcome.PALM_LANDMARK_MISSING)
|
||||
val index = p[5] ?: return reject(HandObservationOutcome.PALM_LANDMARK_MISSING)
|
||||
val middle = p[9] ?: return reject(HandObservationOutcome.PALM_LANDMARK_MISSING)
|
||||
val ring = p[13] ?: return reject(HandObservationOutcome.PALM_LANDMARK_MISSING)
|
||||
val little = p[17] ?: return reject(HandObservationOutcome.PALM_LANDMARK_MISSING)
|
||||
val width = (index-little).length()
|
||||
if (width !in 0.035f..0.13f) return reject(HandObservationOutcome.PALM_WIDTH_INVALID)
|
||||
val forward = (middle-wrist).normalized() ?: return reject(HandObservationOutcome.PALM_BASIS_INVALID)
|
||||
val lateral = (if (rightHand) little-index else index-little).normalized() ?: return reject(HandObservationOutcome.PALM_BASIS_INVALID)
|
||||
val up = lateral.cross(forward).normalized() ?: return reject(HandObservationOutcome.PALM_BASIS_INVALID)
|
||||
val right = forward.cross(up).normalized() ?: return reject(HandObservationOutcome.PALM_BASIS_INVALID)
|
||||
val palm = (wrist+index+middle+ring+little) * 0.2f
|
||||
val pose = worldFromCamera * SpatialPose(palm,SpatialRotation.fromBasis(right,up,forward * -1f))
|
||||
last?.let { old ->
|
||||
val dt=(time-old.timeNs)/1e9f
|
||||
if (dt>0 && dt<0.2f && (pose.position-old.pose.position).length()>0.08f+dt*3f)
|
||||
return reject(HandObservationOutcome.PALM_MOTION_REJECTED)
|
||||
}
|
||||
val stereo = if (leftImage != null || rightImage != null)
|
||||
StereoFingerGestures.estimate(leftImage,rightImage,grip) else null
|
||||
val triggerValid = stereo?.triggerValid ?: (p[4] != null && p[8] != null)
|
||||
val trigger = stereo?.trigger ?: if (triggerValid) ((0.55f-(p[4]!!-p[8]!!).length()/width)/0.35f).coerceIn(0f,1f) else 0f
|
||||
val gripValid = stereo?.gripValid ?: listOf(12,16,20).all { p[it] != null }
|
||||
val closed = if (stereo == null && gripValid) listOf(12 to 9,16 to 13,20 to 17).count { (tip,base) ->
|
||||
(p[tip]!!-wrist).length() < (p[base]!!-wrist).length()*1.35f
|
||||
} else 0
|
||||
grip=stereo?.grip ?: (gripValid && if(grip) closed>=2 else closed==3)
|
||||
lastOutcome = HandObservationOutcome.ACCEPTED
|
||||
return SpatialHand(pose,time,true,trigger,grip,triggerValid,gripValid).also { last=it }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.graphics.Bitmap
|
||||
import com.google.mediapipe.framework.image.ByteBufferImageBuilder
|
||||
import com.google.mediapipe.framework.image.MPImage
|
||||
import java.nio.ByteBuffer
|
||||
|
||||
internal class HandImageBuffer {
|
||||
companion object {
|
||||
const val PREPROCESSING = "raw-grayscale-rgb"
|
||||
}
|
||||
private var pixels = ByteBuffer.allocateDirect(0)
|
||||
private var source = IntArray(0)
|
||||
private var rgb = ByteArray(0)
|
||||
|
||||
private fun allocateRgb(count: Int) {
|
||||
if (rgb.size != count * 3) {
|
||||
rgb = ByteArray(count * 3)
|
||||
pixels = ByteBuffer.allocateDirect(rgb.size)
|
||||
}
|
||||
}
|
||||
|
||||
fun image(grayscale: ByteArray, width: Int, height: Int): MPImage {
|
||||
require(width > 0 && height > 0 && width.toLong() * height == grayscale.size.toLong())
|
||||
allocateRgb(grayscale.size)
|
||||
var index = 0
|
||||
for (value in grayscale) {
|
||||
rgb[index++] = value
|
||||
rgb[index++] = value
|
||||
rgb[index++] = value
|
||||
}
|
||||
return image(width, height)
|
||||
}
|
||||
|
||||
fun image(grayscale: Bitmap): MPImage {
|
||||
val count = grayscale.width * grayscale.height
|
||||
if (source.size != count) source = IntArray(count)
|
||||
allocateRgb(count)
|
||||
grayscale.getPixels(source, 0, grayscale.width, 0, 0, grayscale.width, grayscale.height)
|
||||
var index = 0
|
||||
for (pixel in source) {
|
||||
val value = (pixel ushr 16).toByte()
|
||||
rgb[index++] = value
|
||||
rgb[index++] = value
|
||||
rgb[index++] = value
|
||||
}
|
||||
return image(grayscale.width, grayscale.height)
|
||||
}
|
||||
|
||||
private fun image(width: Int, height: Int): MPImage {
|
||||
pixels.clear()
|
||||
pixels.put(rgb)
|
||||
pixels.rewind()
|
||||
// MediaPipe applies the model metadata's float normalization to these RGB bytes.
|
||||
return ByteBufferImageBuilder(pixels, width, height, MPImage.IMAGE_FORMAT_RGB).build()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal class HandInferenceSchedule(
|
||||
private val stableFrames: Int = 3,
|
||||
private val discoveryIntervalMs: Long = 240,
|
||||
private val maximumGapMs: Long = 400
|
||||
) {
|
||||
init {
|
||||
require(stableFrames > 0)
|
||||
require(discoveryIntervalMs > 0 && maximumGapMs >= discoveryIntervalMs)
|
||||
}
|
||||
|
||||
private var lastFrameMs = -1L
|
||||
private var selectedAtMs = -1L
|
||||
private var selectedMaximum = 2
|
||||
private var lastSearchMs = -1L
|
||||
private var candidateSide = -1
|
||||
private var consistentFrames = 0
|
||||
private var trackingSide = -1
|
||||
|
||||
fun select(timestampMs: Long): Int {
|
||||
if (timestampMs < 0) {
|
||||
reset()
|
||||
return 2
|
||||
}
|
||||
if (selectedAtMs >= 0 || lastFrameMs >= 0 &&
|
||||
(timestampMs <= lastFrameMs || timestampMs - lastFrameMs > maximumGapMs)) reset()
|
||||
val maximum = if (trackingSide >= 0 && lastSearchMs >= 0 &&
|
||||
timestampMs - lastSearchMs < discoveryIntervalMs) 1 else 2
|
||||
lastFrameMs = timestampMs
|
||||
selectedAtMs = timestampMs
|
||||
selectedMaximum = maximum
|
||||
if (maximum == 2) lastSearchMs = timestampMs
|
||||
return maximum
|
||||
}
|
||||
|
||||
fun observe(timestampMs: Long, leftSides: List<Int>, rightSides: List<Int>,
|
||||
leftRawCount: Int = leftSides.size, rightRawCount: Int = rightSides.size) {
|
||||
if (timestampMs < 0 || selectedAtMs != timestampMs) {
|
||||
reset()
|
||||
return
|
||||
}
|
||||
selectedAtMs = -1L
|
||||
val side = leftSides.singleOrNull() ?: -1
|
||||
val consistent = leftRawCount == 1 && rightRawCount == 1 && side in 0..1 &&
|
||||
rightSides.singleOrNull() == side
|
||||
if (!consistent || selectedMaximum == 1 && side != trackingSide) {
|
||||
forgetHand()
|
||||
return
|
||||
}
|
||||
if (candidateSide != side) {
|
||||
candidateSide = side
|
||||
consistentFrames = 0
|
||||
}
|
||||
consistentFrames = minOf(stableFrames, consistentFrames + 1)
|
||||
if (consistentFrames >= stableFrames) trackingSide = side
|
||||
}
|
||||
|
||||
fun reset() {
|
||||
lastFrameMs = -1L
|
||||
selectedAtMs = -1L
|
||||
selectedMaximum = 2
|
||||
lastSearchMs = -1L
|
||||
forgetHand()
|
||||
}
|
||||
|
||||
private fun forgetHand() {
|
||||
candidateSide = -1
|
||||
consistentFrames = 0
|
||||
trackingSide = -1
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal enum class HandModel(val asset: String, val sha256: String) {
|
||||
FULL("models/hand_landmarker.task", "fbc2a30080c3c557093b5ddfc334698132eb341044ccee322ccf8bcf3607cde1"),
|
||||
LITE("models/hand_landmarker_lite.task", "173c0f47ecee7e935bd4a1e93aa580756b4180c1307c07d709313e82c28672c8");
|
||||
|
||||
companion object {
|
||||
val DEFAULT = FULL
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal class HandProcessingRate {
|
||||
var permille = 500
|
||||
private set
|
||||
private var processingMs = Double.NaN
|
||||
private var headroomSamples = 0
|
||||
|
||||
fun observe(processingNs: Long, observationAgeNs: Long, trackingHealthy: Boolean): Int {
|
||||
if (!trackingHealthy || processingNs <= 0 || observationAgeNs < 0) return reset()
|
||||
val measuredMs = processingNs / 1_000_000.0
|
||||
val ageMs = observationAgeNs / 1_000_000.0
|
||||
processingMs = if (processingMs.isNaN()) measuredMs else processingMs + 0.2 * (measuredMs - processingMs)
|
||||
if (measuredMs >= 80 || ageMs >= 140) {
|
||||
permille = 500
|
||||
headroomSamples = 0
|
||||
return permille
|
||||
}
|
||||
val reduced = when {
|
||||
permille > 500 && processingMs > 1000.0 / 15 -> 500
|
||||
permille == 800 && processingMs > 50 -> 600
|
||||
else -> permille
|
||||
}
|
||||
if (reduced != permille) {
|
||||
permille = reduced
|
||||
headroomSamples = 0
|
||||
return permille
|
||||
}
|
||||
val enoughHeadroom = when (permille) {
|
||||
500 -> processingMs <= 55 && ageMs <= 110
|
||||
600 -> processingMs <= 40 && ageMs <= 90
|
||||
else -> false
|
||||
}
|
||||
headroomSamples = if (enoughHeadroom) headroomSamples + 1 else 0
|
||||
if (headroomSamples >= 8) {
|
||||
permille = if (permille == 500) 600 else 800
|
||||
headroomSamples = 0
|
||||
}
|
||||
return permille
|
||||
}
|
||||
|
||||
fun reset(): Int {
|
||||
permille = 500
|
||||
processingMs = Double.NaN
|
||||
headroomSamples = 0
|
||||
return permille
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal enum class HandObservationOutcome {
|
||||
ACCEPTED, DISABLED, CALIBRATION_REQUIRED, CLOCK_NOT_READY, HEAD_TRACKING_INVALID,
|
||||
CAMERA_NOT_SYNCHRONIZED, STALE_OBSERVATION, INFERENCE_UNAVAILABLE,
|
||||
NO_MATCHING_DETECTION, AMBIGUOUS_HANDEDNESS, ONE_CAMERA_ONLY, STEREO_REJECTED,
|
||||
LANDMARK_COUNT_INVALID, PALM_LANDMARK_MISSING, PALM_WIDTH_INVALID, PALM_BASIS_INVALID,
|
||||
PALM_MOTION_REJECTED, WRIST_LANDMARK_MISSING, WRIST_MOTION_REJECTED
|
||||
}
|
||||
|
||||
internal data class HandObservationDiagnostic(
|
||||
val outcome: HandObservationOutcome,
|
||||
val reconstructedPalmLandmarks: List<Int> = emptyList(),
|
||||
val triggerEstimate: Float = 0f,
|
||||
val triggerGeometryValid: Boolean = false,
|
||||
val gripEstimate: Boolean = false,
|
||||
val gripGeometryValid: Boolean = false,
|
||||
val positionSource: String = "none"
|
||||
) {
|
||||
val accepted get() = outcome == HandObservationOutcome.ACCEPTED
|
||||
val triggerValid get() = accepted && triggerGeometryValid
|
||||
val gripValid get() = accepted && gripGeometryValid
|
||||
}
|
||||
|
||||
internal data class HandSideMetrics(
|
||||
val observations: Long,
|
||||
val accepted: Long,
|
||||
val triggerValid: Long,
|
||||
val gripValid: Long,
|
||||
val triggerAboveClickThreshold: Long,
|
||||
val triggerReleased: Long,
|
||||
val gripPressed: Long,
|
||||
val trackingLosses: Long,
|
||||
val triggerLosses: Long,
|
||||
val gripLosses: Long,
|
||||
val maximumConsecutiveRejected: Long,
|
||||
val outcomes: Map<HandObservationOutcome, Long>
|
||||
)
|
||||
|
||||
internal data class HandTrackingMetricSnapshot(
|
||||
val processedPairs: Long,
|
||||
val inferencePairs: Long,
|
||||
val left: HandSideMetrics,
|
||||
val right: HandSideMetrics
|
||||
)
|
||||
|
||||
internal class HandTrackingMetrics {
|
||||
private class Side {
|
||||
val outcomes = LongArray(HandObservationOutcome.entries.size)
|
||||
var triggerValid = 0L
|
||||
var gripValid = 0L
|
||||
var triggerAboveClickThreshold = 0L
|
||||
var triggerReleased = 0L
|
||||
var gripPressed = 0L
|
||||
var trackingLosses = 0L
|
||||
var triggerLosses = 0L
|
||||
var gripLosses = 0L
|
||||
var consecutiveRejected = 0L
|
||||
var maximumConsecutiveRejected = 0L
|
||||
var wasAccepted = false
|
||||
var wasTriggerValid = false
|
||||
var wasGripValid = false
|
||||
|
||||
fun observe(sample: HandObservationDiagnostic) {
|
||||
outcomes[sample.outcome.ordinal]++
|
||||
if (sample.triggerValid) {
|
||||
triggerValid++
|
||||
if (sample.triggerEstimate > 0.65f) triggerAboveClickThreshold++
|
||||
if (sample.triggerEstimate < 0.25f) triggerReleased++
|
||||
}
|
||||
if (sample.gripValid) {
|
||||
gripValid++
|
||||
if (sample.gripEstimate) gripPressed++
|
||||
}
|
||||
if (wasAccepted && !sample.accepted) trackingLosses++
|
||||
if (wasTriggerValid && !sample.triggerValid) triggerLosses++
|
||||
if (wasGripValid && !sample.gripValid) gripLosses++
|
||||
consecutiveRejected = if (sample.accepted) 0 else consecutiveRejected + 1
|
||||
maximumConsecutiveRejected = maxOf(maximumConsecutiveRejected, consecutiveRejected)
|
||||
wasAccepted = sample.accepted
|
||||
wasTriggerValid = sample.triggerValid
|
||||
wasGripValid = sample.gripValid
|
||||
}
|
||||
|
||||
fun snapshot() = HandSideMetrics(outcomes.sum(), outcomes[HandObservationOutcome.ACCEPTED.ordinal],
|
||||
triggerValid, gripValid, triggerAboveClickThreshold, triggerReleased, gripPressed, trackingLosses, triggerLosses, gripLosses,
|
||||
maximumConsecutiveRejected, HandObservationOutcome.entries.filter { outcomes[it.ordinal] > 0 }
|
||||
.associateWith { outcomes[it.ordinal] })
|
||||
}
|
||||
|
||||
private val sides = Array(2) { Side() }
|
||||
private var lastSequence = 0L
|
||||
private var processedPairs = 0L
|
||||
private var inferencePairs = 0L
|
||||
|
||||
fun observe(sequence: Long, inferred: Boolean, left: HandObservationDiagnostic,
|
||||
right: HandObservationDiagnostic) {
|
||||
if (sequence <= lastSequence) return
|
||||
lastSequence = sequence
|
||||
processedPairs++
|
||||
if (inferred) inferencePairs++
|
||||
sides[0].observe(left)
|
||||
sides[1].observe(right)
|
||||
}
|
||||
|
||||
fun snapshot() = HandTrackingMetricSnapshot(processedPairs, inferencePairs, sides[0].snapshot(), sides[1].snapshot())
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import kotlin.math.hypot
|
||||
|
||||
internal object JoyConFlowGate {
|
||||
const val MAX_SEED_AGE_NS = 750_000_000L
|
||||
fun fresh(seed: Long, previous: Long, now: Long) = seed > 0 && previous > 0 &&
|
||||
now > previous && now - previous <= 200_000_000L && now - seed in 0..MAX_SEED_AGE_NS
|
||||
|
||||
fun point(x: Double, y: Double, backX: Double, backY: Double, oldX: Double, oldY: Double,
|
||||
error: Float, forward: Boolean, backward: Boolean, width: Int, height: Int): Boolean =
|
||||
forward && backward && error.isFinite() && error in 0f..20f &&
|
||||
x.isFinite() && y.isFinite() && backX.isFinite() && backY.isFinite() &&
|
||||
x >= 3 && y >= 3 && x < width - 3 && y < height - 3 &&
|
||||
hypot(backX - oldX, backY - oldY) <= 1.5 && hypot(x - oldX, y - oldY) <= 100
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal class JoyConWristPosition {
|
||||
private var last: SpatialHand? = null
|
||||
var lastOutcome = HandObservationOutcome.WRIST_LANDMARK_MISSING; private set
|
||||
|
||||
fun reset() { last = null }
|
||||
|
||||
private fun reject(outcome: HandObservationOutcome): SpatialHand? {
|
||||
lastOutcome = outcome
|
||||
return null
|
||||
}
|
||||
|
||||
fun estimate(points: List<SpatialVector?>, worldFromCamera: SpatialPose, time: Long): SpatialHand? {
|
||||
val wrist = points.firstOrNull()?.takeIf(SpatialVector::finite)
|
||||
?: return reject(HandObservationOutcome.WRIST_LANDMARK_MISSING)
|
||||
val pose = worldFromCamera * SpatialPose(wrist)
|
||||
last?.let { old ->
|
||||
val dt = (time - old.timeNs) / 1e9f
|
||||
if (dt <= 0 || (dt < .2f && (pose.position - old.pose.position).length() > .08f + dt * 3f))
|
||||
return reject(HandObservationOutcome.WRIST_MOTION_REJECTED)
|
||||
}
|
||||
lastOutcome = HandObservationOutcome.ACCEPTED
|
||||
return SpatialHand(pose, time, true, triggerValid = false, gripValid = false).also { last = it }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.Manifest
|
||||
import android.app.PendingIntent
|
||||
import android.app.Service
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.hardware.usb.UsbManager
|
||||
import android.os.IBinder
|
||||
import android.util.AtomicFile
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
|
||||
internal object LumaRgbDiagnostic {
|
||||
private fun file(context: Context) = AtomicFile(File(context.filesDir, "spatial/rgb-diagnostic.json"))
|
||||
private fun save(context: Context, value: JSONObject) {
|
||||
val target = file(context)
|
||||
target.baseFile.parentFile?.mkdirs()
|
||||
val output = target.startWrite()
|
||||
try {
|
||||
output.write(value.toString().toByteArray(Charsets.UTF_8))
|
||||
target.finishWrite(output)
|
||||
} catch (e: Exception) { target.failWrite(output); throw e }
|
||||
}
|
||||
|
||||
private fun expired(value: JSONObject, nowMs: Long) = nowMs-value.optLong("startedMs") !in
|
||||
0..value.optLong("timeoutMs",15_000).coerceIn(15_000,45_000)
|
||||
|
||||
private fun workerAlive(context: Context, pid: Int): Boolean = pid > 0 && pid != android.os.Process.myPid() &&
|
||||
runCatching {
|
||||
File("/proc/$pid/cmdline").readText().substringBefore('\u0000') == "${context.packageName}:rgb_diagnostic"
|
||||
}.getOrDefault(false)
|
||||
|
||||
internal fun resolveStatus(value: JSONObject, nowMs: Long, workerAlive: Boolean): JSONObject {
|
||||
val pending = value.optString("state") == "capturing"
|
||||
if (workerAlive && !expired(value,nowMs)) {
|
||||
if (!pending) value.put("resultState",value.optString("state")).put("phase","finishing")
|
||||
return value.put("state","capturing").put("busy",true)
|
||||
}
|
||||
if (pending && (expired(value,nowMs) || value.optInt("workerPid") > 0 && !workerAlive))
|
||||
value.put("state","failed").put("message","RGB capture worker stopped or timed out")
|
||||
return value.put("busy",workerAlive || value.optString("state") == "capturing")
|
||||
}
|
||||
|
||||
fun status(context: Context): JSONObject = runCatching {
|
||||
val value = JSONObject(file(context).readFully().toString(Charsets.UTF_8))
|
||||
val now = android.os.SystemClock.elapsedRealtime()
|
||||
val pid = value.optInt("workerPid")
|
||||
if (expired(value,now) && workerAlive(context,pid)) android.os.Process.killProcess(pid)
|
||||
resolveStatus(value,now,workerAlive(context,pid))
|
||||
}.getOrElse { JSONObject().put("state", "idle") }
|
||||
|
||||
@Synchronized fun capture(context: Context, transport: String = "sdk", width: Int = 640, height: Int = 480,
|
||||
fps: Int = 30, durationMs: Int = 5000): JSONObject {
|
||||
require(transport in listOf("sdk","uvc") && width in 160..1920 && height in 120..1080 &&
|
||||
fps in 5..30 && durationMs in 1000..30000) { "Invalid RGB capture options" }
|
||||
val app = context.applicationContext
|
||||
val old = status(app)
|
||||
if (old.optBoolean("busy")) return old
|
||||
val usb = app.getSystemService(Context.USB_SERVICE) as UsbManager
|
||||
val device = usb.deviceList.values.singleOrNull { it.vendorId == 0x0c45 && it.productId == 0x636b }
|
||||
fun state(value: String, message: String): JSONObject {
|
||||
val result = JSONObject().put("state", value).put("message", message)
|
||||
save(app, result)
|
||||
return result
|
||||
}
|
||||
if (!Glasses.state.value.spatialConnected || device == null)
|
||||
return state("unavailable", "Connect the Luma Ultra glasses and RGB camera")
|
||||
if (app.checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED)
|
||||
return state("permission_required", "Grant Android camera permission to capture the RGB camera")
|
||||
if (!usb.hasPermission(device)) {
|
||||
val intent = Intent("${app.packageName}.RGB_DIAGNOSTIC_PERMISSION").setPackage(app.packageName)
|
||||
val pending = PendingIntent.getBroadcast(app, 3, intent, PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT)
|
||||
usb.requestPermission(device, pending)
|
||||
return state("permission_required", "Approve the USB camera permission, then request the capture again")
|
||||
}
|
||||
val result = JSONObject().put("state", "capturing").put("startedMs", android.os.SystemClock.elapsedRealtime())
|
||||
.put("timeoutMs",durationMs+15000).put("transport",transport)
|
||||
save(app, result)
|
||||
try { app.startService(Intent(app, LumaRgbCaptureService::class.java).putExtra("device", device.deviceName)
|
||||
.putExtra("transport",transport).putExtra("width",width).putExtra("height",height)
|
||||
.putExtra("fps",fps).putExtra("durationMs",durationMs)) }
|
||||
catch (e: Exception) { return state("failed", e.message ?: e.toString()) }
|
||||
return result
|
||||
}
|
||||
|
||||
internal fun perform(context: Context, deviceName: String, transport: String, width: Int, height: Int,
|
||||
fps: Int, durationMs: Int) {
|
||||
val folder=File(context.filesDir,"spatial/rgb-captures/${System.nanoTime()}").apply { mkdirs() }
|
||||
val startedMs=android.os.SystemClock.elapsedRealtime()
|
||||
val workerPid=android.os.Process.myPid()
|
||||
save(context,JSONObject().put("state","capturing").put("path",folder.path)
|
||||
.put("startedMs",startedMs).put("timeoutMs",durationMs+15000)
|
||||
.put("transport",transport).put("workerPid",workerPid))
|
||||
val result = try {
|
||||
val direct=transport=="uvc"
|
||||
check(if(direct) UvcRgbDiagnostic.available else VitureSdk.available) { "Requested RGB transport is unavailable" }
|
||||
val usb = context.getSystemService(Context.USB_SERVICE) as UsbManager
|
||||
val device = checkNotNull(usb.deviceList[deviceName]) { "RGB camera disconnected" }
|
||||
check(device.vendorId == 0x0c45 && device.productId == 0x636b && usb.hasPermission(device))
|
||||
val connection = checkNotNull(usb.openDevice(device)) { "Could not open RGB camera" }
|
||||
val metadata=LongArray(if(direct) 14 else 7)
|
||||
val jpeg=try {
|
||||
File(folder,"usb-descriptors.bin").writeBytes(connection.rawDescriptors)
|
||||
android.os.ParcelFileDescriptor.open(File(folder,"usb.log"),
|
||||
android.os.ParcelFileDescriptor.MODE_CREATE or android.os.ParcelFileDescriptor.MODE_TRUNCATE or
|
||||
android.os.ParcelFileDescriptor.MODE_WRITE_ONLY).use { log ->
|
||||
if(direct) UvcRgbDiagnostic.nativeSnapshot(connection.fileDescriptor,width,height,fps,durationMs,metadata,log.fd)
|
||||
else VitureSdk.nativeRgbSnapshot(device.vendorId,device.productId,connection.fileDescriptor,metadata,log.fd)
|
||||
}
|
||||
} finally { connection.close() }
|
||||
check(jpeg != null) { "RGB camera returned no frame ($transport status ${metadata[4]}, callbacks ${metadata[5]})" }
|
||||
File(folder,"rgb.jpg").writeBytes(jpeg)
|
||||
val info=JSONObject().put("state","completed").put("path",folder.path)
|
||||
.put("width",metadata[0]).put("height",metadata[1]).put("timestampNs",metadata[2])
|
||||
.put("sequence",metadata[3]).put("callbacks",metadata[5]).put("source","Luma Ultra UVC RGB")
|
||||
.put("receiptNs",metadata[6]).put("timingBasis","Host callback receipt; not exposure time")
|
||||
.put("snapshotAcquired",true)
|
||||
if(direct) {
|
||||
val health=UvcStreamHealth.assess(metadata,fps,durationMs)
|
||||
info.put("firstReceiptNs",metadata[7]).put("maxFrameGapMs",health.maximumGapMs)
|
||||
.put("completeFrames",metadata[9]).put("invalidFrames",metadata[10]).put("bytes",metadata[11])
|
||||
.put("observationEndNs",metadata[12]).put("tailFrameAgeMs",metadata[13]/1e6)
|
||||
.put("observationMs",health.observationMs).put("observedFps",health.observedFps)
|
||||
.put("streamHealthy",health.healthy)
|
||||
}
|
||||
info
|
||||
} catch (e: Exception) { JSONObject().put("state","failed").put("snapshotAcquired",false)
|
||||
.put("message",e.message ?: e.toString()) }
|
||||
result.put("path",folder.path).put("transport",transport).put("requestedWidth",width).put("requestedHeight",height)
|
||||
.put("requestedFps",fps).put("durationMs",durationMs).put("workerPid",workerPid)
|
||||
.put("startedMs",startedMs).put("timeoutMs",durationMs+15000)
|
||||
if (transport == "uvc" && !result.has("streamHealthy")) result.put("streamHealthy",false)
|
||||
File(folder,"frame.json").writeText(result.toString())
|
||||
save(context,result)
|
||||
}
|
||||
}
|
||||
|
||||
class LumaRgbCaptureService : Service() {
|
||||
private var running = false
|
||||
override fun onBind(intent: Intent?): IBinder? = null
|
||||
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
||||
if (!running) {
|
||||
running = true
|
||||
Thread({
|
||||
try { LumaRgbDiagnostic.perform(this,intent?.getStringExtra("device") ?: "",
|
||||
intent?.getStringExtra("transport") ?: "sdk",intent?.getIntExtra("width",640) ?: 640,
|
||||
intent?.getIntExtra("height",480) ?: 480,intent?.getIntExtra("fps",30) ?: 30,
|
||||
intent?.getIntExtra("durationMs",5000) ?: 5000) }
|
||||
finally {
|
||||
stopSelf()
|
||||
// SDK 2.4.0 can crash while cancelling failed USB transfers; isolate each diagnostic.
|
||||
android.os.Process.killProcess(android.os.Process.myPid())
|
||||
}
|
||||
}, "luma-rgb-diagnostic").start()
|
||||
}
|
||||
return START_NOT_STICKY
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import com.droiddeck.launcher.R
|
||||
import java.util.Locale
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
internal object SpatialMeasurements {
|
||||
fun height(centimeters: Int): String {
|
||||
val tenths = (centimeters / 2.54 * 10).roundToInt()
|
||||
return String.format(Locale.getDefault(), "%d cm (%d ft %.1f in)", centimeters, tenths / 120, tenths % 120 / 10.0)
|
||||
}
|
||||
|
||||
fun distance(meters: Float): String = String.format(
|
||||
Locale.getDefault(), "%.1f m (%.1f ft)", meters, meters / 0.3048f)
|
||||
|
||||
fun validation(meters: Float): String = String.format(
|
||||
Locale.getDefault(), "%.1f cm (%.2f in)", meters * 100, meters / 0.0254f)
|
||||
}
|
||||
|
||||
internal enum class CalibrationDistance(val meters: Float, val labelRes: Int) {
|
||||
INCHES_12(0.3048f, R.string.spatial_distance_12in),
|
||||
CENTIMETERS_30(0.3f, R.string.spatial_distance_30cm),
|
||||
INCHES_20(0.508f, R.string.spatial_distance_20in),
|
||||
CENTIMETERS_50(0.5f, R.string.spatial_distance_50cm),
|
||||
}
|
||||
|
||||
internal data class SpatialCalibrationProgress(
|
||||
val collecting: Boolean = false,
|
||||
val views: Int = 0,
|
||||
val fitted: Boolean = false,
|
||||
val solving: Boolean = false,
|
||||
val capturing: Boolean = false,
|
||||
val captureMessage: String? = null,
|
||||
val validating: Boolean = false,
|
||||
val validationMessage: String? = null,
|
||||
)
|
||||
@@ -0,0 +1,278 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import kotlin.math.*
|
||||
|
||||
data class SpatialVector(val x: Float = 0f, val y: Float = 0f, val z: Float = 0f) {
|
||||
operator fun plus(b: SpatialVector) = SpatialVector(x + b.x, y + b.y, z + b.z)
|
||||
operator fun minus(b: SpatialVector) = SpatialVector(x - b.x, y - b.y, z - b.z)
|
||||
operator fun times(s: Float) = SpatialVector(x * s, y * s, z * s)
|
||||
fun dot(b: SpatialVector) = x * b.x + y * b.y + z * b.z
|
||||
fun cross(b: SpatialVector) = SpatialVector(y * b.z - z * b.y, z * b.x - x * b.z, x * b.y - y * b.x)
|
||||
fun length() = sqrt(dot(this))
|
||||
fun normalized() = length().takeIf { it > 1e-6f }?.let { this * (1f / it) }
|
||||
fun finite() = x.isFinite() && y.isFinite() && z.isFinite()
|
||||
fun array() = floatArrayOf(x, y, z)
|
||||
}
|
||||
|
||||
data class SpatialRotation(val x: Float = 0f, val y: Float = 0f, val z: Float = 0f, val w: Float = 1f) {
|
||||
fun normalized(): SpatialRotation? {
|
||||
val n = sqrt(x*x + y*y + z*z + w*w)
|
||||
return if (n.isFinite() && n in 0.5f..1.5f) SpatialRotation(x/n, y/n, z/n, w/n) else null
|
||||
}
|
||||
fun inverse() = SpatialRotation(-x, -y, -z, w)
|
||||
operator fun times(b: SpatialRotation) = SpatialRotation(
|
||||
w*b.x + x*b.w + y*b.z - z*b.y, w*b.y - x*b.z + y*b.w + z*b.x,
|
||||
w*b.z + x*b.y - y*b.x + z*b.w, w*b.w - x*b.x - y*b.y - z*b.z)
|
||||
fun apply(v: SpatialVector): SpatialVector {
|
||||
val q = SpatialVector(x, y, z)
|
||||
val t = q.cross(v) * 2f
|
||||
return v + t * w + q.cross(t)
|
||||
}
|
||||
fun matrix() = floatArrayOf(
|
||||
1-2*(y*y+z*z), 2*(x*y-z*w), 2*(x*z+y*w),
|
||||
2*(x*y+z*w), 1-2*(x*x+z*z), 2*(y*z-x*w),
|
||||
2*(x*z-y*w), 2*(y*z+x*w), 1-2*(x*x+y*y))
|
||||
fun yaw(): SpatialRotation {
|
||||
val forward = apply(SpatialVector(0f, 0f, -1f))
|
||||
val a = if (forward.x*forward.x + forward.z*forward.z < 1e-8f) {
|
||||
val right = apply(SpatialVector(x = 1f))
|
||||
atan2(-right.z, right.x) / 2
|
||||
} else atan2(-forward.x, -forward.z) / 2
|
||||
return SpatialRotation(y = sin(a), w = cos(a))
|
||||
}
|
||||
companion object {
|
||||
fun fromBasis(right: SpatialVector, up: SpatialVector, back: SpatialVector): SpatialRotation {
|
||||
val m = floatArrayOf(right.x, up.x, back.x, right.y, up.y, back.y, right.z, up.z, back.z)
|
||||
val trace = m[0] + m[4] + m[8]
|
||||
val q = if (trace > 0) {
|
||||
val s = sqrt(trace + 1) * 2
|
||||
SpatialRotation((m[7]-m[5])/s, (m[2]-m[6])/s, (m[3]-m[1])/s, s/4)
|
||||
} else {
|
||||
val i = if (m[0] > m[4] && m[0] > m[8]) 0 else if (m[4] > m[8]) 1 else 2
|
||||
val j = (i+1)%3; val k = (i+2)%3
|
||||
val s = sqrt(1 + m[i*3+i] - m[j*3+j] - m[k*3+k]) * 2
|
||||
val v = FloatArray(3)
|
||||
v[i] = s/4; v[j] = (m[j*3+i]+m[i*3+j])/s; v[k] = (m[k*3+i]+m[i*3+k])/s
|
||||
SpatialRotation(v[0], v[1], v[2], (m[k*3+j]-m[j*3+k])/s)
|
||||
}
|
||||
return q.normalized() ?: SpatialRotation()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class SpatialPose(val position: SpatialVector = SpatialVector(), val rotation: SpatialRotation = SpatialRotation()) {
|
||||
fun inverse(): SpatialPose {
|
||||
val r = rotation.inverse()
|
||||
return SpatialPose(r.apply(position * -1f), r)
|
||||
}
|
||||
operator fun times(b: SpatialPose) = SpatialPose(position + rotation.apply(b.position), rotation * b.rotation)
|
||||
companion object {
|
||||
fun fromSdk(v: FloatArray): SpatialPose? {
|
||||
if (v.size != 7 || v.any { !it.isFinite() }) return null
|
||||
val q = SpatialRotation(v[4], v[5], v[6], v[3]).normalized() ?: return null
|
||||
return SpatialPose(SpatialVector(v[0], v[1], v[2]), q)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data class SpatialSample(val pose: SpatialPose, val timeNs: Long, val generation: Long, val valid: Boolean)
|
||||
data class SpatialHand(val pose: SpatialPose = SpatialPose(), val timeNs: Long = 0, val valid: Boolean = false,
|
||||
val trigger: Float = 0f, val grip: Boolean = false,
|
||||
val triggerValid: Boolean = valid, val gripValid: Boolean = valid) {
|
||||
fun flags() = if (!valid) 0 else 1 or (if (triggerValid) 2 else 0) or (if (gripValid) 4 else 0)
|
||||
}
|
||||
|
||||
class SpatialSession {
|
||||
var generation: Long = 0; private set
|
||||
var originGeneration: Long = 0; private set
|
||||
var trackingFrameGeneration: Long = 0; private set
|
||||
var needsRecenter = false; private set
|
||||
val recenterPending: Boolean get() = pendingRecenter != null
|
||||
private var pendingRecenter: Boolean? = null
|
||||
private var recoveryAnchor: SpatialSample? = null
|
||||
private var head: SpatialSample? = null
|
||||
private var lastStable: SpatialSample? = null
|
||||
private var vrOrigin: SpatialPose? = null
|
||||
private var screenOrigin: SpatialPose? = null
|
||||
private var hands = arrayOf(SpatialHand(), SpatialHand())
|
||||
private val gestureEpochs = IntArray(4)
|
||||
private val gestureEligible = BooleanArray(4)
|
||||
private var inputsAfterNs = 0L
|
||||
private var height = 1.6f
|
||||
private val history = ArrayDeque<SpatialSample>()
|
||||
|
||||
@Synchronized fun reset(connection: Long, eyeHeight: Float, requireSetup: Boolean = false) {
|
||||
generation = connection; height = eyeHeight.takeIf { it.isFinite() }?.coerceIn(0.5f, 2.5f) ?: 1.6f
|
||||
originGeneration++; trackingFrameGeneration++; head = null; lastStable = null; vrOrigin = null; screenOrigin = null
|
||||
history.clear()
|
||||
pendingRecenter = null; recoveryAnchor = null
|
||||
hands = arrayOf(SpatialHand(), SpatialHand()); needsRecenter = requireSetup
|
||||
gestureEpochs.fill(0); gestureEligible.fill(false)
|
||||
inputsAfterNs = 0L
|
||||
}
|
||||
@Synchronized fun accept(sample: SpatialSample): Boolean {
|
||||
if (sample.generation != generation || sample.timeNs <= (head?.timeNs ?: 0)) return false
|
||||
val previous = head
|
||||
if (previous != null && sample.timeNs-previous.timeNs > 100_000_000L) invalidateHeadInputs(sample.timeNs)
|
||||
val rotation = sample.pose.rotation.normalized()
|
||||
val current = sample.copy(pose = sample.pose.copy(rotation = rotation ?: SpatialRotation()),
|
||||
valid = sample.valid && sample.pose.position.finite() && rotation != null)
|
||||
head = current
|
||||
if (!current.valid) {
|
||||
recoveryAnchor = null; invalidateHeadInputs(current.timeNs); return false
|
||||
}
|
||||
lastStable?.let {
|
||||
val d = (current.pose.position - it.pose.position).length()
|
||||
val a = current.pose.rotation; val b = it.pose.rotation
|
||||
if (!needsRecenter && (d > 0.5f || abs(a.x*b.x+a.y*b.y+a.z*b.z+a.w*b.w) < 0.9238795f)) {
|
||||
needsRecenter = true; trackingFrameGeneration++
|
||||
}
|
||||
}
|
||||
pendingRecenter?.let { screen ->
|
||||
val anchor = recoveryAnchor
|
||||
val a = current.pose.rotation; val b = anchor?.pose?.rotation ?: a
|
||||
if (anchor == null || previous?.valid != true || current.timeNs-previous.timeNs > 100_000_000L ||
|
||||
(current.pose.position-anchor.pose.position).length() > 0.05f ||
|
||||
abs(a.x*b.x+a.y*b.y+a.z*b.z+a.w*b.w) < 0.9961947f) {
|
||||
recoveryAnchor = current
|
||||
} else if (current.timeNs-anchor.timeNs >= 250_000_000L) {
|
||||
applyRecenter(screen, current)
|
||||
}
|
||||
}
|
||||
if (needsRecenter || recenterPending) { invalidateHeadInputs(current.timeNs); return false }
|
||||
lastStable = current
|
||||
history.addLast(current)
|
||||
while (history.size > 512) history.removeFirst()
|
||||
if (vrOrigin == null) vrOrigin = SpatialPose(current.pose.position - SpatialVector(y = height), current.pose.rotation.yaw())
|
||||
if (screenOrigin == null) screenOrigin = current.pose
|
||||
return true
|
||||
}
|
||||
@Synchronized fun setFloor(eyeHeight: Float, now: Long = System.nanoTime()) {
|
||||
if (!eyeHeight.isFinite()) return
|
||||
height = eyeHeight.coerceIn(0.5f, 2.5f)
|
||||
val p = head?.takeIf { it.valid && now-it.timeNs in 0..100_000_000L }?.pose ?: return
|
||||
vrOrigin = SpatialPose(p.position - SpatialVector(y = height), p.rotation.yaw())
|
||||
if (needsRecenter) { history.clear(); screenOrigin = p }
|
||||
lastStable = head; needsRecenter = false; pendingRecenter = null; recoveryAnchor = null
|
||||
originGeneration++; invalidateHeadInputs(now)
|
||||
}
|
||||
@Synchronized fun recenter(screen: Boolean, now: Long = System.nanoTime()) {
|
||||
val current = head?.takeIf { it.valid && now-it.timeNs in 0..100_000_000L }
|
||||
if (current == null || needsRecenter || recenterPending) {
|
||||
if (pendingRecenter == null) recoveryAnchor = null
|
||||
pendingRecenter = screen
|
||||
invalidateHeadInputs(now)
|
||||
return
|
||||
}
|
||||
applyRecenter(screen, current)
|
||||
}
|
||||
private fun applyRecenter(screen: Boolean, current: SpatialSample) {
|
||||
val p = current.pose
|
||||
if (vrOrigin == null) vrOrigin = SpatialPose(p.position - SpatialVector(y = height), p.rotation.yaw())
|
||||
if (screenOrigin == null) screenOrigin = p
|
||||
if (needsRecenter) {
|
||||
vrOrigin = SpatialPose(p.position - SpatialVector(y = height), p.rotation.yaw())
|
||||
screenOrigin = p; originGeneration++; history.clear()
|
||||
}
|
||||
if (screen) screenOrigin = p else {
|
||||
val floor = vrOrigin?.position?.y ?: (p.position.y-height)
|
||||
vrOrigin = SpatialPose(SpatialVector(p.position.x, floor, p.position.z), p.rotation.yaw())
|
||||
originGeneration++
|
||||
}
|
||||
lastStable = current; needsRecenter = false; pendingRecenter = null; recoveryAnchor = null
|
||||
invalidateHeadInputs(current.timeNs)
|
||||
}
|
||||
@Synchronized fun view(screen: Boolean): SpatialPose? = lastStable?.pose?.let {
|
||||
(if (screen) screenOrigin else vrOrigin)?.inverse()?.times(it)
|
||||
}
|
||||
@Synchronized fun latest() = head
|
||||
@Synchronized fun poseAt(time: Long, connection: Long): SpatialPose? {
|
||||
if (connection != generation || needsRecenter || recenterPending) return null
|
||||
return history.minByOrNull { abs(it.timeNs-time) }?.takeIf { abs(it.timeNs-time) <= 12_000_000L }?.pose
|
||||
}
|
||||
@Synchronized fun valid(now: Long) = !needsRecenter && !recenterPending && head?.let { it.valid && now-it.timeNs in 0..100_000_000L } == true
|
||||
@Synchronized internal fun isCurrentTrackingFrame(connection: Long, frame: Long, clock: () -> Long = System::nanoTime): Boolean =
|
||||
connection == generation && frame == trackingFrameGeneration && valid(clock())
|
||||
@Synchronized fun setHands(connection: Long, left: SpatialHand, right: SpatialHand) {
|
||||
if (connection != generation || needsRecenter || recenterPending) return
|
||||
val next = arrayOf(left, right)
|
||||
for (index in 0..1) {
|
||||
val oldTime = hands[index].timeNs
|
||||
val hand = next[index]
|
||||
if (oldTime > 0 && (hand.timeNs < oldTime || hand.timeNs-oldTime > HandObservation.MAX_AGE_NS))
|
||||
invalidateInputs(index)
|
||||
updateInputs(index, hand, head?.valid == true && usableHand(hand) && hand.timeNs > inputsAfterNs)
|
||||
}
|
||||
hands = next
|
||||
}
|
||||
@Synchronized fun hand(index: Int, now: Long): SpatialHand {
|
||||
val h = hands[index]
|
||||
val origin = vrOrigin
|
||||
val current = origin != null && usableHand(h) && valid(now) && now-h.timeNs in 0..HandObservation.MAX_AGE_NS
|
||||
updateInputs(index, h, current && h.timeNs > inputsAfterNs)
|
||||
return if (origin == null) SpatialHand() else h.copy(pose = origin.inverse() * h.pose, valid = current,
|
||||
triggerValid = h.triggerValid && gestureEligible[index*2], gripValid = h.gripValid && gestureEligible[index*2+1])
|
||||
}
|
||||
@Synchronized fun inputEpochs(): Int {
|
||||
var packed = 0
|
||||
for (index in gestureEpochs.indices) packed = packed or (gestureEpochs[index] shl (index*8))
|
||||
return packed
|
||||
}
|
||||
private fun usableHand(hand: SpatialHand): Boolean {
|
||||
val q = hand.pose.rotation
|
||||
val norm = q.x*q.x+q.y*q.y+q.z*q.z+q.w*q.w
|
||||
return hand.valid && hand.timeNs > 0 && hand.pose.position.finite() && norm > 0.99f && norm < 1.01f
|
||||
}
|
||||
private fun updateInputs(index: Int, hand: SpatialHand, valid: Boolean) {
|
||||
updateInput(index*2, valid && hand.triggerValid && hand.trigger.isFinite() && hand.trigger in 0f..1f)
|
||||
updateInput(index*2+1, valid && hand.gripValid)
|
||||
}
|
||||
private fun updateInput(index: Int, eligible: Boolean) {
|
||||
if (gestureEligible[index] && !eligible) gestureEpochs[index] = (gestureEpochs[index]+1) and 0xff
|
||||
gestureEligible[index] = eligible
|
||||
}
|
||||
private fun invalidateInputs(index: Int? = null) {
|
||||
if (index == null) for (gesture in gestureEligible.indices) updateInput(gesture, false)
|
||||
else { updateInput(index*2, false); updateInput(index*2+1, false) }
|
||||
}
|
||||
private fun invalidateHeadInputs(time: Long) {
|
||||
inputsAfterNs = maxOf(inputsAfterNs, time)
|
||||
hands = arrayOf(SpatialHand(), SpatialHand())
|
||||
invalidateInputs()
|
||||
}
|
||||
}
|
||||
|
||||
internal data class HandObservation(val timeNs: Long, val connection: Long, val trackingFrame: Long) {
|
||||
fun headPose(session: SpatialSession, cameraSynchronized: Boolean, now: Long? = null): SpatialPose? = synchronized(session) {
|
||||
val checkedAt=now ?: System.nanoTime()
|
||||
if (!cameraSynchronized || timeNs <= 0 || checkedAt-timeNs !in 0..MAX_AGE_NS ||
|
||||
!session.isCurrentTrackingFrame(connection, trackingFrame) { checkedAt }) null
|
||||
else session.poseAt(timeNs, connection)
|
||||
}
|
||||
companion object {
|
||||
const val MAX_AGE_NS = 200_000_000L
|
||||
}
|
||||
}
|
||||
|
||||
internal class CameraClock {
|
||||
private val offsets = ArrayDeque<Long>()
|
||||
private var lastSource = 0L
|
||||
private var lastReceipt = 0L
|
||||
fun map(source: Long, receipt: Long): Long? {
|
||||
if (source <= 0 || receipt <= 0) {
|
||||
offsets.clear();lastSource=0;lastReceipt=0
|
||||
return null
|
||||
}
|
||||
val offset = receipt-source
|
||||
val skipped = lastSource > 0 && source-lastSource > 200_000_000L
|
||||
val inconsistent = skipped && offsets.minOrNull()?.let { abs(offset-it) > 20_000_000L } == true
|
||||
if (source <= lastSource || receipt <= lastReceipt || inconsistent) offsets.clear()
|
||||
lastSource = source;lastReceipt = receipt
|
||||
offsets.addLast(offset)
|
||||
while (offsets.size > 64) offsets.removeFirst()
|
||||
if (offsets.size < 16) return null
|
||||
val sorted = offsets.sorted()
|
||||
if (sorted[sorted.size*9/10]-sorted.first() > 20_000_000L) return null
|
||||
return source+sorted.first()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.content.Context
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.wayland.WaylandCompositor
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
import kotlinx.coroutines.flow.update
|
||||
import java.nio.ByteBuffer
|
||||
import java.nio.ByteOrder
|
||||
|
||||
data class SpatialTrackingStatus(val message: String = "", val calibration: Boolean = false,
|
||||
val frames: Long = 0, val dropped: Long = 0, val processingMs: Float = 0f,
|
||||
val frameAgeMs: Float = -1f, val cameraHz: Float = 0f, val synchronized: Boolean = false,
|
||||
val trackingValid: Boolean = false, val needsRecenter: Boolean = false, val recenterPending: Boolean = false,
|
||||
val handStatus: String = "", val headFresh: Boolean = false,
|
||||
val cameraFrameDivisor: Int = 1, val rateLimitedFrames: Long = 0, val cameraCopyPermille: Int = 1000,
|
||||
val calibrationSource: String = "none", val calibrationVerified: Boolean = false)
|
||||
|
||||
object SpatialTracking {
|
||||
val session = SpatialSession()
|
||||
private val mutableStatus = MutableStateFlow(SpatialTrackingStatus())
|
||||
val status = mutableStatus.asStateFlow()
|
||||
@Volatile var enabled = false; private set
|
||||
@Volatile var handsEnabled = false; private set
|
||||
private var context: Context? = null
|
||||
private var engine: StereoHandEngine? = null
|
||||
private var diagnosticsEnabled = false
|
||||
private val buffer = ByteBuffer.allocateDirect(176).order(ByteOrder.LITTLE_ENDIAN)
|
||||
private var lastStatus = 0L
|
||||
@Volatile private var publisherAttached = false
|
||||
@Volatile private var spatialSelected = false
|
||||
private var outputUsesGlasses = true
|
||||
private var handsSelected = false
|
||||
private var steadyAimSelected = false
|
||||
@Volatile private var joyConsSelected = false
|
||||
internal val joyConsEnabled get() = enabled && joyConsSelected
|
||||
private var hadSpatialSession = false
|
||||
internal val selected: Boolean get() = spatialSelected
|
||||
private fun text(id: Int) = context?.getString(id).orEmpty()
|
||||
fun initialize(app: Context) {
|
||||
context = app.applicationContext
|
||||
mutableStatus.update { it.copy(
|
||||
message = it.message.ifEmpty { text(R.string.spatial_disabled) },
|
||||
handStatus = it.handStatus.ifEmpty { text(R.string.spatial_hands_off) },
|
||||
) }
|
||||
}
|
||||
fun attachPublisher(path: String) {
|
||||
publisherAttached = WaylandCompositor.nativeSetSpatialFile(path)
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
|
||||
internal fun connect(token: Long, generation: Long, identity: String?, spatial: Boolean, eyeHeight: Float, hands: Boolean,
|
||||
joyCons: Boolean = false, steadyAim: Boolean = false) {
|
||||
engine?.close(); engine = null
|
||||
enabled = spatial; handsEnabled = spatial && (hands || joyCons)
|
||||
spatialSelected = spatial; handsSelected = spatial && hands && !joyCons
|
||||
joyConsSelected = spatial && joyCons
|
||||
steadyAimSelected = steadyAim
|
||||
session.reset(generation, eyeHeight, spatial && hadSpatialSession)
|
||||
hadSpatialSession = hadSpatialSession || spatial
|
||||
if (spatial) context?.let { engine = StereoHandEngine(it, token, generation, identity) }
|
||||
mutableStatus.value = SpatialTrackingStatus(if (spatial) text(R.string.spatial_wait_stable) else text(R.string.spatial_disabled))
|
||||
engine?.setEnabled(handsEnabled)
|
||||
engine?.diagnostics(diagnosticsEnabled)
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
internal fun disconnect() {
|
||||
enabled = false; handsEnabled = false
|
||||
session.reset(session.generation + 1, 1.6f)
|
||||
publish(System.nanoTime())
|
||||
engine?.close(); engine = null
|
||||
mutableStatus.value = SpatialTrackingStatus(text(R.string.spatial_disconnected))
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
@Synchronized internal fun useLegacyTracking() {
|
||||
spatialSelected = false; handsSelected = false; joyConsSelected = false
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
@Synchronized internal fun setOutputUsesGlasses(on: Boolean) {
|
||||
if (outputUsesGlasses == on) return
|
||||
outputUsesGlasses = on
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
@Synchronized internal fun accept(values: FloatArray, time: Long, generation: Long, valid: Boolean) {
|
||||
if (!enabled || generation != session.generation) return
|
||||
val pose = SpatialPose.fromSdk(values)
|
||||
session.accept(SpatialSample(pose ?: SpatialPose(), time, generation, valid && pose != null))
|
||||
publish(time)
|
||||
if (time-lastStatus > 250_000_000L) {
|
||||
lastStatus = time
|
||||
refreshStatus(time)
|
||||
}
|
||||
}
|
||||
@Synchronized fun refreshStatus(now: Long? = null) {
|
||||
val checkedAt = now ?: System.nanoTime()
|
||||
val valid = session.valid(checkedAt)
|
||||
val pending = session.recenterPending
|
||||
val recenter = session.needsRecenter
|
||||
val fresh = session.latest()?.let { it.valid && checkedAt-it.timeNs in 0..100_000_000L } == true
|
||||
mutableStatus.update { it.copy(trackingValid = valid, needsRecenter = recenter, recenterPending = pending, headFresh = fresh,
|
||||
handStatus = engine?.handStatus?.message ?: text(R.string.spatial_hands_off),
|
||||
message = when {
|
||||
!enabled -> text(R.string.spatial_tracking_off)
|
||||
pending -> text(R.string.spatial_recenter_pending)
|
||||
!fresh -> text(R.string.spatial_tracking_lost)
|
||||
recenter -> text(R.string.spatial_origin_changed)
|
||||
else -> text(R.string.spatial_active)
|
||||
}) }
|
||||
}
|
||||
@Synchronized fun setControllers(hands: Boolean, joyCons: Boolean) {
|
||||
handsSelected = hands && !joyCons
|
||||
joyConsSelected = joyCons
|
||||
handsEnabled = enabled && (hands || joyCons)
|
||||
session.setHands(session.generation, SpatialHand(), SpatialHand())
|
||||
engine?.setEnabled(handsEnabled)
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
@Synchronized fun setSteadyHandAim(on: Boolean) {
|
||||
steadyAimSelected = on
|
||||
publish(System.nanoTime())
|
||||
}
|
||||
fun setFloor(height: Float) { session.setFloor(height); refreshStatus(); publish(System.nanoTime()) }
|
||||
fun recenter(screen: Boolean) { session.recenter(screen); refreshStatus(); publish(System.nanoTime()) }
|
||||
fun diagnostics(on: Boolean) { diagnosticsEnabled = on; engine?.diagnostics(on) }
|
||||
fun calibrationStart() { engine?.startCalibration() }
|
||||
fun calibrationCapture() { engine?.captureCalibration() }
|
||||
fun calibrationSolve() { engine?.solveCalibration() }
|
||||
fun calibrationValidate(distanceMeters: Float) { engine?.validateCalibration(distanceMeters) }
|
||||
fun captureDiagnostic() { engine?.recordDiagnostic() }
|
||||
fun setGpu(gpu: Boolean) { engine?.setGpu(gpu) }
|
||||
fun setHandModel(lite: Boolean) { engine?.setModel(lite) }
|
||||
fun setHandHalfRate(halfRate: Boolean) { engine?.setHalfRate(halfRate) }
|
||||
fun setHandCpuParallel(parallel: Boolean) { engine?.setCpuParallel(parallel) }
|
||||
fun setHandAdaptiveInput(adaptive: Boolean) { engine?.setAdaptiveInput(adaptive) }
|
||||
fun setHandScheduledTracking(scheduled: Boolean) { engine?.setScheduledTracking(scheduled) }
|
||||
internal fun handLiteEnabled() = (engine?.model ?: HandModel.DEFAULT) == HandModel.LITE
|
||||
internal fun handHalfRateEnabled() = engine?.halfRateInput ?: true
|
||||
internal fun handAdaptiveInputEnabled() = engine?.adaptiveInput ?: true
|
||||
fun benchmarkSavedCapture(id: String) { checkNotNull(engine) { "Connect glasses and enable positional tracking" }.benchmarkSavedCapture(id) }
|
||||
internal fun diagnosticFrame() = engine?.preview
|
||||
internal fun diagnosticMessage() = engine?.message ?: text(R.string.spatial_connect_hint)
|
||||
internal fun calibrationProgress() = engine?.calibrationProgress ?: SpatialCalibrationProgress()
|
||||
internal fun gpuEnabled() = engine?.gpu ?: false
|
||||
fun diagnosticStatus(): org.json.JSONObject {
|
||||
val now=System.nanoTime();val sample=session.latest();val state=status.value;val progress=calibrationProgress()
|
||||
return org.json.JSONObject().put("enabled",enabled).put("valid",session.valid(now)).put("needsRecenter",session.needsRecenter)
|
||||
.put("joyConTracking",joyConsSelected)
|
||||
.put("steadyHandAim",handsSelected && steadyAimSelected)
|
||||
.put("positionEstimator",if (joyConsSelected) "joycon-wrist" else "full-palm")
|
||||
.put("recenterPending",session.recenterPending)
|
||||
.put("handMaxAgeMs",HandObservation.MAX_AGE_NS / 1_000_000)
|
||||
.put("handGpuEnabled",gpuEnabled()).put("handInferenceBackend",engine?.inferenceBackend ?: "not initialized")
|
||||
.put("handModel",(engine?.model ?: HandModel.DEFAULT).name).put("handHalfRate",handHalfRateEnabled())
|
||||
.put("handImagePreprocessing",HandImageBuffer.PREPROCESSING)
|
||||
.put("handLensCorrection",engine?.lensCorrection ?: "none")
|
||||
.put("handCpuParallel",engine?.cpuParallel ?: true)
|
||||
.put("handAdaptiveInput",handAdaptiveInputEnabled()).put("handScheduledTracking",engine?.scheduledTracking ?: true)
|
||||
.put("handInputRatePermille",engine?.inputRatePermille ?: 500)
|
||||
.put("handBenchmark",engine?.benchmark)
|
||||
.put("calibrationViews",progress.views).put("calibrationCollecting",progress.collecting)
|
||||
.put("capturePending",progress.capturing).put("captureMessage",progress.captureMessage)
|
||||
.put("calibrationFitted",progress.fitted).put("validationPending",progress.validating).put("validationMessage",progress.validationMessage)
|
||||
.put("calibrationIdentityAvailable",engine?.hasStableIdentity == true).put("handTracking",engine?.handStatus?.json())
|
||||
.put("calibrationSource",engine?.calibrationSource ?: "none").put("calibrationVerified",engine?.calibrationVerified == true)
|
||||
.put("generation",session.generation).put("origin",session.originGeneration)
|
||||
.put("ageMs",sample?.let { (now-it.timeNs)/1e6 } ?: -1.0)
|
||||
.put("position",sample?.pose?.position?.array()?.let { org.json.JSONArray(it.toList()) })
|
||||
.put("message",state.message).put("camera",diagnosticMessage()).put("frames",state.frames).put("dropped",state.dropped)
|
||||
.put("processingMs",state.processingMs.toDouble()).put("calibrated",state.calibration)
|
||||
.put("frameAgeMs",state.frameAgeMs.toDouble()).put("cameraHz",state.cameraHz.toDouble()).put("synchronized",state.synchronized)
|
||||
.put("cameraFrameDivisor",state.cameraFrameDivisor).put("rateLimitedFrames",state.rateLimitedFrames)
|
||||
.put("cameraCopyPermille",state.cameraCopyPermille)
|
||||
.put("leftValid",session.hand(0,now).valid).put("rightValid",session.hand(1,now).valid)
|
||||
}
|
||||
internal fun statistics(calibrated: Boolean, frames: Long, dropped: Long, ms: Float, age: Float, hz: Float, synchronized: Boolean,
|
||||
frameDivisor: Int, rateLimited: Long, copyPermille: Int = 1000) {
|
||||
mutableStatus.update { it.copy(calibration = calibrated, frames = frames, dropped = dropped, processingMs = ms,
|
||||
frameAgeMs=age,cameraHz=hz,synchronized=synchronized,cameraFrameDivisor=frameDivisor,
|
||||
rateLimitedFrames=rateLimited,cameraCopyPermille=copyPermille,
|
||||
calibrationSource=engine?.calibrationSource ?: "none",calibrationVerified=engine?.calibrationVerified == true) }
|
||||
}
|
||||
@Synchronized internal fun publish(now: Long) {
|
||||
synchronized(session) {
|
||||
val hands = Array(2) { session.hand(it, now) }
|
||||
buffer.clear()
|
||||
buffer.putInt(0x53504444).putInt(4).putInt(176).putInt(0)
|
||||
buffer.putLong(session.generation).putLong(session.originGeneration)
|
||||
val modes = (if (spatialSelected) 1 else 0) or (if (handsSelected) 2 else 0) or
|
||||
(if (joyConsSelected) 4 else 0) or (if (handsSelected && steadyAimSelected) 8 else 0)
|
||||
buffer.putInt(if (outputUsesGlasses) modes else 0).putInt(session.inputEpochs())
|
||||
putPose(session.view(false) ?: SpatialPose(), session.latest()?.timeNs ?: 0, if(session.valid(now)) 1 else 0)
|
||||
for (i in 0..1) {
|
||||
val hand = hands[i]
|
||||
putPose(hand.pose, hand.timeNs, hand.flags())
|
||||
buffer.putFloat(if (hand.valid && hand.triggerValid) hand.trigger else 0f)
|
||||
.putInt(if (hand.valid && hand.gripValid && hand.grip) 1 else 0)
|
||||
}
|
||||
buffer.flip()
|
||||
if (publisherAttached) WaylandCompositor.nativePublishSpatial(buffer)
|
||||
}
|
||||
}
|
||||
private fun putPose(p: SpatialPose, time: Long, flags: Int) {
|
||||
buffer.putLong(time).putFloat(p.position.x).putFloat(p.position.y).putFloat(p.position.z)
|
||||
buffer.putFloat(p.rotation.x).putFloat(p.rotation.y).putFloat(p.rotation.z).putFloat(p.rotation.w)
|
||||
buffer.putInt(flags)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal class SteamVrSpatialGate(private val settleMs: Long = 2_000) {
|
||||
private var waiting = false
|
||||
private var running = false
|
||||
var deadline: Long? = null
|
||||
private set
|
||||
|
||||
fun prepare(): Boolean {
|
||||
if (running || waiting) return false
|
||||
waiting = true
|
||||
deadline = null
|
||||
return true
|
||||
}
|
||||
|
||||
fun setRunning(on: Boolean, now: Long): Boolean {
|
||||
if (running == on) return false
|
||||
running = on
|
||||
waiting = false
|
||||
deadline = if (on) now + settleMs else null
|
||||
return true
|
||||
}
|
||||
|
||||
fun cancel() {
|
||||
waiting = false
|
||||
running = false
|
||||
deadline = null
|
||||
}
|
||||
|
||||
fun spatialEnabled(requested: Boolean, now: Long) =
|
||||
requested && !waiting && (deadline?.let { now >= it } ?: true)
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.content.Context
|
||||
import com.droiddeck.launcher.R
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
import org.opencv.calib3d.Calib3d
|
||||
import org.opencv.core.*
|
||||
import org.opencv.objdetect.CharucoBoard
|
||||
import org.opencv.objdetect.CharucoDetector
|
||||
import org.opencv.objdetect.Objdetect
|
||||
import org.opencv.imgproc.Imgproc
|
||||
import java.io.File
|
||||
import kotlin.math.*
|
||||
|
||||
internal data class BoardObservation(val objects: Mat, val left: Mat, val right: Mat, val head: SpatialPose)
|
||||
|
||||
internal class CalibrationAlignmentException(context: Context, val deviation: Float) : IllegalArgumentException(
|
||||
context.getString(R.string.spatial_alignment_error, SpatialMeasurements.validation(deviation)))
|
||||
|
||||
internal class StereoCalibration(val identity: String, val width: Int, val height: Int, val fisheye: Boolean,
|
||||
val k1: Mat, val d1: Mat, val k2: Mat, val d2: Mat, val r: Mat, val t: Mat,
|
||||
val cameraToHead: SpatialPose, val error: Double, var verified: Boolean = false,
|
||||
val bundledBase: Boolean = false) : AutoCloseable {
|
||||
val readyForTracking get() = verified || bundledBase
|
||||
val source get() = if (bundledBase && !verified) "luma-ultra-base" else "device"
|
||||
private val p1 = Mat.zeros(3, 4, CvType.CV_64F)
|
||||
private val p2 = Mat.zeros(3, 4, CvType.CV_64F)
|
||||
private val undistortionMaps = Array(2) { arrayOf(Mat(),Mat()) }
|
||||
init {
|
||||
val a = Mat.eye(3, 4, CvType.CV_64F)
|
||||
val b = Mat.zeros(3, 4, CvType.CV_64F)
|
||||
r.copyTo(b.colRange(0, 3)); t.copyTo(b.col(3))
|
||||
val empty = Mat()
|
||||
Core.gemm(k1, a, 1.0, empty, 0.0, p1); Core.gemm(k2, b, 1.0, empty, 0.0, p2)
|
||||
a.release(); b.release(); empty.release()
|
||||
}
|
||||
fun undistort(src: Mat, dst: Mat, right: Boolean) {
|
||||
val k = if (right) k2 else k1; val d = if (right) d2 else d1
|
||||
val maps=undistortionMaps[if(right) 1 else 0]
|
||||
if(maps[0].empty()) {
|
||||
val identity=Mat.eye(3,3,CvType.CV_64F)
|
||||
try {
|
||||
if(fisheye) Calib3d.fisheye_initUndistortRectifyMap(k,d,identity,k,src.size(),CvType.CV_16SC2,maps[0],maps[1])
|
||||
else Calib3d.initUndistortRectifyMap(k,d,identity,k,src.size(),CvType.CV_16SC2,maps[0],maps[1])
|
||||
} finally { identity.release() }
|
||||
}
|
||||
Imgproc.remap(src,dst,maps[0],maps[1],Imgproc.INTER_LINEAR)
|
||||
}
|
||||
fun triangulateRaw(left: List<Point>, right: List<Point>): List<SpatialVector>? {
|
||||
val points = triangulateLandmarksRaw(left,right) ?: return null
|
||||
return if(points.any { it == null }) null else points.filterNotNull()
|
||||
}
|
||||
fun triangulateLandmarksRaw(left: List<Point>, right: List<Point>): List<SpatialVector?>? {
|
||||
if (left.size != right.size || left.isEmpty()) return null
|
||||
val l=MatOfPoint2f(*left.toTypedArray()); val r=MatOfPoint2f(*right.toTypedArray())
|
||||
try { return triangulateRawPoints(l,r) } finally { l.release();r.release() }
|
||||
}
|
||||
fun triangulateRaw(left: Mat, right: Mat): List<SpatialVector>? {
|
||||
val points = triangulateRawPoints(left,right) ?: return null
|
||||
return if(points.any { it == null }) null else points.filterNotNull()
|
||||
}
|
||||
private fun triangulateRawPoints(left: Mat, right: Mat): List<SpatialVector?>? {
|
||||
val l=MatOfPoint2f(); val r=MatOfPoint2f(); val identity=Mat.eye(3,3,CvType.CV_64F)
|
||||
try {
|
||||
if (fisheye) {
|
||||
Calib3d.fisheye_undistortPoints(left,l,k1,d1,identity,k1)
|
||||
Calib3d.fisheye_undistortPoints(right,r,k2,d2,identity,k2)
|
||||
} else {
|
||||
val criteria=TermCriteria(TermCriteria.COUNT+TermCriteria.EPS,50,1e-8)
|
||||
Calib3d.undistortPointsIter(left,l,k1,d1,identity,k1,criteria)
|
||||
Calib3d.undistortPointsIter(right,r,k2,d2,identity,k2,criteria)
|
||||
}
|
||||
return triangulatePoints(l.toList(),r.toList())
|
||||
} finally { l.release(); r.release(); identity.release() }
|
||||
}
|
||||
fun triangulate(left: List<Point>, right: List<Point>): List<SpatialVector>? {
|
||||
val points = triangulatePoints(left,right) ?: return null
|
||||
return if(points.any { it == null }) null else points.filterNotNull()
|
||||
}
|
||||
private fun triangulatePoints(left: List<Point>, right: List<Point>): List<SpatialVector?>? {
|
||||
if (left.size != right.size || left.isEmpty()) return null
|
||||
val a = Mat(2, left.size, CvType.CV_64F); val b = Mat(2, right.size, CvType.CV_64F); val out = Mat()
|
||||
try {
|
||||
a.put(0, 0, *left.map { it.x }.toDoubleArray()); a.put(1, 0, *left.map { it.y }.toDoubleArray())
|
||||
b.put(0, 0, *right.map { it.x }.toDoubleArray()); b.put(1, 0, *right.map { it.y }.toDoubleArray())
|
||||
Calib3d.triangulatePoints(p1, p2, a, b, out)
|
||||
val projection1 = DoubleArray(12); val projection2 = DoubleArray(12)
|
||||
p1.get(0,0,projection1); p2.get(0,0,projection2)
|
||||
return left.indices.map { i ->
|
||||
StereoPointValidation.reconstruct(DoubleArray(4) { out.get(it,i)[0] },
|
||||
doubleArrayOf(left[i].x,left[i].y),doubleArrayOf(right[i].x,right[i].y),projection1,projection2)
|
||||
}
|
||||
} finally { a.release(); b.release(); out.release() }
|
||||
}
|
||||
fun save(file: File) {
|
||||
fun mat(m: Mat) = JSONArray((0 until m.rows()).flatMap { row -> (0 until m.cols()).map { m.get(row,it)[0] } })
|
||||
val j = JSONObject().put("version",1).put("identity",identity).put("width",width).put("height",height)
|
||||
.put("fisheye",fisheye).put("error",error).put("verified",verified)
|
||||
for ((name,value) in listOf("k1" to k1,"d1" to d1,"k2" to k2,"d2" to d2,"r" to r,"t" to t))
|
||||
j.put(name, JSONObject().put("rows",value.rows()).put("cols",value.cols()).put("data",mat(value)))
|
||||
val p = cameraToHead.position; val q = cameraToHead.rotation
|
||||
j.put("cameraToHead",JSONArray(listOf(p.x,p.y,p.z,q.w,q.x,q.y,q.z)))
|
||||
file.parentFile?.mkdirs()
|
||||
val tmp = File(file.parentFile,file.name+".tmp"); tmp.writeText(j.toString())
|
||||
java.nio.file.Files.move(tmp.toPath(),file.toPath(),java.nio.file.StandardCopyOption.REPLACE_EXISTING)
|
||||
}
|
||||
override fun close() { (listOf(k1,d1,k2,d2,r,t,p1,p2)+undistortionMaps.flatMap { it.toList() }).forEach { it.release() } }
|
||||
companion object {
|
||||
fun load(file: File, identity: String, width: Int, height: Int): StereoCalibration? =
|
||||
runCatching { StereoCalibrationProfile.parse(file.readText(), identity, width, height)?.let(::fromProfile) }.getOrNull()
|
||||
|
||||
fun fromProfile(profile: StereoCalibrationProfile): StereoCalibration {
|
||||
val matrices = mutableListOf<Mat>()
|
||||
try {
|
||||
profile.matrices.forEach { data ->
|
||||
val matrix = Mat(data.rows, data.columns, CvType.CV_64F)
|
||||
matrices.add(matrix)
|
||||
matrix.put(0, 0, *data.values)
|
||||
}
|
||||
return StereoCalibration(profile.identity, profile.width, profile.height, profile.fisheye,
|
||||
matrices[0], matrices[1], matrices[2], matrices[3], matrices[4], matrices[5],
|
||||
profile.cameraToHead, profile.error, profile.verified, profile.bundledBase)
|
||||
} catch (e: Exception) {
|
||||
matrices.forEach(Mat::release)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal class StereoCalibrator(private val context: Context) : AutoCloseable {
|
||||
val board = CharucoBoard(Size(5.0,7.0),0.03f,0.022f,Objdetect.getPredefinedDictionary(Objdetect.DICT_4X4_50))
|
||||
private val detector = CharucoDetector(board)
|
||||
val observations = mutableListOf<BoardObservation>()
|
||||
var commonCorners: Int = 0; private set
|
||||
fun detect(left: Mat, right: Mat, head: SpatialPose): BoardObservation? {
|
||||
val lc=Mat(); val li=Mat(); val rc=Mat(); val ri=Mat()
|
||||
try {
|
||||
detector.detectBoard(left,lc,li); detector.detectBoard(right,rc,ri)
|
||||
val l=(0 until li.rows()).associate { li.get(it,0)[0].toInt() to lc.get(it,0) }
|
||||
val r=(0 until ri.rows()).associate { ri.get(it,0)[0].toInt() to rc.get(it,0) }
|
||||
val ids=l.keys.intersect(r.keys).sorted()
|
||||
commonCorners = ids.size
|
||||
if (ids.size<12) return null
|
||||
val objects=MatOfPoint3f(*ids.map { Point3((it%4+1)*0.03,(it/4+1)*0.03,0.0) }.toTypedArray())
|
||||
return BoardObservation(objects,MatOfPoint2f(*ids.map { Point(l[it]!![0],l[it]!![1]) }.toTypedArray()),
|
||||
MatOfPoint2f(*ids.map { Point(r[it]!![0],r[it]!![1]) }.toTypedArray()),head)
|
||||
} finally { listOf(lc,li,rc,ri).forEach { it.release() } }
|
||||
}
|
||||
fun solve(identity: String,width: Int,height: Int): StereoCalibration {
|
||||
require(observations.size>=24) { context.getString(R.string.spatial_need_views) }
|
||||
val initial=observations.first().head.rotation
|
||||
val axes=observations.map { initial.inverse()*it.head.rotation }.map { SpatialVector(it.x,it.y,it.z) }
|
||||
require(axes.any { a -> axes.any { b -> a.cross(b).length()>0.02f } }) { context.getString(R.string.spatial_need_rotation) }
|
||||
val training=observations.filterIndexed { i,_ -> i%5!=0 }; val held=observations.filterIndexed { i,_ -> i%5==0 }
|
||||
val failures = mutableListOf<Pair<String, Exception>>()
|
||||
val candidates=listOf(false,true).mapNotNull { fish ->
|
||||
try { fit(identity,width,height,fish,training,held) }
|
||||
catch (e: Exception) {
|
||||
val model = if (fish) "Fisheye" else "Pinhole"
|
||||
android.util.Log.w("StereoCalibration", "$model calibration failed", e)
|
||||
failures.add(model to e)
|
||||
null
|
||||
}
|
||||
}
|
||||
val best=candidates.minByOrNull { it.error } ?: run {
|
||||
val alignment=failures.map { it.second }.filterIsInstance<CalibrationAlignmentException>().minByOrNull { it.deviation }
|
||||
val reason=alignment?.message ?: failures.joinToString(" ") { (model,e) ->
|
||||
"$model: ${e.message?.substringBefore('\n') ?: e.javaClass.simpleName}"
|
||||
}
|
||||
error(context.getString(R.string.spatial_calibration_error, reason))
|
||||
}
|
||||
candidates.filter { it !== best }.forEach { it.close() }
|
||||
if(best.error>2.0) { best.close(); error(context.getString(R.string.spatial_reprojection_error, best.error)) }
|
||||
return best
|
||||
}
|
||||
private fun fit(identity: String,w: Int,h: Int,fish: Boolean,train: List<BoardObservation>,held: List<BoardObservation>): StereoCalibration {
|
||||
val k1=Mat.eye(3,3,CvType.CV_64F); val k2=Mat.eye(3,3,CvType.CV_64F)
|
||||
val d1=Mat.zeros(if(fish) 4 else 5,1,CvType.CV_64F); val d2=d1.clone()
|
||||
val r=Mat(); val t=Mat(); val objects=train.map { it.objects }; val size=Size(w.toDouble(),h.toDouble())
|
||||
fun intrinsic(k: Mat,d: Mat,images: List<Mat>) {
|
||||
val rv=mutableListOf<Mat>(); val tv=mutableListOf<Mat>()
|
||||
try {
|
||||
if(fish) Calib3d.fisheye_calibrate(objects,images,size,k,d,rv,tv,Calib3d.fisheye_CALIB_RECOMPUTE_EXTRINSIC or Calib3d.fisheye_CALIB_FIX_SKEW)
|
||||
else Calib3d.calibrateCamera(objects,images,size,k,d,rv,tv)
|
||||
} finally { (rv+tv).forEach { it.release() } }
|
||||
}
|
||||
try {
|
||||
intrinsic(k1,d1,train.map { it.left }); intrinsic(k2,d2,train.map { it.right })
|
||||
stereoGeometry(objects,train.map{it.left},train.map{it.right},k1,d1,k2,d2,size,fish,r,t)
|
||||
require(Core.norm(t) in 0.02..0.3) { context.getString(R.string.spatial_bad_baseline) }
|
||||
val hr=mutableListOf<Mat>(); val ht=mutableListOf<Mat>(); val br=mutableListOf<Mat>(); val bt=mutableListOf<Mat>()
|
||||
val camR=Mat(); val camT=Mat()
|
||||
try {
|
||||
for(o in train) {
|
||||
val rv=Mat(); val tv=Mat(); val rotation=Mat()
|
||||
boardPose(o.objects,o.left,k1,d1,fish,rv,tv)
|
||||
Calib3d.Rodrigues(rv,rotation); rv.release(); br.add(rotation); bt.add(tv)
|
||||
hr.add(matrix(o.head.rotation)); ht.add(vector(o.head.position))
|
||||
}
|
||||
Calib3d.calibrateHandEye(hr,ht,br,bt,camR,camT,Calib3d.CALIB_HAND_EYE_PARK)
|
||||
require(Core.checkRange(camR) && Core.checkRange(camT) && Core.norm(camT)<0.3) { context.getString(R.string.spatial_need_orientations) }
|
||||
val cameraToHead=pose(camR,camT)
|
||||
require(abs(Core.determinant(camR)-1.0)<0.01) { context.getString(R.string.spatial_bad_rotation) }
|
||||
val boardWorld=held.map { o ->
|
||||
val rv=Mat();val tv=Mat();val rotation=Mat()
|
||||
try {
|
||||
boardPose(o.objects,o.left,k1,d1,fish,rv,tv);Calib3d.Rodrigues(rv,rotation)
|
||||
o.head*cameraToHead*pose(rotation,tv)
|
||||
} finally { rv.release();tv.release();rotation.release() }
|
||||
}
|
||||
val center=boardWorld.map { it.position }.reduce { a,b -> a+b }*(1f/boardWorld.size)
|
||||
val drift = boardWorld.maxOf { (it.position-center).length() }
|
||||
if (drift>=0.05f) throw CalibrationAlignmentException(context,drift)
|
||||
val orientation=boardWorld.first().rotation
|
||||
require(boardWorld.all { val q=orientation.inverse()*it.rotation;abs(q.w)>cos(Math.toRadians(8.0)/2) }) { context.getString(R.string.spatial_bad_alignment) }
|
||||
val error=held.map { reprojection(it,k1,d1,k2,d2,r,t,fish) }.average()
|
||||
require(error.isFinite())
|
||||
return StereoCalibration(identity,w,h,fish,k1,d1,k2,d2,r,t,cameraToHead,error)
|
||||
} finally { (hr+ht+br+bt+listOf(camR,camT)).forEach { it.release() } }
|
||||
} catch(e: Exception) { listOf(k1,k2,d1,d2,r,t).forEach { it.release() }; throw e }
|
||||
}
|
||||
private fun boardPose(objects: Mat,pixels: Mat,k: Mat,d: Mat,fish: Boolean,rv: Mat,tv: Mat) {
|
||||
val ok=if(fish) Calib3d.fisheye_solvePnP(objects,pixels,k,d,rv,tv)
|
||||
else {
|
||||
val obj=MatOfPoint3f(objects); val img=MatOfPoint2f(pixels); val dist=MatOfDouble(d)
|
||||
try { Calib3d.solvePnP(obj,img,k,dist,rv,tv) } finally { obj.release(); img.release(); dist.release() }
|
||||
}
|
||||
require(ok)
|
||||
}
|
||||
private fun reprojection(o: BoardObservation,k1: Mat,d1: Mat,k2: Mat,d2: Mat,r: Mat,t: Mat,fish: Boolean): Double {
|
||||
val rv=Mat(); val tv=Mat(); val rot=Mat(); val rightR=Mat(); val rightT=Mat(); val rightV=Mat(); val empty=Mat()
|
||||
val predicted=MatOfPoint2f(); val objectPoints=MatOfPoint3f(o.objects); val dist=MatOfDouble(d2)
|
||||
try {
|
||||
boardPose(o.objects,o.left,k1,d1,fish,rv,tv); Calib3d.Rodrigues(rv,rot)
|
||||
Core.gemm(r,rot,1.0,empty,0.0,rightR); Core.gemm(r,tv,1.0,t,1.0,rightT); Calib3d.Rodrigues(rightR,rightV)
|
||||
if(fish) Calib3d.fisheye_projectPoints(o.objects,predicted,rightV,rightT,k2,d2)
|
||||
else Calib3d.projectPoints(objectPoints,rightV,rightT,k2,dist,predicted)
|
||||
return Core.norm(predicted,o.right,Core.NORM_L2)/sqrt(o.right.rows().toDouble())
|
||||
} finally { listOf(rv,tv,rot,rightR,rightT,rightV,empty,predicted,objectPoints,dist).forEach { it.release() } }
|
||||
}
|
||||
override fun close() { observations.forEach { listOf(it.objects,it.left,it.right).forEach(Mat::release) }; observations.clear(); detector.clear() }
|
||||
fun save(file: File, identity: String, width: Int, height: Int) {
|
||||
fun points(m: Mat) = JSONArray((0 until m.rows()).map { row -> JSONArray(m.get(row,0).toList()) })
|
||||
val views = JSONArray(observations.map { o ->
|
||||
val p=o.head.position; val q=o.head.rotation
|
||||
JSONObject().put("objects",points(o.objects)).put("left",points(o.left)).put("right",points(o.right))
|
||||
.put("head",JSONArray(listOf(p.x,p.y,p.z,q.w,q.x,q.y,q.z)))
|
||||
})
|
||||
val data=JSONObject().put("version",1).put("identity",identity).put("width",width).put("height",height).put("views",views)
|
||||
file.parentFile?.mkdirs()
|
||||
val temporary=File(file.parentFile,file.name+".tmp")
|
||||
temporary.writeText(data.toString())
|
||||
java.nio.file.Files.move(temporary.toPath(),file.toPath(),java.nio.file.StandardCopyOption.REPLACE_EXISTING)
|
||||
}
|
||||
companion object {
|
||||
fun load(context: Context, file: File, identity: String, width: Int, height: Int): StereoCalibrator? {
|
||||
if (!file.isFile) return null
|
||||
val restored=StereoCalibrator(context)
|
||||
try {
|
||||
val data=JSONObject(file.readText())
|
||||
require(data.getInt("version")==1 && data.getString("identity")==identity && data.getInt("width")==width && data.getInt("height")==height)
|
||||
val views=data.getJSONArray("views")
|
||||
require(views.length() in 1..1000)
|
||||
for (i in 0 until views.length()) {
|
||||
val view=views.getJSONObject(i)
|
||||
val points=CalibrationObservationData.parse(view)
|
||||
restored.observations.add(BoardObservation(
|
||||
MatOfPoint3f(*points.objects.map { Point3(it[0],it[1],it[2]) }.toTypedArray()),
|
||||
MatOfPoint2f(*points.left.map { Point(it[0],it[1]) }.toTypedArray()),
|
||||
MatOfPoint2f(*points.right.map { Point(it[0],it[1]) }.toTypedArray()),points.head))
|
||||
}
|
||||
return restored
|
||||
} catch (e: Exception) {
|
||||
restored.close()
|
||||
android.util.Log.w("StereoCalibration", "Saved captures could not be restored", e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
internal fun stereoGeometry(objects: List<Mat>, left: List<Mat>, right: List<Mat>, k1: Mat,d1: Mat,k2: Mat,d2: Mat,
|
||||
size: Size,fish: Boolean,r: Mat,t: Mat) {
|
||||
val e=Mat(); val f=Mat(); val identity=Mat.eye(3,3,CvType.CV_64F); val zero=Mat.zeros(5,1,CvType.CV_64F)
|
||||
val corrected=mutableListOf<Mat>()
|
||||
fun images(input: List<Mat>, k: Mat,d: Mat) = if (!fish) input else input.map { pixels ->
|
||||
Mat().also { output -> corrected.add(output); Calib3d.fisheye_undistortPoints(pixels,output,k,d,identity,k) }
|
||||
}
|
||||
try {
|
||||
// OpenCV 4.13 fisheye stereo fitting assumes equal corner counts in every view.
|
||||
Calib3d.stereoCalibrate(objects,images(left,k1,d1),images(right,k2,d2),k1,if(fish)zero else d1,k2,if(fish)zero else d2,
|
||||
size,r,t,e,f,Calib3d.CALIB_FIX_INTRINSIC)
|
||||
} finally { (corrected+listOf(e,f,identity,zero)).forEach(Mat::release) }
|
||||
}
|
||||
fun matrix(q: SpatialRotation)=Mat(3,3,CvType.CV_64F).also { it.put(0,0,*q.matrix().map(Float::toDouble).toDoubleArray()) }
|
||||
fun vector(p: SpatialVector)=Mat(3,1,CvType.CV_64F).also { it.put(0,0,p.x.toDouble(),p.y.toDouble(),p.z.toDouble()) }
|
||||
fun pose(r: Mat,t: Mat)=SpatialPose(SpatialVector(t.get(0,0)[0].toFloat(),t.get(1,0)[0].toFloat(),t.get(2,0)[0].toFloat()),
|
||||
SpatialRotation.fromBasis(column(r,0),column(r,1),column(r,2)))
|
||||
private fun column(r: Mat,c: Int)=SpatialVector(r.get(0,c)[0].toFloat(),r.get(1,c)[0].toFloat(),r.get(2,c)[0].toFloat())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import org.json.JSONObject
|
||||
|
||||
internal data class CalibrationMatrix(val rows: Int, val columns: Int, val values: DoubleArray)
|
||||
|
||||
internal data class StereoCalibrationProfile(
|
||||
val identity: String, val width: Int, val height: Int, val fisheye: Boolean,
|
||||
val matrices: List<CalibrationMatrix>, val cameraToHead: SpatialPose, val error: Double,
|
||||
val verified: Boolean, val bundledBase: Boolean = false,
|
||||
) {
|
||||
companion object {
|
||||
const val BASE_ASSET = "calibration/luma-ultra-640x480.json"
|
||||
const val BASE_IDENTITY = "luma-ultra-base-v1"
|
||||
|
||||
fun select(saved: String?, base: () -> String?, identity: String?, width: Int, height: Int): StereoCalibrationProfile? {
|
||||
if (identity != null && saved != null) parse(saved, identity, width, height)?.let { return it }
|
||||
return base()?.let { parse(it, identity ?: BASE_IDENTITY, width, height, bundledBase = true) }
|
||||
}
|
||||
|
||||
fun parse(text: String, identity: String, width: Int, height: Int,
|
||||
bundledBase: Boolean = false): StereoCalibrationProfile? = runCatching {
|
||||
val data = JSONObject(text)
|
||||
require(data.getInt("version") == 1 && data.getInt("width") == width && data.getInt("height") == height)
|
||||
if (bundledBase) {
|
||||
require(data.getString("identity") == BASE_IDENTITY && data.getString("deviceModel") == "VITURE Luma Ultra")
|
||||
require(data.getString("cameraPair") == "left0-right0" && width == 640 && height == 480)
|
||||
} else require(data.getString("identity") == identity)
|
||||
val fisheye = data.getBoolean("fisheye")
|
||||
val matrices = listOf("k1", "d1", "k2", "d2", "r", "t").map { name ->
|
||||
val matrix = data.getJSONObject(name)
|
||||
val rows = matrix.getInt("rows"); val columns = matrix.getInt("cols")
|
||||
val values = matrix.getJSONArray("data")
|
||||
require(rows in 1..14 && columns in 1..14 && values.length() == rows * columns)
|
||||
CalibrationMatrix(rows, columns, DoubleArray(values.length()) { values.getDouble(it).also { n -> require(n.isFinite()) } })
|
||||
}
|
||||
require(listOf(0, 2, 4).all { matrices[it].rows == 3 && matrices[it].columns == 3 })
|
||||
require(matrices[5].rows == 3 && matrices[5].columns == 1)
|
||||
require(listOf(1, 3).all {
|
||||
val matrix = matrices[it]
|
||||
(matrix.rows == 1 || matrix.columns == 1) && matrix.values.size in if (fisheye) listOf(4) else listOf(4, 5, 8, 12, 14)
|
||||
})
|
||||
require(listOf(0, 2).all { matrices[it].values[0] > 0 && matrices[it].values[4] > 0 })
|
||||
val pose = data.getJSONArray("cameraToHead"); require(pose.length() == 7)
|
||||
val cameraToHead = SpatialPose.fromSdk(FloatArray(7) { pose.getDouble(it).toFloat() }) ?: error("Invalid camera transform")
|
||||
val error = data.getDouble("error"); require(error.isFinite() && error in 0.0..2.0)
|
||||
StereoCalibrationProfile(identity, width, height, fisheye, matrices, cameraToHead, error,
|
||||
verified = !bundledBase && data.getBoolean("verified"), bundledBase = bundledBase)
|
||||
}.getOrNull()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.min
|
||||
|
||||
internal object StereoFingerGestures {
|
||||
data class Result(val trigger: Float = 0f, val triggerValid: Boolean = false,
|
||||
val grip: Boolean = false, val gripValid: Boolean = false)
|
||||
|
||||
fun estimate(left: List<SpatialVector>?, right: List<SpatialVector>?, gripping: Boolean): Result {
|
||||
val a = View.create(left) ?: return Result()
|
||||
val b = View.create(right) ?: return Result()
|
||||
val triggerA = a.pinch()
|
||||
val triggerB = b.pinch()
|
||||
val triggerValid = triggerA != null && triggerB != null && abs(triggerA-triggerB) <= 0.30f
|
||||
val curls = listOf(9,13,17).map { base ->
|
||||
val ca = a.curl(base)
|
||||
val cb = b.curl(base)
|
||||
if (ca != null && cb != null && abs(ca-cb) <= 0.45f) min(ca,cb) else null
|
||||
}
|
||||
val gripValid = curls.all { it != null }
|
||||
val closed = curls.count { it != null && it >= (if (gripping) 0.40f else 0.65f) }
|
||||
return Result(if (triggerValid) min(triggerA!!,triggerB!!) else 0f,triggerValid,
|
||||
gripValid && closed >= (if (gripping) 2 else 3),gripValid)
|
||||
}
|
||||
|
||||
private class View(val points: List<SpatialVector>,val width: Float) {
|
||||
private fun point(index: Int) = points[index].takeIf(SpatialVector::finite)
|
||||
?.let { SpatialVector(it.x,it.y) }
|
||||
|
||||
fun pinch(): Float? {
|
||||
val thumb = point(4) ?: return null
|
||||
val index = point(8) ?: return null
|
||||
val wrist = point(0) ?: return null
|
||||
if ((thumb-wrist).length() > width*3.5f || (index-wrist).length() > width*3.5f) return null
|
||||
return ((0.55f-(thumb-index).length()/width)/0.35f).coerceIn(0f,1f)
|
||||
}
|
||||
|
||||
fun curl(base: Int): Float? {
|
||||
val wrist = point(0) ?: return null
|
||||
val joints = (base..base+3).map { point(it) ?: return null }
|
||||
val lengths = joints.zipWithNext { a,b -> (b-a).length() }
|
||||
if (lengths.any { it < width*0.005f }) return null
|
||||
val path = lengths.sum()
|
||||
val reach = (joints[0]-wrist).length()
|
||||
if (path !in width*0.15f..width*3f || reach < width*0.20f) return null
|
||||
val shortness = ((1.7f-(joints[3]-wrist).length()/reach)/0.55f).coerceIn(0f,1f)
|
||||
val bend = ((1f-(joints[3]-joints[0]).length()/path)/0.35f).coerceIn(0f,1f)
|
||||
return min(shortness,bend)
|
||||
}
|
||||
|
||||
companion object {
|
||||
fun create(points: List<SpatialVector>?): View? {
|
||||
if (points?.size != 21) return null
|
||||
if (listOf(0,5,9,13,17).any { !points[it].finite() }) return null
|
||||
fun xy(index: Int) = SpatialVector(points[index].x,points[index].y)
|
||||
val across = xy(5)-xy(17)
|
||||
val along = xy(9)-xy(0)
|
||||
val width = across.length()
|
||||
val length = along.length()
|
||||
if (!width.isFinite() || !length.isFinite() || width < 1e-3f || length < 1e-3f) return null
|
||||
if (width/length !in 0.25f..3f || abs(across.cross(along).z)/(width*length) < 0.15f) return null
|
||||
return View(points,width)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,650 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import android.content.Context
|
||||
import com.droiddeck.launcher.R
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import com.google.mediapipe.framework.image.MPImage
|
||||
import com.google.mediapipe.tasks.core.BaseOptions
|
||||
import com.google.mediapipe.tasks.core.Delegate
|
||||
import com.google.mediapipe.tasks.vision.core.RunningMode
|
||||
import com.google.mediapipe.tasks.vision.handlandmarker.HandLandmarker
|
||||
import org.opencv.android.OpenCVLoader
|
||||
import org.opencv.android.Utils
|
||||
import org.opencv.core.*
|
||||
import org.opencv.imgproc.Imgproc
|
||||
import org.opencv.imgcodecs.Imgcodecs
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
import java.nio.ByteBuffer
|
||||
import java.security.MessageDigest
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
|
||||
internal data class HandTrackingDiagnostics(val message: String, val leftCameraHands: Int = 0,
|
||||
val rightCameraHands: Int = 0, val matched: Int = 0, val reconstructed: Int = 0, val accepted: Int = 0,
|
||||
val preprocessMs: Float = 0f, val inferenceMs: Float = 0f,
|
||||
val triggerValid: Int = 0, val gripValid: Int = 0,
|
||||
val leftRawHands: Int = 0, val rightRawHands: Int = 0,
|
||||
val previousLeftRawHands: Int = -1, val previousRightRawHands: Int = -1,
|
||||
val leftInferenceMs: Float = 0f, val rightInferenceMs: Float = 0f,
|
||||
val inferenceHandLimit: Int = 2,
|
||||
val observationTimeNs: Long = 0, val observationAgeMs: Float = -1f,
|
||||
val left: HandObservationDiagnostic? = null, val right: HandObservationDiagnostic? = null,
|
||||
val observations: HandTrackingMetricSnapshot? = null) {
|
||||
fun json() = JSONObject().put("message",message).put("experimental",true).put("leftCameraHands",leftCameraHands)
|
||||
.put("rightCameraHands",rightCameraHands).put("matched",matched).put("reconstructed",reconstructed).put("accepted",accepted)
|
||||
.put("preprocessMs",preprocessMs.toDouble()).put("inferenceMs",inferenceMs.toDouble())
|
||||
.put("triggerValid",triggerValid).put("gripValid",gripValid)
|
||||
.put("leftRawHands",leftRawHands).put("rightRawHands",rightRawHands)
|
||||
.put("previousLeftRawHands",previousLeftRawHands).put("previousRightRawHands",previousRightRawHands)
|
||||
.put("leftInferenceMs",leftInferenceMs.toDouble()).put("rightInferenceMs",rightInferenceMs.toDouble())
|
||||
.put("inferenceHandLimit",inferenceHandLimit)
|
||||
.put("observationTimeNs",observationTimeNs).put("observationAgeMs",observationAgeMs.toDouble())
|
||||
.put("left",left?.json()).put("right",right?.json()).put("observations",observations?.json())
|
||||
}
|
||||
|
||||
private fun HandObservationDiagnostic.json() = JSONObject().put("outcome",outcome.name.lowercase())
|
||||
.put("positionSource",positionSource)
|
||||
.put("reconstructedPalmLandmarks",org.json.JSONArray(reconstructedPalmLandmarks))
|
||||
.put("accepted",accepted).put("triggerEstimate",triggerEstimate.toDouble())
|
||||
.put("triggerGeometryValid",triggerGeometryValid).put("triggerValid",triggerValid)
|
||||
.put("gripEstimate",gripEstimate).put("gripGeometryValid",gripGeometryValid).put("gripValid",gripValid)
|
||||
|
||||
private fun HandSideMetrics.json() = JSONObject().put("observations",observations).put("accepted",accepted)
|
||||
.put("triggerValid",triggerValid).put("gripValid",gripValid)
|
||||
.put("triggerAboveClickThreshold",triggerAboveClickThreshold).put("triggerReleased",triggerReleased)
|
||||
.put("gripPressed",gripPressed).put("trackingLosses",trackingLosses)
|
||||
.put("triggerLosses",triggerLosses).put("gripLosses",gripLosses)
|
||||
.put("maximumConsecutiveRejected",maximumConsecutiveRejected)
|
||||
.put("outcomes",JSONObject().also { json -> outcomes.forEach { (outcome,count) -> json.put(outcome.name.lowercase(),count) } })
|
||||
|
||||
private fun HandTrackingMetricSnapshot.json() = JSONObject().put("processedPairs",processedPairs).put("inferencePairs",inferencePairs)
|
||||
.put("scope","Processed camera pairs in this tracking connection, including absent hands and setup; not detection accuracy")
|
||||
.put("left",left.json()).put("right",right.json())
|
||||
|
||||
internal class StereoHandEngine(private val context: Context, private val token: Long,
|
||||
private val generation: Long, private val identity: String?) : AutoCloseable {
|
||||
@Volatile private var running=true
|
||||
@Volatile private var enabled=false
|
||||
@Volatile private var debug=false
|
||||
@Volatile var preview: Bitmap? = null; private set
|
||||
@Volatile var message=context.getString(R.string.spatial_wait_cameras); private set
|
||||
@Volatile var calibrationProgress = SpatialCalibrationProgress(); private set
|
||||
@Volatile var handStatus = HandTrackingDiagnostics(context.getString(R.string.spatial_wait_hands)); private set
|
||||
val hasStableIdentity get() = identity != null
|
||||
private val commands=ConcurrentLinkedQueue<() -> Unit>()
|
||||
private var calibrator: StereoCalibrator?=null
|
||||
private var calibrationFrame = 0L
|
||||
private var calibration: StereoCalibration?=null
|
||||
val calibrationSource get() = calibration?.source ?: "none"
|
||||
val calibrationVerified get() = calibration?.verified == true
|
||||
val lensCorrection get() = calibration?.let { if(it.fisheye) "fisheye-points-only" else "pinhole-points-only" } ?: "none"
|
||||
private var capture=false
|
||||
private var record=false
|
||||
private var verify: Float?=null
|
||||
private var validationRequestedAt=0L
|
||||
@Volatile var gpu=false; private set
|
||||
@Volatile var model=HandModel.DEFAULT; private set
|
||||
@Volatile var halfRateInput=true; private set
|
||||
@Volatile var adaptiveInput=true; private set
|
||||
@Volatile var scheduledTracking=true; private set
|
||||
@Volatile var inputRatePermille=500; private set
|
||||
@Volatile var cpuParallel=true; private set
|
||||
@Volatile var benchmark: JSONObject?=null; private set
|
||||
@Volatile var inferenceBackend: String? = null; private set
|
||||
private var detectors: DetectorBank?=null
|
||||
private var fastDetectors: DetectorBank?=null
|
||||
private var rightWorker: StereoInferenceRunner?=null
|
||||
private val inferenceSchedule=HandInferenceSchedule()
|
||||
private val processingRate=HandProcessingRate()
|
||||
private val handMetrics=HandTrackingMetrics()
|
||||
private val gestures=Array(2) { HandGestures(rightHand=it==1) }
|
||||
private val joyConPositions=Array(2) { JoyConWristPosition() }
|
||||
private var joyConFlow: StereoJoyConFlow? = null
|
||||
private val profile: File get() {
|
||||
val hash=MessageDigest.getInstance("SHA-256").digest((identity ?: "session:$generation").toByteArray()).joinToString("") { "%02x".format(it) }
|
||||
return File(context.filesDir,"spatial/calibration-$hash.json")
|
||||
}
|
||||
private val captures: File get() = File(profile.parentFile,profile.nameWithoutExtension+"-captures.json")
|
||||
private fun saveCaptures() {
|
||||
try { calibrator?.save(captures,identity ?: "session:$generation",width,height) }
|
||||
catch (e: Exception) { android.util.Log.w("StereoCalibration", "Could not save captured views", e) }
|
||||
}
|
||||
fun setEnabled(on: Boolean) { enabled=on }
|
||||
fun diagnostics(on: Boolean) { debug=on }
|
||||
fun setGpu(on: Boolean) { commands.add { clearHands();closeDetectors();gpu=on } }
|
||||
fun setModel(lite: Boolean) { commands.add {
|
||||
clearHands();closeDetectors();model=if(lite) HandModel.LITE else HandModel.FULL
|
||||
} }
|
||||
fun setHalfRate(on: Boolean) { commands.add { adaptiveInput=false;halfRateInput=on;processingRate.reset() } }
|
||||
fun setAdaptiveInput(on: Boolean) { commands.add { adaptiveInput=on;processingRate.reset() } }
|
||||
fun setScheduledTracking(on: Boolean) { commands.add { clearHands();closeDetectors();scheduledTracking=on } }
|
||||
fun setCpuParallel(on: Boolean) { commands.add { clearHands();closeDetectors();cpuParallel=on } }
|
||||
fun benchmarkSavedCapture(id: String) {
|
||||
require(id.matches(Regex("[0-9]{1,20}")))
|
||||
benchmark=JSONObject().put("state","pending").put("capture",id)
|
||||
commands.add {
|
||||
try { benchmark=benchmarkCapture(id) }
|
||||
catch(e: Exception) { benchmark=JSONObject().put("state","failed").put("error",e.message) }
|
||||
}
|
||||
}
|
||||
fun startCalibration() { debug=true; commands.add {
|
||||
check(width > 0 && height > 0) { context.getString(R.string.spatial_wait_previews) }
|
||||
calibrator?.close();calibrator=StereoCalibrator(context)
|
||||
calibrationFrame = SpatialTracking.session.trackingFrameGeneration
|
||||
calibrationProgress = SpatialCalibrationProgress(collecting = true)
|
||||
saveCaptures()
|
||||
message=context.getString(R.string.spatial_collect_instructions)
|
||||
} }
|
||||
fun captureCalibration() { commands.add {
|
||||
check(calibrationProgress.collecting && calibrationFrame==SpatialTracking.session.trackingFrameGeneration) {
|
||||
context.getString(R.string.spatial_saved_views_hint)
|
||||
}
|
||||
capture=true
|
||||
calibrationProgress = calibrationProgress.copy(capturing = true, captureMessage = context.getString(R.string.spatial_capture_pending))
|
||||
} }
|
||||
fun recordDiagnostic() { commands.add { record=true } }
|
||||
fun validateCalibration(distance: Float) { commands.add {
|
||||
require(distance.isFinite() && distance in 0.15f..1.5f)
|
||||
verify=distance;validationRequestedAt=System.nanoTime()
|
||||
calibrationProgress=calibrationProgress.copy(validating=true,
|
||||
validationMessage=context.getString(R.string.spatial_validation_pending))
|
||||
} }
|
||||
private var width=0; private var height=0
|
||||
private val imageBuffer=HandImageBuffer()
|
||||
private val rightImageBuffer=HandImageBuffer()
|
||||
private val worker=Thread({ loop() },"viture-stereo-hands").apply { isDaemon=true; start() }
|
||||
fun solveCalibration() { commands.add {
|
||||
val c=calibrator ?: error(context.getString(R.string.spatial_start_first))
|
||||
saveCaptures()
|
||||
calibrationProgress = calibrationProgress.copy(solving = true, validationMessage = null)
|
||||
try {
|
||||
val fitted=c.solve(identity ?: "session:$generation",width,height)
|
||||
clearHands();closeDetectors()
|
||||
calibration?.close();calibration=fitted
|
||||
calibrationProgress = calibrationProgress.copy(fitted = true)
|
||||
if(identity!=null) fitted.save(profile)
|
||||
message=context.getString(R.string.spatial_fit_ready, fitted.error)
|
||||
} finally { calibrationProgress = calibrationProgress.copy(solving = false) }
|
||||
} }
|
||||
private fun detector(selected: HandModel=model, useGpu: Boolean=gpu, maxHands: Int=2): HandLandmarker {
|
||||
val bytes=context.assets.open(selected.asset).use { it.readBytes() }
|
||||
val hash=MessageDigest.getInstance("SHA-256").digest(bytes).joinToString("") { "%02x".format(it) }
|
||||
check(hash==selected.sha256) { context.getString(R.string.spatial_model_checksum_error) }
|
||||
return HandLandmarker.createFromOptions(context,HandLandmarker.HandLandmarkerOptions.builder()
|
||||
.setBaseOptions(BaseOptions.builder().setModelAssetPath(selected.asset)
|
||||
.setDelegate(if(useGpu) Delegate.GPU else Delegate.CPU).build())
|
||||
.setRunningMode(RunningMode.VIDEO).setNumHands(maxHands).setMinHandDetectionConfidence(0.6f)
|
||||
.setMinHandPresenceConfidence(0.6f).setMinTrackingConfidence(0.6f).build())
|
||||
}
|
||||
private data class Detection(val side: Int,val points: List<Point>)
|
||||
private data class Detections(val hands: List<Detection>,val rawCount: Int,val inferenceMs: Float)
|
||||
private class DetectorBank(val left: HandLandmarker, val right: HandLandmarker, val maxHands: Int) {
|
||||
var previousLeftRaw=-1
|
||||
var previousRightRaw=-1
|
||||
}
|
||||
private fun createBank(selected: HandModel, useGpu: Boolean, maxHands: Int,
|
||||
worker: StereoInferenceRunner?): DetectorBank {
|
||||
val left=detector(selected,useGpu,maxHands)
|
||||
try {
|
||||
val right=worker?.onRight { detector(selected,useGpu,maxHands) } ?: detector(selected,useGpu,maxHands)
|
||||
return DetectorBank(left,right,maxHands)
|
||||
} catch(e: Throwable) { left.close();throw e }
|
||||
}
|
||||
private fun closeBank(bank: DetectorBank?, worker: StereoInferenceRunner?) {
|
||||
if(bank==null) return
|
||||
try { bank.left.close() } finally {
|
||||
if(worker!=null) worker.onRight { bank.right.close() } else bank.right.close()
|
||||
}
|
||||
}
|
||||
private fun imagePoints(detection: Detection?) = detection?.points?.map {
|
||||
SpatialVector(it.x.toFloat(),it.y.toFloat(),0f)
|
||||
}
|
||||
private fun detect(detector: HandLandmarker, bitmap: Bitmap, stamp: Long, buffer: HandImageBuffer,
|
||||
wristOnly: Boolean): Detections =
|
||||
detect(detector,stamp,wristOnly) { buffer.image(bitmap) }
|
||||
private fun detect(detector: HandLandmarker, stamp: Long, wristOnly: Boolean,
|
||||
prepareImage: () -> MPImage): Detections {
|
||||
val image=prepareImage()
|
||||
try {
|
||||
val start=System.nanoTime()
|
||||
val result=detector.detectForVideo(image,stamp)
|
||||
val elapsed=(System.nanoTime()-start)/1e6f
|
||||
val hands=result.landmarks().mapIndexedNotNull { i, hand ->
|
||||
val category=result.handedness()[i].firstOrNull() ?: return@mapIndexedNotNull null
|
||||
if(category.score()<0.6f) return@mapIndexedNotNull null
|
||||
val points=if(wristOnly) hand.take(1) else hand
|
||||
Detection(if(category.categoryName()=="Left") 0 else 1,points.map { Point(it.x()*image.width.toDouble(),it.y()*image.height.toDouble()) })
|
||||
}
|
||||
return Detections(hands,result.landmarks().size,elapsed)
|
||||
} finally { image.close() }
|
||||
}
|
||||
private fun closeDetectors() {
|
||||
inferenceBackend=null
|
||||
inferenceSchedule.reset();processingRate.reset()
|
||||
val search=detectors;val fast=fastDetectors;val worker=rightWorker
|
||||
detectors=null;fastDetectors=null;rightWorker=null
|
||||
try { closeBank(search,worker) } finally {
|
||||
try { closeBank(fast,worker) } finally { worker?.close() }
|
||||
}
|
||||
}
|
||||
private fun detectPair(left: HandLandmarker, right: HandLandmarker, leftImage: Bitmap,
|
||||
rightImage: Bitmap, stamp: Long, worker: StereoInferenceRunner?, wristOnly: Boolean): Pair<Detections,Detections> {
|
||||
val l={ detect(left,leftImage,stamp,imageBuffer,wristOnly) };val r={ detect(right,rightImage,stamp,rightImageBuffer,wristOnly) }
|
||||
return worker?.pair(l,r) ?: (l() to r())
|
||||
}
|
||||
private fun detectRawPair(left: HandLandmarker, right: HandLandmarker, leftGray: ByteArray,
|
||||
rightGray: ByteArray, stamp: Long, worker: StereoInferenceRunner?, wristOnly: Boolean): Pair<Detections,Detections> {
|
||||
val l={ detect(left,stamp,wristOnly) { imageBuffer.image(leftGray,width,height) } }
|
||||
val r={ detect(right,stamp,wristOnly) { rightImageBuffer.image(rightGray,width,height) } }
|
||||
return worker?.pair(l,r) ?: (l() to r())
|
||||
}
|
||||
private fun benchmarkCapture(id: String): JSONObject {
|
||||
val folder=File(context.filesDir,"spatial/captures/$id")
|
||||
val c=calibration ?: error("Load the saved calibration before benchmarking")
|
||||
val wristOnly=SpatialTracking.joyConsEnabled
|
||||
val gray=ArrayList<Mat>();val images=ArrayList<Bitmap>();val rows=org.json.JSONArray()
|
||||
clearHands();closeDetectors()
|
||||
try {
|
||||
for(name in listOf("left","right")) {
|
||||
val source=Imgcodecs.imread(File(folder,"$name.png").path,Imgcodecs.IMREAD_GRAYSCALE)
|
||||
gray.add(source)
|
||||
check(source.cols()==width && source.rows()==height) { "Saved pair has incompatible dimensions" }
|
||||
val rgba=Mat()
|
||||
try {
|
||||
Imgproc.cvtColor(source,rgba,Imgproc.COLOR_GRAY2RGBA)
|
||||
val bitmap=Bitmap.createBitmap(width,height,Bitmap.Config.ARGB_8888)
|
||||
images.add(bitmap);Utils.matToBitmap(rgba,bitmap)
|
||||
} finally { rgba.release() }
|
||||
}
|
||||
for((selected,parallel,scheduled) in listOf(Triple(HandModel.FULL,true,false),Triple(HandModel.FULL,true,true),
|
||||
Triple(HandModel.LITE,true,true),Triple(HandModel.FULL,true,false))) {
|
||||
var search: DetectorBank?=null;var fast: DetectorBank?=null
|
||||
val schedule=HandInferenceSchedule()
|
||||
val worker=if(parallel) StereoInferenceRunner() else null
|
||||
val times=ArrayList<Double>();val samples=org.json.JSONArray()
|
||||
try {
|
||||
search=createBank(selected,false,2,worker)
|
||||
if(scheduled) fast=createBank(selected,false,1,worker)
|
||||
repeat(24) { frame ->
|
||||
check(running) { "Tracking stopped during benchmark" }
|
||||
val start=System.nanoTime();val stamp=1000L+frame*80L
|
||||
val bank=if(scheduled && schedule.select(stamp)==1) fast!! else search
|
||||
val (leftResult,rightResult)=detectPair(bank.left,bank.right,images[0],images[1],stamp,worker,wristOnly)
|
||||
val l=leftResult.hands;val r=rightResult.hands
|
||||
if(scheduled) schedule.observe(stamp,l.map { it.side },r.map { it.side },leftResult.rawCount,rightResult.rawCount)
|
||||
val ms=(System.nanoTime()-start)/1e6
|
||||
val hands=org.json.JSONArray()
|
||||
val joyCons=org.json.JSONArray()
|
||||
val wrists=org.json.JSONArray()
|
||||
for(side in 0..1) {
|
||||
val a=l.singleOrNull { it.side==side };val b=r.singleOrNull { it.side==side }
|
||||
val points=if(a!=null && b!=null) c.triangulateLandmarksRaw(a.points,b.points) else null
|
||||
points?.get(0)?.takeIf { it.finite() }?.let { wrist ->
|
||||
wrists.put(JSONObject().put("side",side).put("position",org.json.JSONArray(wrist.array().toList())))
|
||||
}
|
||||
val hand=if(wristOnly) null else points?.let { HandGestures(rightHand=side==1).estimate(it,SpatialPose(),stamp*1_000_000,imagePoints(a),imagePoints(b)) }
|
||||
val joyCon=if(wristOnly) points?.let { JoyConWristPosition().estimate(it,SpatialPose(),stamp*1_000_000) } else null
|
||||
if(joyCon!=null) joyCons.put(JSONObject().put("side",side)
|
||||
.put("position",org.json.JSONArray(joyCon.pose.position.array().toList())))
|
||||
if(hand!=null) hands.put(JSONObject().put("side",side).put("flags",hand.flags())
|
||||
.put("position",org.json.JSONArray(hand.pose.position.array().toList()))
|
||||
.put("trigger",hand.trigger.toDouble()).put("grip",hand.grip))
|
||||
}
|
||||
if(frame>=4) {
|
||||
times.add(ms);samples.put(JSONObject().put("left",l.size).put("right",r.size).put("hands",hands)
|
||||
.put("joyCons",joyCons).put("wrists",wrists)
|
||||
.put("inferenceHandLimit",bank.maxHands)
|
||||
.put("leftRawHands",leftResult.rawCount).put("rightRawHands",rightResult.rawCount)
|
||||
.put("leftInferenceMs",leftResult.inferenceMs.toDouble()).put("rightInferenceMs",rightResult.inferenceMs.toDouble()))
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { closeBank(search,worker) } finally {
|
||||
try { closeBank(fast,worker) } finally { worker?.close() }
|
||||
}
|
||||
}
|
||||
val sorted=times.sorted()
|
||||
rows.put(JSONObject().put("model",selected.name).put("parallel",parallel).put("medianMs",(sorted[9]+sorted[10])/2)
|
||||
.put("scheduled",scheduled)
|
||||
.put("p95Ms",sorted[18]).put("samples",samples))
|
||||
}
|
||||
return JSONObject().put("state","completed").put("capture",id).put("backend","CPU")
|
||||
.put("wristOnly",wristOnly).put("imagePreprocessing",HandImageBuffer.PREPROCESSING)
|
||||
.put("lensCorrection",lensCorrection)
|
||||
.put("scope","Saved stereo pair replay; not live tracking or external latency").put("results",rows)
|
||||
} finally { gray.forEach { it.release() };images.forEach { it.recycle() };clearHands() }
|
||||
}
|
||||
private fun clearHands() {
|
||||
gestures.forEach { it.reset() }
|
||||
joyConPositions.forEach { it.reset() }
|
||||
joyConFlow?.reset()
|
||||
inferenceSchedule.reset();processingRate.reset()
|
||||
SpatialTracking.session.setHands(generation,SpatialHand(),SpatialHand())
|
||||
}
|
||||
private fun loop() {
|
||||
try { check(OpenCVLoader.initLocal()) } catch(e: Throwable) { message=context.getString(R.string.spatial_opencv_error, e.message.orEmpty());return }
|
||||
var grayL=Mat();var grayR=Mat();val rgba=Mat()
|
||||
var bmpL: Bitmap?=null;var bmpR: Bitmap?=null
|
||||
var leftGray=ByteArray(0);var rightGray=ByteArray(0)
|
||||
val left=ByteBuffer.allocateDirect(2048*2048);val right=ByteBuffer.allocateDirect(2048*2048)
|
||||
val metadata=DoubleArray(9);val headData=FloatArray(7)
|
||||
var lastSeq=0L;var dropped=0L;var frames=0L;var previewTime=0L
|
||||
val clock=CameraClock()
|
||||
var lastSource=0L;var lastStamp=0L;var captureOn=false;var captureRate=1000;var lastInputSeq=0L
|
||||
var rate=0f
|
||||
try {
|
||||
check(OpenCVLoader.initLocal()) { context.getString(R.string.spatial_opencv_load_error) }
|
||||
joyConFlow=StereoJoyConFlow()
|
||||
while(running) {
|
||||
while(true) {
|
||||
val command=commands.poll() ?: break
|
||||
try { command() } catch(e: Exception) { message=e.message ?: e.toString() }
|
||||
}
|
||||
if (verify!=null && System.nanoTime()-validationRequestedAt>2_000_000_000L) {
|
||||
verify=null
|
||||
calibrationProgress=calibrationProgress.copy(validating=false,
|
||||
validationMessage=context.getString(R.string.spatial_no_camera_frame))
|
||||
}
|
||||
val want=enabled || debug
|
||||
val copyPermille=if(enabled && !calibrationProgress.collecting && !capture && verify==null && !record)
|
||||
if(adaptiveInput && !gpu) processingRate.permille else if(halfRateInput) 500 else 1000
|
||||
else 1000
|
||||
inputRatePermille=copyPermille
|
||||
if(want!=captureOn || copyPermille!=captureRate) {
|
||||
VitureSdk.nativeCapture(token,want,copyPermille);captureOn=want;captureRate=copyPermille
|
||||
if(!want) clearHands()
|
||||
}
|
||||
if(!want) { Thread.sleep(20);continue }
|
||||
val seq=VitureSdk.nativeStereo(token,left,right,metadata,headData,!enabled || capture)
|
||||
if(seq<=0) { Thread.sleep(5);continue }
|
||||
val sequenceDelta=if(lastSeq>0) (seq-lastSeq).coerceAtLeast(1) else 1
|
||||
if(lastSeq>0) dropped+=(sequenceDelta-1)
|
||||
lastSeq=seq;frames++
|
||||
val start=System.nanoTime();val source=(metadata[2]*1e9).toLong();val receipt=metadata[3].toLong()
|
||||
val inputSequence=metadata[7].toLong()
|
||||
if(source<=lastSource) { clearHands();message=context.getString(R.string.spatial_clock_reset);closeDetectors();lastStamp=0 }
|
||||
if(lastSource>0 && source>lastSource) {
|
||||
val hz=1e9f*(inputSequence-lastInputSeq).coerceAtLeast(1)/(source-lastSource)
|
||||
rate=if(rate==0f)hz else rate*.9f+hz*.1f
|
||||
}
|
||||
lastSource=source;lastInputSeq=inputSequence
|
||||
val time=clock.map(source,receipt) ?: 0L
|
||||
val trackingFrame = SpatialTracking.session.trackingFrameGeneration
|
||||
if (calibrator != null && calibrationProgress.collecting && calibrationFrame != trackingFrame) {
|
||||
capture=false
|
||||
message=context.getString(R.string.spatial_frame_changed)
|
||||
calibrationProgress = calibrationProgress.copy(collecting = false, capturing = false, captureMessage = message)
|
||||
}
|
||||
// The callback pose's basis is undocumented; use canonical GL pose history.
|
||||
val observation=HandObservation(time,generation,trackingFrame)
|
||||
var captureHead=observation.headPose(SpatialTracking.session,metadata[6]>0)
|
||||
val w=metadata[0].toInt();val h=metadata[1].toInt()
|
||||
if(w!=width || h!=height) {
|
||||
clearHands();closeDetectors()
|
||||
width=w;height=h;leftGray=ByteArray(w*h);rightGray=ByteArray(w*h)
|
||||
grayL.release();grayR.release();grayL=Mat(h,w,CvType.CV_8UC1);grayR=Mat(h,w,CvType.CV_8UC1)
|
||||
bmpL?.recycle();bmpR?.recycle();bmpL=null;bmpR=null
|
||||
calibration?.close()
|
||||
val selected=StereoCalibrationProfile.select(
|
||||
if(identity!=null) runCatching { profile.readText() }.getOrNull() else null,
|
||||
{ runCatching { context.assets.open(StereoCalibrationProfile.BASE_ASSET).bufferedReader().use { it.readText() } }.getOrNull() },
|
||||
identity,w,h)
|
||||
calibration=selected?.let(StereoCalibration::fromProfile)
|
||||
calibrator?.close();calibrator=null
|
||||
calibrationProgress = SpatialCalibrationProgress(fitted = calibration != null)
|
||||
message=context.getString(R.string.spatial_stereo_loaded, w, h, when {
|
||||
calibration?.verified==true -> context.getString(R.string.spatial_device_loaded)
|
||||
calibration?.bundledBase==true -> context.getString(R.string.spatial_base_loaded)
|
||||
else -> context.getString(R.string.spatial_calibration_needed)
|
||||
})
|
||||
if (identity!=null) calibrator=StereoCalibrator.load(context,captures,identity,w,h)
|
||||
calibrator?.let {
|
||||
calibrationFrame=-1
|
||||
val restored=context.getString(R.string.spatial_restored, it.observations.size)
|
||||
calibrationProgress=calibrationProgress.copy(views=it.observations.size,captureMessage=restored)
|
||||
message=restored
|
||||
}
|
||||
}
|
||||
left.position(0);left.get(leftGray)
|
||||
right.position(0);right.get(rightGray)
|
||||
val joyConMode=SpatialTracking.joyConsEnabled
|
||||
val drawPreview=debug && start-previewTime>150_000_000L
|
||||
if(capture || verify!=null || record || drawPreview || joyConMode) {
|
||||
grayL.put(0,0,leftGray);grayR.put(0,0,rightGray)
|
||||
}
|
||||
if(capture) {
|
||||
capture=false
|
||||
val observation=captureHead?.let { calibrator?.detect(grayL,grayR,it) }
|
||||
val currentTracking = SpatialTracking.session.isCurrentTrackingFrame(generation, trackingFrame)
|
||||
if(observation!=null && currentTracking) {
|
||||
calibrator!!.observations.add(observation)
|
||||
calibrationProgress = calibrationProgress.copy(views = calibrator!!.observations.size)
|
||||
saveCaptures()
|
||||
message=context.getString(R.string.spatial_captured, calibrator!!.observations.size)
|
||||
} else {
|
||||
observation?.let { listOf(it.objects,it.left,it.right).forEach(Mat::release) }
|
||||
message = when {
|
||||
calibrator == null -> context.getString(R.string.spatial_tap_start)
|
||||
SpatialTracking.session.recenterPending -> context.getString(R.string.spatial_capture_recenter_pending)
|
||||
SpatialTracking.session.needsRecenter -> context.getString(R.string.spatial_capture_recenter)
|
||||
captureHead == null || !currentTracking -> context.getString(R.string.spatial_capture_unsynchronized)
|
||||
else -> context.getString(R.string.spatial_capture_corners, calibrator?.commonCorners ?: 0)
|
||||
}
|
||||
}
|
||||
calibrationProgress = calibrationProgress.copy(capturing = false, captureMessage = message)
|
||||
}
|
||||
val c=calibration
|
||||
val distance=verify
|
||||
if(distance!=null) {
|
||||
verify=null
|
||||
var temporary: StereoCalibrator?=null
|
||||
var observed: BoardObservation?=null
|
||||
try {
|
||||
check(c!=null) { context.getString(R.string.spatial_align_first) }
|
||||
temporary=StereoCalibrator(context)
|
||||
observed=temporary.detect(grayL,grayR,SpatialPose())
|
||||
val points=observed?.let { c.triangulateRaw(it.left,it.right) }
|
||||
val result=CalibrationDistanceCheck.evaluate(context,distance,temporary.commonCorners,points)
|
||||
if(result.accepted) {
|
||||
c.verified=true;if(identity!=null)c.save(profile)
|
||||
calibrationProgress=calibrationProgress.copy(validationMessage=result.message+
|
||||
if(identity==null) " " + context.getString(R.string.spatial_validation_session) else " " + context.getString(R.string.spatial_validation_saved))
|
||||
} else calibrationProgress=calibrationProgress.copy(validationMessage=result.message)
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.w("StereoCalibration","Distance validation failed",e)
|
||||
calibrationProgress=calibrationProgress.copy(validationMessage=context.getString(R.string.spatial_validation_error, e.message?.substringBefore('\n') ?: context.getString(R.string.spatial_processing_failed)))
|
||||
} finally {
|
||||
calibrationProgress=calibrationProgress.copy(validating=false)
|
||||
observed?.let { listOf(it.objects,it.left,it.right).forEach(Mat::release) };temporary?.close()
|
||||
}
|
||||
}
|
||||
var dl=emptyList<Detection>();var dr=emptyList<Detection>()
|
||||
var trackingHealthy=false
|
||||
var inferred=false
|
||||
val sideDiagnostics=Array(2) { HandObservationDiagnostic(HandObservationOutcome.NO_MATCHING_DETECTION) }
|
||||
val nextStatus: HandTrackingDiagnostics
|
||||
if(enabled && c?.readyForTracking==true && time>0 && SpatialTracking.session.isCurrentTrackingFrame(generation, trackingFrame) && System.nanoTime()-time in 0..HandObservation.MAX_AGE_NS) {
|
||||
val preprocessMs=(System.nanoTime()-start)/1e6f
|
||||
if(detectors==null) {
|
||||
try {
|
||||
rightWorker=if(cpuParallel && !gpu) StereoInferenceRunner() else null
|
||||
detectors=createBank(model,gpu,2,rightWorker)
|
||||
if(scheduledTracking && !gpu) fastDetectors=createBank(model,false,1,rightWorker)
|
||||
inferenceBackend=if(gpu) "GPU" else "CPU"
|
||||
}
|
||||
catch(e: Exception) {
|
||||
closeDetectors()
|
||||
enabled=false;clearHands();message=context.getString(R.string.spatial_inference_error, e.message.orEmpty())
|
||||
val unavailable=HandObservationDiagnostic(HandObservationOutcome.INFERENCE_UNAVAILABLE)
|
||||
handMetrics.observe(seq,false,unavailable,unavailable)
|
||||
handStatus=HandTrackingDiagnostics(message,observationTimeNs=time,
|
||||
observationAgeMs=(System.nanoTime()-time)/1e6f,left=unavailable,right=unavailable,observations=handMetrics.snapshot())
|
||||
continue
|
||||
}
|
||||
}
|
||||
val stamp=maxOf(time/1_000_000,lastStamp+1);lastStamp=stamp
|
||||
val inferenceStart=System.nanoTime()
|
||||
val scheduled=scheduledTracking && !gpu
|
||||
val bank=if(scheduled && inferenceSchedule.select(stamp)==1) fastDetectors!! else detectors!!
|
||||
val pair=detectRawPair(bank.left,bank.right,leftGray,rightGray,stamp,rightWorker,joyConMode)
|
||||
inferred=true
|
||||
dl=pair.first.hands;dr=pair.second.hands
|
||||
if(scheduled) inferenceSchedule.observe(stamp,dl.map { it.side },dr.map { it.side },pair.first.rawCount,pair.second.rawCount)
|
||||
val inferenceMs=(System.nanoTime()-inferenceStart)/1e6f
|
||||
// VIO callbacks can arrive during inference; verify this exact camera stamp before publishing.
|
||||
captureHead=observation.headPose(SpatialTracking.session,
|
||||
metadata[6]>0 || VitureSdk.nativeCameraSynchronized(token,metadata[2]))
|
||||
val hands=Array(2) { SpatialHand() }
|
||||
if (joyConMode) joyConFlow?.begin(grayL,grayR,time) else joyConFlow?.reset()
|
||||
var matched=0;var reconstructed=0
|
||||
for(side in 0..1) {
|
||||
val leftCandidates=dl.filter { it.side==side };val rightCandidates=dr.filter { it.side==side }
|
||||
val l=leftCandidates.singleOrNull();val r=rightCandidates.singleOrNull()
|
||||
val detectedPoints=if(l!=null && r!=null) {
|
||||
matched++
|
||||
c.triangulateLandmarksRaw(l.points,r.points)
|
||||
} else null
|
||||
var points=detectedPoints
|
||||
if(points?.any { it != null }==true) reconstructed++
|
||||
var hand=points?.let { p -> captureHead?.let { head ->
|
||||
if (joyConMode) joyConPositions[side].estimate(p,head*c.cameraToHead,time)
|
||||
else gestures[side].estimate(p,head*c.cameraToHead,time,imagePoints(l),imagePoints(r))
|
||||
} }
|
||||
var positionSource=if(hand!=null) "landmarks" else "none"
|
||||
if (joyConMode && captureHead!=null) {
|
||||
if (hand!=null && l!=null && r!=null && detectedPoints!=null)
|
||||
joyConFlow?.seed(side,l.points.first(),r.points.first(),time)
|
||||
else if (leftCandidates.size<=1 && rightCandidates.size<=1) {
|
||||
joyConFlow?.points(side)?.let { tracked ->
|
||||
val flowed=c.triangulateLandmarksRaw(tracked.left,tracked.right)
|
||||
val position=flowed?.let { joyConPositions[side].estimate(it,captureHead!!*c.cameraToHead,time) }
|
||||
if(position!=null) { hand=position;points=flowed;positionSource="stereo_flow" }
|
||||
}
|
||||
}
|
||||
}
|
||||
val outcome=when {
|
||||
hand!=null -> HandObservationOutcome.ACCEPTED
|
||||
leftCandidates.size>1 || rightCandidates.size>1 -> HandObservationOutcome.AMBIGUOUS_HANDEDNESS
|
||||
l==null && r==null -> HandObservationOutcome.NO_MATCHING_DETECTION
|
||||
l==null || r==null -> HandObservationOutcome.ONE_CAMERA_ONLY
|
||||
points?.any { it != null }!=true -> HandObservationOutcome.STEREO_REJECTED
|
||||
captureHead==null -> HandObservationOutcome.CAMERA_NOT_SYNCHRONIZED
|
||||
else -> if (joyConMode) joyConPositions[side].lastOutcome else gestures[side].lastOutcome
|
||||
}
|
||||
sideDiagnostics[side]=HandObservationDiagnostic(outcome,
|
||||
listOf(0,5,9,13,17).filter { points?.getOrNull(it)?.finite()==true },
|
||||
hand?.trigger ?: 0f,hand?.triggerValid ?: false,hand?.grip ?: false,hand?.gripValid ?: false,positionSource)
|
||||
if (hand!=null) hands[side]=hand
|
||||
else if (joyConMode) gestures[side].reset()
|
||||
else gestures[side].resetAfterRejection(outcome)
|
||||
}
|
||||
if (joyConMode) joyConFlow?.end(grayL,grayR,time)
|
||||
val current=SpatialTracking.session.isCurrentTrackingFrame(generation, trackingFrame) &&
|
||||
joyConMode==SpatialTracking.joyConsEnabled
|
||||
val fresh=System.nanoTime()-time in 0..HandObservation.MAX_AGE_NS
|
||||
if (current && fresh)
|
||||
SpatialTracking.session.setHands(generation,hands[0],hands[1])
|
||||
else {
|
||||
clearHands()
|
||||
for(side in 0..1) sideDiagnostics[side]=sideDiagnostics[side].copy(outcome=
|
||||
if(!current) HandObservationOutcome.HEAD_TRACKING_INVALID else HandObservationOutcome.STALE_OBSERVATION)
|
||||
}
|
||||
val accepted=if(current && fresh) hands.count { it.valid } else 0
|
||||
trackingHealthy=accepted>0
|
||||
nextStatus=HandTrackingDiagnostics(when {
|
||||
!current -> context.getString(R.string.spatial_hand_head_changed)
|
||||
!fresh -> context.getString(R.string.spatial_hand_timeout, HandObservation.MAX_AGE_NS / 1_000_000)
|
||||
captureHead==null -> context.getString(R.string.spatial_hand_wait_sync)
|
||||
accepted>0 && joyConMode -> context.getString(R.string.spatial_wrists_tracked, accepted)
|
||||
accepted>0 -> context.getString(R.string.spatial_palms_tracked, accepted, hands.count { it.valid && it.triggerValid }, hands.count { it.valid && it.gripValid })
|
||||
dl.isEmpty() && dr.isEmpty() && joyConMode -> context.getString(R.string.spatial_no_wrists)
|
||||
dl.isEmpty() && dr.isEmpty() -> context.getString(R.string.spatial_no_hands)
|
||||
(dl.isEmpty() || dr.isEmpty()) && joyConMode -> context.getString(R.string.spatial_one_camera_wrists)
|
||||
dl.isEmpty() || dr.isEmpty() -> context.getString(R.string.spatial_one_camera_hands)
|
||||
matched==0 -> context.getString(R.string.spatial_ambiguous_hands)
|
||||
reconstructed==0 -> context.getString(R.string.spatial_inconsistent_depth)
|
||||
joyConMode -> context.getString(R.string.spatial_rejected_wrist)
|
||||
else -> context.getString(R.string.spatial_inconsistent_palm)
|
||||
},dl.size,dr.size,matched,reconstructed,accepted,preprocessMs,inferenceMs,
|
||||
if(current && fresh) hands.count { it.valid && it.triggerValid } else 0,
|
||||
if(current && fresh) hands.count { it.valid && it.gripValid } else 0,
|
||||
pair.first.rawCount,pair.second.rawCount,bank.previousLeftRaw,bank.previousRightRaw,
|
||||
pair.first.inferenceMs,pair.second.inferenceMs,bank.maxHands)
|
||||
bank.previousLeftRaw=pair.first.rawCount;bank.previousRightRaw=pair.second.rawCount
|
||||
} else {
|
||||
clearHands()
|
||||
val outcome=when {
|
||||
!enabled -> HandObservationOutcome.DISABLED
|
||||
c?.readyForTracking!=true -> HandObservationOutcome.CALIBRATION_REQUIRED
|
||||
time<=0 -> HandObservationOutcome.CLOCK_NOT_READY
|
||||
!SpatialTracking.session.isCurrentTrackingFrame(generation,trackingFrame) -> HandObservationOutcome.HEAD_TRACKING_INVALID
|
||||
else -> HandObservationOutcome.STALE_OBSERVATION
|
||||
}
|
||||
for(side in 0..1) sideDiagnostics[side]=HandObservationDiagnostic(outcome)
|
||||
nextStatus=HandTrackingDiagnostics(when {
|
||||
!enabled -> context.getString(R.string.spatial_hands_off)
|
||||
c?.readyForTracking!=true -> context.getString(R.string.spatial_hands_need_calibration)
|
||||
captureHead==null -> context.getString(R.string.spatial_hand_wait_sync)
|
||||
!SpatialTracking.session.isCurrentTrackingFrame(generation,trackingFrame) -> context.getString(R.string.spatial_hands_wait_stable)
|
||||
else -> context.getString(R.string.spatial_old_frame)
|
||||
})
|
||||
}
|
||||
handMetrics.observe(seq,inferred,sideDiagnostics[0],sideDiagnostics[1])
|
||||
handStatus=nextStatus.copy(observationTimeNs=time,
|
||||
observationAgeMs=if(time>0) (System.nanoTime()-time)/1e6f else -1f,
|
||||
left=sideDiagnostics[0],right=sideDiagnostics[1],observations=handMetrics.snapshot())
|
||||
if(drawPreview) {
|
||||
previewTime=start
|
||||
if(bmpL==null) bmpL=Bitmap.createBitmap(w,h,Bitmap.Config.ARGB_8888)
|
||||
if(bmpR==null) bmpR=Bitmap.createBitmap(w,h,Bitmap.Config.ARGB_8888)
|
||||
Imgproc.cvtColor(grayL,rgba,Imgproc.COLOR_GRAY2RGBA);Utils.matToBitmap(rgba,bmpL)
|
||||
Imgproc.cvtColor(grayR,rgba,Imgproc.COLOR_GRAY2RGBA);Utils.matToBitmap(rgba,bmpR)
|
||||
val combined=Bitmap.createBitmap(w*2,h,Bitmap.Config.ARGB_8888);val canvas=Canvas(combined)
|
||||
canvas.drawBitmap(bmpL!!,0f,0f,null);canvas.drawBitmap(bmpR!!,w.toFloat(),0f,null)
|
||||
val paint=Paint().apply { color=Color.GREEN;strokeWidth=3f }
|
||||
for((index,detections) in listOf(dl,dr).withIndex()) for(d in detections) for(p in d.points)
|
||||
canvas.drawCircle(p.x.toFloat()+index*w,p.y.toFloat(),3f,paint)
|
||||
preview=combined
|
||||
}
|
||||
if(record) {
|
||||
record=false
|
||||
val folder=File(context.filesDir,"spatial/captures/$receipt").apply { mkdirs() }
|
||||
Imgcodecs.imwrite(File(folder,"left.png").path,grayL);Imgcodecs.imwrite(File(folder,"right.png").path,grayR)
|
||||
File(folder,"frame.json").writeText(JSONObject().put("sourceSeconds",metadata[2]).put("receiptNs",receipt)
|
||||
.put("connection",generation).put("width",w).put("height",h).put("pairedPose",captureHead!=null).toString())
|
||||
message=context.getString(R.string.spatial_capture_saved, folder.path)
|
||||
}
|
||||
if(running && generation==SpatialTracking.session.generation) {
|
||||
val finished=System.nanoTime()
|
||||
if(adaptiveInput && !gpu) processingRate.observe(finished-start,if(time>0) finished-time else -1,trackingHealthy)
|
||||
SpatialTracking.statistics(c?.readyForTracking==true,frames,dropped,(finished-start)/1e6f,
|
||||
if(time>0)(finished-time)/1e6f else -1f,rate,captureHead!=null,
|
||||
when(copyPermille) { 500 -> 2;1000 -> 1;else -> 0 },metadata[8].toLong(),copyPermille)
|
||||
SpatialTracking.publish(System.nanoTime())
|
||||
}
|
||||
}
|
||||
} catch(e: InterruptedException) { /* Shutdown interrupts the idle wait. */ }
|
||||
catch(e: Throwable) {
|
||||
message=context.getString(R.string.spatial_hand_error, e.message.orEmpty())
|
||||
val unavailable=HandObservationDiagnostic(HandObservationOutcome.INFERENCE_UNAVAILABLE)
|
||||
handStatus=HandTrackingDiagnostics(message,left=unavailable,right=unavailable,observations=handMetrics.snapshot())
|
||||
calibrationProgress = calibrationProgress.copy(capturing = false, captureMessage = message,
|
||||
validating = false, validationMessage = if(calibrationProgress.validating) message else calibrationProgress.validationMessage)
|
||||
clearHands()
|
||||
}
|
||||
finally {
|
||||
VitureSdk.nativeCapture(token,false,1000)
|
||||
joyConFlow?.close();joyConFlow=null
|
||||
closeDetectors()
|
||||
calibration?.close();calibrator?.close()
|
||||
listOf(grayL,grayR,rgba).forEach(Mat::release);bmpL?.recycle();bmpR?.recycle()
|
||||
}
|
||||
}
|
||||
override fun close() {
|
||||
running=false;worker.interrupt()
|
||||
if(Thread.currentThread()!==worker) worker.join(2000)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import java.util.concurrent.ExecutionException
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.Future
|
||||
|
||||
internal class StereoInferenceRunner : AutoCloseable {
|
||||
private val worker = Executors.newSingleThreadExecutor { task ->
|
||||
Thread(task, "viture-right-hand").apply { isDaemon = true }
|
||||
}
|
||||
|
||||
fun <T> onRight(action: () -> T): T = worker.submit<T> { action() }.completed()
|
||||
|
||||
fun <T> pair(left: () -> T, right: () -> T): Pair<T, T> {
|
||||
val pending = worker.submit<T> { right() }
|
||||
val result = try { left() } catch (error: Throwable) {
|
||||
try { pending.completed() } catch (other: Throwable) { if (other !== error) error.addSuppressed(other) }
|
||||
throw error
|
||||
}
|
||||
return result to pending.completed()
|
||||
}
|
||||
|
||||
override fun close() { worker.shutdown() }
|
||||
|
||||
private fun <T> Future<T>.completed(): T {
|
||||
var interrupted = false
|
||||
try {
|
||||
while (true) {
|
||||
try { return get() }
|
||||
catch (e: InterruptedException) { interrupted = true }
|
||||
catch (e: ExecutionException) { throw e.cause ?: e }
|
||||
}
|
||||
} finally {
|
||||
// Native inference must finish before either camera's pixels or detector can be reused.
|
||||
if (interrupted) Thread.currentThread().interrupt()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import org.opencv.core.*
|
||||
import org.opencv.video.Video
|
||||
|
||||
internal class StereoJoyConFlow : AutoCloseable {
|
||||
data class Points(val left: List<Point>, val right: List<Point>, val seedNs: Long)
|
||||
private val previousLeft = Mat()
|
||||
private val previousRight = Mat()
|
||||
private val tracks = arrayOfNulls<Points>(2)
|
||||
private var previousTime = 0L
|
||||
|
||||
fun reset() {
|
||||
previousTime = 0
|
||||
tracks.fill(null)
|
||||
previousLeft.release(); previousRight.release()
|
||||
}
|
||||
|
||||
fun begin(left: Mat, right: Mat, time: Long) {
|
||||
for (side in 0..1) {
|
||||
val old = tracks[side] ?: continue
|
||||
if (!JoyConFlowGate.fresh(old.seedNs, previousTime, time) ||
|
||||
previousLeft.size() != left.size() || previousRight.size() != right.size()) {
|
||||
tracks[side] = null
|
||||
continue
|
||||
}
|
||||
tracks[side] = try {
|
||||
Points(follow(previousLeft, left, old.left), follow(previousRight, right, old.right), old.seedNs)
|
||||
} catch (e: CvException) {
|
||||
android.util.Log.w("StereoJoyConFlow", "Discarding failed optical flow", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fun points(side: Int): Points? = tracks[side]
|
||||
|
||||
fun seed(side: Int, left: Point, right: Point, time: Long) {
|
||||
tracks[side] = Points(listOf(Point(left.x, left.y)), listOf(Point(right.x, right.y)), time)
|
||||
}
|
||||
|
||||
fun end(left: Mat, right: Mat, time: Long) {
|
||||
left.copyTo(previousLeft); right.copyTo(previousRight); previousTime = time
|
||||
}
|
||||
|
||||
private fun follow(previous: Mat, current: Mat, points: List<Point>): List<Point> {
|
||||
val indices = points.indices.filter { points[it].x.isFinite() && points[it].y.isFinite() }
|
||||
val result = MutableList(points.size) { missing() }
|
||||
if (indices.isEmpty()) return result
|
||||
val before = MatOfPoint2f(*indices.map { points[it] }.toTypedArray())
|
||||
val after = MatOfPoint2f(); val back = MatOfPoint2f()
|
||||
val forwardStatus = MatOfByte(); val backStatus = MatOfByte()
|
||||
val error = MatOfFloat(); val backError = MatOfFloat()
|
||||
try {
|
||||
Video.calcOpticalFlowPyrLK(previous, current, before, after, forwardStatus, error)
|
||||
Video.calcOpticalFlowPyrLK(current, previous, after, back, backStatus, backError)
|
||||
val next = after.toArray(); val returned = back.toArray()
|
||||
val valid = forwardStatus.toArray(); val validBack = backStatus.toArray(); val errors = error.toArray()
|
||||
for ((i, index) in indices.withIndex()) {
|
||||
val n = next[i]; val b = returned[i]; val p = points[index]
|
||||
if (JoyConFlowGate.point(n.x,n.y,b.x,b.y,p.x,p.y,errors[i],valid[i].toInt()!=0,
|
||||
validBack[i].toInt()!=0,current.cols(),current.rows())) result[index] = n
|
||||
}
|
||||
} finally {
|
||||
listOf(before, after, back, forwardStatus, backStatus, error, backError).forEach(Mat::release)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private fun missing() = Point(Double.NaN, Double.NaN)
|
||||
override fun close() = reset()
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
import kotlin.math.abs
|
||||
import kotlin.math.hypot
|
||||
|
||||
internal object StereoPointValidation {
|
||||
fun reconstruct(homogeneous: DoubleArray, left: DoubleArray, right: DoubleArray,
|
||||
p1: DoubleArray, p2: DoubleArray): SpatialVector? {
|
||||
if (homogeneous.size != 4 || left.size != 2 || right.size != 2 || p1.size != 12 || p2.size != 12) return null
|
||||
val w = homogeneous[3]
|
||||
if (!w.isFinite() || abs(w) < 1e-8) return null
|
||||
val v = DoubleArray(3) { homogeneous[it]/w }
|
||||
if (v.any { !it.isFinite() } || v[2] !in 0.15..1.5) return null
|
||||
for ((projection,pixel) in listOf(p1 to left,p2 to right)) {
|
||||
val q = DoubleArray(3) { row -> (0..2).sumOf { projection[row*4+it]*v[it] } + projection[row*4+3] }
|
||||
if (q.any { !it.isFinite() } || pixel.any { !it.isFinite() } || q[2] <= 0 ||
|
||||
hypot(q[0]/q[2]-pixel[0],q[1]/q[2]-pixel[1]) > 3.0) return null
|
||||
}
|
||||
return SpatialVector(v[0].toFloat(),v[1].toFloat(),v[2].toFloat())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal enum class TrackingRecovery {
|
||||
DISCONNECTED, LOST, RECENTER, STABILIZING;
|
||||
|
||||
companion object {
|
||||
fun forSession(active: Boolean, connected: Boolean, status: SpatialTrackingStatus): TrackingRecovery? = when {
|
||||
!active -> null
|
||||
!connected -> DISCONNECTED
|
||||
status.trackingValid -> null
|
||||
!status.headFresh -> LOST
|
||||
status.recenterPending -> STABILIZING
|
||||
status.needsRecenter -> RECENTER
|
||||
else -> LOST
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package com.droiddeck.launcher.glasses
|
||||
|
||||
internal object UvcRgbDiagnostic {
|
||||
val available: Boolean by lazy {
|
||||
runCatching { System.loadLibrary("droiddeck_uvc"); true }
|
||||
.onFailure { android.util.Log.w("UvcRgbDiagnostic", "Direct UVC transport unavailable", it) }.getOrDefault(false)
|
||||
}
|
||||
external fun nativeSnapshot(fd: Int, width: Int, height: Int, fps: Int, durationMs: Int,
|
||||
metadata: LongArray, logFd: Int): ByteArray?
|
||||
}
|
||||
|
||||
internal data class UvcStreamHealth(val observationMs: Double, val observedFps: Double,
|
||||
val maximumGapMs: Double, val healthy: Boolean) {
|
||||
companion object {
|
||||
// Health requires the full requested window, 80% of requested FPS, and no gap
|
||||
// (including the final silent interval) above max(250 ms, three frame periods).
|
||||
fun assess(metadata: LongArray, requestedFps: Int, durationMs: Int): UvcStreamHealth {
|
||||
require(metadata.size == 14 && requestedFps > 0 && durationMs > 0)
|
||||
val first = metadata[7]
|
||||
val last = metadata[6]
|
||||
val end = metadata[12]
|
||||
val window = if (first > 0 && end > first) end-first else 0
|
||||
val frames = metadata[9]
|
||||
val fps = if (window > 0 && frames > 1) (frames-1)*1e9/window else 0.0
|
||||
val maximumGapMs = maxOf(metadata[8],metadata[13],(end-last).coerceAtLeast(0))/1e6
|
||||
val healthy = metadata[4] == 0L && first > 0 && last in first..end && frames >= 2 &&
|
||||
window >= durationMs*1_000_000L && fps >= requestedFps*0.8 &&
|
||||
maximumGapMs <= maxOf(250.0,3000.0/requestedFps)
|
||||
return UvcStreamHealth(window/1e6,fps,maximumGapMs,healthy)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -48,11 +48,21 @@ object VitureProtocol {
|
||||
Family.BEAST -> 58f
|
||||
}
|
||||
|
||||
override fun mode(hz: Int, sbs: Boolean): Int = when {
|
||||
tall && sbs -> if (hz >= 90 && sbs90) 0x45 else 0x42
|
||||
tall -> when { hz >= 120 -> 0x44; hz >= 90 -> 0x43; else -> 0x41 }
|
||||
sbs -> if (hz >= 90 && sbs90) 0x35 else 0x32
|
||||
else -> when { hz >= 120 -> 0x34; hz >= 90 -> 0x33; else -> 0x31 }
|
||||
override val resolutions: List<GlassesResolution> get() = buildList {
|
||||
add(GlassesResolution(1920, 1080))
|
||||
if (tall) add(GlassesResolution(1920, 1200))
|
||||
}
|
||||
|
||||
override fun mode(hz: Int, sbs: Boolean): Int = mode(hz, sbs, null)
|
||||
|
||||
override fun mode(hz: Int, sbs: Boolean, resolution: GlassesResolution?): Int {
|
||||
val useTall = tall && (resolution == null || resolution !in resolutions || resolution.height == 1200)
|
||||
return when {
|
||||
useTall && sbs -> if (hz >= 90 && sbs90) 0x45 else 0x42
|
||||
useTall -> when { hz >= 120 -> 0x44; hz >= 90 -> 0x43; else -> 0x41 }
|
||||
sbs -> if (hz >= 90 && sbs90) 0x35 else 0x32
|
||||
else -> when { hz >= 120 -> 0x34; hz >= 90 -> 0x33; else -> 0x31 }
|
||||
}
|
||||
}
|
||||
|
||||
override fun modeForStep(step: Int): Int? = if (tall) null else when (step) {
|
||||
|
||||
@@ -15,8 +15,16 @@ internal object VitureSdk {
|
||||
fun marketName(pid: Int): String? = if (available) nativeMarketName(pid) else null
|
||||
external fun nativeAvailable(): Boolean
|
||||
external fun nativeMarketName(pid: Int): String?
|
||||
external fun nativeOpen(pid: Int, fd: Int, cacheDirectory: String): Long
|
||||
external fun nativeOpen(pid: Int, fd: Int, cacheDirectory: String, spatial: Boolean = false): Long
|
||||
external fun nativeSpatialPose(token: Long, pose: FloatArray, state: IntArray): Long
|
||||
external fun nativeCapture(token: Long, enabled: Boolean, ratePermille: Int)
|
||||
external fun nativeStereo(token: Long, left: java.nio.ByteBuffer, right: java.nio.ByteBuffer, metadata: DoubleArray, head: FloatArray, waitForPose: Boolean): Long
|
||||
external fun nativeCameraSynchronized(token: Long, sourceSeconds: Double): Boolean
|
||||
external fun nativeRgbSnapshot(vid: Int, pid: Int, fd: Int, metadata: LongArray, logFd: Int): ByteArray?
|
||||
external fun nativeType(token: Long): Int
|
||||
external fun nativeFirmware(token: Long): String?
|
||||
external fun nativeSerialHash(token: Long): ByteArray?
|
||||
external fun nativeSpatialDiagnostics(token: Long): DoubleArray?
|
||||
external fun nativeTracking(token: Long, enabled: Boolean): Int
|
||||
external fun nativeCommand(token: Long, command: Int, value: Int): Int
|
||||
/** Returns CLOCK_BOOTTIME receipt time, zero for no new pose, or a negative SDK error. */
|
||||
@@ -34,6 +42,11 @@ internal object VitureSdk {
|
||||
const val SET_FILM = 7
|
||||
}
|
||||
|
||||
internal object VitureSpatialIdentity {
|
||||
fun fromSerialHash(hash: ByteArray?): String? = hash?.takeIf { it.size == 32 && it.any { b -> b != 0.toByte() } }
|
||||
?.joinToString("", prefix = "viture:sn-sha256:") { "%02x".format(it.toInt() and 0xff) }
|
||||
}
|
||||
|
||||
internal object VitureSdkPose {
|
||||
// Pro 2 hardware captures use OpenGL axes despite the generic Gen2 callback's
|
||||
// NWU documentation. Keep that model-specific exception out of other Gen2 devices.
|
||||
|
||||
@@ -13,13 +13,21 @@ internal class VitureSdkLink private constructor(
|
||||
private val token: Long,
|
||||
private val listener: Listener,
|
||||
family: VitureProtocol.Family?,
|
||||
private val spatial: Boolean,
|
||||
) : GlassesLink {
|
||||
interface Listener {
|
||||
fun onValue(msgId: Int, value: Int)
|
||||
fun onPose(q: FloatArray, timeNs: Long)
|
||||
fun onSpatial(pose: FloatArray, timeNs: Long, valid: Boolean) {}
|
||||
fun onLost()
|
||||
}
|
||||
|
||||
internal val sdkToken get() = token
|
||||
internal val spatialEnabled get() = spatial
|
||||
internal val firmware = VitureSdk.nativeFirmware(token)
|
||||
internal fun spatialDiagnostics() = if (running && spatial) VitureSdk.nativeSpatialDiagnostics(token) else null
|
||||
internal val isCarina = VitureSdk.nativeType(token) == VitureSdk.CARINA
|
||||
internal val spatialIdentity = if (isCarina) VitureSpatialIdentity.fromSerialHash(VitureSdk.nativeSerialHash(token)) else null
|
||||
private val lock = Any()
|
||||
@Volatile private var running = true
|
||||
@Volatile private var tracking = false
|
||||
@@ -71,6 +79,8 @@ internal class VitureSdkLink private constructor(
|
||||
|
||||
private fun poll() {
|
||||
val pose = FloatArray(4)
|
||||
val spatialPose = FloatArray(7)
|
||||
val spatialStatus = IntArray(1)
|
||||
val event = IntArray(2)
|
||||
var failures = 0
|
||||
while (running) {
|
||||
@@ -84,7 +94,14 @@ internal class VitureSdkLink private constructor(
|
||||
3 -> listener.onValue(VitureProtocol.MSG_FILM, event[1])
|
||||
}
|
||||
}
|
||||
if (!running || !tracking) 0L else VitureSdk.nativePose(token, pose)
|
||||
if (!running || !tracking) 0L else if (spatial && isCarina) {
|
||||
val time = VitureSdk.nativeSpatialPose(token, spatialPose, spatialStatus)
|
||||
if (time > 0) listener.onSpatial(spatialPose, time, spatialStatus[0] == 0)
|
||||
if (spatialStatus[0] == 0) {
|
||||
spatialPose.copyInto(pose, 0, 3, 7)
|
||||
time
|
||||
} else if (spatialStatus[0] < 0) spatialStatus[0].toLong() else 0L
|
||||
} else VitureSdk.nativePose(token, pose)
|
||||
}
|
||||
if (timestamp > 0) {
|
||||
failures = 0
|
||||
@@ -113,15 +130,15 @@ internal class VitureSdkLink private constructor(
|
||||
|
||||
companion object {
|
||||
fun open(usb: UsbManager, device: UsbDevice, cacheDirectory: File, listener: Listener,
|
||||
family: VitureProtocol.Family? = null): VitureSdkLink? {
|
||||
family: VitureProtocol.Family? = null, spatial: Boolean = false): VitureSdkLink? {
|
||||
if (!VitureSdk.available) return null
|
||||
val conn = usb.openDevice(device) ?: return null
|
||||
val token = runCatching {
|
||||
cacheDirectory.mkdirs()
|
||||
VitureSdk.nativeOpen(device.productId, conn.fileDescriptor, cacheDirectory.absolutePath)
|
||||
VitureSdk.nativeOpen(device.productId, conn.fileDescriptor, cacheDirectory.absolutePath, spatial)
|
||||
}.onFailure { Log.w("VitureSdkLink", "SDK initialization failed", it) }.getOrDefault(0)
|
||||
if (token <= 0) { conn.close(); return null }
|
||||
return runCatching { VitureSdkLink(conn, token, listener, family) }.getOrElse {
|
||||
return runCatching { VitureSdkLink(conn, token, listener, family, spatial) }.getOrElse {
|
||||
VitureSdk.nativeClose(token)
|
||||
conn.close()
|
||||
Log.w("VitureSdkLink", "SDK worker could not start", it)
|
||||
|
||||
@@ -42,12 +42,13 @@ data class Cardboard(
|
||||
companion object {
|
||||
const val SOURCE_SCREEN = 0
|
||||
const val SOURCE_SBS = 1
|
||||
const val SOURCE_FLAT = 2
|
||||
|
||||
val OFF = Cardboard()
|
||||
|
||||
fun decode(json: JSONObject): Cardboard = Cardboard(
|
||||
enabled = json.optBoolean("enabled", false),
|
||||
source = if (json.optInt("source", SOURCE_SCREEN) == SOURCE_SBS) SOURCE_SBS else SOURCE_SCREEN,
|
||||
source = json.optInt("source", SOURCE_SCREEN).takeIf { it in SOURCE_SCREEN..SOURCE_FLAT } ?: SOURCE_SCREEN,
|
||||
viewer = Viewer.byId(json.optString("viewer", Viewer.CARDBOARD_V2.id)).id,
|
||||
ipdMm = json.optInt("ipdMm", 64).coerceIn(50, 80),
|
||||
screenWidthDm = json.optInt("screenWidthDm", 40).coerceIn(5, 200),
|
||||
@@ -102,7 +103,8 @@ object HeadPose {
|
||||
|
||||
class Reference(val basis: FloatArray, val tilt: FloatArray)
|
||||
|
||||
fun recenter(screenToWorld: FloatArray): Reference {
|
||||
fun recenter(screenToWorld: FloatArray, centerView: Boolean = false): Reference {
|
||||
if (centerView) return Reference(screenToWorld.copyOf(), IDENTITY)
|
||||
val fz = -screenToWorld[8]
|
||||
val flat = kotlin.math.abs(fz) > FLAT_SIN
|
||||
var fx = if (flat) screenToWorld[1] else -screenToWorld[2]
|
||||
|
||||
@@ -8,9 +8,11 @@ import android.hardware.SensorManager
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.util.Log
|
||||
import com.droiddeck.launcher.glasses.SpatialTracking
|
||||
import com.droiddeck.launcher.wayland.WaylandCompositor
|
||||
|
||||
class HeadTracker(context: Context, private val glasses: Boolean = false, private val rotation: () -> Int) : SensorEventListener {
|
||||
class HeadTracker(context: Context, private val glasses: Boolean = false, val centerView: Boolean = false,
|
||||
private val rotation: () -> Int) : SensorEventListener {
|
||||
private val sensors = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
|
||||
private val sensor: Sensor? = sensors.getDefaultSensor(Sensor.TYPE_GAME_ROTATION_VECTOR)
|
||||
?: sensors.getDefaultSensor(Sensor.TYPE_ROTATION_VECTOR)
|
||||
@@ -26,7 +28,12 @@ class HeadTracker(context: Context, private val glasses: Boolean = false, privat
|
||||
synchronized(glassesLock) {
|
||||
if (!started) return@PoseSink
|
||||
latest = m
|
||||
apply(m)
|
||||
if (SpatialTracking.enabled) {
|
||||
SpatialTracking.session.view(centerView)?.let { pose ->
|
||||
WaylandCompositor.nativeSetCardboardSpatialPose(pose.rotation.matrix(), pose.position.array(),
|
||||
SpatialTracking.session.valid(System.nanoTime()))
|
||||
}
|
||||
} else apply(m)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,6 +71,7 @@ class HeadTracker(context: Context, private val glasses: Boolean = false, privat
|
||||
synchronized(glassesLock) {
|
||||
if (!started) return
|
||||
started = false
|
||||
SpatialTracking.publish(System.nanoTime())
|
||||
WaylandCompositor.nativeSetCardboardPose(HeadPose.IDENTITY, false)
|
||||
}
|
||||
com.droiddeck.launcher.glasses.Glasses.clearPoseSink(glassesSink)
|
||||
@@ -77,6 +85,7 @@ class HeadTracker(context: Context, private val glasses: Boolean = false, privat
|
||||
}
|
||||
|
||||
fun recenter() {
|
||||
if (glasses && SpatialTracking.enabled) { SpatialTracking.recenter(centerView); return }
|
||||
recenterPending = true
|
||||
if (glasses) synchronized(glassesLock) { if (started) latest?.let { apply(it) } }
|
||||
else latest?.let { apply(it) }
|
||||
@@ -90,7 +99,7 @@ class HeadTracker(context: Context, private val glasses: Boolean = false, privat
|
||||
}
|
||||
|
||||
private fun apply(screenToWorld: FloatArray) {
|
||||
val ref = reference.takeUnless { recenterPending } ?: HeadPose.recenter(screenToWorld).also {
|
||||
val ref = reference.takeUnless { recenterPending } ?: HeadPose.recenter(screenToWorld, centerView).also {
|
||||
reference = it
|
||||
recenterPending = false
|
||||
}
|
||||
|
||||
@@ -91,6 +91,10 @@ class JoyConVr(
|
||||
worker.post { hands.forEach { it?.release() } }
|
||||
}
|
||||
|
||||
fun recenter() {
|
||||
worker.post { hands.forEach { it?.recenter() } }
|
||||
}
|
||||
|
||||
fun close() {
|
||||
worker.post {
|
||||
closed = true
|
||||
@@ -195,9 +199,7 @@ class JoyConVr(
|
||||
private val hold = Runnable {
|
||||
if (!systemDown || held) return@Runnable
|
||||
held = true
|
||||
recenter++
|
||||
confirm()
|
||||
publish()
|
||||
recenter()
|
||||
}
|
||||
private val tapEnd = Runnable {
|
||||
systemTap = false
|
||||
@@ -281,6 +283,13 @@ class JoyConVr(
|
||||
publish()
|
||||
}
|
||||
|
||||
fun recenter() {
|
||||
if (!live) return
|
||||
recenter++
|
||||
confirm()
|
||||
publish()
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
live = false
|
||||
runCatching { sensors.unregisterListener(this) }
|
||||
|
||||
@@ -36,7 +36,7 @@ object BwrapSpawner {
|
||||
return
|
||||
}
|
||||
server = socket
|
||||
FileUtils.delete(File(app.cacheDir, "bwrap"))
|
||||
FileUtils.delete(File(LinuxRuntime.liveDir(app), "bwrap"))
|
||||
Thread({
|
||||
while (true) {
|
||||
val client = try { socket.accept() } catch (e: IOException) { Log.e(TAG, "accept", e); break }
|
||||
@@ -47,7 +47,7 @@ object BwrapSpawner {
|
||||
|
||||
private fun serve(context: Context, client: LocalSocket) {
|
||||
val id = seq.incrementAndGet()
|
||||
val dir = File(context.cacheDir, "bwrap/$id")
|
||||
val dir = File(LinuxRuntime.liveDir(context), "bwrap/$id")
|
||||
client.use {
|
||||
val out = DataOutputStream(client.outputStream)
|
||||
try {
|
||||
@@ -239,12 +239,12 @@ object BwrapSpawner {
|
||||
}
|
||||
|
||||
private fun genericCpuinfo(context: Context): File? = try {
|
||||
val out = File(context.cacheDir, "cpuinfo-generic")
|
||||
val out = File(LinuxRuntime.liveDir(context), "cpuinfo-generic")
|
||||
val text = File("/proc/cpuinfo").readLines()
|
||||
.filterNot { it.startsWith("CPU implementer") || it.startsWith("CPU part") || it.startsWith("CPU variant") || it.startsWith("CPU revision") }
|
||||
.joinToString("\n", postfix = "\n")
|
||||
if (!out.isFile || out.readText() != text) {
|
||||
val staged = File.createTempFile("cpuinfo", null, context.cacheDir)
|
||||
val staged = File.createTempFile("cpuinfo", null, LinuxRuntime.liveDir(context))
|
||||
staged.writeText(text)
|
||||
if (!staged.renameTo(out)) staged.delete()
|
||||
}
|
||||
@@ -290,7 +290,7 @@ object BwrapSpawner {
|
||||
builder.environment().apply {
|
||||
clear()
|
||||
put("PROOT_LOADER", LinuxRuntime.prootLoader(context).path)
|
||||
put("PROOT_TMP_DIR", context.cacheDir.path)
|
||||
put("PROOT_TMP_DIR", LinuxRuntime.prootTempDir(context).path)
|
||||
if (SessionPrefs.prootNoSeccomp(context)) put("PROOT_NO_SECCOMP", "1")
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { put("LD_LIBRARY_PATH", it) }
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ object GuestCommand {
|
||||
val builder = ProcessBuilder(cmd).directory(root).redirectErrorStream(true)
|
||||
builder.environment().apply {
|
||||
put("PROOT_LOADER", LinuxRuntime.prootLoader(context).path)
|
||||
put("PROOT_TMP_DIR", context.cacheDir.path)
|
||||
put("PROOT_TMP_DIR", LinuxRuntime.prootTempDir(context).path)
|
||||
if (linkDir != null) put("PROOT_L2S_DIR", linkDir.path)
|
||||
if (SessionPrefs.prootNoSeccomp(context)) put("PROOT_NO_SECCOMP", "1")
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { put("LD_LIBRARY_PATH", it) }
|
||||
|
||||
@@ -48,6 +48,19 @@ public final class LinuxRuntime {
|
||||
return new File(context.getFilesDir(), "session");
|
||||
}
|
||||
|
||||
public static File liveDir(Context context) {
|
||||
// Android may evict cache files even while guest processes are using them.
|
||||
File dir = new File(context.getNoBackupFilesDir(), "guest-runtime");
|
||||
dir.mkdirs();
|
||||
return dir;
|
||||
}
|
||||
|
||||
public static File prootTempDir(Context context) {
|
||||
File dir = new File(liveDir(context), "proot");
|
||||
dir.mkdirs();
|
||||
return dir;
|
||||
}
|
||||
|
||||
private static final String HOST_DIR = "opt/android-host";
|
||||
|
||||
public static File prootBinary(Context context) {
|
||||
@@ -151,6 +164,7 @@ public final class LinuxRuntime {
|
||||
bind(cmd, new File(root, "etc/droiddeck/empty").getPath() + ":/sys/fs/selinux");
|
||||
bind(cmd, context.getFilesDir().getPath());
|
||||
bind(cmd, context.getCacheDir().getPath());
|
||||
bind(cmd, liveDir(context).getPath());
|
||||
if (runtimeDir != null) {
|
||||
bind(cmd, runtimeDir.getPath());
|
||||
bind(cmd, runtimeDir.getPath() + ":" + GUEST_RUNTIME_DIR);
|
||||
@@ -159,7 +173,7 @@ public final class LinuxRuntime {
|
||||
if (externalStorage != null && externalStorage.isDirectory()) {
|
||||
bind(cmd, externalStorage.getPath());
|
||||
}
|
||||
File shm = new File(context.getCacheDir(), "shm");
|
||||
File shm = new File(liveDir(context), "shm");
|
||||
shm.mkdirs();
|
||||
bind(cmd, shm.getPath() + ":/dev/shm");
|
||||
bindSystemFonts(cmd, root);
|
||||
@@ -222,7 +236,7 @@ public final class LinuxRuntime {
|
||||
long major = ((dev >> 8) & 0xfff) | ((dev >> 32) & ~0xfffL);
|
||||
long minor = (dev & 0xff) | ((dev >> 12) & ~0xffL);
|
||||
String node = "renderD" + minor;
|
||||
File base = new File(context.getCacheDir(), "drm");
|
||||
File base = new File(liveDir(context), "drm");
|
||||
File dri = new File(base, "dri");
|
||||
File device = new File(base, "sys/" + major + ":" + minor + "/device");
|
||||
File drm = new File(device, "drm/" + node);
|
||||
|
||||
@@ -367,12 +367,17 @@ public final class LinuxRuntimeInstaller {
|
||||
}
|
||||
|
||||
public static Removal beginUninstall(Context context) {
|
||||
if (com.droiddeck.launcher.session.SessionState.INSTANCE.getRunning()) return null;
|
||||
com.droiddeck.launcher.session.SessionPhase phase = com.droiddeck.launcher.session.SessionState.INSTANCE.getPhase();
|
||||
if (phase != com.droiddeck.launcher.session.SessionPhase.IDLE && phase != com.droiddeck.launcher.session.SessionPhase.FAILED) return null;
|
||||
if (sessionInUse()) return null;
|
||||
return beginUninstall(context.getApplicationContext(), LinuxRuntime.rootDir(context));
|
||||
}
|
||||
|
||||
private static boolean sessionInUse() {
|
||||
com.droiddeck.launcher.session.SessionState session = com.droiddeck.launcher.session.SessionState.INSTANCE;
|
||||
com.droiddeck.launcher.session.SessionPhase phase = session.getPhase();
|
||||
return session.getRunning() ||
|
||||
(phase != com.droiddeck.launcher.session.SessionPhase.IDLE && phase != com.droiddeck.launcher.session.SessionPhase.FAILED);
|
||||
}
|
||||
|
||||
static Removal beginUninstall(Context app, File root) {
|
||||
synchronized (JOB_LOCK) {
|
||||
if (running != null) return null;
|
||||
@@ -397,6 +402,7 @@ public final class LinuxRuntimeInstaller {
|
||||
if (!started.compareAndSet(false, true)) return join(job, listener);
|
||||
if (listener != null) job.listeners.add(listener);
|
||||
try {
|
||||
requireStopped();
|
||||
report(text(app, R.string.rtinst_removing), -1);
|
||||
final long[] removed = {0};
|
||||
final long[] lastShown = {0};
|
||||
@@ -423,6 +429,8 @@ public final class LinuxRuntimeInstaller {
|
||||
if (!discard(leftover, progress)) throw new IOException("Cannot remove " + leftover);
|
||||
}
|
||||
emptyTrash(root.getParentFile());
|
||||
requireStopped();
|
||||
if (app != null) RuntimeFileTree.delete(new File(app.getNoBackupFilesDir(), "guest-runtime"), null);
|
||||
job.ok = true;
|
||||
report(text(app, R.string.rtinst_removed), 100);
|
||||
return true;
|
||||
@@ -436,6 +444,11 @@ public final class LinuxRuntimeInstaller {
|
||||
job.done.countDown();
|
||||
}
|
||||
}
|
||||
|
||||
private void requireStopped() throws IOException {
|
||||
if (Thread.currentThread().isInterrupted()) throw new IOException("Runtime removal was interrupted");
|
||||
if (app != null && sessionInUse()) throw new IOException("A session is still using the runtime");
|
||||
}
|
||||
private void report(String stage, int percent) {
|
||||
job.stage = stage;
|
||||
job.percent = percent;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package com.droiddeck.launcher.session
|
||||
|
||||
import android.content.Context
|
||||
import android.util.Log
|
||||
import com.droiddeck.launcher.core.DownloadSource
|
||||
import com.droiddeck.launcher.core.Downloader
|
||||
import org.json.JSONObject
|
||||
import java.io.File
|
||||
|
||||
/**
|
||||
* The FEX build DroidDeck-Components' fex.json names for the x86-64 client. A session uses the last copy read.
|
||||
* Components' build-fex.yml replaces the archive on its single `fex` release and records the build's version;
|
||||
* a release per build (fex-2610-r5) is still read.
|
||||
*/
|
||||
object FexCatalog {
|
||||
private const val TAG = "FexCatalog"
|
||||
const val CATALOG_URL = "https://raw.githubusercontent.com/Droid-Deck/DroidDeck-Components/main/fex.json"
|
||||
private const val RELEASES = "https://github.com/Droid-Deck/DroidDeck-Components/releases/download/"
|
||||
private const val REFRESH_MS = 10 * 60 * 1000L
|
||||
private val TAG_FORMAT = Regex("""fex-\d+-r\d+""")
|
||||
private val VERSION_FORMAT = Regex("""\d+-r\d+""")
|
||||
private val SHA256 = Regex("[0-9a-f]{64}")
|
||||
|
||||
data class Build(val tag: String, val url: String, val sha256: String)
|
||||
|
||||
@Volatile private var refreshedAt = 0L
|
||||
|
||||
private fun file(context: Context) = File(context.filesDir, "fex-catalog.json")
|
||||
|
||||
internal fun parse(text: String): Build? = runCatching {
|
||||
val o = JSONObject(text)
|
||||
val release = o.getString("release")
|
||||
val url = o.getString("url")
|
||||
val sha = o.getString("sha256").lowercase()
|
||||
val version = o.optString("version")
|
||||
val tag = when {
|
||||
TAG_FORMAT.matches(release) -> release
|
||||
release == "fex" && VERSION_FORMAT.matches(version) -> "fex-$version"
|
||||
else -> return@runCatching null
|
||||
}
|
||||
Build(tag, url, sha).takeIf { url == "$RELEASES$release/fex.tzst" && SHA256.matches(sha) }
|
||||
}.getOrNull()
|
||||
|
||||
fun cached(context: Context): Build? = runCatching { parse(file(context).readText()) }.getOrNull()
|
||||
|
||||
/** Reads fex.json straight from GitHub (a digest never comes from a mirror), at most every ten minutes. */
|
||||
@Synchronized
|
||||
fun refresh(context: Context, now: Long = System.currentTimeMillis()): Build? {
|
||||
if (now - refreshedAt < REFRESH_MS) return cached(context)
|
||||
refreshedAt = now
|
||||
val text = Downloader.downloadDirect(CATALOG_URL) ?: return cached(context)
|
||||
val build = parse(text) ?: run {
|
||||
Log.w(TAG, "fex.json not usable; keeping the last one")
|
||||
return cached(context)
|
||||
}
|
||||
runCatching {
|
||||
val f = file(context)
|
||||
val tmp = File(f.path + ".tmp")
|
||||
tmp.writeText(text)
|
||||
if (!tmp.renameTo(f)) tmp.delete()
|
||||
}
|
||||
return build
|
||||
}
|
||||
|
||||
/** For droiddeck-steam-x64: the build and every source its archive may come from (it checks the sha256). */
|
||||
fun environment(build: Build, sources: List<String> = DownloadSource.candidates(build.url)): List<String> = listOf(
|
||||
"BL_FEX_TAG=${build.tag}",
|
||||
"BL_FEX_SHA256=${build.sha256}",
|
||||
"BL_FEX_URLS=" + sources.joinToString(" "),
|
||||
)
|
||||
}
|
||||
@@ -225,7 +225,7 @@ object ProtonExtras {
|
||||
.redirectErrorStream(true)
|
||||
val hostEnv = process.environment()
|
||||
hostEnv["PROOT_LOADER"] = LinuxRuntime.prootLoader(context).path
|
||||
hostEnv["PROOT_TMP_DIR"] = context.cacheDir.path
|
||||
hostEnv["PROOT_TMP_DIR"] = LinuxRuntime.prootTempDir(context).path
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { hostEnv["LD_LIBRARY_PATH"] = it }
|
||||
val child = process.start()
|
||||
var placed: String? = null
|
||||
|
||||
@@ -121,13 +121,14 @@ object SessionFiles {
|
||||
"droiddeck/driver.vrdrivermanifest",
|
||||
"droiddeck/resources/settings/default.vrsettings",
|
||||
"layer/libVkLayer_droiddeck_vr.so",
|
||||
"layer/libdroiddeck_vr_scheduling.so",
|
||||
"layer/VK_LAYER_DROIDDECK_vr.json",
|
||||
) + handModels).map { "usr/local/lib/droiddeck-vr/$it" to "usr/local/lib/droiddeck-vr/$it" } +
|
||||
listOf("usr/local/lib/droiddeck-vr/layer/VK_LAYER_DROIDDECK_vr_implicit.json" to
|
||||
"usr/share/vulkan/implicit_layer.d/droiddeck_vr.json")
|
||||
val fexPreloads = listOf("x86_64", "i386").flatMap { arch ->
|
||||
listOf("libblsession.so", "libfakeinput.so", "libfaultreport.so").map { "$arch/$it" to "usr/local/lib/droiddeck-fex/$arch/$it" }
|
||||
} + listOf("libthunkaudit.so", "libvulkan-thunk.so", "libwebkitinput.so").map { "x86_64/$it" to "usr/local/lib/droiddeck-fex/x86_64/$it" }
|
||||
} + listOf("libvulkan-thunk.so", "libwebkitinput.so").map { "x86_64/$it" to "usr/local/lib/droiddeck-fex/x86_64/$it" }
|
||||
val optional = (arrayOf(
|
||||
"usr/local/bin/gamescope" to "usr/local/bin/gamescope",
|
||||
"usr/local/lib/droiddeck/uruntime" to "usr/local/lib/droiddeck/uruntime",
|
||||
|
||||
@@ -415,7 +415,7 @@ class SessionService : Service() {
|
||||
Log.i(TAG, "run: $program (${SessionState.programArgs.size} arguments) under gamescope")
|
||||
}
|
||||
|
||||
FileUtils.clear(File(cacheDir, "shm"))
|
||||
FileUtils.clear(File(LinuxRuntime.liveDir(this), "shm"))
|
||||
|
||||
val binds = sessionBinds(controllersOn, fakeInputDir, sessionDir, guest, sessionRoot, runtimeDir)
|
||||
val fastPathKey = if (ProotFastPath.enabled(this)) {
|
||||
@@ -440,7 +440,7 @@ class SessionService : Service() {
|
||||
|
||||
val hostEnv = HostEnvironment()
|
||||
hostEnv["PROOT_LOADER"] = LinuxRuntime.prootLoader(this).path
|
||||
hostEnv["PROOT_TMP_DIR"] = cacheDir.path
|
||||
hostEnv["PROOT_TMP_DIR"] = LinuxRuntime.prootTempDir(this).path
|
||||
if (SessionPrefs.prootNoSeccomp(this)) {
|
||||
hostEnv["PROOT_NO_SECCOMP"] = "1"
|
||||
Log.i(TAG, "proot: seccomp acceleration off by request")
|
||||
@@ -591,6 +591,7 @@ class SessionService : Service() {
|
||||
SessionState.steamClientArch?.let { arch ->
|
||||
guest.add("BL_STEAM_CLIENT_ARCH=$arch")
|
||||
SessionPaths.current()?.let { SessionArtifacts.recordClient(it, arch) }
|
||||
if (arch == "x86_64") FexCatalog.cached(this)?.let { guest.addAll(FexCatalog.environment(it)) }
|
||||
}
|
||||
if (SessionState.mode == MODE_STEAM) {
|
||||
guest.add("BL_GAMESCOPE_STRETCH_16X9=" + (if (SessionPrefs.stretch16x9(this)) "1" else "0"))
|
||||
@@ -1375,6 +1376,9 @@ class SessionService : Service() {
|
||||
val failureStatus = SessionState.failureStatus ?: status
|
||||
SessionEvents.fail(code, message, failureStatus)
|
||||
}
|
||||
SessionState.steamVr = false
|
||||
com.droiddeck.launcher.glasses.Glasses.cancelSteamVrStart()
|
||||
com.droiddeck.launcher.glasses.Glasses.setVrActive(false)
|
||||
com.droiddeck.launcher.glasses.Glasses.set3dForVr(false)
|
||||
SessionState.notifyEnded(status)
|
||||
stopForeground(STOP_FOREGROUND_REMOVE)
|
||||
|
||||
@@ -155,7 +155,7 @@ object SteamDeckPad {
|
||||
|
||||
val listing = listingDir(sessionRoot)
|
||||
for (name in listOf("sys/bus", "sys/class", "sys/devices", "class/hidraw", "bus")) File(listing, name).mkdirs()
|
||||
link(File(context.cacheDir, "drm/sys/$MAJOR:$MINOR"), HIDRAW)
|
||||
link(File(LinuxRuntime.liveDir(context), "drm/sys/$MAJOR:$MINOR"), HIDRAW)
|
||||
|
||||
val udevData = File(LinuxRuntime.rootDir(context), "run/udev/data").apply { mkdirs() }
|
||||
File(udevData, "c$MAJOR:$MINOR").writeText("I:1\nE:ID_INPUT=1\nE:ID_INPUT_JOYSTICK=1\n")
|
||||
@@ -183,7 +183,7 @@ object SteamDeckPad {
|
||||
link(File(dir(TOUCH_HIDRAW), "subsystem"), "/sys/class/hidraw")
|
||||
link(File(dir(TOUCH_HIDRAW), "device"), TOUCH_HID)
|
||||
link(File(hidrawClass, TOUCH_NODE), TOUCH_HIDRAW)
|
||||
link(File(context.cacheDir, "drm/sys/$MAJOR:$TOUCH_MINOR"), TOUCH_HIDRAW)
|
||||
link(File(LinuxRuntime.liveDir(context), "drm/sys/$MAJOR:$TOUCH_MINOR"), TOUCH_HIDRAW)
|
||||
File(udevData, "c$MAJOR:$TOUCH_MINOR").writeText("I:1\nE:ID_INPUT=1\nE:ID_INPUT_JOYSTICK=1\n")
|
||||
Log.i(TAG, "deck pad: /dev/$TOUCH_NODE described as Steam's touch controller (0000:11fb)")
|
||||
}
|
||||
@@ -220,7 +220,7 @@ object SteamDeckPad {
|
||||
link(File(dir(hidraw), "subsystem"), "/sys/class/hidraw")
|
||||
link(File(dir(hidraw), "device"), hid)
|
||||
if (wired) padNodes[minor] = hidraw else link(File(hidrawClass, node), hidraw)
|
||||
link(File(context.cacheDir, "drm/sys/$MAJOR:$minor"), hidraw)
|
||||
link(File(LinuxRuntime.liveDir(context), "drm/sys/$MAJOR:$minor"), hidraw)
|
||||
File(udevData, "c$MAJOR:$minor").writeText("I:1\nE:ID_INPUT=1\nE:ID_INPUT_JOYSTICK=1\n")
|
||||
Log.i(TAG, "steam controller: /dev/$node described as Triton (28de:$product)")
|
||||
}
|
||||
@@ -255,7 +255,7 @@ object SteamDeckPad {
|
||||
link(File(dir(hidraw), "subsystem"), "/sys/class/hidraw")
|
||||
link(File(dir(hidraw), "device"), hid)
|
||||
padNodes[minor] = hidraw
|
||||
link(File(context.cacheDir, "drm/sys/$MAJOR:$minor"), hidraw)
|
||||
link(File(LinuxRuntime.liveDir(context), "drm/sys/$MAJOR:$minor"), hidraw)
|
||||
File(udevData, "c$MAJOR:$minor").writeText("I:1\nE:ID_INPUT=1\nE:ID_INPUT_JOYSTICK=1\n")
|
||||
Log.i(TAG, "playstation pad: /dev/$node described as 054c:$product")
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ object GogComet {
|
||||
pb.environment().apply {
|
||||
remove("LD_PRELOAD"); remove("LD_AUDIT"); remove("LD_LIBRARY_PATH")
|
||||
put("PROOT_LOADER", LinuxRuntime.prootLoader(context).path)
|
||||
put("PROOT_TMP_DIR", context.cacheDir.path)
|
||||
put("PROOT_TMP_DIR", LinuxRuntime.prootTempDir(context).path)
|
||||
if (SessionPrefs.prootNoSeccomp(context)) put("PROOT_NO_SECCOMP", "1")
|
||||
LinuxRuntime.prootLibraryPath(context).takeIf { it.isNotEmpty() }?.let { put("LD_LIBRARY_PATH", it) }
|
||||
}
|
||||
|
||||
@@ -86,6 +86,7 @@ import com.droiddeck.launcher.R
|
||||
private enum class CheckState { OK, WARN, BUSY }
|
||||
|
||||
private const val STORAGE_TAB = 3
|
||||
private const val GLASSES_TAB = 5
|
||||
|
||||
@Composable
|
||||
private fun CheckRow(state: CheckState, title: String, detail: String?, divider: Boolean = true, action: (@Composable () -> Unit)? = null) {
|
||||
@@ -175,7 +176,7 @@ internal fun SetupPanel(
|
||||
onDismiss = { editDownloadPrefix = false },
|
||||
onSave = { a.onDownloadPrefix(it); editDownloadPrefix = false },
|
||||
)
|
||||
val tabs = listOf(stringResource(R.string.setup_tab_overview), stringResource(R.string.setup_tab_controller), stringResource(R.string.setup_tab_launcher), stringResource(R.string.setup_tab_storage), stringResource(R.string.setup_tab_diagnostics))
|
||||
val tabs = listOf(stringResource(R.string.setup_tab_overview), stringResource(R.string.setup_tab_controller), stringResource(R.string.setup_tab_launcher), stringResource(R.string.setup_tab_storage), stringResource(R.string.setup_tab_diagnostics), stringResource(R.string.setup_tab_glasses))
|
||||
var tab by rememberSaveable { mutableStateOf(0) }
|
||||
val chipFocus = remember { List(tabs.size) { FocusRequester() } }
|
||||
var tabTurned by remember { mutableStateOf(false) }
|
||||
@@ -451,6 +452,7 @@ internal fun SetupPanel(
|
||||
stringResource(R.string.setup_reset_settings_button), a.onResetSettings)
|
||||
}
|
||||
}
|
||||
GLASSES_TAB -> GlassesSetupPanel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,10 @@ import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.interaction.MutableInteractionSource
|
||||
import androidx.compose.foundation.interaction.collectIsPressedAsState
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.Spacer
|
||||
import androidx.compose.foundation.layout.WindowInsets
|
||||
import androidx.compose.foundation.layout.displayCutout
|
||||
@@ -42,8 +44,14 @@ import androidx.compose.runtime.collectAsState
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.glasses.Glasses
|
||||
import com.droiddeck.launcher.glasses.GlassesRendering
|
||||
import com.droiddeck.launcher.glasses.GlassesDisplayModes
|
||||
import com.droiddeck.launcher.glasses.GlassesDisplayProfile
|
||||
import com.droiddeck.launcher.glasses.GlassesSettings
|
||||
import com.droiddeck.launcher.glasses.GlassesState
|
||||
import android.content.Intent
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import com.droiddeck.launcher.glasses.SpatialTracking
|
||||
import com.droiddeck.launcher.glasses.SpatialMeasurements
|
||||
import com.droiddeck.launcher.handtracking.HandTracking
|
||||
import java.util.Locale
|
||||
|
||||
@@ -68,6 +76,7 @@ internal fun rememberGlasses(): Pair<GlassesState, GlassesSettings> {
|
||||
|
||||
internal object GlassesTab {
|
||||
const val DISPLAY = "display"
|
||||
const val VR = "vr"
|
||||
const val HARDWARE = "hardware"
|
||||
const val TRACKING = "tracking"
|
||||
}
|
||||
@@ -75,9 +84,10 @@ internal object GlassesTab {
|
||||
@Composable
|
||||
internal fun glassesTabs(g: GlassesState): List<Pair<String, String>> = buildList {
|
||||
add(GlassesTab.DISPLAY to stringResource(R.string.glasses_display))
|
||||
if (g.canControl || g.hasDisplay) add(GlassesTab.VR to stringResource(R.string.glasses_vr))
|
||||
if (g.canControl) {
|
||||
if (g.hardwareControls) add(GlassesTab.HARDWARE to stringResource(R.string.glasses_hardware))
|
||||
add(GlassesTab.TRACKING to stringResource(R.string.glasses_tracking))
|
||||
if (g.hardwareControls) add(GlassesTab.HARDWARE to stringResource(R.string.glasses_hardware))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -95,10 +105,22 @@ internal fun GlassesGroups(
|
||||
steamVr: @Composable () -> Unit = {},
|
||||
) {
|
||||
val display = tab == null || tab == GlassesTab.DISPLAY
|
||||
val vr = tab == null || tab == GlassesTab.VR
|
||||
val picture = GlassesRendering.picture(s, g.sbs, steamVrRunning)
|
||||
val usesGlasses = GlassesRendering.usesGlasses(g, outputActive, steamVrRunning)
|
||||
val spatialTracking = g.spatialConnected && s.positionalTracking
|
||||
val spatialVr = spatialTracking && usesGlasses && steamVrRunning
|
||||
val flatPicture = picture == GlassesRendering.Picture.FLAT || picture == GlassesRendering.Picture.DUPLICATED
|
||||
val virtualScreen = usesGlasses && picture == GlassesRendering.Picture.VIRTUAL_SCREEN
|
||||
val directView = usesGlasses && picture == GlassesRendering.Picture.STEREO
|
||||
val menuScreen = !inSession && g.usable && s.gameOnGlasses && s.screen3dof
|
||||
if (g.canControl && onRecenter != null && s.headTracking && (virtualScreen || menuScreen || (directView && steamVrRunning))) {
|
||||
SettingsGroup(stringResource(R.string.glasses_quick_actions), compact = true) {
|
||||
ActionRow(stringResource(R.string.glasses_recenter),
|
||||
if (virtualScreen || menuScreen) stringResource(R.string.glasses_recenter_screen_hint) else null,
|
||||
stringResource(R.string.cardboard_recenter_button), onRecenter)
|
||||
}
|
||||
}
|
||||
if (display) SettingsGroup(g.model) {
|
||||
if (g.experimental) SettingsRow(stringResource(R.string.glasses_control), stringResource(R.string.glasses_xreal_experimental)) {}
|
||||
val status = if (g.hasDisplay) stringResource(R.string.glasses_status, g.width, g.height, kotlin.math.round(g.hz).toInt(),
|
||||
@@ -119,35 +141,38 @@ internal fun GlassesGroups(
|
||||
ToggleRow(host, "glasses-game", stringResource(R.string.glasses_game_on), stringResource(R.string.glasses_game_on_hint),
|
||||
s.gameOnGlasses, chipModifier = track("glasses-game")) { Glasses.setGameOnGlasses(it) }
|
||||
}
|
||||
if (display && (g.canControl || g.hasDisplay)) SettingsGroup(stringResource(R.string.glasses_display)) {
|
||||
if (g.canControl) {
|
||||
if (!g.sbs) ChoiceRow(host, "glasses-hz", stringResource(R.string.glasses_refresh), null,
|
||||
listOf(60, 90, 120).map { it to "$it Hz" }, s.refreshHz, chipModifier = track("glasses-hz")) { Glasses.setRefreshHz(it) }
|
||||
if (display && g.canControl && (inSession || g.sbs || g.sbsSupport == GlassesState.SBS_FAILS)) {
|
||||
SettingsGroup(stringResource(R.string.glasses_display_mode)) {
|
||||
if (g.sbsSupport == GlassesState.SBS_FAILS && !g.sbs) {
|
||||
ActionRow(stringResource(R.string.glasses_3d), stringResource(R.string.glasses_3d_failed),
|
||||
stringResource(R.string.stores_dl_retry), { Glasses.set3d(true) })
|
||||
} else {
|
||||
if (inSession || g.sbs) ToggleRow(host, "glasses-3d", stringResource(R.string.glasses_3d), stringResource(R.string.glasses_3d_hint),
|
||||
g.sbs, chipModifier = track("glasses-3d")) { Glasses.set3d(it) }
|
||||
if (g.sbs) {
|
||||
if (g.sbs90) ChoiceRow(host, "glasses-3d-hz", stringResource(R.string.glasses_3d_refresh), null,
|
||||
listOf(60, 90).map { it to "$it Hz" }, s.sbsHz, chipModifier = track("glasses-3d-hz")) { Glasses.setSbsHz(it) }
|
||||
else SettingsRow(stringResource(R.string.glasses_3d_refresh), "60 Hz") {}
|
||||
}
|
||||
}
|
||||
} else ToggleRow(host, "glasses-3d", stringResource(R.string.glasses_3d), stringResource(R.string.glasses_3d_hint),
|
||||
g.sbs, chipModifier = track("glasses-3d")) { Glasses.set3d(it) }
|
||||
}
|
||||
ToggleRow(host, "glasses-auto3d", stringResource(R.string.glasses_auto3d), stringResource(R.string.glasses_auto3d_hint),
|
||||
s.auto3d, chipModifier = track("glasses-auto3d")) { Glasses.setAuto3d(it) }
|
||||
}
|
||||
if (display && g.canControl) {
|
||||
val profiles = if (inSession) listOf(GlassesDisplayModes.profile(g.sbs, false))
|
||||
else listOf(GlassesDisplayProfile.TWO_D, GlassesDisplayProfile.THREE_D)
|
||||
for (profile in profiles) SettingsGroup(stringResource(if (profile == GlassesDisplayProfile.TWO_D)
|
||||
R.string.glasses_output_2d else R.string.glasses_output_3d)) {
|
||||
GlassesResolutionRow(host, g, s, profile, track)
|
||||
if ((profile == GlassesDisplayProfile.THREE_D) == g.sbs) GlassesRefreshRow(host, g, s, track)
|
||||
}
|
||||
}
|
||||
if (display && (g.canControl || g.hasDisplay)) SettingsGroup(stringResource(R.string.glasses_picture_group)) {
|
||||
ChoiceRow(host, "glasses-picture", stringResource(R.string.glasses_picture), stringResource(R.string.glasses_picture_hint),
|
||||
listOf(
|
||||
GlassesSettings.PICTURE_AUTO to stringResource(R.string.glasses_picture_auto),
|
||||
GlassesSettings.PICTURE_SCREEN to stringResource(R.string.glasses_picture_screen),
|
||||
GlassesSettings.PICTURE_SBS to stringResource(R.string.glasses_picture_sbs),
|
||||
), s.picture3d, chipModifier = track("glasses-picture"), hintLines = 2) { Glasses.setPicture3d(it) }
|
||||
), s.picture3d, chipModifier = track("glasses-picture")) { Glasses.setPicture3d(it) }
|
||||
if (g.canControl && g.supportsScreen3dof) ToggleRow(host, "glasses-3dof",
|
||||
stringResource(if (g.spatialConnected && s.positionalTracking) R.string.glasses_6dof else R.string.glasses_3dof),
|
||||
stringResource(R.string.glasses_3dof_hint), s.screen3dof, chipModifier = track("glasses-3dof")) { Glasses.set3dof(it) }
|
||||
when {
|
||||
!inSession -> SettingsRow(stringResource(R.string.glasses_picture_screen), stringResource(R.string.glasses_screen_next_session)) {}
|
||||
!inSession -> SettingsRow(stringResource(R.string.glasses_picture_screen), stringResource(R.string.glasses_screen_app_wide)) {}
|
||||
!usesGlasses -> SettingsRow(stringResource(R.string.glasses_picture_screen), stringResource(R.string.glasses_screen_no_output)) {}
|
||||
picture == GlassesRendering.Picture.FLAT -> ActionRow(
|
||||
flatPicture -> ActionRow(
|
||||
stringResource(R.string.glasses_picture_screen), stringResource(R.string.glasses_screen_flat_hint),
|
||||
stringResource(R.string.glasses_screen_use), { Glasses.setPicture3d(GlassesSettings.PICTURE_SCREEN) })
|
||||
directView -> SettingsRow(stringResource(R.string.glasses_picture_screen),
|
||||
@@ -155,15 +180,44 @@ internal fun GlassesGroups(
|
||||
}
|
||||
if (virtualScreen || (!inSession && s.picture3d != GlassesSettings.PICTURE_SBS)) {
|
||||
SliderRow(stringResource(R.string.glasses_screen_size), null, s.screenWidthDm, 5..60, step = 1,
|
||||
format = { String.format(Locale.US, "%.1f m", it / 10f) }, modifier = track("glasses-width")) { Glasses.setScreenWidthDm(it) }
|
||||
format = { SpatialMeasurements.distance(it / 10f) }, modifier = track("glasses-width")) { Glasses.setScreenWidthDm(it) }
|
||||
SliderRow(stringResource(R.string.glasses_screen_distance), null, s.screenDistanceDm, 5..100, step = 1,
|
||||
format = { String.format(Locale.US, "%.1f m", it / 10f) }, modifier = track("glasses-distance")) { Glasses.setScreenDistanceDm(it) }
|
||||
format = { SpatialMeasurements.distance(it / 10f) }, modifier = track("glasses-distance")) { Glasses.setScreenDistanceDm(it) }
|
||||
}
|
||||
}
|
||||
if (vr && (g.canControl || g.hasDisplay)) SettingsGroup(stringResource(R.string.cardboard_steamvr)) {
|
||||
steamVr()
|
||||
if ((directView && steamVrRunning) || !inSession) SliderRow(stringResource(R.string.glasses_vr_fov), stringResource(R.string.glasses_vr_fov_hint), s.vrFovDeg,
|
||||
ToggleRow(host, "glasses-auto3d", stringResource(R.string.glasses_auto3d), stringResource(R.string.glasses_auto3d_hint),
|
||||
s.auto3d, chipModifier = track("glasses-auto3d")) { Glasses.setAuto3d(it) }
|
||||
if (g.canControl) {
|
||||
GlassesResolutionRow(host, g, s, GlassesDisplayProfile.VR, track)
|
||||
if (steamVrRunning && tab != null) GlassesRefreshRow(host, g, s, track)
|
||||
}
|
||||
SliderRow(stringResource(R.string.glasses_vr_fov), stringResource(R.string.glasses_vr_fov_hint), s.vrFovDeg,
|
||||
GlassesSettings.VR_FOV_MIN..GlassesSettings.VR_FOV_MAX, step = 1,
|
||||
format = { "$it°" }, modifier = track("glasses-vr-fov")) { Glasses.setVrFovDeg(it) }
|
||||
}
|
||||
if ((tab == null || tab == GlassesTab.TRACKING) && g.canControl) {
|
||||
SettingsGroup(stringResource(R.string.glasses_head_tracking)) {
|
||||
val trackingHint = when {
|
||||
inSession && !usesGlasses -> R.string.glasses_screen_no_output
|
||||
spatialVr -> R.string.glasses_head_tracking_spatial_vr_hint
|
||||
spatialTracking -> R.string.glasses_head_tracking_spatial_hint
|
||||
inSession && flatPicture -> R.string.glasses_tracking_flat_hint
|
||||
inSession && directView && !steamVrRunning -> R.string.glasses_tracking_sbs_hint
|
||||
else -> R.string.glasses_head_tracking_hint
|
||||
}
|
||||
ToggleRow(host, "glasses-tracking", stringResource(R.string.glasses_head_tracking), stringResource(trackingHint),
|
||||
s.headTracking || spatialVr, enabled = !spatialVr,
|
||||
chipModifier = track("glasses-tracking")) { Glasses.setHeadTracking(it) }
|
||||
}
|
||||
val calibration by Glasses.calibration.collectAsState()
|
||||
if (calibration.supported) SettingsGroup(stringResource(R.string.glasses_calibration)) {
|
||||
GlassesCalibrationControls(calibration,
|
||||
onStart = Glasses::startCalibration, onCancel = Glasses::cancelCalibration,
|
||||
onReset = Glasses::resetCalibration, onDone = { onRecenter?.invoke() })
|
||||
}
|
||||
}
|
||||
if ((tab == null || tab == GlassesTab.HARDWARE) && g.canControl && g.hardwareControls) SettingsGroup(stringResource(R.string.glasses_hardware)) {
|
||||
if (g.brightnessMax > 0) SliderRow(stringResource(R.string.glasses_brightness), null, g.brightness.coerceAtLeast(0), 0..g.brightnessMax, step = 1,
|
||||
modifier = track("glasses-brightness")) { Glasses.setBrightness(it) }
|
||||
@@ -172,28 +226,71 @@ internal fun GlassesGroups(
|
||||
if (g.hasSunblock) ToggleRow(host, "glasses-sunblock", stringResource(R.string.glasses_sunblock), stringResource(R.string.glasses_sunblock_hint),
|
||||
g.sunblock, chipModifier = track("glasses-sunblock")) { Glasses.setSunblock(it) }
|
||||
}
|
||||
if ((tab == null || tab == GlassesTab.TRACKING) && g.canControl) SettingsGroup(stringResource(R.string.glasses_tracking)) {
|
||||
val trackingHint = when {
|
||||
inSession && !usesGlasses -> R.string.glasses_screen_no_output
|
||||
inSession && picture == GlassesRendering.Picture.FLAT -> R.string.glasses_tracking_flat_hint
|
||||
inSession && directView && !steamVrRunning -> R.string.glasses_tracking_sbs_hint
|
||||
else -> R.string.glasses_head_tracking_hint
|
||||
}
|
||||
ToggleRow(host, "glasses-tracking", stringResource(R.string.glasses_head_tracking), stringResource(trackingHint),
|
||||
s.headTracking, chipModifier = track("glasses-tracking")) { Glasses.setHeadTracking(it) }
|
||||
if (onRecenter != null && s.headTracking && (virtualScreen || (directView && steamVrRunning))) ActionRow(stringResource(R.string.glasses_recenter), null,
|
||||
stringResource(R.string.cardboard_recenter_button), onRecenter)
|
||||
val calibration by Glasses.calibration.collectAsState()
|
||||
if (calibration.supported) GlassesCalibrationControls(calibration,
|
||||
onStart = Glasses::startCalibration, onCancel = Glasses::cancelCalibration,
|
||||
onReset = Glasses::resetCalibration, onDone = { onRecenter?.invoke() })
|
||||
}
|
||||
val context = androidx.compose.ui.platform.LocalContext.current
|
||||
val hands by HandTracking.settings.collectAsState()
|
||||
androidx.compose.runtime.LaunchedEffect(Unit) { HandTracking.load(context) }
|
||||
if ((tab == null || tab == GlassesTab.TRACKING) && g.canControl && hands.usable) SettingsGroup(stringResource(R.string.hand_camera_group)) {
|
||||
HandCameraRows(host, track, inSession)
|
||||
if ((tab == null || tab == GlassesTab.TRACKING) && g.canControl) {
|
||||
if (g.spatialConnected) {
|
||||
SettingsGroup(stringResource(R.string.glasses_experimental)) {
|
||||
val spatial by SpatialTracking.status.collectAsState()
|
||||
ToggleRow(host, "glasses-position", stringResource(R.string.glasses_position), stringResource(R.string.glasses_position_hint),
|
||||
s.positionalTracking, chipModifier = track("glasses-position")) { Glasses.setPositionalTracking(it) }
|
||||
if (s.positionalTracking) {
|
||||
SettingsRow(stringResource(R.string.glasses_tracking_status), spatial.message) {}
|
||||
SliderRow(stringResource(R.string.glasses_eye_height), null, s.eyeHeightCm, 50..250, step = 1,
|
||||
format = SpatialMeasurements::height, modifier = track("glasses-eye-height")) { Glasses.setEyeHeight(it) }
|
||||
ActionRow(stringResource(R.string.glasses_floor), null, stringResource(R.string.glasses_floor_set), Glasses::setSpatialFloor)
|
||||
HandCalibrationStatus(spatial)
|
||||
ActionRow(stringResource(R.string.glasses_spatial_setup), stringResource(R.string.glasses_spatial_setup_hint),
|
||||
stringResource(R.string.glasses_spatial_open), { context.startActivity(Intent(context, SpatialTrackingActivity::class.java)) })
|
||||
ToggleRow(host, "glasses-hands", stringResource(R.string.glasses_hands), stringResource(R.string.glasses_hands_hint),
|
||||
s.handControllers, chipModifier = track("glasses-hands")) { Glasses.setHandControllers(it) }
|
||||
if (s.handControllers) ToggleRow(host, "glasses-steady-hand-aim",
|
||||
stringResource(R.string.glasses_steady_hand_aim), stringResource(R.string.glasses_steady_hand_aim_hint),
|
||||
s.steadyHandAim, chipModifier = track("glasses-steady-hand-aim")) { Glasses.setSteadyHandAim(it) }
|
||||
}
|
||||
ToggleRow(host, "glasses-joycons", stringResource(R.string.glasses_joycons),
|
||||
stringResource(if (s.positionalTracking) R.string.glasses_joycons_hint else R.string.glasses_joycons_requires_position),
|
||||
s.joyConTracking, enabled = s.positionalTracking, chipModifier = track("glasses-joycons")) { Glasses.setJoyConTracking(it) }
|
||||
}
|
||||
}
|
||||
if (hands.usable) SettingsGroup(stringResource(R.string.phone_camera_tracking_group)) {
|
||||
HandCameraRows(host, track, inSession)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun GlassesResolutionRow(host: MenuHost, g: GlassesState, s: GlassesSettings,
|
||||
profile: GlassesDisplayProfile, track: (String) -> Modifier) {
|
||||
val label = when (profile) {
|
||||
GlassesDisplayProfile.TWO_D -> R.string.glasses_resolution_2d
|
||||
GlassesDisplayProfile.THREE_D -> R.string.glasses_resolution_3d
|
||||
GlassesDisplayProfile.VR -> R.string.glasses_resolution_vr
|
||||
}
|
||||
val choices = listOf("" to stringResource(if (profile == GlassesDisplayProfile.VR)
|
||||
R.string.glasses_resolution_inherit else R.string.glasses_resolution_auto)) +
|
||||
g.resolutions.map { it.key to "${it.width} × ${it.height}" }
|
||||
val saved = GlassesDisplayModes.choice(s, profile)
|
||||
val key = "glasses-resolution-${profile.name.lowercase()}"
|
||||
ChoiceRow(host, key, stringResource(label), stringResource(R.string.glasses_resolution_hint), choices,
|
||||
saved.takeIf { value -> choices.any { it.first == value } } ?: "",
|
||||
enabled = !g.switching3d, chipModifier = track(key)) { Glasses.setResolution(profile, it) }
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun GlassesRefreshRow(host: MenuHost, g: GlassesState, s: GlassesSettings, track: (String) -> Modifier) {
|
||||
val rates = GlassesRendering.refreshRates(s, g).map { it to "$it Hz" }
|
||||
val hint = if (g.supports30Hz && !s.gameOnGlasses) stringResource(R.string.glasses_refresh_30_hint) else null
|
||||
val label = stringResource(if (g.sbs) R.string.glasses_3d_refresh else R.string.glasses_refresh)
|
||||
if (rates.size > 1) {
|
||||
val key = if (g.sbs) "glasses-3d-hz" else "glasses-hz"
|
||||
ChoiceRow(host, key, label, hint, rates, GlassesRendering.outputHz(s, g),
|
||||
enabled = !g.switching3d, chipModifier = track(key)) {
|
||||
if (g.sbs) Glasses.setSbsHz(it) else Glasses.setRefreshHz(it)
|
||||
}
|
||||
} else SettingsRow(label, hint ?: "${rates.single().first} Hz") {}
|
||||
}
|
||||
|
||||
private fun padButtonColors(tint: Int, opacity: Int, pressed: Boolean): Pair<Color, Color> {
|
||||
@@ -231,28 +328,37 @@ internal fun GlassesCornerButton(slot: Int, tint: Int, opacity: Int, onClick: ()
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun GlassesRecenterButton(tint: Int, opacity: Int, onClick: () -> Unit) {
|
||||
internal fun GlassesRecenterButton(tint: Int, opacity: Int, onRecenterJoyCons: (() -> Unit)? = null, onClick: () -> Unit) {
|
||||
Box(
|
||||
Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.displayCutout).padding(top = 12.dp),
|
||||
contentAlignment = Alignment.TopCenter,
|
||||
) {
|
||||
Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||
GlassesRecenterAction(stringResource(R.string.glasses_recenter_button), "glasses-recenter", tint, opacity, onClick)
|
||||
if (onRecenterJoyCons != null) GlassesRecenterAction(
|
||||
stringResource(R.string.glasses_recenter_joycons_button), "glasses-recenter-joycons", tint, opacity, onRecenterJoyCons)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Composable
|
||||
private fun GlassesRecenterAction(label: String, tag: String, tint: Int, opacity: Int, onClick: () -> Unit) {
|
||||
val source = remember { MutableInteractionSource() }
|
||||
val pressed by source.collectIsPressedAsState()
|
||||
val (fill, line) = padButtonColors(tint, opacity, pressed)
|
||||
val shape = RoundedCornerShape(50)
|
||||
Box(
|
||||
Modifier.fillMaxSize().windowInsetsPadding(WindowInsets.displayCutout).padding(top = 12.dp),
|
||||
contentAlignment = Alignment.TopCenter,
|
||||
contentAlignment = Alignment.Center,
|
||||
modifier = Modifier
|
||||
.height(36.dp)
|
||||
.clip(shape)
|
||||
.background(fill)
|
||||
.border(1.5.dp, line, shape)
|
||||
.clickable(interactionSource = source, indication = null, role = Role.Button, onClick = onClick)
|
||||
.padding(horizontal = 18.dp)
|
||||
.testTag(tag),
|
||||
) {
|
||||
Box(
|
||||
contentAlignment = Alignment.Center,
|
||||
modifier = Modifier
|
||||
.height(36.dp)
|
||||
.clip(shape)
|
||||
.background(fill)
|
||||
.border(1.5.dp, line, shape)
|
||||
.clickable(interactionSource = source, indication = null, role = Role.Button, onClick = onClick)
|
||||
.padding(horizontal = 18.dp)
|
||||
.testTag("glasses-recenter"),
|
||||
) {
|
||||
Text(stringResource(R.string.glasses_recenter_button), color = line, fontSize = 14.sp)
|
||||
}
|
||||
Text(label, color = line, fontSize = 14.sp)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package com.droiddeck.launcher.ui
|
||||
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.platform.testTag
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.glasses.GlassesMenuMirror
|
||||
|
||||
@Composable
|
||||
internal fun GlassesSetupPanel() {
|
||||
val (glasses, settings) = rememberGlasses()
|
||||
val host = rememberMenuHost()
|
||||
val tabs = glassesTabs(glasses)
|
||||
var selectedTab by rememberSaveable { mutableStateOf(GlassesTab.DISPLAY) }
|
||||
val tab = selectedTab.takeIf { selected -> tabs.any { it.first == selected } } ?: GlassesTab.DISPLAY
|
||||
LaunchedEffect(tab, glasses.present, glasses.canControl, glasses.model) { host.open = null }
|
||||
|
||||
Column(Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(12.dp)) {
|
||||
Note(stringResource(R.string.setup_glasses_scope))
|
||||
if (!glasses.present && !glasses.hasDisplay && !glasses.canControl) {
|
||||
Note(stringResource(R.string.setup_glasses_connect))
|
||||
} else {
|
||||
SegmentedTabs(tabs, tab, modifier = Modifier.fillMaxWidth(),
|
||||
tabModifier = { Modifier.testTag("setup-glasses-tab-$it") }) { selectedTab = it }
|
||||
GlassesGroups(host, glasses, settings, onRecenter = GlassesMenuMirror::recenter, inSession = false, tab = tab)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,11 +79,12 @@ import java.util.Locale
|
||||
internal fun HandCameraHost(check: Boolean?, onAskPermission: () -> Unit, onPlay: () -> Unit, onClose: () -> Unit) {
|
||||
if (!HandTracking.supported) return
|
||||
val context = LocalContext.current
|
||||
val (glasses, _) = rememberGlasses()
|
||||
val (glasses, tracking) = rememberGlasses()
|
||||
val settings by HandTracking.settings.collectAsState()
|
||||
LaunchedEffect(Unit) { HandTracking.load(context) }
|
||||
var prompted by rememberSaveable { mutableStateOf(false) }
|
||||
if (!prompted && HandTracking.shouldAsk(settings, glasses.present && glasses.canControl)) {
|
||||
if (!prompted && !glasses.usesSpatialControllers(tracking) &&
|
||||
HandTracking.shouldAsk(settings, glasses.present && glasses.canControl)) {
|
||||
HandCameraPrompt(settings.sides,
|
||||
onAllow = { side ->
|
||||
prompted = true
|
||||
@@ -168,7 +169,7 @@ internal fun HandCameraRows(host: MenuHost, track: (String) -> Modifier, inSessi
|
||||
HandTracking.refreshPermission(context)
|
||||
denied = !granted
|
||||
}
|
||||
ToggleRow(host, "hand-camera", stringResource(R.string.hand_camera_setting), stringResource(R.string.hand_camera_setting_hint),
|
||||
ToggleRow(host, "hand-camera", stringResource(R.string.phone_camera_tracking_setting), stringResource(R.string.hand_camera_setting_hint),
|
||||
s.enabled, chipModifier = track("hand-camera")) { on ->
|
||||
HandTracking.setEnabled(context, on)
|
||||
if (on && !s.permitted) ask.launch(Manifest.permission.CAMERA)
|
||||
|
||||
@@ -152,13 +152,21 @@ class ModeSettingsActions(
|
||||
|
||||
/** "2610 r3" from droiddeck-steam-x64's FEX_TAG line (fex-2610-r3), the build the x86-64 client downloads. */
|
||||
internal fun fexVersionOf(script: String): String? =
|
||||
Regex("""(?m)^FEX_TAG=fex-(\d+)-r(\d+)\s*$""").find(script)?.let { "${it.groupValues[1]} r${it.groupValues[2]}" }
|
||||
Regex("""(?m)^FEX_TAG=(fex-\d+-r\d+)\s*$""").find(script)?.let { fexVersionOfTag(it.groupValues[1]) }
|
||||
|
||||
/** "2610 r5" from a tag such as fex-2610-r5. */
|
||||
internal fun fexVersionOfTag(tag: String): String? =
|
||||
Regex("""fex-(\d+)-r(\d+)""").matchEntire(tag.trim())?.let { "${it.groupValues[1]} r${it.groupValues[2]}" }
|
||||
|
||||
/** The build installed in the runtime, else the one fex.json names, else the one the launcher pins. */
|
||||
@Composable
|
||||
private fun x64FexVersion(context: android.content.Context): String? = remember {
|
||||
runCatching {
|
||||
context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) }
|
||||
}.getOrNull()
|
||||
val root = com.droiddeck.launcher.runtime.LinuxRuntime.rootDir(context)
|
||||
runCatching { java.io.File(root, "usr/local/share/fex-emu/.droiddeck-tag").readText() }.getOrNull()?.let(::fexVersionOfTag)
|
||||
?: com.droiddeck.launcher.session.FexCatalog.cached(context)?.tag?.let(::fexVersionOfTag)
|
||||
?: runCatching {
|
||||
context.assets.open("linuxfs/usr/local/bin/droiddeck-steam-x64").bufferedReader().use { fexVersionOf(it.readText()) }
|
||||
}.getOrNull()
|
||||
}
|
||||
|
||||
@Composable
|
||||
|
||||
@@ -306,6 +306,10 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
val colors = MaterialTheme.colorScheme
|
||||
val pal = LocalPalette.current
|
||||
val host = rememberMenuHost()
|
||||
val (glasses, glassesSettings) = rememberGlasses()
|
||||
val glassesPages = glassesTabs(glasses)
|
||||
var selectedGlassesTab by rememberSaveable { mutableStateOf(GlassesTab.DISPLAY) }
|
||||
val glassesTab = selectedGlassesTab.takeIf { key -> glassesPages.any { it.first == key } } ?: GlassesTab.DISPLAY
|
||||
var androidAppsExpanded by rememberSaveable { mutableStateOf(false) }
|
||||
var advancedEffectsExpanded by rememberSaveable { mutableStateOf(false) }
|
||||
var appToChooseDisplay by remember { mutableStateOf<HomeApp.LaunchableApp?>(null) }
|
||||
@@ -313,6 +317,10 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
var sheetCoords by remember { mutableStateOf<LayoutCoordinates?>(null) }
|
||||
var stopCoords by remember { mutableStateOf<LayoutCoordinates?>(null) }
|
||||
val pageScroll = remember { SessionDrawerPage.entries.associateWith { ScrollState(0) } }
|
||||
LaunchedEffect(glassesTab) {
|
||||
if (page == SessionDrawerPage.GLASSES) host.open = null
|
||||
pageScroll.getValue(SessionDrawerPage.GLASSES).scrollTo(0)
|
||||
}
|
||||
val veil by animateFloatAsState(if (open) 1f else 0f, Motion.tw(260), label = "veil")
|
||||
val focus = remember { DrawerFocus() }
|
||||
val inputModeManager = LocalInputModeManager.current
|
||||
@@ -339,7 +347,7 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
if (open && !controllerActive) focusManager.clearFocus(force = true)
|
||||
}
|
||||
var focusedPage by remember { mutableStateOf(page) }
|
||||
LaunchedEffect(open, page, controllerActive, host.open, appToChooseDisplay, confirmStop) {
|
||||
LaunchedEffect(open, page, glassesTab, controllerActive, host.open, appToChooseDisplay, confirmStop) {
|
||||
if (!open) {
|
||||
host.open = null
|
||||
confirmStop = false
|
||||
@@ -348,7 +356,11 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
if (host.open != null || appToChooseDisplay != null || confirmStop) return@LaunchedEffect
|
||||
val turned = page != focusedPage
|
||||
focusedPage = page
|
||||
val first = if (page == SessionDrawerPage.SESSION && a.isHomeApp) "apps" else page.firstControl
|
||||
val first = when {
|
||||
page == SessionDrawerPage.GLASSES -> "glasses-tab-$glassesTab"
|
||||
page == SessionDrawerPage.SESSION && a.isHomeApp -> "apps"
|
||||
else -> page.firstControl
|
||||
}
|
||||
if (turned) focus.forget(page)
|
||||
val target = focus.target(page) ?: first
|
||||
repeat(24) {
|
||||
@@ -437,6 +449,10 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
host.open = null; onPageChange(pages.step(page, 1))
|
||||
}
|
||||
}
|
||||
if (page == SessionDrawerPage.GLASSES) SegmentedTabs(
|
||||
glassesPages, glassesTab, modifier = Modifier.fillMaxWidth().padding(top = 8.dp),
|
||||
tabModifier = { key -> focus.track(page, "glasses-tab-$key").testTag("glasses-tab-$key") },
|
||||
) { selectedGlassesTab = it }
|
||||
Column(
|
||||
modifier = Modifier.weight(1f).fillMaxWidth().verticalScroll(pageScroll.getValue(page)),
|
||||
) { CompositionLocalProvider(LocalChipMinWidth provides 120.dp) {
|
||||
@@ -538,9 +554,8 @@ internal fun SessionDrawer(open: Boolean, requestedPage: SessionDrawerPage, cont
|
||||
SecondScreenGroup(host, a) { key -> focus.track(page, key) }
|
||||
}
|
||||
SessionDrawerPage.GLASSES -> {
|
||||
val (g, s) = rememberGlasses()
|
||||
GlassesGroups(host, g, s, { key -> focus.track(page, key) }, onRecenter = a.onGlassesRecenter,
|
||||
outputActive = a.glassesOutput, steamVrRunning = a.steamVr == SteamVr.Status.RUNNING,
|
||||
GlassesGroups(host, glasses, glassesSettings, { key -> focus.track(page, key) }, onRecenter = a.onGlassesRecenter,
|
||||
tab = glassesTab, outputActive = a.glassesOutput, steamVrRunning = a.steamVr == SteamVr.Status.RUNNING,
|
||||
steamVr = { SteamVrRow(a) })
|
||||
}
|
||||
SessionDrawerPage.GAMES -> {
|
||||
|
||||
@@ -761,7 +761,8 @@ fun SettingsPage(
|
||||
}
|
||||
|
||||
@Composable
|
||||
fun <T> SegmentedTabs(options: List<Pair<T, String>>, selected: T?, modifier: Modifier = Modifier, onPick: (T) -> Unit) {
|
||||
fun <T> SegmentedTabs(options: List<Pair<T, String>>, selected: T?, modifier: Modifier = Modifier,
|
||||
tabModifier: (T) -> Modifier = { Modifier }, onPick: (T) -> Unit) {
|
||||
val colors = MaterialTheme.colorScheme
|
||||
val pal = LocalPalette.current
|
||||
val index = options.indexOfFirst { it.first == selected }
|
||||
@@ -798,7 +799,7 @@ fun <T> SegmentedTabs(options: List<Pair<T, String>>, selected: T?, modifier: Mo
|
||||
val pick = { onPick(value) }
|
||||
Box(
|
||||
contentAlignment = Alignment.Center,
|
||||
modifier = Modifier.width(segment).fillMaxHeight()
|
||||
modifier = tabModifier(value).width(segment).fillMaxHeight()
|
||||
.clip(pillShape)
|
||||
.glideBorder(hot, pillShape, if (chosen) pal.onSignal else pal.signal)
|
||||
.semantics { role = Role.Tab; this.selected = chosen }
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
package com.droiddeck.launcher.ui
|
||||
|
||||
import android.os.Bundle
|
||||
import android.os.CancellationSignal
|
||||
import android.os.ParcelFileDescriptor
|
||||
import android.print.*
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.pdf.PdfDocument
|
||||
import android.graphics.Rect
|
||||
import android.graphics.RectF
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.text.Layout
|
||||
import android.text.StaticLayout
|
||||
import android.text.TextPaint
|
||||
import androidx.activity.ComponentActivity
|
||||
import androidx.activity.compose.setContent
|
||||
import androidx.compose.foundation.Image
|
||||
import androidx.compose.foundation.layout.*
|
||||
import androidx.compose.material3.*
|
||||
import androidx.compose.runtime.*
|
||||
import androidx.compose.runtime.saveable.rememberSaveable
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.graphics.asImageBitmap
|
||||
import androidx.compose.ui.platform.testTag
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.text.style.TextOverflow
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.core.AppUiPrefs
|
||||
import com.droiddeck.launcher.glasses.CalibrationDistance
|
||||
import com.droiddeck.launcher.glasses.Glasses
|
||||
import com.droiddeck.launcher.glasses.SpatialCalibrationProgress
|
||||
import com.droiddeck.launcher.glasses.SpatialTracking
|
||||
import com.droiddeck.launcher.glasses.SpatialTrackingStatus
|
||||
import com.droiddeck.launcher.glasses.StereoCalibrator
|
||||
import kotlinx.coroutines.delay
|
||||
import org.opencv.android.OpenCVLoader
|
||||
import org.opencv.android.Utils
|
||||
import org.opencv.core.Mat
|
||||
import org.opencv.core.Size
|
||||
|
||||
class SpatialTrackingActivity : ComponentActivity() {
|
||||
override fun attachBaseContext(newBase: android.content.Context) {
|
||||
super.attachBaseContext(newBase)
|
||||
com.droiddeck.launcher.core.AppLanguage.applyTo(this, newBase)
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
if (!Glasses.state.value.spatialConnected) { finish(); return }
|
||||
requestedOrientation = com.droiddeck.launcher.session.SessionPrefs.orientation(this)
|
||||
setContent {
|
||||
DroidDeckTheme(appScale = AppUiPrefs.scale(this)) {
|
||||
val glasses by Glasses.state.collectAsState()
|
||||
if (!glasses.spatialConnected) {
|
||||
LaunchedEffect(Unit) { SpatialTracking.diagnostics(false); finish() }
|
||||
return@DroidDeckTheme
|
||||
}
|
||||
var preview by remember { mutableStateOf<Bitmap?>(null) }
|
||||
var message by remember { mutableStateOf("") }
|
||||
var progress by remember { mutableStateOf(SpatialCalibrationProgress()) }
|
||||
var gpu by remember { mutableStateOf(SpatialTracking.gpuEnabled()) }
|
||||
var lite by remember { mutableStateOf(SpatialTracking.handLiteEnabled()) }
|
||||
var halfRate by remember { mutableStateOf(SpatialTracking.handHalfRateEnabled()) }
|
||||
var adaptive by remember { mutableStateOf(SpatialTracking.handAdaptiveInputEnabled()) }
|
||||
val status by SpatialTracking.status.collectAsState()
|
||||
LaunchedEffect(Unit) {
|
||||
while(true) {
|
||||
SpatialTracking.refreshStatus()
|
||||
preview = SpatialTracking.diagnosticFrame()
|
||||
message = SpatialTracking.diagnosticMessage()
|
||||
progress = SpatialTracking.calibrationProgress()
|
||||
gpu = SpatialTracking.gpuEnabled()
|
||||
lite = SpatialTracking.handLiteEnabled()
|
||||
halfRate = SpatialTracking.handHalfRateEnabled()
|
||||
adaptive = SpatialTracking.handAdaptiveInputEnabled()
|
||||
delay(200)
|
||||
}
|
||||
}
|
||||
SpatialSetupScreen(status, progress, preview, message, SpatialSetupActions(
|
||||
recenter = { SpatialTracking.recenter(false) }, restart = Glasses::restartSpatialTracking,
|
||||
print = ::printBoard, start = SpatialTracking::calibrationStart,
|
||||
capture = SpatialTracking::calibrationCapture, solve = SpatialTracking::calibrationSolve,
|
||||
validate = SpatialTracking::calibrationValidate, gpu = SpatialTracking::setGpu,
|
||||
lite = SpatialTracking::setHandModel, halfRate = SpatialTracking::setHandHalfRate,
|
||||
adaptive = SpatialTracking::setHandAdaptiveInput,
|
||||
saveFrame = SpatialTracking::captureDiagnostic, done = { finish() }), gpu, lite, halfRate, adaptive)
|
||||
}
|
||||
}
|
||||
}
|
||||
override fun onResume() {
|
||||
super.onResume()
|
||||
if (Glasses.state.value.spatialConnected) SpatialTracking.diagnostics(true) else finish()
|
||||
}
|
||||
override fun onPause() { SpatialTracking.diagnostics(false);super.onPause() }
|
||||
private fun printBoard() {
|
||||
if(!OpenCVLoader.initLocal()) return
|
||||
val c=StereoCalibrator(this);val mat=Mat()
|
||||
c.board.generateImage(Size(1500.0,2100.0),mat,0,1)
|
||||
val bitmap=Bitmap.createBitmap(1500,2100,Bitmap.Config.ARGB_8888)
|
||||
Utils.matToBitmap(mat,bitmap);mat.release();c.close()
|
||||
val manager=getSystemService(PRINT_SERVICE) as PrintManager
|
||||
manager.print(getString(R.string.spatial_print_job),object:PrintDocumentAdapter() {
|
||||
override fun onLayout(old: PrintAttributes?, new: PrintAttributes, signal: CancellationSignal, callback: LayoutResultCallback, extras: Bundle?) {
|
||||
callback.onLayoutFinished(PrintDocumentInfo.Builder("droiddeck-charuco.pdf").setPageCount(1).setContentType(PrintDocumentInfo.CONTENT_TYPE_DOCUMENT).build(),true)
|
||||
}
|
||||
override fun onWrite(pages: Array<out PageRange>, destination: ParcelFileDescriptor, signal: CancellationSignal, callback: WriteResultCallback) {
|
||||
if(signal.isCanceled) { callback.onWriteCancelled();return }
|
||||
try {
|
||||
val pdf=PdfDocument()
|
||||
try {
|
||||
val page=pdf.startPage(PdfDocument.PageInfo.Builder(595,842,1).create())
|
||||
val scale=72f/25.4f
|
||||
page.canvas.drawBitmap(bitmap,Rect(0,0,1500,2100),RectF(30*scale,35*scale,180*scale,245*scale),null)
|
||||
val ink = Paint(Paint.ANTI_ALIAS_FLAG).apply { color = Color.BLACK; textSize = 11f; textAlign = Paint.Align.CENTER }
|
||||
printText(page.canvas, getString(R.string.spatial_print_title), 12*scale, ink)
|
||||
ink.textSize = 9f
|
||||
printText(page.canvas, getString(R.string.spatial_print_scale), 21*scale, ink)
|
||||
printText(page.canvas, getString(R.string.spatial_print_stationary), 254*scale, ink)
|
||||
page.canvas.drawLine(55*scale, 270*scale, 155*scale, 270*scale, ink)
|
||||
page.canvas.drawLine(55*scale, 268*scale, 55*scale, 272*scale, ink)
|
||||
page.canvas.drawLine(155*scale, 268*scale, 155*scale, 272*scale, ink)
|
||||
printText(page.canvas, getString(R.string.spatial_print_check), 276*scale, ink)
|
||||
pdf.finishPage(page)
|
||||
java.io.FileOutputStream(destination.fileDescriptor).use { pdf.writeTo(it) }
|
||||
} finally { pdf.close() }
|
||||
callback.onWriteFinished(arrayOf(PageRange.ALL_PAGES))
|
||||
} catch(e: Exception) { callback.onWriteFailed(e.message) }
|
||||
}
|
||||
override fun onFinish() { bitmap.recycle() }
|
||||
},PrintAttributes.Builder().setMediaSize(PrintAttributes.MediaSize.ISO_A4).setColorMode(PrintAttributes.COLOR_MODE_MONOCHROME)
|
||||
.setMinMargins(PrintAttributes.Margins.NO_MARGINS).build())
|
||||
}
|
||||
|
||||
private fun printText(canvas: android.graphics.Canvas, text: String, y: Float, ink: Paint) {
|
||||
val paint = TextPaint(ink).apply { textAlign = Paint.Align.LEFT }
|
||||
val layout = StaticLayout.Builder.obtain(text, 0, text.length, paint, 535)
|
||||
.setAlignment(Layout.Alignment.ALIGN_CENTER).setIncludePad(false).build()
|
||||
val saved = canvas.save()
|
||||
canvas.translate(30f, y)
|
||||
layout.draw(canvas)
|
||||
canvas.restoreToCount(saved)
|
||||
}
|
||||
}
|
||||
|
||||
internal data class SpatialSetupActions(
|
||||
val recenter: () -> Unit = {}, val restart: () -> Unit = {}, val print: () -> Unit = {},
|
||||
val start: () -> Unit = {}, val capture: () -> Unit = {}, val solve: () -> Unit = {},
|
||||
val validate: (Float) -> Unit = {}, val gpu: (Boolean) -> Unit = {},
|
||||
val lite: (Boolean) -> Unit = {}, val halfRate: (Boolean) -> Unit = {},
|
||||
val adaptive: (Boolean) -> Unit = {},
|
||||
val saveFrame: () -> Unit = {}, val done: () -> Unit = {},
|
||||
)
|
||||
|
||||
@Composable
|
||||
internal fun SpatialSetupScreen(
|
||||
status: SpatialTrackingStatus, progress: SpatialCalibrationProgress, preview: Bitmap?, message: String,
|
||||
actions: SpatialSetupActions, gpu: Boolean = false, lite: Boolean = false, halfRate: Boolean = true,
|
||||
adaptive: Boolean = true,
|
||||
) {
|
||||
val host = rememberMenuHost()
|
||||
var distance by rememberSaveable { mutableStateOf(CalibrationDistance.INCHES_12) }
|
||||
var diagnostics by rememberSaveable { mutableStateOf(false) }
|
||||
var confirmRestart by remember { mutableStateOf(false) }
|
||||
val ready = status.trackingValid && status.synchronized && !progress.solving && !progress.validating
|
||||
Surface(Modifier.fillMaxSize().systemBarsPadding(), color = MaterialTheme.colorScheme.background) {
|
||||
BoxWithConstraints(Modifier.fillMaxSize()) {
|
||||
CompositionLocalProvider(LocalNarrowPane provides (maxWidth < NarrowPaneWidth)) {
|
||||
SettingsPage(host, null, actions.done, compactLayout = true,
|
||||
modifier = Modifier.testTag("spatial-setup-page"), header = {
|
||||
Text(stringResource(R.string.spatial_title), style = MaterialTheme.typography.titleLarge,
|
||||
maxLines = 1, overflow = TextOverflow.Ellipsis)
|
||||
}) {
|
||||
SettingsGroup(stringResource(R.string.glasses_tracking)) {
|
||||
SettingsRow(stringResource(R.string.glasses_head_tracking), status.message.ifEmpty { stringResource(R.string.spatial_disabled) }, stacked = true) {
|
||||
SecondaryButton(if (status.recenterPending) stringResource(R.string.spatial_wait_tracking) else stringResource(R.string.cardboard_recenter_button),
|
||||
enabled = !status.recenterPending, onClick = actions.recenter)
|
||||
}
|
||||
preview?.let {
|
||||
Image(it.asImageBitmap(), stringResource(R.string.spatial_cameras_description),
|
||||
Modifier.fillMaxWidth().padding(12.dp).aspectRatio(it.width.toFloat() / it.height))
|
||||
}
|
||||
SettingsRow(stringResource(R.string.spatial_camera_sync), if (status.synchronized) stringResource(R.string.spatial_ready) else
|
||||
stringResource(R.string.spatial_sync_hint)) {}
|
||||
}
|
||||
SettingsGroup(stringResource(R.string.spatial_prepare)) {
|
||||
ActionRow(stringResource(R.string.spatial_print_actual), stringResource(R.string.spatial_print_hint),
|
||||
stringResource(R.string.spatial_print), actions.print)
|
||||
}
|
||||
SettingsGroup(stringResource(R.string.spatial_capture_group)) {
|
||||
SettingsRow(stringResource(R.string.spatial_board_stationary), stringResource(R.string.spatial_capture_hint), stacked = true) {
|
||||
SecondaryButton(stringResource(R.string.spatial_start), enabled = ready, onClick = actions.start)
|
||||
}
|
||||
SettingsRow(stringResource(R.string.spatial_views_count, progress.views), when {
|
||||
!status.trackingValid -> stringResource(R.string.spatial_capture_paused, status.message)
|
||||
!status.synchronized -> stringResource(R.string.spatial_wait_sync)
|
||||
!progress.collecting -> progress.captureMessage ?: stringResource(R.string.spatial_start_hint)
|
||||
else -> progress.captureMessage ?: stringResource(R.string.spatial_views_hint)
|
||||
}, stacked = true) {
|
||||
SecondaryButton(if (progress.capturing) stringResource(R.string.spatial_capturing) else stringResource(R.string.spatial_capture),
|
||||
enabled = ready && progress.collecting && !progress.capturing, onClick = actions.capture)
|
||||
}
|
||||
SettingsRow(stringResource(R.string.spatial_alignment), message, stacked = true) {
|
||||
SecondaryButton(if (progress.solving) stringResource(R.string.spatial_calculating) else stringResource(R.string.spatial_calculate),
|
||||
enabled = !progress.solving && !progress.validating && progress.views >= 24, onClick = actions.solve)
|
||||
}
|
||||
}
|
||||
SettingsGroup(stringResource(R.string.spatial_distance_group)) {
|
||||
ChoiceRow(host, "calibration-distance", stringResource(R.string.spatial_distance), stringResource(R.string.spatial_distance_hint),
|
||||
CalibrationDistance.entries.map { it to stringResource(it.labelRes) }, distance, onPick = { distance = it })
|
||||
SettingsRow(stringResource(R.string.spatial_camera_board_distance), progress.validationMessage ?:
|
||||
stringResource(R.string.spatial_measure_hint, stringResource(distance.labelRes)), stacked = true) {
|
||||
SecondaryButton(if(progress.validating) stringResource(R.string.spatial_checking_distance) else stringResource(R.string.spatial_validate),
|
||||
enabled = ready && progress.fitted, onClick = { actions.validate(distance.meters) })
|
||||
}
|
||||
HandCalibrationStatus(status)
|
||||
SettingsRow(stringResource(R.string.glasses_hands_group), status.handStatus.ifEmpty { stringResource(R.string.spatial_hands_off) }) {}
|
||||
}
|
||||
SettingsGroup(stringResource(R.string.spatial_diagnostics)) {
|
||||
ToggleRow(host, "spatial-diagnostics", stringResource(R.string.spatial_show_diagnostics), stringResource(R.string.spatial_diagnostics_hint), diagnostics) { diagnostics = it }
|
||||
if (diagnostics) {
|
||||
SettingsRow(stringResource(R.string.spatial_camera_timing), stringResource(R.string.spatial_timing_values, status.cameraHz, status.frameAgeMs, status.processingMs)) {}
|
||||
SettingsRow(stringResource(R.string.spatial_hand_input), if (status.cameraHz > 0)
|
||||
stringResource(R.string.spatial_input_values, status.cameraHz*status.cameraCopyPermille/1000f, status.cameraCopyPermille/10f)
|
||||
else stringResource(R.string.spatial_wait_timing)) {}
|
||||
SettingsRow(stringResource(R.string.spatial_frames), stringResource(R.string.spatial_frame_counts, status.frames, status.dropped, status.rateLimitedFrames)) {}
|
||||
ToggleRow(host, "spatial-gpu", stringResource(R.string.spatial_gpu), stringResource(R.string.spatial_gpu_hint), gpu, onChange = actions.gpu)
|
||||
ToggleRow(host, "spatial-lite", stringResource(R.string.spatial_lite), stringResource(R.string.spatial_lite_hint), lite, onChange = actions.lite)
|
||||
ToggleRow(host, "spatial-adaptive-rate", stringResource(R.string.spatial_adaptive), stringResource(R.string.spatial_adaptive_hint), adaptive, onChange = actions.adaptive)
|
||||
if (!adaptive) ToggleRow(host, "spatial-half-rate", stringResource(R.string.spatial_half_rate), stringResource(R.string.spatial_half_rate_hint), halfRate, onChange = actions.halfRate)
|
||||
ActionRow(stringResource(R.string.spatial_diagnostic_capture), stringResource(R.string.spatial_diagnostic_capture_hint), stringResource(R.string.spatial_save_frame), actions.saveFrame)
|
||||
ActionRow(stringResource(R.string.spatial_restart_tracking), stringResource(R.string.spatial_restart_hint), stringResource(R.string.spatial_restart), { confirmRestart = true })
|
||||
}
|
||||
}
|
||||
Spacer(Modifier.height(16.dp))
|
||||
SecondaryButton(stringResource(R.string.setup_check_done), onClick = actions.done)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (confirmRestart) AlertDialog(
|
||||
onDismissRequest = { confirmRestart = false }, title = { Text(stringResource(R.string.spatial_restart_title)) },
|
||||
text = { Text(stringResource(R.string.spatial_restart_confirm)) },
|
||||
confirmButton = { TextButton(onClick = { confirmRestart = false; actions.restart() }) { Text(stringResource(R.string.spatial_restart)) } },
|
||||
dismissButton = { TextButton(onClick = { confirmRestart = false }) { Text(stringResource(R.string.common_cancel)) } },
|
||||
)
|
||||
}
|
||||
|
||||
@Composable
|
||||
internal fun HandCalibrationStatus(status: SpatialTrackingStatus) {
|
||||
val (label, hint) = when {
|
||||
status.calibrationSource == "luma-ultra-base" -> R.string.glasses_hands_base to R.string.glasses_hands_base_hint
|
||||
status.calibrationVerified -> R.string.glasses_hands_calibration_verified to R.string.glasses_hands_ready_hint
|
||||
else -> R.string.glasses_hands_calibration_required to R.string.glasses_hands_calibration_required_hint
|
||||
}
|
||||
SettingsRow(stringResource(label), stringResource(hint)) {}
|
||||
}
|
||||
@@ -49,6 +49,7 @@ import androidx.compose.ui.graphics.Color
|
||||
import androidx.compose.ui.graphics.graphicsLayer
|
||||
import androidx.compose.ui.unit.dp
|
||||
import com.droiddeck.launcher.frontend.Library
|
||||
import com.droiddeck.launcher.glasses.GlassesMenuMirror
|
||||
import androidx.compose.foundation.layout.width
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.key
|
||||
@@ -120,7 +121,8 @@ internal fun SteamHome(s: FrontEndState, a: FrontEndActions, modifier: Modifier)
|
||||
modifier = Modifier.fillMaxWidth().testTag("play-steamvr")) { if (enabled) { drop.open = false; a.onPlaySteamVr() } }
|
||||
},
|
||||
) { tab ->
|
||||
GlassesGroups(rememberMenuHost(), glasses, glassesSettings, inSession = false, tab = tab)
|
||||
GlassesGroups(rememberMenuHost(), glasses, glassesSettings,
|
||||
onRecenter = GlassesMenuMirror::recenter, inSession = false, tab = tab)
|
||||
}
|
||||
},
|
||||
onOpen = a.onSteamSettings, onPrimary = a.onPlay,
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package com.droiddeck.launcher.ui
|
||||
|
||||
import android.content.Context
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Paint
|
||||
import android.graphics.Typeface
|
||||
import android.text.Layout
|
||||
import android.text.StaticLayout
|
||||
import android.text.TextPaint
|
||||
import android.view.View
|
||||
import androidx.compose.ui.graphics.toArgb
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.glasses.TrackingRecovery
|
||||
import com.droiddeck.launcher.session.SessionPrefs
|
||||
|
||||
internal class TrackingRecoveryView(context: Context) : View(context) {
|
||||
private var recovery: TrackingRecovery? = null
|
||||
private var stereo = false
|
||||
private var palette = Themes.byId(SessionPrefs.theme(context))
|
||||
private val paint = Paint(Paint.ANTI_ALIAS_FLAG)
|
||||
private var blocks = emptyList<StaticLayout>()
|
||||
|
||||
init {
|
||||
visibility = GONE
|
||||
isClickable = true
|
||||
importantForAccessibility = IMPORTANT_FOR_ACCESSIBILITY_YES
|
||||
}
|
||||
|
||||
fun update(state: TrackingRecovery?, stereo: Boolean, palette: Palette = Themes.byId(SessionPrefs.theme(context))) {
|
||||
if (recovery == state && this.stereo == stereo && this.palette == palette) return
|
||||
recovery = state
|
||||
this.stereo = stereo
|
||||
this.palette = palette
|
||||
visibility = if (state == null) GONE else VISIBLE
|
||||
blocks = state?.let {
|
||||
val (title, body, help) = when (it) {
|
||||
TrackingRecovery.DISCONNECTED -> Triple(R.string.tracking_recovery_disconnected, R.string.tracking_recovery_connect, R.string.tracking_recovery_usb_help)
|
||||
TrackingRecovery.LOST -> Triple(R.string.tracking_recovery_lost, R.string.tracking_recovery_look, R.string.tracking_recovery_restart_help)
|
||||
TrackingRecovery.RECENTER -> Triple(R.string.tracking_recovery_recenter, R.string.tracking_recovery_recenter_steps, R.string.tracking_recovery_recenter_help)
|
||||
TrackingRecovery.STABILIZING -> Triple(R.string.tracking_recovery_stabilizing, R.string.tracking_recovery_hold, R.string.tracking_recovery_automatic)
|
||||
}
|
||||
val text = listOf(context.getString(R.string.glasses_head_tracking), context.getString(title), context.getString(body), context.getString(help))
|
||||
contentDescription = text.joinToString(". ")
|
||||
text.mapIndexed { index, value ->
|
||||
val ink = TextPaint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
textSize = floatArrayOf(16f, 38f, 24f, 19f)[index]
|
||||
color = (when (index) { 0 -> palette.primary; 3 -> palette.onSurfaceVariant; else -> palette.onBackground }).toArgb()
|
||||
typeface = if (index < 2) Typeface.create("sans-serif-medium", Typeface.NORMAL) else Typeface.create("sans-serif", Typeface.NORMAL)
|
||||
}
|
||||
StaticLayout.Builder.obtain(value, 0, value.length, ink, 660)
|
||||
.setAlignment(Layout.Alignment.ALIGN_NORMAL).setIncludePad(false).setLineSpacing(4f, 1f).build()
|
||||
}
|
||||
} ?: emptyList()
|
||||
if (state == null) contentDescription = null
|
||||
invalidate()
|
||||
}
|
||||
|
||||
override fun onDraw(canvas: Canvas) {
|
||||
super.onDraw(canvas)
|
||||
if (recovery == null) return
|
||||
canvas.drawColor(palette.background.toArgb())
|
||||
val eyes = if (stereo) 2 else 1
|
||||
val eyeWidth = width.toFloat() / eyes
|
||||
val gaps = floatArrayOf(14f, 24f, 32f, 0f)
|
||||
val cardHeight = blocks.sumOf { it.height } + gaps.sum() + 80f
|
||||
val scale = minOf(eyeWidth / 860f, height / (cardHeight + 120f))
|
||||
repeat(eyes) { eye ->
|
||||
val saved = canvas.save()
|
||||
canvas.clipRect(eye * eyeWidth, 0f, (eye + 1) * eyeWidth, height.toFloat())
|
||||
canvas.translate(eye * eyeWidth + (eyeWidth - 740f * scale) / 2f, (height - cardHeight * scale) / 2f)
|
||||
canvas.scale(scale, scale)
|
||||
paint.style = Paint.Style.FILL; paint.color = palette.surface.toArgb()
|
||||
canvas.drawRoundRect(0f, 0f, 740f, cardHeight, 24f, 24f, paint)
|
||||
paint.style = Paint.Style.STROKE; paint.strokeWidth = 1.5f; paint.color = palette.line.toArgb()
|
||||
canvas.drawRoundRect(0f, 0f, 740f, cardHeight, 24f, 24f, paint)
|
||||
canvas.translate(40f, 40f)
|
||||
blocks.forEachIndexed { index, block ->
|
||||
block.draw(canvas)
|
||||
canvas.translate(0f, block.height + gaps[index])
|
||||
}
|
||||
canvas.restoreToCount(saved)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -276,6 +276,9 @@ public final class WaylandCompositor {
|
||||
float lensInset, float chroma, boolean mono, float displayHz);
|
||||
|
||||
public static native void nativeSetCardboardPose(float[] rotation, boolean live);
|
||||
public static native void nativeSetCardboardSpatialPose(float[] rotation, float[] position, boolean live);
|
||||
public static native boolean nativeSetSpatialFile(String path);
|
||||
public static native void nativePublishSpatial(java.nio.ByteBuffer snapshot);
|
||||
|
||||
public static native void nativeSetCardboardPoseFile(String path);
|
||||
|
||||
|
||||
Reference in new issue
Block a user