mirror of
https://github.com/Droid-Deck/DroidDeck.git
synced 2026-11-19 04:46:08 +00:00
feat(agent): hold to talk, slide up to lock, let go to send, with guided voice setup (#2796)
This commit is contained in:
1 parent
384eb78215
commit
c1cf0f3a9c
6 files changed
+471
-46
No files matched your search
@@ -4,6 +4,9 @@
|
||||
|
||||
<queries>
|
||||
<package android:name="cn.nubia.gamelauncher" />
|
||||
<intent>
|
||||
<action android:name="android.speech.RecognitionService" />
|
||||
</intent>
|
||||
<intent>
|
||||
<action android:name="android.content.pm.action.CONFIRM_INSTALL" />
|
||||
</intent>
|
||||
|
||||
@@ -235,6 +235,17 @@ object WirelessAdbFix {
|
||||
|
||||
fun hasSavedPairing(context: Context): Boolean = savedConnection(context.applicationContext) != null
|
||||
|
||||
fun savedPairingIsLocal(context: Context): Boolean = savedConnection(context.applicationContext)?.host
|
||||
?.let { host ->
|
||||
// Unlike isLocalAddress, an address that cannot be matched to this device's own counts as remote.
|
||||
runCatching {
|
||||
val address = InetAddress.getByName(host)
|
||||
address.isLoopbackAddress || NetworkInterface.getNetworkInterfaces().toList().any { network ->
|
||||
network.inetAddresses.toList().any { it.address.contentEquals(address.address) }
|
||||
}
|
||||
}.getOrDefault(false)
|
||||
} == true
|
||||
|
||||
fun clearCachedApps(context: Context) {
|
||||
shell(context, "am kill-all")
|
||||
}
|
||||
|
||||
@@ -19,9 +19,11 @@ import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.gestures.awaitEachGesture
|
||||
import androidx.compose.foundation.gestures.awaitFirstDown
|
||||
import androidx.compose.foundation.gestures.waitForUpOrCancellation
|
||||
import androidx.compose.foundation.horizontalScroll
|
||||
import androidx.compose.foundation.layout.Arrangement
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.BoxWithConstraints
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.WindowInsets
|
||||
@@ -55,6 +57,7 @@ import androidx.compose.material.icons.outlined.Build
|
||||
import androidx.compose.material.icons.outlined.Close
|
||||
import androidx.compose.material.icons.outlined.ExpandMore
|
||||
import androidx.compose.material.icons.outlined.History
|
||||
import androidx.compose.material.icons.outlined.Lock
|
||||
import androidx.compose.material.icons.outlined.Memory
|
||||
import androidx.compose.material.icons.outlined.MoreHoriz
|
||||
import androidx.compose.material.icons.outlined.Pets
|
||||
@@ -95,6 +98,7 @@ import androidx.compose.ui.input.key.onPreviewKeyEvent
|
||||
import androidx.compose.ui.input.nestedscroll.NestedScrollConnection
|
||||
import androidx.compose.ui.input.nestedscroll.NestedScrollSource
|
||||
import androidx.compose.ui.input.nestedscroll.nestedScroll
|
||||
import androidx.compose.ui.graphics.graphicsLayer
|
||||
import androidx.compose.ui.input.pointer.PointerEventPass
|
||||
import androidx.compose.ui.input.pointer.pointerInput
|
||||
import androidx.compose.ui.layout.onGloballyPositioned
|
||||
@@ -305,7 +309,7 @@ private fun AgentPageContent(
|
||||
}
|
||||
}
|
||||
androidx.compose.animation.AnimatedVisibility(!hidden, enter = expandVertically() + fadeIn(), exit = shrinkVertically() + fadeOut()) {
|
||||
AgentComposer(composer, draft, visible, inlineControls, onPanel = { panel = it }, onSent = toLatest)
|
||||
AgentComposer(composer, draft, visible && started, inlineControls, onPanel = { panel = it }, onSent = toLatest)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -704,9 +708,23 @@ private fun AgentEditor(composer: ComposerState, draftState: MutableState<String
|
||||
val pal = LocalPalette.current
|
||||
var draft by draftState
|
||||
val voice = remember { AgentVoice(context.applicationContext, { draftState.value }) { draftState.value = it } }
|
||||
val voiceAvailable = remember { AgentVoice.available(context) }
|
||||
DisposableEffect(voice) { onDispose { voice.stop() } }
|
||||
LaunchedEffect(visible) { if (!visible) voice.stop() }
|
||||
DisposableEffect(voice) { onDispose { voice.settle() } }
|
||||
var voiceSetup by remember { mutableStateOf(false) }
|
||||
LaunchedEffect(voice.serviceMicBlocked) {
|
||||
if (voice.serviceMicBlocked) {
|
||||
voice.serviceMicBlocked = false
|
||||
voiceSetup = true
|
||||
}
|
||||
}
|
||||
LaunchedEffect(visible) { if (!visible) voice.settle() }
|
||||
// Sends once dictation has written its last words; while a send already waits on them, there is nothing more to do.
|
||||
val sendNow = {
|
||||
when {
|
||||
voice.listening -> voice.finish(send)
|
||||
voice.finishing -> Unit
|
||||
else -> send()
|
||||
}
|
||||
}
|
||||
val working = AgentHost.state == AgentHost.State.WORKING
|
||||
if (draft.startsWith("/") && !draft.contains(' ')) {
|
||||
val matches = (LOCAL_COMMANDS + CLI_COMMANDS).filter { it.startsWith(draft) && it != draft }
|
||||
@@ -726,23 +744,35 @@ private fun AgentEditor(composer: ComposerState, draftState: MutableState<String
|
||||
) {
|
||||
for (path in AgentAttachments.pending.toList()) AttachmentChip(path) { AgentAttachments.pending.remove(path) }
|
||||
}
|
||||
if (voiceSetup) AgentVoiceSetup(onReady = { voiceSetup = false }, onClose = { voiceSetup = false })
|
||||
var focused by remember { mutableStateOf(false) }
|
||||
val field = remember { FocusRequester() }
|
||||
LaunchedEffect(composer.focusRequests) { if (composer.hasUnconsumedFocusRequest()) runCatching { field.requestFocus() } }
|
||||
Column(
|
||||
val attach = LocalActivityResultRegistryOwner.current != null
|
||||
BoxWithConstraints(
|
||||
Modifier.fillMaxWidth().padding(top = 8.dp).clip(Shape16).background(colors.surface)
|
||||
.border(if (focused) 2.dp else 1.dp, if (focused) pal.primary else pal.line, Shape16).padding(6.dp),
|
||||
) {
|
||||
Row(verticalAlignment = Alignment.Bottom) {
|
||||
if (LocalActivityResultRegistryOwner.current != null) AttachButton()
|
||||
if (AgentAttachments.gameFrame != null) GameFrameButton()
|
||||
Box(Modifier.weight(1f).padding(start = 8.dp, top = 10.dp, bottom = 10.dp)) {
|
||||
// A narrow editor gives the buttons a row of their own so the text keeps room to be tapped.
|
||||
val stacked = maxWidth < (if (controls != null) 560.dp else 320.dp)
|
||||
val text: @Composable (Modifier) -> Unit = { modifier -> Box(modifier.padding(start = 8.dp, top = 10.dp, bottom = 10.dp)) {
|
||||
if (draft.isEmpty()) Text(
|
||||
stringResource(if (working) R.string.agent_hint_working else R.string.agent_hint), fontSize = 16.sp,
|
||||
stringResource(
|
||||
when {
|
||||
voice.listening && !voice.locked -> R.string.agent_voice_hold
|
||||
voice.listening -> R.string.agent_voice_listening
|
||||
working -> R.string.agent_hint_working
|
||||
else -> R.string.agent_hint
|
||||
},
|
||||
),
|
||||
fontSize = 16.sp,
|
||||
color = pal.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis,
|
||||
)
|
||||
BasicTextField(
|
||||
draft, { draft = it },
|
||||
draft, {
|
||||
draft = it
|
||||
if (voice.listening) voice.stop()
|
||||
},
|
||||
Modifier.fillMaxWidth().focusRequester(field).onFocusChanged {
|
||||
focused = it.isFocused
|
||||
composer.focused = it.isFocused
|
||||
@@ -751,50 +781,132 @@ private fun AgentEditor(composer: ComposerState, draftState: MutableState<String
|
||||
textStyle = TextStyle(fontSize = 16.sp, color = colors.onSurface),
|
||||
cursorBrush = SolidColor(pal.primary),
|
||||
keyboardOptions = KeyboardOptions(imeAction = ImeAction.Send),
|
||||
keyboardActions = KeyboardActions(onSend = {
|
||||
voice.stop()
|
||||
send()
|
||||
}),
|
||||
keyboardActions = KeyboardActions(onSend = { sendNow() }),
|
||||
minLines = 1,
|
||||
maxLines = 8,
|
||||
)
|
||||
}
|
||||
if (controls != null && !focused) Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.height(42.dp)) { controls() }
|
||||
} }
|
||||
val actions: @Composable () -> Unit = {
|
||||
if (working) RoundButton(Icons.Outlined.Stop, stringResource(R.string.agent_stop), pal.error.copy(alpha = 0.22f), colors.error, "agent-stop") {
|
||||
AgentHost.stop(context)
|
||||
}
|
||||
val canSend = draft.isNotBlank() || AgentAttachments.pending.isNotEmpty()
|
||||
// An empty message has nothing to send, so the button there dictates one instead.
|
||||
if ((!canSend || voice.listening) && voiceAvailable) VoiceButton(voice) { voice.start(draft) }
|
||||
else RoundButton(
|
||||
if ((!canSend || voice.listening) && !voice.finishing) VoiceButton(voice, onStart = { voice.start(draft) }, onSend = send, onSetup = { voiceSetup = true })
|
||||
if (voice.finishing || (voice.listening && voice.locked) || (canSend && !voice.listening)) RoundButton(
|
||||
Icons.AutoMirrored.Outlined.Send, stringResource(if (working) R.string.agent_queue else R.string.agent_send),
|
||||
if (canSend) pal.primary else pal.surfaceVariant, if (canSend) pal.onPrimary else pal.onSurfaceVariant, "agent-send", enabled = canSend,
|
||||
) {
|
||||
voice.stop()
|
||||
send()
|
||||
pal.primary, pal.onPrimary, "agent-send", enabled = !voice.finishing,
|
||||
) { sendNow() }
|
||||
}
|
||||
if (stacked) Column {
|
||||
text(Modifier.fillMaxWidth())
|
||||
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.fillMaxWidth()) {
|
||||
if (attach) AttachButton()
|
||||
if (AgentAttachments.gameFrame != null) GameFrameButton()
|
||||
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.weight(1f).height(42.dp)) { if (controls != null && !focused) controls() }
|
||||
actions()
|
||||
}
|
||||
} else Row(verticalAlignment = Alignment.Bottom) {
|
||||
if (attach) AttachButton()
|
||||
if (AgentAttachments.gameFrame != null) GameFrameButton()
|
||||
text(Modifier.weight(1f))
|
||||
if (controls != null && !focused) Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.height(42.dp)) { controls() }
|
||||
actions()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Held, it listens until let go and then sends; slid up, it keeps listening hands-free; slid left, it puts the text back.
|
||||
* A tap or a controller press also listens hands-free.
|
||||
*/
|
||||
@Composable
|
||||
private fun VoiceButton(voice: AgentVoice, onStart: () -> Unit) {
|
||||
private fun VoiceButton(voice: AgentVoice, onStart: () -> Unit, onSend: () -> Unit, onSetup: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val pal = LocalPalette.current
|
||||
val ask = if (LocalActivityResultRegistryOwner.current != null) rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
if (granted) onStart() else android.widget.Toast.makeText(context, R.string.agent_voice_denied, android.widget.Toast.LENGTH_SHORT).show()
|
||||
} else null
|
||||
RoundButton(
|
||||
if (voice.listening) Icons.Outlined.MicOff else Icons.Outlined.Mic, stringResource(if (voice.listening) R.string.agent_voice_stop else R.string.agent_voice),
|
||||
if (voice.listening) pal.primary else pal.surfaceVariant, if (voice.listening) pal.onPrimary else pal.onBackground, "agent-voice",
|
||||
) {
|
||||
val start by rememberUpdatedState(onStart)
|
||||
val send by rememberUpdatedState(onSend)
|
||||
val setup by rememberUpdatedState(onSetup)
|
||||
val handsFree = {
|
||||
start()
|
||||
if (voice.listening) voice.locked = true
|
||||
}
|
||||
val toggle = {
|
||||
when {
|
||||
voice.listening -> voice.finish()
|
||||
context.checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) == android.content.pm.PackageManager.PERMISSION_GRANTED -> onStart()
|
||||
ask != null -> ask.launch(android.Manifest.permission.RECORD_AUDIO)
|
||||
else -> android.widget.Toast.makeText(context, R.string.agent_voice_denied, android.widget.Toast.LENGTH_SHORT).show()
|
||||
!voiceReady(context) -> setup()
|
||||
else -> handsFree()
|
||||
}
|
||||
}
|
||||
val density = LocalDensity.current
|
||||
val lockPx = with(density) { 56.dp.toPx() }
|
||||
val cancelPx = with(density) { 72.dp.toPx() }
|
||||
var drag by remember { mutableStateOf(Offset.Zero) }
|
||||
val holding = voice.listening && !voice.locked
|
||||
val icon = when {
|
||||
voice.locked -> Icons.Outlined.MicOff
|
||||
holding && -drag.x > cancelPx / 2 && -drag.x > -drag.y -> Icons.Outlined.Close
|
||||
holding && -drag.y > lockPx / 2 -> Icons.Outlined.Lock
|
||||
else -> Icons.Outlined.Mic
|
||||
}
|
||||
val scale = if (holding) 1.12f + 0.12f * voice.level else 1f
|
||||
Box(
|
||||
contentAlignment = Alignment.Center,
|
||||
modifier = Modifier.padding(start = 6.dp).size(42.dp).graphicsLayer {
|
||||
scaleX = scale
|
||||
scaleY = scale
|
||||
}.clip(CircleShape).background(if (voice.listening) pal.primary else pal.surfaceVariant)
|
||||
.pointerInput(voice) {
|
||||
awaitEachGesture {
|
||||
val down = awaitFirstDown(pass = PointerEventPass.Initial)
|
||||
down.consume()
|
||||
if (voice.listening || !voiceReady(context)) {
|
||||
waitForUpOrCancellation(PointerEventPass.Initial)?.let {
|
||||
it.consume()
|
||||
toggle()
|
||||
}
|
||||
return@awaitEachGesture
|
||||
}
|
||||
start()
|
||||
var resolved = !voice.listening
|
||||
try {
|
||||
while (!resolved) {
|
||||
val change = awaitPointerEvent(PointerEventPass.Initial).changes.firstOrNull { it.id == down.id } ?: break
|
||||
// Compose ends a cancelled touch stream with a lift that arrives already consumed; that is not a release.
|
||||
if (!change.pressed && change.isConsumed) break
|
||||
change.consume()
|
||||
drag = change.position - down.position
|
||||
when {
|
||||
!voice.listening -> resolved = true
|
||||
drag.y < -lockPx -> {
|
||||
voice.locked = true
|
||||
resolved = true
|
||||
}
|
||||
drag.x < -cancelPx -> {
|
||||
voice.cancel()
|
||||
resolved = true
|
||||
}
|
||||
!change.pressed -> {
|
||||
val tap = change.uptimeMillis - down.uptimeMillis < viewConfiguration.longPressTimeoutMillis / 2 &&
|
||||
drag.getDistance() < viewConfiguration.touchSlop
|
||||
if (tap) voice.locked = true else voice.finish(send)
|
||||
resolved = true
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
drag = Offset.Zero
|
||||
if (!resolved && voice.listening && !voice.locked) voice.finish()
|
||||
}
|
||||
}
|
||||
}
|
||||
.clickable(onClick = toggle).testTag("agent-voice"),
|
||||
) {
|
||||
Icon(
|
||||
icon, stringResource(if (voice.listening) R.string.agent_voice_stop else R.string.agent_voice),
|
||||
tint = if (voice.listening) pal.onPrimary else pal.onBackground, modifier = Modifier.size(21.dp),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Attaches what the running game shows, taken from its own surface so nothing drawn over it is in the picture. */
|
||||
|
||||
@@ -1,44 +1,93 @@
|
||||
package com.droiddeck.launcher.ui
|
||||
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.ServiceConnection
|
||||
import android.os.Build
|
||||
import android.os.IBinder
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognitionService
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableFloatStateOf
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.setValue
|
||||
|
||||
/** Dictation into the Agent editor: what is heard so far follows the text already typed, until the user edits the draft. */
|
||||
/**
|
||||
* Dictation into the Agent editor: what is heard so far follows the text already typed, until the user edits the draft.
|
||||
* Listening carries on across pauses until [finish], which can send once the last words are written.
|
||||
*/
|
||||
internal class AgentVoice(private val context: Context, private val draft: () -> String, private val onText: (String) -> Unit) {
|
||||
var listening by mutableStateOf(false)
|
||||
private set
|
||||
/** Whether listening goes on without the button held. */
|
||||
var locked by mutableStateOf(false)
|
||||
/** How loud the speech is, from 0 to 1. */
|
||||
var level by mutableFloatStateOf(0f)
|
||||
private set
|
||||
/** Set when the speech service could not open the microphone, which a fresh install of Speech Services by Google cannot until allowed. */
|
||||
var serviceMicBlocked by mutableStateOf(false)
|
||||
private var recognizer: SpeechRecognizer? = null
|
||||
private var typed = ""
|
||||
private var base = ""
|
||||
private var written = ""
|
||||
/** Set between [finish] and the final result, while a send may still be waiting on the last words. */
|
||||
var finishing by mutableStateOf(false)
|
||||
private set
|
||||
private var then: (() -> Unit)? = null
|
||||
private val main = Handler(Looper.getMainLooper())
|
||||
|
||||
fun start(typed: String) {
|
||||
stop()
|
||||
val speech = runCatching { SpeechRecognizer.createSpeechRecognizer(context) }.getOrNull() ?: return
|
||||
val service = component(context)
|
||||
val speech = runCatching { SpeechRecognizer.createSpeechRecognizer(context, service) }.getOrNull() ?: return
|
||||
recognizer = speech
|
||||
base = typed.trimEnd().let { if (it.isEmpty()) it else "$it " }
|
||||
if (service != null && !isDefault(context)) lend(service)
|
||||
this.typed = typed
|
||||
base = continued(typed)
|
||||
written = typed
|
||||
finishing = false
|
||||
listening = true
|
||||
speech.setRecognitionListener(object : RecognitionListener {
|
||||
override fun onReadyForSpeech(params: Bundle?) { listening = true }
|
||||
override fun onReadyForSpeech(params: Bundle?) {}
|
||||
override fun onBeginningOfSpeech() {}
|
||||
override fun onRmsChanged(rmsdB: Float) {}
|
||||
override fun onRmsChanged(rmsdB: Float) { if (recognizer === speech) level = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f) }
|
||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||
override fun onEndOfSpeech() {}
|
||||
override fun onError(error: Int) { if (recognizer === speech) stop() }
|
||||
override fun onEndOfSpeech() { level = 0f }
|
||||
override fun onError(error: Int) {
|
||||
if (recognizer !== speech) return
|
||||
// The recognizer gives up on a pause; the user is still talking until they let go or stop.
|
||||
if (!finishing && (error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT)) {
|
||||
base = continued(written)
|
||||
relisten(speech)
|
||||
} else {
|
||||
done()
|
||||
if (error == SpeechRecognizer.ERROR_AUDIO || error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) serviceMicBlocked = true
|
||||
}
|
||||
}
|
||||
override fun onResults(results: Bundle?) {
|
||||
if (recognizer !== speech) return
|
||||
heard(results)
|
||||
stop()
|
||||
if (recognizer !== speech) return
|
||||
if (finishing) done() else {
|
||||
base = continued(written)
|
||||
relisten(speech)
|
||||
}
|
||||
}
|
||||
override fun onPartialResults(partialResults: Bundle?) { if (recognizer === speech) heard(partialResults) }
|
||||
override fun onEvent(eventType: Int, params: Bundle?) {}
|
||||
})
|
||||
listen(speech)
|
||||
}
|
||||
|
||||
private fun continued(text: String) = text.trimEnd().let { if (it.isEmpty()) it else "$it " }
|
||||
|
||||
private fun listen(speech: SpeechRecognizer) {
|
||||
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH)
|
||||
.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
||||
@@ -46,22 +95,71 @@ internal class AgentVoice(private val context: Context, private val draft: () ->
|
||||
runCatching { speech.startListening(intent) }.onFailure { stop() }
|
||||
}
|
||||
|
||||
private fun relisten(speech: SpeechRecognizer) {
|
||||
main.post { if (recognizer === speech && !finishing) listen(speech) }
|
||||
}
|
||||
|
||||
private fun heard(results: Bundle?) {
|
||||
val spoken = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()?.takeIf { it.isNotBlank() } ?: return
|
||||
if (draft() != written) return stop()
|
||||
if (draft() != written) return if (finishing) done() else stop()
|
||||
written = base + spoken
|
||||
onText(written)
|
||||
}
|
||||
|
||||
/** Ends listening but keeps the recognizer until its final result arrives. */
|
||||
fun finish() {
|
||||
/** Ends listening but keeps the recognizer until its final result arrives, then runs [then]. */
|
||||
fun finish(then: (() -> Unit)? = null) {
|
||||
val speech = recognizer ?: return
|
||||
finishing = true
|
||||
this.then = then
|
||||
listening = false
|
||||
runCatching { speech.stopListening() }.onFailure { stop() }
|
||||
locked = false
|
||||
level = 0f
|
||||
runCatching { speech.stopListening() }.onFailure { done() }
|
||||
// Some recognizers never answer a stop with nothing heard.
|
||||
main.postDelayed({ if (recognizer === speech) done() }, FINAL_RESULT_TIMEOUT_MS)
|
||||
}
|
||||
|
||||
private fun done() {
|
||||
val next = then
|
||||
stop()
|
||||
next?.invoke()
|
||||
}
|
||||
|
||||
/** Stops listening, but a send already waiting on the last words goes ahead with what was heard. */
|
||||
fun settle() {
|
||||
if (finishing) done() else stop()
|
||||
}
|
||||
|
||||
/** Stops and puts back the text typed before listening began. */
|
||||
fun cancel() {
|
||||
val restore = draft() == written
|
||||
stop()
|
||||
if (restore) onText(typed)
|
||||
}
|
||||
|
||||
private var lent: ServiceConnection? = null
|
||||
|
||||
// Android passes the microphone only to the default speech service on its own; any other one records silence unless
|
||||
// the app in front binds it with its own capabilities for as long as it listens.
|
||||
private fun lend(service: ComponentName) {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) return
|
||||
val connection = object : ServiceConnection {
|
||||
override fun onServiceConnected(name: ComponentName?, binder: IBinder?) {}
|
||||
override fun onServiceDisconnected(name: ComponentName?) {}
|
||||
}
|
||||
val intent = Intent(RecognitionService.SERVICE_INTERFACE).setComponent(service)
|
||||
if (runCatching { context.bindService(intent, connection, Context.BIND_AUTO_CREATE or Context.BIND_INCLUDE_CAPABILITIES) }.getOrDefault(false)) lent = connection
|
||||
else runCatching { context.unbindService(connection) }
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
lent?.let { runCatching { context.unbindService(it) } }
|
||||
lent = null
|
||||
then = null
|
||||
finishing = false
|
||||
listening = false
|
||||
locked = false
|
||||
level = 0f
|
||||
val speech = recognizer ?: return
|
||||
recognizer = null
|
||||
runCatching { speech.cancel() }
|
||||
@@ -69,6 +167,38 @@ internal class AgentVoice(private val context: Context, private val draft: () ->
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val FINAL_RESULT_TIMEOUT_MS = 3000L
|
||||
|
||||
fun available(context: Context) = SpeechRecognizer.isRecognitionAvailable(context)
|
||||
|
||||
/** The package that turns speech into text, if one is installed. */
|
||||
fun service(context: Context): String? = component(context)?.packageName
|
||||
|
||||
const val DEFAULT_SETTING = "voice_recognition_service"
|
||||
|
||||
private fun chosen(context: Context) = android.provider.Settings.Secure.getString(context.contentResolver, DEFAULT_SETTING)
|
||||
?.takeIf { it.isNotBlank() }?.let(ComponentName::unflattenFromString)
|
||||
|
||||
private fun installed(context: Context) = context.packageManager.queryIntentServices(Intent(RecognitionService.SERVICE_INTERFACE), 0)
|
||||
.map { ComponentName(it.serviceInfo.packageName, it.serviceInfo.name) }
|
||||
|
||||
// Installing a speech service does not always make it the system's choice, and without one Android refuses to listen.
|
||||
fun component(context: Context): ComponentName? {
|
||||
val chosen = chosen(context)
|
||||
val installed = installed(context)
|
||||
return installed.firstOrNull { it == chosen } ?: installed.firstOrNull()
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the installed service is the system's chosen one. Android only passes DroidDeck's right to the microphone on to that one;
|
||||
* any other service hears silence once DroidDeck is in front.
|
||||
*/
|
||||
fun isDefault(context: Context): Boolean {
|
||||
val chosen = chosen(context) ?: return false
|
||||
return chosen in installed(context)
|
||||
}
|
||||
|
||||
fun serviceHears(context: Context, service: String) =
|
||||
context.packageManager.checkPermission(android.Manifest.permission.RECORD_AUDIO, service) == android.content.pm.PackageManager.PERMISSION_GRANTED
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package com.droiddeck.launcher.ui
|
||||
|
||||
import android.Manifest
|
||||
import android.app.Activity
|
||||
import android.content.Context
|
||||
import android.content.ContextWrapper
|
||||
import android.content.Intent
|
||||
import android.content.pm.PackageManager
|
||||
import android.net.Uri
|
||||
import android.provider.Settings
|
||||
import androidx.activity.compose.LocalActivityResultRegistryOwner
|
||||
import androidx.activity.compose.rememberLauncherForActivityResult
|
||||
import androidx.activity.result.contract.ActivityResultContracts
|
||||
import androidx.compose.foundation.background
|
||||
import androidx.compose.foundation.border
|
||||
import androidx.compose.foundation.clickable
|
||||
import androidx.compose.foundation.layout.Box
|
||||
import androidx.compose.foundation.layout.Column
|
||||
import androidx.compose.foundation.layout.Row
|
||||
import androidx.compose.foundation.layout.fillMaxWidth
|
||||
import androidx.compose.foundation.layout.padding
|
||||
import androidx.compose.foundation.layout.size
|
||||
import androidx.compose.foundation.shape.CircleShape
|
||||
import androidx.compose.material.icons.Icons
|
||||
import androidx.compose.material.icons.outlined.Close
|
||||
import androidx.compose.material3.Icon
|
||||
import androidx.compose.material3.MaterialTheme
|
||||
import androidx.compose.material3.Text
|
||||
import androidx.compose.runtime.Composable
|
||||
import androidx.compose.runtime.DisposableEffect
|
||||
import androidx.compose.runtime.LaunchedEffect
|
||||
import androidx.compose.runtime.getValue
|
||||
import androidx.compose.runtime.mutableIntStateOf
|
||||
import androidx.compose.runtime.mutableStateOf
|
||||
import androidx.compose.runtime.remember
|
||||
import androidx.compose.runtime.setValue
|
||||
import androidx.compose.ui.Alignment
|
||||
import androidx.compose.ui.Modifier
|
||||
import androidx.compose.ui.draw.clip
|
||||
import androidx.compose.ui.platform.LocalContext
|
||||
import androidx.compose.ui.platform.LocalLifecycleOwner
|
||||
import androidx.compose.ui.platform.testTag
|
||||
import androidx.compose.ui.res.stringResource
|
||||
import androidx.compose.ui.text.font.FontWeight
|
||||
import androidx.compose.ui.unit.dp
|
||||
import androidx.compose.ui.unit.sp
|
||||
import androidx.lifecycle.Lifecycle
|
||||
import androidx.lifecycle.LifecycleEventObserver
|
||||
import com.droiddeck.launcher.R
|
||||
import com.droiddeck.launcher.core.WirelessAdbFix
|
||||
|
||||
private const val SPEECH_SERVICES = "com.google.android.tts"
|
||||
|
||||
/** Whether the mic can turn speech into text: a speech service is installed, and it and DroidDeck may both use the microphone. */
|
||||
internal fun voiceReady(context: Context): Boolean {
|
||||
val service = AgentVoice.service(context) ?: return false
|
||||
return appHears(context) && AgentVoice.serviceHears(context, service)
|
||||
}
|
||||
|
||||
/** Runs [command] over the saved Wireless debugging pairing off the main thread, then [done] on it with whether it worked. */
|
||||
private fun adb(context: Context, command: String, done: (Boolean) -> Unit) {
|
||||
val app = context.applicationContext
|
||||
Thread({
|
||||
// The pairing can be with another device, where the grant would land instead of here.
|
||||
val ok = runCatching { WirelessAdbFix.savedPairingIsLocal(app) }.getOrDefault(false) && runCatching { WirelessAdbFix.shell(app, command) }.isSuccess
|
||||
android.os.Handler(android.os.Looper.getMainLooper()).post { done(ok) }
|
||||
}, "agent-voice-setup").start()
|
||||
}
|
||||
|
||||
private fun appHears(context: Context) = context.checkSelfPermission(Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
|
||||
|
||||
/** Opens [intent] over DroidDeck when it can, so Back comes straight back to the setup. */
|
||||
private fun open(context: Context, intent: Intent): Boolean {
|
||||
var base = context
|
||||
while (base !is Activity && base is ContextWrapper) base = base.baseContext
|
||||
return runCatching {
|
||||
if (base is Activity) base.startActivity(intent) else context.startActivity(intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK))
|
||||
}.isSuccess
|
||||
}
|
||||
|
||||
private fun openAppInfo(context: Context, pkg: String) {
|
||||
open(context, Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", pkg, null)))
|
||||
}
|
||||
|
||||
private fun openStore(context: Context) {
|
||||
if (!open(context, Intent(Intent.ACTION_VIEW, Uri.parse("market://details?id=$SPEECH_SERVICES")))) {
|
||||
open(context, Intent(Intent.ACTION_VIEW, Uri.parse("https://play.google.com/store/apps/details?id=$SPEECH_SERVICES")))
|
||||
}
|
||||
}
|
||||
|
||||
/** What talking to the agent still needs, each step one button away, checked again whenever DroidDeck comes back. */
|
||||
@Composable
|
||||
internal fun AgentVoiceSetup(onReady: () -> Unit, onClose: () -> Unit) {
|
||||
val context = LocalContext.current
|
||||
val colors = MaterialTheme.colorScheme
|
||||
val pal = LocalPalette.current
|
||||
var checks by remember { mutableIntStateOf(0) }
|
||||
val owner = LocalLifecycleOwner.current
|
||||
DisposableEffect(owner) {
|
||||
val observer = LifecycleEventObserver { _, event -> if (event == Lifecycle.Event.ON_RESUME) checks++ }
|
||||
owner.lifecycle.addObserver(observer)
|
||||
onDispose { owner.lifecycle.removeObserver(observer) }
|
||||
}
|
||||
val service = remember(checks) { AgentVoice.service(context) }
|
||||
val appMic = remember(checks) { appHears(context) }
|
||||
val serviceMic = remember(checks, service) { service != null && AgentVoice.serviceHears(context, service) }
|
||||
val paired = remember(checks) { WirelessAdbFix.hasSavedPairing(context) }
|
||||
var working by remember { mutableStateOf(false) }
|
||||
val shown = remember { booleanArrayOf(true) }
|
||||
DisposableEffect(Unit) { onDispose { shown[0] = false } }
|
||||
// A saved pairing does not mean Wireless debugging is on now, so a refusal falls back to doing it by hand.
|
||||
val viaAdb: (String, () -> Unit) -> Unit = { command, byHand ->
|
||||
working = true
|
||||
adb(context, command) { ok ->
|
||||
working = false
|
||||
checks++
|
||||
if (!ok && shown[0]) byHand()
|
||||
}
|
||||
}
|
||||
LaunchedEffect(service, appMic, serviceMic) { if (service != null && appMic && serviceMic) onReady() }
|
||||
val ask = if (LocalActivityResultRegistryOwner.current != null) rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
|
||||
// Android stops asking after the user refuses twice; the switch is then only in App info.
|
||||
if (granted) checks++ else openAppInfo(context, context.packageName)
|
||||
} else null
|
||||
Column(
|
||||
Modifier.fillMaxWidth().padding(top = 8.dp).clip(Shape14).background(colors.surface).border(1.dp, pal.line, Shape14).testTag("agent-voice-setup"),
|
||||
) {
|
||||
Row(verticalAlignment = Alignment.CenterVertically, modifier = Modifier.padding(start = 14.dp, end = 6.dp, top = 6.dp)) {
|
||||
Text(
|
||||
stringResource(R.string.agent_voice_setup), fontSize = 14.sp, fontWeight = FontWeight.SemiBold, color = colors.onBackground,
|
||||
modifier = Modifier.weight(1f),
|
||||
)
|
||||
Box(contentAlignment = Alignment.Center, modifier = Modifier.size(36.dp).clip(CircleShape).clickable(onClick = onClose)) {
|
||||
Icon(Icons.Outlined.Close, stringResource(R.string.agent_voice_setup_close), tint = pal.onSurfaceVariant, modifier = Modifier.size(18.dp))
|
||||
}
|
||||
}
|
||||
CheckRow(
|
||||
if (service != null) CheckState.OK else CheckState.WARN, stringResource(R.string.agent_voice_setup_service),
|
||||
if (service == null) stringResource(R.string.agent_voice_setup_service_detail) else null,
|
||||
action = if (service == null) ({ SecondaryButton(stringResource(R.string.setup_install), compact = true) { openStore(context) } }) else null,
|
||||
)
|
||||
CheckRow(
|
||||
if (appMic) CheckState.OK else CheckState.WARN, stringResource(R.string.agent_voice_setup_app_mic), null,
|
||||
action = if (!appMic) ({
|
||||
SecondaryButton(stringResource(R.string.libraries_access_allow), compact = true) {
|
||||
if (ask != null) ask.launch(Manifest.permission.RECORD_AUDIO) else openAppInfo(context, context.packageName)
|
||||
}
|
||||
}) else null,
|
||||
)
|
||||
CheckRow(
|
||||
if (serviceMic) CheckState.OK else CheckState.WARN, stringResource(R.string.agent_voice_setup_service_mic),
|
||||
if (service != null && !serviceMic && !paired) stringResource(R.string.agent_voice_setup_service_mic_detail) else null, divider = false,
|
||||
action = if (service != null && !serviceMic) ({
|
||||
SecondaryButton(stringResource(R.string.libraries_access_allow), enabled = !working, compact = true) {
|
||||
if (paired) viaAdb("pm grant $service ${Manifest.permission.RECORD_AUDIO}") { openAppInfo(context, service) } else openAppInfo(context, service)
|
||||
}
|
||||
}) else null,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1238,6 +1238,15 @@
|
||||
<string name="agent_tools_add">Add</string>
|
||||
<string name="agent_voice">Dictate</string>
|
||||
<string name="agent_voice_stop">Stop dictating</string>
|
||||
<string name="agent_voice_hold">Slide up to lock · left to cancel</string>
|
||||
<string name="agent_voice_listening">Listening…</string>
|
||||
<string name="agent_voice_setup">Set up talking to the agent</string>
|
||||
<string name="agent_voice_setup_close">Close</string>
|
||||
<string name="agent_voice_setup_service">Speech service</string>
|
||||
<string name="agent_voice_setup_service_detail">Speech Services by Google turns your voice into text</string>
|
||||
<string name="agent_voice_setup_app_mic">Microphone for DroidDeck</string>
|
||||
<string name="agent_voice_setup_service_mic">Microphone for the speech service</string>
|
||||
<string name="agent_voice_setup_service_mic_detail">Permissions › Microphone › Allow</string>
|
||||
<string name="agent_voice_denied">DroidDeck needs the microphone to dictate</string>
|
||||
<string name="agent_attach_game">Attach what the game shows</string>
|
||||
<string name="agent_attach_game_failed">Could not capture the game</string>
|
||||
|
||||
Reference in new issue
Block a user