diff --git a/app/src/main/java/io/homeassistant/companion/android/assist/AssistActivity.kt b/app/src/main/java/io/homeassistant/companion/android/assist/AssistActivity.kt index 9ad895dee06..85a5e2e5797 100644 --- a/app/src/main/java/io/homeassistant/companion/android/assist/AssistActivity.kt +++ b/app/src/main/java/io/homeassistant/companion/android/assist/AssistActivity.kt @@ -74,6 +74,7 @@ class AssistActivity : BaseActivity() { if (savedInstanceState == null) { viewModel.onCreate( + hasPermission = hasRecordingPermission(), serverId = if (intent.hasExtra(EXTRA_SERVER)) { intent.getIntExtra(EXTRA_SERVER, ServerManager.SERVER_ID_ACTIVE) } else { @@ -137,9 +138,7 @@ class AssistActivity : BaseActivity() { override fun onResume() { super.onResume() - viewModel.setPermissionInfo( - ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - ) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) } + viewModel.setPermissionInfo(hasRecordingPermission()) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) } } override fun onPause() { @@ -152,4 +151,7 @@ class AssistActivity : BaseActivity() { this.intent = intent viewModel.onNewIntent(intent) } + + private fun hasRecordingPermission() = + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED } diff --git a/app/src/main/java/io/homeassistant/companion/android/assist/AssistViewModel.kt b/app/src/main/java/io/homeassistant/companion/android/assist/AssistViewModel.kt index d42869a1ea7..73e76beeb36 100644 --- a/app/src/main/java/io/homeassistant/companion/android/assist/AssistViewModel.kt +++ b/app/src/main/java/io/homeassistant/companion/android/assist/AssistViewModel.kt @@ -53,22 +53,37 @@ class AssistViewModel @Inject constructor( var inputMode by mutableStateOf(null) private set - fun onCreate(serverId: Int?, pipelineId: String?, startListening: Boolean?) { + fun onCreate(hasPermission: Boolean, serverId: Int?, pipelineId: String?, startListening: Boolean?) { viewModelScope.launch { + this@AssistViewModel.hasPermission = hasPermission serverId?.let { filteredServerId = serverId selectedServerId = serverId } startListening?.let { recorderAutoStart = it } - val supported = checkSupport() if (!serverManager.isRegistered()) { inputMode = AssistInputMode.BLOCKED _conversation.clear() _conversation.add( AssistMessage(app.getString(commonR.string.not_registered), isInput = false) ) - } else if (supported == null) { // Couldn't get config + return@launch + } + + if ( + pipelineId == PIPELINE_LAST_USED && recorderAutoStart && + hasPermission && hasMicrophone && + serverManager.getServer(selectedServerId) != null && + serverManager.integrationRepository(selectedServerId).getLastUsedPipelineSttSupport() + ) { + // Start microphone recording to prevent missing voice input while doing network checks + onMicrophoneInput(proactive = true) + } + + val supported = checkSupport() + if (supported != true) stopRecording() + if (supported == null) { // Couldn't get config inputMode = AssistInputMode.BLOCKED _conversation.clear() _conversation.add( @@ -86,7 +101,7 @@ class AssistViewModel @Inject constructor( } else { setPipeline( when { - pipelineId == PIPELINE_LAST_USED -> serverManager.integrationRepository(selectedServerId).getLastUsedPipeline() + pipelineId == PIPELINE_LAST_USED -> serverManager.integrationRepository(selectedServerId).getLastUsedPipelineId() pipelineId == PIPELINE_PREFERRED -> null pipelineId?.isNotBlank() == true -> pipelineId else -> null @@ -169,7 +184,7 @@ class AssistViewModel @Inject constructor( id = it.id, name = it.name ) - serverManager.integrationRepository(selectedServerId).setLastUsedPipeline(it.id) + serverManager.integrationRepository(selectedServerId).setLastUsedPipeline(it.id, it.sttEngine != null) _conversation.clear() _conversation.add(startMessage) @@ -177,7 +192,7 @@ class AssistViewModel @Inject constructor( if (hasMicrophone && it.sttEngine != null) { if (recorderAutoStart && (hasPermission || requestSilently)) { inputMode = AssistInputMode.VOICE_INACTIVE - onMicrophoneInput() + onMicrophoneInput(proactive = null) } else { // already requested permission once and was denied inputMode = AssistInputMode.TEXT } @@ -219,31 +234,37 @@ class AssistViewModel @Inject constructor( fun onTextInput(input: String) = runAssistPipeline(input) - fun onMicrophoneInput() { + /** + * Start/stop microphone input for Assist, depending on the current state. + * @param proactive true if proactive, null if not important, false if not + */ + fun onMicrophoneInput(proactive: Boolean? = false) { if (!hasPermission) { requestPermission?.let { it() } return } - if (inputMode == AssistInputMode.VOICE_ACTIVE) { + if (inputMode == AssistInputMode.VOICE_ACTIVE && proactive == false) { stopRecording() return } val recording = try { - audioRecorder.startRecording() + recorderProactive || audioRecorder.startRecording() } catch (e: Exception) { Log.e(TAG, "Exception while starting recording", e) false } if (recording) { - setupRecorderQueue() + if (!recorderProactive) setupRecorderQueue() inputMode = AssistInputMode.VOICE_ACTIVE - runAssistPipeline(null) + if (proactive == true) _conversation.add(AssistMessage("…", isInput = true)) + if (proactive != true) runAssistPipeline(null) } else { _conversation.add(AssistMessage(app.getString(commonR.string.assist_error), isInput = false, isError = true)) } + recorderProactive = recording && proactive == true } private fun runAssistPipeline(text: String?) { @@ -269,6 +290,9 @@ class AssistViewModel @Inject constructor( _conversation.add(haMessage) message = haMessage } + if (isError && inputMode == AssistInputMode.VOICE_ACTIVE) { + stopRecording() + } } } } @@ -280,15 +304,16 @@ class AssistViewModel @Inject constructor( fun onPermissionResult(granted: Boolean) { hasPermission = granted + val proactive = currentPipeline == null if (granted) { inputMode = AssistInputMode.VOICE_INACTIVE - onMicrophoneInput() - } else if (requestSilently) { // Don't notify the user if they haven't explicitly requested + onMicrophoneInput(proactive = proactive) + } else if (requestSilently && !proactive) { // Don't notify the user if they haven't explicitly requested inputMode = AssistInputMode.TEXT - } else { + } else if (!requestSilently) { _conversation.add(AssistMessage(app.getString(commonR.string.assist_permission), isInput = false)) } - requestSilently = false + if (!proactive) requestSilently = false } fun onPause() { diff --git a/common/src/main/java/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt b/common/src/main/java/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt index a49606056f1..b0ae1d43779 100644 --- a/common/src/main/java/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt +++ b/common/src/main/java/io/homeassistant/companion/android/common/assist/AssistViewModelBase.kt @@ -43,6 +43,7 @@ abstract class AssistViewModelBase( protected var selectedServerId = ServerManager.SERVER_ID_ACTIVE + protected var recorderProactive = false private var recorderJob: Job? = null private var recorderQueue: MutableList? = null protected val hasMicrophone = app.packageManager.hasSystemFeature(PackageManager.FEATURE_MICROPHONE) @@ -99,8 +100,11 @@ abstract class AssistViewModelBase( } AssistPipelineEventType.STT_START -> { viewModelScope.launch { - recorderQueue?.forEach { item -> - sendVoiceData(item) + binaryHandlerId?.let { id -> + // Manually loop here to avoid the queue being reset too soon + recorderQueue?.forEach { data -> + serverManager.webSocketRepository(selectedServerId).sendVoiceData(id, data) + } } recorderQueue = null } @@ -156,7 +160,7 @@ abstract class AssistViewModelBase( binaryHandlerId?.let { viewModelScope.launch { // Launch to prevent blocking the output flow if the network is slow - serverManager.webSocketRepository().sendVoiceData(it, data) + serverManager.webSocketRepository(selectedServerId).sendVoiceData(it, data) } } } @@ -186,8 +190,9 @@ abstract class AssistViewModelBase( recorderQueue = null } if (getInput() == AssistInputMode.VOICE_ACTIVE) { - setInput(AssistInputMode.VOICE_INACTIVE) + setInput(if (recorderProactive) AssistInputMode.BLOCKED else AssistInputMode.VOICE_INACTIVE) } + recorderProactive = false } protected fun stopPlayback() = audioUrlPlayer.stop() diff --git a/common/src/main/java/io/homeassistant/companion/android/common/data/integration/IntegrationRepository.kt b/common/src/main/java/io/homeassistant/companion/android/common/data/integration/IntegrationRepository.kt index 87c46c59ec9..43100b1cf13 100644 --- a/common/src/main/java/io/homeassistant/companion/android/common/data/integration/IntegrationRepository.kt +++ b/common/src/main/java/io/homeassistant/companion/android/common/data/integration/IntegrationRepository.kt @@ -63,9 +63,11 @@ interface IntegrationRepository { conversationId: String? = null ): Flow? - suspend fun getLastUsedPipeline(): String? + suspend fun getLastUsedPipelineId(): String? - suspend fun setLastUsedPipeline(pipelineId: String) + suspend fun getLastUsedPipelineSttSupport(): Boolean + + suspend fun setLastUsedPipeline(pipelineId: String, supportsStt: Boolean) } @AssistedFactory diff --git a/common/src/main/java/io/homeassistant/companion/android/common/data/integration/impl/IntegrationRepositoryImpl.kt b/common/src/main/java/io/homeassistant/companion/android/common/data/integration/impl/IntegrationRepositoryImpl.kt index f67021548f0..99504db3c37 100644 --- a/common/src/main/java/io/homeassistant/companion/android/common/data/integration/impl/IntegrationRepositoryImpl.kt +++ b/common/src/main/java/io/homeassistant/companion/android/common/data/integration/impl/IntegrationRepositoryImpl.kt @@ -62,7 +62,8 @@ class IntegrationRepositoryImpl @AssistedInject constructor( private const val PREF_SESSION_EXPIRE = "session_expire" private const val PREF_TRUSTED = "trusted" private const val PREF_SEC_WARNING_NEXT = "sec_warning_last" - private const val PREF_LAST_USED_PIPELINE = "last_used_pipeline" + private const val PREF_LAST_USED_PIPELINE_ID = "last_used_pipeline" + private const val PREF_LAST_USED_PIPELINE_STT = "last_used_pipeline_stt" private const val TAG = "IntegrationRepository" private const val RATE_LIMIT_URL = BuildConfig.RATE_LIMIT_URL @@ -166,7 +167,8 @@ class IntegrationRepositoryImpl @AssistedInject constructor( localStorage.remove("${serverId}_$PREF_SESSION_EXPIRE") localStorage.remove("${serverId}_$PREF_TRUSTED") localStorage.remove("${serverId}_$PREF_SEC_WARNING_NEXT") - localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE") + localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE_ID") + localStorage.remove("${serverId}_$PREF_LAST_USED_PIPELINE_STT") // app version and push token is device-specific } @@ -552,11 +554,16 @@ class IntegrationRepositoryImpl @AssistedInject constructor( } } - override suspend fun getLastUsedPipeline(): String? = - localStorage.getString("${serverId}_$PREF_LAST_USED_PIPELINE") + override suspend fun getLastUsedPipelineId(): String? = + localStorage.getString("${serverId}_$PREF_LAST_USED_PIPELINE_ID") - override suspend fun setLastUsedPipeline(pipelineId: String) = - localStorage.putString("${serverId}_$PREF_LAST_USED_PIPELINE", pipelineId) + override suspend fun getLastUsedPipelineSttSupport(): Boolean = + localStorage.getBoolean("${serverId}_$PREF_LAST_USED_PIPELINE_STT") + + override suspend fun setLastUsedPipeline(pipelineId: String, supportsStt: Boolean) { + localStorage.putString("${serverId}_$PREF_LAST_USED_PIPELINE_ID", pipelineId) + localStorage.putBoolean("${serverId}_$PREF_LAST_USED_PIPELINE_STT", supportsStt) + } override suspend fun getEntities(): List>? { val response = webSocketRepository.getStates() diff --git a/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationActivity.kt b/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationActivity.kt index 3d884b30247..adae55d4daf 100755 --- a/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationActivity.kt +++ b/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationActivity.kt @@ -48,7 +48,7 @@ class ConversationActivity : ComponentActivity() { super.onCreate(savedInstanceState) lifecycleScope.launch { - val launchIntent = conversationViewModel.onCreate() + val launchIntent = conversationViewModel.onCreate(hasRecordingPermission()) if (launchIntent) { launchVoiceInputIntent() } @@ -64,9 +64,7 @@ class ConversationActivity : ComponentActivity() { override fun onResume() { super.onResume() - conversationViewModel.setPermissionInfo( - ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED - ) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) } + conversationViewModel.setPermissionInfo(hasRecordingPermission()) { requestPermission.launch(Manifest.permission.RECORD_AUDIO) } } override fun onPause() { @@ -88,6 +86,9 @@ class ConversationActivity : ComponentActivity() { } } + private fun hasRecordingPermission() = + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED + private fun launchVoiceInputIntent() { val searchIntent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { putExtra( diff --git a/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationViewModel.kt b/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationViewModel.kt index ee0b206851b..0012afb36ee 100755 --- a/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationViewModel.kt +++ b/wear/src/main/java/io/homeassistant/companion/android/conversation/ConversationViewModel.kt @@ -56,14 +56,25 @@ class ConversationViewModel @Inject constructor( val conversation: List = _conversation /** @return `true` if the voice input intent should be fired */ - suspend fun onCreate(): Boolean { - val supported = checkAssistSupport() + suspend fun onCreate(hasPermission: Boolean): Boolean { + this.hasPermission = hasPermission + if (!serverManager.isRegistered()) { _conversation.clear() _conversation.add( AssistMessage(app.getString(commonR.string.not_registered), isInput = false) ) - } else if (supported == null) { // Couldn't get config + return false + } + + if (hasPermission && hasMicrophone && serverManager.integrationRepository().getLastUsedPipelineSttSupport()) { + // Start microphone recording to prevent missing voice input while doing network checks + onMicrophoneInput(proactive = true) + } + + val supported = checkAssistSupport() + if (supported != true) stopRecording() + if (supported == null) { // Couldn't get config _conversation.clear() _conversation.add( AssistMessage(app.getString(commonR.string.assist_connnect), isInput = false) @@ -90,7 +101,7 @@ class ConversationViewModel @Inject constructor( return setPipeline( if (useAssistPipeline) { - serverManager.integrationRepository().getLastUsedPipeline() + serverManager.integrationRepository().getLastUsedPipelineId() } else { null } @@ -168,7 +179,7 @@ class ConversationViewModel @Inject constructor( if (pipeline != null || !useAssistPipeline) { currentPipeline = pipeline currentPipeline?.let { - serverManager.integrationRepository().setLastUsedPipeline(it.id) + serverManager.integrationRepository().setLastUsedPipeline(it.id, pipeline?.sttEngine != null) } _conversation.clear() @@ -178,7 +189,7 @@ class ConversationViewModel @Inject constructor( if (hasPermission || requestSilently) { inputMode = AssistInputMode.VOICE_INACTIVE useAssistPipelineStt = true - onMicrophoneInput() + onMicrophoneInput(proactive = null) } else { inputMode = AssistInputMode.TEXT } @@ -198,31 +209,37 @@ class ConversationViewModel @Inject constructor( fun updateSpeechResult(commonResult: String) = runAssistPipeline(commonResult) - fun onMicrophoneInput() { + /** + * Start/stop microphone input for Assist, depending on the current state. + * @param proactive true if proactive, null if not important, false if not + */ + fun onMicrophoneInput(proactive: Boolean? = false) { if (!hasPermission) { requestPermission?.let { it() } return } - if (inputMode == AssistInputMode.VOICE_ACTIVE) { + if (inputMode == AssistInputMode.VOICE_ACTIVE && proactive == false) { stopRecording() return } val recording = try { - audioRecorder.startRecording() + recorderProactive || audioRecorder.startRecording() } catch (e: Exception) { Log.e(TAG, "Exception while starting recording", e) false } if (recording) { - setupRecorderQueue() + if (!recorderProactive) setupRecorderQueue() inputMode = AssistInputMode.VOICE_ACTIVE - runAssistPipeline(null) + if (proactive == true) _conversation.add(AssistMessage("…", isInput = true)) + if (proactive != true) runAssistPipeline(null) } else { _conversation.add(AssistMessage(app.getString(commonR.string.assist_error), isInput = false, isError = true)) } + recorderProactive = recording && proactive == true } private fun runAssistPipeline(text: String?) { @@ -248,6 +265,9 @@ class ConversationViewModel @Inject constructor( _conversation.add(haMessage) message = haMessage } + if (isError && inputMode == AssistInputMode.VOICE_ACTIVE) { + stopRecording() + } } } } @@ -260,14 +280,15 @@ class ConversationViewModel @Inject constructor( fun onPermissionResult(granted: Boolean, voiceInputIntent: (() -> Unit)) { hasPermission = granted useAssistPipelineStt = currentPipeline?.sttEngine != null && granted + val proactive = currentPipeline == null if (granted) { inputMode = AssistInputMode.VOICE_INACTIVE - onMicrophoneInput() - } else if (requestSilently) { // Don't notify the user if they haven't explicitly requested + onMicrophoneInput(proactive = proactive) + } else if (requestSilently && !proactive) { // Don't notify the user if they haven't explicitly requested inputMode = AssistInputMode.TEXT voiceInputIntent() } - requestSilently = false + if (!proactive) requestSilently = false } fun onConversationScreenHidden() {