Speech Recognition v2 - StartListening/StopListening

samples/CommunityToolkit.Maui.Sample/Pages/Essentials/SpeechToTextPage.xaml

@@ -14,34 +14,28 @@
     </ContentPage.Resources>
 
     <ScrollView>
-        <Grid
-            RowDefinitions="64, 24, 64, 24, auto, 60, 60, 60"
+        <VerticalStackLayout
             Padding="30,0">
 
             <Label
-                Grid.Row="0"
                 Text="SpeechToText allows the user to convert speech to text in real time"/>
 
             <Label
-                Grid.Row="1"
                 Text="Locale"
                 FontAttributes="Bold"/>
 
             <Picker
-                Grid.Row="2"
                 ItemsSource="{Binding Locales}"
                 SelectedItem="{Binding CurrentLocale}"
                 ItemDisplayBinding="{Binding ., Converter={StaticResource PickerLocaleDisplayConverter}}"
                 Margin="0,0,0,20">
             </Picker>
 
             <Label
-                Grid.Row="3"
                 Text="Language Output"
                 FontAttributes="Bold"/>
 
             <Label
-                Grid.Row="4"
                 Text="{Binding RecognitionText}"
                 FontSize="18"
                 HorizontalOptions="Center"
@@ -50,26 +44,34 @@
                 Margin="0,0,0,20" />
 
             <Button
-                Grid.Row="5"
                 Text="Play"
                 Command="{Binding PlayCommand}"
                 HorizontalOptions="Center"
                 Margin="0,0,0,20"/>
 
             <Button
-                Grid.Row="6"
                 Text="Listen"
                 Command="{Binding ListenCommand}"
                 HorizontalOptions="Center"
                 Margin="0,0,0,20"/>
 
             <Button
-                Grid.Row="7"
                 Text="Stop Listening"
                 Command="{Binding ListenCancelCommand}"
                 HorizontalOptions="Center"/>
 
-        </Grid>
+            <Button
+                Text="Start Listening"
+                Command="{Binding StartListenCommand}"
+                HorizontalOptions="Center"
+                Margin="0,0,0,20"/>
+
+            <Button
+                Text="Stop Listening"
+                Command="{Binding StopListenCommand}"
+                HorizontalOptions="Center"/>
+
+        </VerticalStackLayout>
     </ScrollView>
 
 </pages:BasePage>

samples/CommunityToolkit.Maui.Sample/ViewModels/Essentials/SpeechToTextViewModel.cs

@@ -22,7 +22,7 @@ public partial class SpeechToTextViewModel : BaseViewModel
 
 	[ObservableProperty]
 	string? recognitionText = "Welcome to .NET MAUI Community Toolkit!";
-
+	
 	public SpeechToTextViewModel(ITextToSpeech textToSpeech, ISpeechToText speechToText)
 	{
 		this.textToSpeech = textToSpeech;
@@ -99,6 +99,48 @@ async Task Listen(CancellationToken cancellationToken)
 			RecognitionText = string.Empty;
 		}
 	}
+
+	[RelayCommand]
+	async Task StartListen(CancellationToken cancellationToken)
+	{
+		var isGranted = await speechToText.RequestPermissions(cancellationToken);
+		if (!isGranted)
+		{
+			await Toast.Make("Permission not granted").Show(CancellationToken.None);
+			return;
+		}
+
+		const string beginSpeakingPrompt = "Begin speaking...";
+
+		RecognitionText = beginSpeakingPrompt;
+
+		await speechToText.StartListeningAsync(CultureInfo.GetCultureInfo(CurrentLocale?.Language ?? defaultLanguage), cancellationToken);
+		speechToText.RecognitionResultUpdated += SpeechToTextOnRecognitionResultUpdated;
+		speechToText.RecognitionResultCompleted += SpeechToTextOnRecognitionResultCompleted;
+
+		if (RecognitionText is beginSpeakingPrompt)
+		{
+			RecognitionText = string.Empty;
+		}
+	}
+
+	[RelayCommand]
+	async Task StopListen(CancellationToken cancellationToken)
+	{
+		speechToText.RecognitionResultUpdated -= SpeechToTextOnRecognitionResultUpdated;
+		speechToText.RecognitionResultCompleted -= SpeechToTextOnRecognitionResultCompleted;
+		await speechToText.StopListeningAsync( cancellationToken);
+	}
+
+	void SpeechToTextOnRecognitionResultUpdated(object? sender, OnSpeechToTextRecognitionResultUpdated e)
+	{
+		RecognitionText += e.RecognitionResult;
+	}
+
+	void SpeechToTextOnRecognitionResultCompleted(object? sender, OnSpeechToTextRecognitionResultCompleted e)
+	{
+		RecognitionText = e.RecognitionResult;
+	}
 
 	void HandleLocalesCollectionChanged(object? sender, NotifyCollectionChangedEventArgs e)
 	{

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/ISpeechToText.shared.cs

@@ -16,10 +16,54 @@ public interface ISpeechToText : IAsyncDisposable
 	/// <returns>Final recognition result</returns>
 	Task<SpeechToTextResult> ListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken);
 
+	/// <summary>
+	/// Converts speech to text in real time.
+	/// </summary>
+	/// <param name="culture">Speak language</param>
+	/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
+	Task StartListeningAsync(CultureInfo culture, CancellationToken cancellationToken);
+
+	/// <summary>
+	/// Stop listening.
+	/// </summary>
+	/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
+	Task StopListeningAsync(CancellationToken cancellationToken);
+
+	/// <summary>
+	/// Converts speech to text in real time.
+	/// </summary>
+	event EventHandler<OnSpeechToTextRecognitionResultUpdated> RecognitionResultUpdated;
+
+	/// <summary>
+	/// Final recognition result.
+	/// </summary>
+	event EventHandler<OnSpeechToTextRecognitionResultCompleted> RecognitionResultCompleted;
+
 	/// <summary>
 	/// Request permissions for speech to text.
 	/// </summary>
 	/// <param name="cancellationToken"><see cref="CancellationToken"/></param>
 	/// <returns>True if permissions granted</returns>
 	Task<bool> RequestPermissions(CancellationToken cancellationToken);
+
+	/// <summary>
+	/// Current listening state
+	/// </summary>
+	SpeechToTextState State { get; }
+}
+
+/// <summary>
+/// Speech To Text listening state
+/// </summary>
+public enum SpeechToTextState
+{
+	/// <summary>
+	/// Listening is active
+	/// </summary>
+	Listening,
+
+	/// <summary>
+	/// Listening is stopped
+	/// </summary>
+	Stopped
 }

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/SharedSpeechToTextImplementation.macios.cs

@@ -10,6 +10,11 @@ public sealed partial class SpeechToTextImplementation
 	SFSpeechRecognitionTask? recognitionTask;
 	SFSpeechAudioBufferRecognitionRequest? liveSpeechRequest;
 
+	/// <inheritdoc/>
+	public SpeechToTextState State => recognitionTask?.State == SFSpeechRecognitionTaskState.Running
+		? SpeechToTextState.Listening
+		: SpeechToTextState.Stopped;
+
 	/// <inheritdoc />
 	public ValueTask DisposeAsync()
 	{
@@ -48,4 +53,10 @@ void StopRecording()
 		liveSpeechRequest?.EndAudio();
 		recognitionTask?.Cancel();
 	}
+
+	Task InternalStopListeningAsync(CancellationToken cancellationToken)
+	{
+		StopRecording();
+		return Task.CompletedTask;
+	}
 }

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/SpeechToTextImplementation.android.cs

@@ -12,6 +12,12 @@ public sealed partial class SpeechToTextImplementation
 {
 	SpeechRecognizer? speechRecognizer;
 	SpeechRecognitionListener? listener;
+	TaskCompletionSource<string>? speechRecognitionListenerTaskCompletionSource;
+	IProgress<string>? recognitionProgress;
+	CultureInfo? cultureInfo;
+
+	/// <inheritdoc/>
+	public SpeechToTextState State { get; private set; }
 
 	/// <inheritdoc />
 	public ValueTask DisposeAsync()
@@ -26,8 +32,9 @@ public ValueTask DisposeAsync()
 	}
 
 	[MemberNotNull(nameof(speechRecognizer), nameof(listener))]
-	async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken)
+	Task InternalStartListeningAsync(CultureInfo culture, CancellationToken cancellationToken)
 	{
+		cultureInfo = culture;
 		var isSpeechRecognitionAvailable = IsSpeechRecognitionAvailable();
 		if (!isSpeechRecognitionAvailable)
 		{
@@ -40,7 +47,6 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 			throw new FeatureNotSupportedException("Speech recognizer is not available on this device");
 		}
 
-		var speechRecognitionListenerTaskCompletionSource = new TaskCompletionSource<string>();
 
 		listener = new SpeechRecognitionListener
 		{
@@ -49,7 +55,22 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 			Results = HandleListenerResults
 		};
 		speechRecognizer.SetRecognitionListener(listener);
-		speechRecognizer.StartListening(CreateSpeechIntent(culture));
+		speechRecognizer.StartListening(CreateSpeechIntent(cultureInfo));
+		State = SpeechToTextState.Listening;
+		return Task.CompletedTask;
+	}
+
+	Task InternalStopListeningAsync(CancellationToken cancellationToken)
+	{
+		StopRecording();
+		return Task.CompletedTask;
+	}
+
+	async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken)
+	{
+		recognitionProgress = recognitionResult;
+		speechRecognitionListenerTaskCompletionSource = new();
+		await InternalStartListeningAsync(culture, CancellationToken.None);
 
 		await using (cancellationToken.Register(() =>
 		{
@@ -59,15 +80,23 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 		{
 			return await speechRecognitionListenerTaskCompletionSource.Task;
 		}
+	}
 
-		void HandleListenerError(SpeechRecognizerError error)
-			=> speechRecognitionListenerTaskCompletionSource.TrySetException(new Exception("Failure in speech engine - " + error));
+	void HandleListenerError(SpeechRecognizerError error)
+	{
+		speechRecognitionListenerTaskCompletionSource?.TrySetException(new Exception("Failure in speech engine - " + error));
+	}
 
-		void HandleListenerPartialResults(string sentence)
-			=> recognitionResult?.Report(sentence);
+	void HandleListenerPartialResults(string sentence)
+	{
+		OnRecognitionResultUpdated(sentence);
+		recognitionProgress?.Report(sentence);
+	}
 
-		void HandleListenerResults(string result)
-			=> speechRecognitionListenerTaskCompletionSource.TrySetResult(result);
+	void HandleListenerResults(string result)
+	{
+		OnRecognitionResultCompleted(result);
+		speechRecognitionListenerTaskCompletionSource?.TrySetResult(result);
 	}
 
 	static Intent CreateSpeechIntent(CultureInfo culture)
@@ -90,6 +119,7 @@ void StopRecording()
 	{
 		speechRecognizer?.StopListening();
 		speechRecognizer?.Destroy();
+		State = SpeechToTextState.Stopped;
 	}
 
 	class SpeechRecognitionListener : Java.Lang.Object, IRecognitionListener

src/CommunityToolkit.Maui.Core/Essentials/SpeechToText/SpeechToTextImplementation.ios.cs

@@ -1,17 +1,20 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
 using AVFoundation;
-using Microsoft.Maui.ApplicationModel;
 using Speech;
 
 namespace CommunityToolkit.Maui.Media;
 
 /// <inheritdoc />
 public sealed partial class SpeechToTextImplementation
 {
+	IProgress<string>? recognitionProgress;
+	TaskCompletionSource<string>? getRecognitionTaskCompletionSource;
+
 	[MemberNotNull(nameof(audioEngine), nameof(recognitionTask), nameof(liveSpeechRequest))]
-	async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken)
+	Task InternalStartListeningAsync(CultureInfo culture, CancellationToken cancellationToken)
 	{
+		getRecognitionTaskCompletionSource = new TaskCompletionSource<string>();
 		speechRecognizer = new SFSpeechRecognizer(NSLocale.FromLocaleIdentifier(culture.Name));
 
 		if (!speechRecognizer.Available)
@@ -43,7 +46,6 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 		}
 
 		var currentIndex = 0;
-		var getRecognitionTaskCompletionSource = new TaskCompletionSource<string>();
 
 		recognitionTask = speechRecognizer.GetRecognitionTask(liveSpeechRequest, (result, err) =>
 		{
@@ -58,6 +60,7 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 				{
 					currentIndex = 0;
 					StopRecording();
+					OnRecognitionResultCompleted(result.BestTranscription.FormattedString);
 					getRecognitionTaskCompletionSource.TrySetResult(result.BestTranscription.FormattedString);
 				}
 				else
@@ -66,12 +69,20 @@ async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? r
 					{
 						var s = result.BestTranscription.Segments[i].Substring;
 						currentIndex++;
-						recognitionResult?.Report(s);
+						recognitionProgress?.Report(s);
+						OnRecognitionResultUpdated(s);
 					}
 				}
 			}
 		});
+		return Task.CompletedTask;
+	}
 
+	async Task<string> InternalListenAsync(CultureInfo culture, IProgress<string>? recognitionResult, CancellationToken cancellationToken)
+	{
+		recognitionProgress = recognitionResult;
+		getRecognitionTaskCompletionSource ??= new TaskCompletionSource<string>();
+		await StartListeningAsync(culture, CancellationToken.None);
 		await using (cancellationToken.Register(() =>
 		{
 			StopRecording();