From d383d4befadcc1aab1fab39adfc50cda3a7f02ce Mon Sep 17 00:00:00 2001 From: Googler Date: Wed, 8 May 2024 12:04:05 -0700 Subject: [PATCH] Fix for ShadowSpeechRecognizer on Android V With Android V, the internal implementation for SpeechRecognizer has changed. Prepare ShadowSpeechRecognizer to be extended in the future to support V's changed implementations. PiperOrigin-RevId: 631880010 --- .../shadows/ShadowSpeechRecognizerTest.java | 12 + .../shadows/ShadowSpeechRecognizer.java | 306 ++++++++++++------ 2 files changed, 218 insertions(+), 100 deletions(-) diff --git a/robolectric/src/test/java/org/robolectric/shadows/ShadowSpeechRecognizerTest.java b/robolectric/src/test/java/org/robolectric/shadows/ShadowSpeechRecognizerTest.java index c0ff1622fe6..25c7ed722cc 100644 --- a/robolectric/src/test/java/org/robolectric/shadows/ShadowSpeechRecognizerTest.java +++ b/robolectric/src/test/java/org/robolectric/shadows/ShadowSpeechRecognizerTest.java @@ -18,6 +18,7 @@ import android.util.Log; import androidx.test.core.app.ApplicationProvider; import java.util.ArrayList; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,6 +43,11 @@ public void setUp() { supportCallback = new TestRecognitionSupportCallback(); } + @After + public void tearDown() { + speechRecognizer.destroy(); + } + @Test public void onErrorCalled() { startListening(); @@ -118,6 +124,7 @@ public void startAndStopListening() { shadowOf(speechRecognizer).triggerOnResults(new Bundle()); speechRecognizer.stopListening(); + shadowOf(getMainLooper()).idle(); assertNoErrorLogs(); } @@ -136,6 +143,8 @@ public void startListeningThenDestroyAndStartListening() { /** Verify the startlistening flow works when using custom component name. */ @Test public void startListeningWithCustomComponent() { + speechRecognizer.destroy(); + speechRecognizer = SpeechRecognizer.createSpeechRecognizer( ApplicationProvider.getApplicationContext(), @@ -157,6 +166,7 @@ public void getLatestSpeechRecognizer() { shadowOf(getMainLooper()).idle(); assertThat(ShadowSpeechRecognizer.getLatestSpeechRecognizer()) .isSameInstanceAs(newSpeechRecognizer); + newSpeechRecognizer.destroy(); } @Test @@ -170,6 +180,7 @@ public void getLastRecognizerIntent() { newSpeechRecognizer.startListening(intent); shadowOf(getMainLooper()).idle(); assertThat(shadowOf(newSpeechRecognizer).getLastRecognizerIntent()).isEqualTo(intent); + newSpeechRecognizer.destroy(); } private void startListening() { @@ -236,6 +247,7 @@ public void onRmsChanged(float rmsdB) { @Config(minSdk = TIRAMISU) @Test public void onCreateOnDeviceRecognizer_setsLatestSpeechRecognizer() { + speechRecognizer.destroy(); speechRecognizer = SpeechRecognizer.createOnDeviceSpeechRecognizer(applicationContext); assertThat(speechRecognizer) diff --git a/shadows/framework/src/main/java/org/robolectric/shadows/ShadowSpeechRecognizer.java b/shadows/framework/src/main/java/org/robolectric/shadows/ShadowSpeechRecognizer.java index ec5a63cc4fb..b7375e557a0 100644 --- a/shadows/framework/src/main/java/org/robolectric/shadows/ShadowSpeechRecognizer.java +++ b/shadows/framework/src/main/java/org/robolectric/shadows/ShadowSpeechRecognizer.java @@ -11,10 +11,13 @@ import android.os.Build.VERSION_CODES; import android.os.Bundle; import android.os.Handler; +import android.os.IBinder; import android.os.Looper; import android.os.Message; import android.speech.IRecognitionService; import android.speech.RecognitionListener; +import android.speech.RecognitionSupport; +import android.speech.RecognitionSupportCallback; import android.speech.SpeechRecognizer; import com.google.common.base.Preconditions; import java.util.Queue; @@ -35,34 +38,16 @@ @Implements(value = SpeechRecognizer.class, looseSignatures = true) public class ShadowSpeechRecognizer { - @RealObject SpeechRecognizer realSpeechRecognizer; - protected static SpeechRecognizer latestSpeechRecognizer; - private Intent recognizerIntent; - private RecognitionListener recognitionListener; - private static boolean isOnDeviceRecognitionAvailable = true; - private boolean isRecognizerDestroyed = false; + @SuppressWarnings("NonFinalStaticField") + private static SpeechRecognizer latestSpeechRecognizer; - private /*RecognitionSupportCallback*/ Object recognitionSupportCallback; - private Executor recognitionSupportExecutor; - @Nullable private Intent latestModelDownloadIntent; - - /** - * Returns the latest SpeechRecognizer. This method can only be called after {@link - * SpeechRecognizer#createSpeechRecognizer(Context)} is called. - */ - public static SpeechRecognizer getLatestSpeechRecognizer() { - return latestSpeechRecognizer; - } + @SuppressWarnings("NonFinalStaticField") + private static boolean isOnDeviceRecognitionAvailable = true; - /** Returns the argument passed to the last call to {@link SpeechRecognizer#startListening}. */ - public Intent getLastRecognizerIntent() { - return recognizerIntent; - } + @RealObject SpeechRecognizer realSpeechRecognizer; - /** Returns true iff the destroy method of was invoked for the recognizer. */ - public boolean isDestroyed() { - return isRecognizerDestroyed; - } + // NOTE: Do not manipulate state directly in this class. Call {@link #getState()} instead. + private final ShadowSpeechRecognizerState state = new ShadowSpeechRecognizerState(); @Resetter public static void reset() { @@ -70,10 +55,12 @@ public static void reset() { isOnDeviceRecognitionAvailable = true; } - @Implementation - protected void destroy() { - isRecognizerDestroyed = true; - reflector(SpeechRecognizerReflector.class, realSpeechRecognizer).destroy(); + /** + * Returns the latest SpeechRecognizer. This method can only be called after {@link + * SpeechRecognizer#createSpeechRecognizer(Context)} is called. + */ + public static SpeechRecognizer getLatestSpeechRecognizer() { + return latestSpeechRecognizer; } @Implementation @@ -86,145 +73,264 @@ protected static SpeechRecognizer createSpeechRecognizer( return result; } - @Implementation - protected void startListening(Intent recognizerIntent) { - this.recognizerIntent = recognizerIntent; - // from the implementation of {@link SpeechRecognizer#startListening} it seems that it allows - // running the method on an already destroyed object, so we replicate the same by resetting - // isRecognizerDestroyed - isRecognizerDestroyed = false; - // the real implementation connects to a service - // simulate the resulting behavior once the service is connected - Handler mainHandler = new Handler(Looper.getMainLooper()); - // perform the onServiceConnected logic - mainHandler.post( - () -> { - SpeechRecognizerReflector recognizerReflector = - reflector(SpeechRecognizerReflector.class, realSpeechRecognizer); - recognizerReflector.setService( - ReflectionHelpers.createNullProxy(IRecognitionService.class)); - Queue pendingTasks = recognizerReflector.getPendingTasks(); - while (!pendingTasks.isEmpty()) { - recognizerReflector.getHandler().sendMessage(pendingTasks.poll()); - } - }); + @Implementation(minSdk = VERSION_CODES.TIRAMISU) + protected static SpeechRecognizer createOnDeviceSpeechRecognizer(final Context context) { + SpeechRecognizer result = + reflector(SpeechRecognizerReflector.class).createOnDeviceSpeechRecognizer(context); + latestSpeechRecognizer = result; + return result; + } + + public static void setIsOnDeviceRecognitionAvailable(boolean available) { + isOnDeviceRecognitionAvailable = available; + } + + @Implementation(minSdk = VERSION_CODES.TIRAMISU) + protected static boolean isOnDeviceRecognitionAvailable(final Context context) { + return isOnDeviceRecognitionAvailable; + } + + /** + * Returns the state of this shadow instance. + * + *

Subclasses may override this function to customize which state is returned. + */ + protected ShadowSpeechRecognizerState getState() { + return state; } /** - * Handles changing the listener and allows access to the internal listener to trigger events and - * sets the latest SpeechRecognizer. + * Returns the {@link ShadowSpeechRecognizerDirectAccessors} implementation that can handle direct + * access to functions/variables of a real {@link SpeechRecognizer}. + * + *

Subclasses may override this function to customize access in case they are shadowing a + * subclass of {@link SpeechRecognizer} that functions differently than the parent class. */ + protected ShadowSpeechRecognizerDirectAccessors getDirectAccessors() { + return reflector(SpeechRecognizerReflector.class, realSpeechRecognizer); + } + + /** Returns true iff the destroy method of was invoked for the recognizer. */ + public boolean isDestroyed() { + return getState().isRecognizerDestroyed; + } + + @Implementation(maxSdk = U.SDK_INT) + protected void destroy() { + getState().isRecognizerDestroyed = true; + getDirectAccessors().destroy(); + } + + /** Returns the argument passed to the last call to {@link SpeechRecognizer#startListening}. */ + public Intent getLastRecognizerIntent() { + return getState().recognizerIntent; + } + + @Implementation(maxSdk = U.SDK_INT) + protected void startListening(Intent recognizerIntent) { + // Record the most recent requested intent. + ShadowSpeechRecognizerState shadowState = getState(); + shadowState.recognizerIntent = recognizerIntent; + + // From the implementation of {@link SpeechRecognizer#startListening} it seems that it allows + // running the method on an already destroyed object, so we replicate the same by resetting + // isRecognizerDestroyed. + shadowState.isRecognizerDestroyed = false; + + // The real implementation connects to a service simulate the resulting behavior once + // the service is connected. + new Handler(Looper.getMainLooper()) + .post( + () -> { + ShadowSpeechRecognizerDirectAccessors directAccessors = getDirectAccessors(); + directAccessors.setService(createFakeSpeechRecognitionService()); + + Handler taskHandler = directAccessors.getHandler(); + Queue pendingTasks = directAccessors.getPendingTasks(); + while (!pendingTasks.isEmpty()) { + taskHandler.sendMessage(pendingTasks.poll()); + } + }); + } + + /** Handles changing the listener and allows access to the internal listener to trigger events. */ @Implementation(maxSdk = U.SDK_INT) // TODO(hoisie): Update this to support Android V @InDevelopment protected void handleChangeListener(RecognitionListener listener) { - recognitionListener = listener; + getState().recognitionListener = listener; } public void triggerOnEndOfSpeech() { - recognitionListener.onEndOfSpeech(); + getState().recognitionListener.onEndOfSpeech(); } public void triggerOnError(int error) { - recognitionListener.onError(error); + getState().recognitionListener.onError(error); } public void triggerOnReadyForSpeech(Bundle bundle) { - recognitionListener.onReadyForSpeech(bundle); + getState().recognitionListener.onReadyForSpeech(bundle); } public void triggerOnPartialResults(Bundle bundle) { - recognitionListener.onPartialResults(bundle); + getState().recognitionListener.onPartialResults(bundle); } public void triggerOnResults(Bundle bundle) { - recognitionListener.onResults(bundle); + getState().recognitionListener.onResults(bundle); } public void triggerOnRmsChanged(float rmsdB) { - recognitionListener.onRmsChanged(rmsdB); - } - - @Implementation(minSdk = VERSION_CODES.TIRAMISU) - protected static SpeechRecognizer createOnDeviceSpeechRecognizer(final Context context) { - SpeechRecognizer result = - reflector(SpeechRecognizerReflector.class).createOnDeviceSpeechRecognizer(context); - latestSpeechRecognizer = result; - return result; - } - - @Implementation(minSdk = VERSION_CODES.TIRAMISU) - protected static boolean isOnDeviceRecognitionAvailable(final Context context) { - return isOnDeviceRecognitionAvailable; + getState().recognitionListener.onRmsChanged(rmsdB); } @RequiresApi(api = VERSION_CODES.TIRAMISU) - @Implementation(minSdk = VERSION_CODES.TIRAMISU) + @Implementation(minSdk = VERSION_CODES.TIRAMISU, maxSdk = U.SDK_INT) protected void checkRecognitionSupport( @NonNull /*Intent*/ Object recognizerIntent, @NonNull /*Executor*/ Object executor, @NonNull /*RecognitionSupportCallback*/ Object supportListener) { Preconditions.checkArgument(recognizerIntent instanceof Intent); Preconditions.checkArgument(executor instanceof Executor); - Preconditions.checkArgument( - supportListener instanceof android.speech.RecognitionSupportCallback); - recognitionSupportExecutor = (Executor) executor; - recognitionSupportCallback = supportListener; + Preconditions.checkArgument(supportListener instanceof RecognitionSupportCallback); + + ShadowSpeechRecognizerState shadowState = getState(); + shadowState.recognitionSupportExecutor = (Executor) executor; + shadowState.recognitionSupportCallback = supportListener; } - @Implementation(minSdk = VERSION_CODES.TIRAMISU) - protected void triggerModelDownload(Intent recognizerIntent) { - latestModelDownloadIntent = recognizerIntent; + @RequiresApi(VERSION_CODES.TIRAMISU) + @Nullable + public Intent getLatestModelDownloadIntent() { + return getState().latestModelDownloadIntent; } - public static void setIsOnDeviceRecognitionAvailable(boolean available) { - isOnDeviceRecognitionAvailable = available; + @Implementation(minSdk = VERSION_CODES.TIRAMISU, maxSdk = U.SDK_INT) + protected void triggerModelDownload(Intent recognizerIntent) { + getState().latestModelDownloadIntent = recognizerIntent; } @RequiresApi(VERSION_CODES.TIRAMISU) public void triggerSupportResult(/*RecognitionSupport*/ Object recognitionSupport) { - Preconditions.checkArgument(recognitionSupport instanceof android.speech.RecognitionSupport); - recognitionSupportExecutor.execute( + Preconditions.checkArgument(recognitionSupport instanceof RecognitionSupport); + + ShadowSpeechRecognizerState shadowState = getState(); + shadowState.recognitionSupportExecutor.execute( () -> - ((android.speech.RecognitionSupportCallback) recognitionSupportCallback) - .onSupportResult((android.speech.RecognitionSupport) recognitionSupport)); + ((RecognitionSupportCallback) shadowState.recognitionSupportCallback) + .onSupportResult((RecognitionSupport) recognitionSupport)); } @RequiresApi(VERSION_CODES.TIRAMISU) public void triggerSupportError(int error) { - recognitionSupportExecutor.execute( - () -> - ((android.speech.RecognitionSupportCallback) recognitionSupportCallback) - .onError(error)); + ShadowSpeechRecognizerState shadowState = getState(); + shadowState.recognitionSupportExecutor.execute( + () -> ((RecognitionSupportCallback) shadowState.recognitionSupportCallback).onError(error)); } - @RequiresApi(VERSION_CODES.TIRAMISU) - @Nullable - public Intent getLatestModelDownloadIntent() { - return latestModelDownloadIntent; + /** + * {@link SpeechRecognizer} implementation now checks if the service's binder is alive whenever + * {@link SpeechRecognizer#checkOpenConnection} is called. This means that we need to return a + * deeper proxy that returns a delegating proxy that always reports the binder as alive. + */ + private static IRecognitionService createFakeSpeechRecognitionService() { + return ReflectionHelpers.createDelegatingProxy( + IRecognitionService.class, new AlwaysAliveSpeechRecognitionServiceDelegate()); + } + + /** + * A proxy delegate for {@link IRecognitionService} that always returns a delegating proxy that + * returns an {@link AlwaysAliveBinderDelegate} when {@link IRecognitionService#asBinder()} is + * called. + * + * @see #createFakeSpeechRecognitionService() for more details + */ + private static class AlwaysAliveSpeechRecognitionServiceDelegate { + public IBinder asBinder() { + return ReflectionHelpers.createDelegatingProxy( + IBinder.class, new AlwaysAliveBinderDelegate()); + } + } + + /** + * A proxy delegate for {@link IBinder} that always returns when {@link IBinder#isBinderAlive()} + * is called. + * + * @see #createFakeSpeechRecognitionService() for more details + */ + private static class AlwaysAliveBinderDelegate { + public boolean isBinderAlive() { + return true; + } + } + + /** + * The state of a specific instance of {@link ShadowSpeechRecognizer}. + * + *

NOTE: Not stored as variables in the parent class itself since subclasses may need to return + * a different instance of this class to operate on. + * + *

NOTE: This class is public since custom shadows may reside in a different package. + */ + public static class ShadowSpeechRecognizerState { + private boolean isRecognizerDestroyed = false; + private Intent recognizerIntent; + private RecognitionListener recognitionListener; + private Executor recognitionSupportExecutor; + private /*RecognitionSupportCallback*/ Object recognitionSupportCallback; + @Nullable private Intent latestModelDownloadIntent; + } + + /** + * An interface to access direct functions/variables of an instance of {@link SpeechRecognizer}. + * + *

Abstracted to allow subclasses to return customized accessors. + */ + protected interface ShadowSpeechRecognizerDirectAccessors { + /** + * Invokes {@link SpeechRecognizer#destroy()} on a real instance of {@link SpeechRecognizer}. + */ + void destroy(); + + /** Sets the {@link IRecognitionService} used by a real {@link SpeechRecognizer}. */ + void setService(IRecognitionService service); + + /** Returns a {@link Queue} of pending async tasks of a real {@link SpeechRecognizer}. */ + Queue getPendingTasks(); + + /** + * Returns the {@link Handler} of a real {@link SpeechRecognizer} that it uses to process any + * pending async tasks returned by {@link #getPendingTasks()}. + */ + Handler getHandler(); } /** Reflector interface for {@link SpeechRecognizer}'s internals. */ @ForType(SpeechRecognizer.class) - interface SpeechRecognizerReflector { + interface SpeechRecognizerReflector extends ShadowSpeechRecognizerDirectAccessors { @Static @Direct SpeechRecognizer createSpeechRecognizer(Context context, ComponentName serviceComponent); + @Static @Direct + SpeechRecognizer createOnDeviceSpeechRecognizer(Context context); + + @Direct + @Override void destroy(); @Accessor("mService") + @Override void setService(IRecognitionService service); @Accessor("mPendingTasks") + @Override Queue getPendingTasks(); @Accessor("mHandler") + @Override Handler getHandler(); - - @Static - @Direct - SpeechRecognizer createOnDeviceSpeechRecognizer(Context context); } }