diff --git a/src/main/java/io/spokestack/spokestack/SpeechPipeline.java b/src/main/java/io/spokestack/spokestack/SpeechPipeline.java index 49f5009..7ba119f 100644 --- a/src/main/java/io/spokestack/spokestack/SpeechPipeline.java +++ b/src/main/java/io/spokestack/spokestack/SpeechPipeline.java @@ -67,6 +67,7 @@ public final class SpeechPipeline implements AutoCloseable { */ public static final int DEFAULT_BUFFER_WIDTH = 20; + private final Object lock = new Object(); private final String inputClass; private final List stageClasses; private final SpeechConfig config; @@ -75,6 +76,7 @@ public final class SpeechPipeline implements AutoCloseable { private List stages; private Thread thread; private boolean running; + private boolean paused; private boolean managed; /** @@ -117,10 +119,18 @@ public SpeechContext getContext() { } /** - * @return true if the pipeline has been started, false otherwise. + * @return true if the pipeline has been started and is not paused, + * false otherwise. */ public boolean isRunning() { - return this.running; + return this.running && !isPaused(); + } + + /** + * @return true if the pipeline has been paused, false otherwise. + */ + public boolean isPaused() { + return this.paused; } /** manually activate the speech pipeline. */ @@ -222,6 +232,40 @@ private void startThread() throws Exception { this.thread.start(); } + /** + * Pauses the speech pipeline, temporarily stopping passive listening. + * + *

+ * Note that active listening (an active ASR stage) cannot be paused, so + * the pipeline is deactivated before it is paused. This may prevent the + * delivery of a + * {@link io.spokestack.spokestack.SpeechContext.Event#RECOGNIZE} event if + * an ASR request is currently in progress. + *

+ * + *

+ * While paused, the pipeline will not respond to the wakeword, but + * in order to support a quick {@link #resume()}, it will retain control + * of the microphone. No audio is explicitly read or analyzed. To fully + * release the pipeline's resources, see {@link #stop()}. + *

+ */ + public void pause() { + deactivate(); + this.paused = true; + } + + /** + * Resumes a paused speech pipeline, returning the pipeline to a passive + * listening state. + */ + public void resume() { + this.paused = false; + synchronized (lock) { + lock.notify(); + } + } + /** * stops the speech pipeline and releases all resources. */ @@ -238,9 +282,24 @@ public void stop() { } private void run() { - while (this.running) + synchronized (lock) { + while (this.running) { + step(); + } + cleanup(); + } + } + + private void step() { + if (this.paused) { + try { + lock.wait(); + } catch (InterruptedException e) { + this.running = false; + } + } else { dispatch(); - cleanup(); + } } private void dispatch() { diff --git a/src/main/java/io/spokestack/spokestack/Spokestack.java b/src/main/java/io/spokestack/spokestack/Spokestack.java index f06b21e..1b19911 100644 --- a/src/main/java/io/spokestack/spokestack/Spokestack.java +++ b/src/main/java/io/spokestack/spokestack/Spokestack.java @@ -9,6 +9,7 @@ import io.spokestack.spokestack.nlu.tensorflow.parsers.IntegerParser; import io.spokestack.spokestack.nlu.tensorflow.parsers.SelsetParser; import io.spokestack.spokestack.tts.SynthesisRequest; +import io.spokestack.spokestack.tts.TTSEvent; import io.spokestack.spokestack.tts.TTSManager; import io.spokestack.spokestack.util.AsyncResult; import io.spokestack.spokestack.util.EventTracer; @@ -171,30 +172,90 @@ public SpeechPipeline getSpeechPipeline() { } /** - * Starts the speech pipeline in order to process user input via the - * microphone (or chosen input class). + * Prepares all registered Spokestack modules for use and starts the + * speech pipeline in passive listening mode. * - * @throws Exception if there is an error configuring or starting the speech - * pipeline. + * @throws Exception if there is an error configuring or starting a module. */ public void start() throws Exception { if (this.speechPipeline != null) { this.speechPipeline.start(); } + if (this.nlu != null) { + this.nlu.prepare(); + } + if (this.tts != null) { + this.tts.prepare(); + } } /** - * Stops the speech pipeline and releases all its internal resources. + * Pauses the speech pipeline, suspending passive listening. This can be + * useful for scenarios where you expect false positives for the wakeword + * to be possible. + * + *

+ * This method will implicitly deactivate the pipeline, canceling any + * in-flight ASR requests. + *

* *

- * This is useful for stopping passive listening (listening for wakeword - * activation); for fully releasing all internal resources held by - * Spokestack, see {@link #release()}. + * This method is called automatically when Spokestack is playing a TTS + * prompt if Spokestack is managing audio playback. *

+ * + * @see SpeechPipeline#pause() */ - public void stop() { + public void pause() { if (this.speechPipeline != null) { - this.speechPipeline.stop(); + this.speechPipeline.pause(); + } + } + + /** + * Resumes a paused speech pipeline, returning it to a passive listening + * state. + * + *

+ * This method is called automatically when Spokestack finishes playing a + * TTS prompt if Spokestack is managing audio playback. + *

+ * + * @see SpeechPipeline#resume() + */ + public void resume() { + if (this.speechPipeline != null) { + this.speechPipeline.resume(); + } + } + + /** + * Stops the speech pipeline and releases internal resources held by all + * registered Spokestack modules. + * + *

+ * In order to support restarting Spokestack (calling {@link #start()} after + * this method), this method does not clear registered + * listeners. To do this, close then destroy the current Spokestack + * instance and build a new one. + *

+ */ + public void stop() { + closeSafely(this.speechPipeline); + closeSafely(this.nlu); + closeSafely(this.tts); + } + + private void closeSafely(AutoCloseable module) { + if (module == null) { + return; + } + try { + module.close(); + } catch (Exception e) { + for (SpokestackAdapter listener : this.listeners) { + listener.onError(e); + } } } @@ -365,29 +426,18 @@ private AsyncResult classifyInternal(String text) { return result; } - /** - * Prepares all registered Spokestack modules for use. - * - *

- * Calling this method is only necessary if internal resources have been - * released via {@link #close()} or {@link #release()}. - *

- * - *

- * The speech pipeline is not modified by this method since it - * manages its own resources via {@link #start()} and {@link #stop()}, - * and some of its components are designed to be used immediately after - * construction. - *

- * - * @throws Exception if there is an error configuring or starting a module. - */ - public void prepare() throws Exception { - if (this.nlu != null) { - this.nlu.prepare(); - } - if (this.tts != null) { - this.tts.prepare(); + @Override + public void eventReceived(@NotNull TTSEvent event) { + switch (event.type) { + case PLAYBACK_STARTED: + pause(); + break; + case PLAYBACK_STOPPED: + resume(); + break; + default: + // do nothing + break; } } @@ -395,52 +445,15 @@ public void prepare() throws Exception { * Release internal resources held by all registered Spokestack modules. * *

- * If Spokestack is needed again after this method is called, - * {@link #prepare()} must be called to reconstruct the modules. - *

- * - *

- * In order to support such restarts, this method does not clear registered + * In order to support restarting Spokestack (calling {@link #start()} after + * this method), this method does not clear registered * listeners. To do this, close then destroy the current Spokestack * instance and build a new one. *

*/ @Override public void close() { - release(); - } - - /** - * Release internal resources held by all registered Spokestack modules. - * - *

- * If Spokestack is needed again after this method is called, - * {@link #prepare()} must be called to reconstruct the modules. - *

- * - *

- * In order to support such restarts, this method does not clear registered - * listeners. To do this, close then destroy the current Spokestack - * instance and build a new one. - *

- */ - public void release() { - closeSafely(this.speechPipeline); - closeSafely(this.nlu); - closeSafely(this.tts); - } - - private void closeSafely(AutoCloseable module) { - if (module == null) { - return; - } - try { - module.close(); - } catch (Exception e) { - for (SpokestackAdapter listener : this.listeners) { - listener.onError(e); - } - } + stop(); } /** diff --git a/src/test/java/io/spokestack/spokestack/SpeechPipelineTest.java b/src/test/java/io/spokestack/spokestack/SpeechPipelineTest.java index b9af3f7..8c8f1e0 100644 --- a/src/test/java/io/spokestack/spokestack/SpeechPipelineTest.java +++ b/src/test/java/io/spokestack/spokestack/SpeechPipelineTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import org.junit.jupiter.api.function.Executable; import static org.junit.jupiter.api.Assertions.*; +import static io.spokestack.spokestack.SpeechTestUtils.FreeInput; public class SpeechPipelineTest implements OnSpeechEventListener { private static final List> PROFILES = Arrays.asList( @@ -162,6 +163,41 @@ public void testStartStop() throws Exception { assertFalse(Stage.open); } + @Test + public void testPause() throws Exception { + final SpeechPipeline pipeline = new SpeechPipeline.Builder() + .setInputClass("io.spokestack.spokestack.SpeechTestUtils$FreeInput") + .setProperty("sample-rate", 16000) + .setProperty("frame-width", 20) + .setProperty("buffer-width", 300) + .setProperty("trace-level", EventTracer.Level.DEBUG.value()) + .build(); + + // startup + int frames = FreeInput.counter; + assertEquals(frames, 0); + pipeline.start(); + assertTrue(pipeline.isRunning()); + Thread.sleep(5); + assertTrue(FreeInput.counter > frames); + + // we won't get any more frames if we're paused + pipeline.pause(); + + // wait for the pause to take effect + Thread.sleep(10); + frames = FreeInput.counter; + + // wait some more to make sure we don't get any more frames + Thread.sleep(15); + assertEquals(FreeInput.counter, frames); + + // after resuming, frames should start increasing almost immediately + pipeline.resume(); + Thread.sleep(5); + assertTrue(FreeInput.counter > frames); + } + @Test public void testInputFailure() throws Exception { SpeechPipeline pipeline = new SpeechPipeline.Builder() diff --git a/src/test/java/io/spokestack/spokestack/SpeechTestUtils.java b/src/test/java/io/spokestack/spokestack/SpeechTestUtils.java new file mode 100644 index 0000000..19685c2 --- /dev/null +++ b/src/test/java/io/spokestack/spokestack/SpeechTestUtils.java @@ -0,0 +1,26 @@ +package io.spokestack.spokestack; + +import java.nio.ByteBuffer; + +/** + * Test classes related to the speech pipeline used in more than one test + * suite. + */ +public class SpeechTestUtils { + + public static class FreeInput implements SpeechInput { + public static int counter; + + public FreeInput(SpeechConfig config) { + counter = 0; + } + + public void close() { + counter = -1; + } + + public void read(SpeechContext context, ByteBuffer frame) { + frame.putInt(0, ++counter); + } + } +} diff --git a/src/test/java/io/spokestack/spokestack/SpokestackTest.java b/src/test/java/io/spokestack/spokestack/SpokestackTest.java index b49fb38..a7ecd01 100644 --- a/src/test/java/io/spokestack/spokestack/SpokestackTest.java +++ b/src/test/java/io/spokestack/spokestack/SpokestackTest.java @@ -16,6 +16,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -23,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static io.spokestack.spokestack.SpeechTestUtils.FreeInput; @RunWith(PowerMockRunner.class) @PrepareForTest({SystemClock.class}) @@ -285,7 +287,7 @@ public void testListenerManagement() throws Exception { } @Test - public void testClose() throws Exception { + public void testRestart() throws Exception { mockStatic(SystemClock.class); TestAdapter listener = new TestAdapter(); @@ -305,8 +307,8 @@ public void testClose() throws Exception { listener.speechEvents.poll(1, TimeUnit.SECONDS); assertEquals(SpeechContext.Event.ACTIVATE, event); - // modules don't work after close() - spokestack.close(); + // modules don't work after stop() + spokestack.stop(); assertNull(spokestack.getTts().getTtsService()); assertNull(spokestack.getNlu().getNlu()); assertThrows( @@ -319,13 +321,53 @@ public void testClose() throws Exception { () -> spokestack.synthesize(request)); // restart supported - spokestack.prepare(); + spokestack.start(); assertNotNull(spokestack.getTts().getTtsService()); assertNotNull(spokestack.getNlu().getNlu()); assertDoesNotThrow(() -> spokestack.classify("test")); assertDoesNotThrow(() -> spokestack.synthesize(request)); } + @Test + public void testPause() throws Exception { + mockStatic(SystemClock.class); + TestAdapter listener = new TestAdapter(); + + Spokestack.Builder builder = new Spokestack + .Builder(new SpeechPipeline.Builder(), mockTts()) + .withoutWakeword() + .setConfig(testConfig()) + .addListener(listener); + + builder = mockAndroidComponents(builder); + builder.getPipelineBuilder() + .setInputClass("io.spokestack.spokestack.SpeechTestUtils$FreeInput"); + Spokestack spokestack = new Spokestack(builder, mockNlu()); + + // startup + int frames = FreeInput.counter; + assertEquals(frames, 0); + spokestack.start(); + Thread.sleep(10); + assertTrue(FreeInput.counter > frames); + + // we won't get any more frames if we're paused + spokestack.pause(); + + // wait for the pause to take effect + Thread.sleep(10); + frames = FreeInput.counter; + + // wait some more to make sure we don't get any more frames + Thread.sleep(15); + assertEquals(FreeInput.counter, frames); + + // after resuming, frames should start increasing almost immediately + spokestack.resume(); + Thread.sleep(5); + assertTrue(FreeInput.counter > frames); + } + private NLUManager mockNlu() throws Exception { return NLUTestUtils.mockManager(); }