Skip to content
This repository has been archived by the owner on May 6, 2022. It is now read-only.

Commit

Permalink
breaking: refactor to make NLU interchangeable
Browse files Browse the repository at this point in the history
This introduces an NLUManager component capable of dispatching
classification to different backends. It constitutes a breaking
change for the `Spokestack` class since the type of its `nlu`
field has changed.
  • Loading branch information
space-pope committed Nov 2, 2020
1 parent 6e1cf3f commit 531d6e3
Show file tree
Hide file tree
Showing 7 changed files with 371 additions and 73 deletions.
94 changes: 72 additions & 22 deletions src/main/java/io/spokestack/spokestack/Spokestack.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,8 @@

import android.content.Context;
import androidx.lifecycle.Lifecycle;
import io.spokestack.spokestack.nlu.NLUManager;
import io.spokestack.spokestack.nlu.NLUResult;
import io.spokestack.spokestack.nlu.NLUService;
import io.spokestack.spokestack.nlu.tensorflow.TensorflowNLU;
import io.spokestack.spokestack.tts.SynthesisRequest;
import io.spokestack.spokestack.tts.TTSManager;
import io.spokestack.spokestack.util.AsyncResult;
Expand Down Expand Up @@ -65,7 +64,7 @@
* </p>
*
* @see SpeechPipeline
* @see TensorflowNLU
* @see io.spokestack.spokestack.nlu.tensorflow.TensorflowNLU
* @see TTSManager
*/
public final class Spokestack extends SpokestackAdapter
Expand All @@ -75,7 +74,7 @@ public final class Spokestack extends SpokestackAdapter
private final boolean autoClassify;
private final TranscriptEditor transcriptEditor;
private SpeechPipeline speechPipeline;
private TensorflowNLU nlu;
private NLUManager nlu;
private TTSManager tts;

/**
Expand Down Expand Up @@ -109,6 +108,35 @@ private Spokestack(Builder builder) throws Exception {
}
}

/**
* Package-private constructor used for testing with an injected NLU.
*
* @param builder The builder to use for everything but NLU.
* @param nluManager The NLU manager to inject.
*/
Spokestack(Builder builder, NLUManager nluManager) throws Exception {
this.listeners = new ArrayList<>();
this.listeners.addAll(builder.listeners);
this.autoClassify = builder.autoClassify;
this.transcriptEditor = builder.transcriptEditor;
if (builder.useAsr) {
this.speechPipeline = builder.getPipelineBuilder()
.addOnSpeechEventListener(this)
.build();
}
if (builder.useNLU) {
this.nlu = nluManager;
}
if (builder.useTTS) {
if (!builder.useTTSPlayback) {
builder.ttsBuilder.setOutputClass(null);
}
this.tts = builder.getTtsBuilder()
.addTTSListener(this)
.build();
}
}

// speech pipeline

/**
Expand All @@ -126,14 +154,18 @@ public SpeechPipeline getSpeechPipeline() {
* pipeline.
*/
public void start() throws Exception {
this.speechPipeline.start();
if (this.speechPipeline != null) {
this.speechPipeline.start();
}
}

/**
* Stops the speech pipeline and releases all its internal resources.
*/
public void stop() {
this.speechPipeline.stop();
if (this.speechPipeline != null) {
this.speechPipeline.stop();
}
}

/**
Expand All @@ -142,7 +174,9 @@ public void stop() {
* conjunction with a microphone button.
*/
public void activate() {
this.speechPipeline.activate();
if (this.speechPipeline != null) {
this.speechPipeline.activate();
}
}

/**
Expand All @@ -156,15 +190,17 @@ public void activate() {
* </p>
*/
public void deactivate() {
this.speechPipeline.deactivate();
if (this.speechPipeline != null) {
this.speechPipeline.deactivate();
}
}

// NLU

/**
* @return The NLU service currently in use.
* @return The NLU manager currently in use.
*/
public NLUService getNlu() {
public NLUManager getNlu() {
return nlu;
}

Expand All @@ -184,7 +220,10 @@ public NLUService getNlu() {
* classification.
*/
public AsyncResult<NLUResult> classify(String utterance) {
return classifyInternal(utterance);
if (this.nlu != null) {
return classifyInternal(utterance);
}
return null;
}

// TTS
Expand All @@ -205,7 +244,9 @@ public TTSManager getTts() {
* @throws Exception If there is an error constructing TTS components.
*/
public void prepareTts() throws Exception {
this.tts.prepare();
if (this.tts != null) {
this.tts.prepare();
}
}

/**
Expand All @@ -219,7 +260,9 @@ public void prepareTts() throws Exception {
* </p>
*/
public void releaseTts() {
this.tts.release();
if (this.tts != null) {
this.tts.release();
}
}

/**
Expand All @@ -229,14 +272,18 @@ public void releaseTts() {
* @param request The synthesis request data.
*/
public void synthesize(SynthesisRequest request) {
this.tts.synthesize(request);
if (this.tts != null) {
this.tts.synthesize(request);
}
}

/**
* Stops playback of any playing or queued synthesis results.
*/
public void stopPlayback() {
this.tts.stopPlayback();
if (this.tts != null) {
this.tts.stopPlayback();
}
}

// listeners
Expand Down Expand Up @@ -294,6 +341,11 @@ public void onEvent(@NotNull SpeechContext.Event event,
}
}

@Override
public void nluResult(@NotNull NLUResult result) {
super.nluResult(result);
}

private AsyncResult<NLUResult> classifyInternal(String text) {
AsyncResult<NLUResult> result =
this.nlu.classify(text);
Expand Down Expand Up @@ -322,7 +374,7 @@ public void close() {
*/
public static class Builder {
private final SpeechPipeline.Builder pipelineBuilder;
private final TensorflowNLU.Builder nluBuilder;
private final NLUManager.Builder nluBuilder;
private final TTSManager.Builder ttsBuilder;
private final List<SpokestackAdapter> listeners = new ArrayList<>();

Expand Down Expand Up @@ -434,7 +486,7 @@ public Builder() {
.setConfig(this.speechConfig)
.useProfile(profileClass);
this.nluBuilder =
new TensorflowNLU.Builder().setConfig(this.speechConfig);
new NLUManager.Builder().setConfig(this.speechConfig);
String ttsServiceClass =
"io.spokestack.spokestack.tts.SpokestackTTSService";
String ttsOutputClass =
Expand All @@ -461,14 +513,12 @@ private void setDefaults(SpeechConfig config) {
* for testing.
*
* @param pipeline the speech pipeline builder
* @param nlu the NLU builder
* @param tts the TTS builder
*/
Builder(SpeechPipeline.Builder pipeline, TensorflowNLU.Builder nlu,
TTSManager.Builder tts) {
Builder(SpeechPipeline.Builder pipeline, TTSManager.Builder tts) {
this.speechConfig = new SpeechConfig();
this.pipelineBuilder = pipeline;
this.nluBuilder = nlu;
this.nluBuilder = new NLUManager.Builder();
this.ttsBuilder = tts;
}

Expand All @@ -482,7 +532,7 @@ public SpeechPipeline.Builder getPipelineBuilder() {
/**
* @return The builder used to configure the NLU subsystem.
*/
public TensorflowNLU.Builder getNluBuilder() {
public NLUManager.Builder getNluBuilder() {
return nluBuilder;
}

Expand Down
157 changes: 157 additions & 0 deletions src/main/java/io/spokestack/spokestack/nlu/NLUManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
package io.spokestack.spokestack.nlu;

import io.spokestack.spokestack.SpeechConfig;
import io.spokestack.spokestack.nlu.tensorflow.TensorflowNLU;
import io.spokestack.spokestack.util.AsyncResult;
import io.spokestack.spokestack.util.EventTracer;
import io.spokestack.spokestack.util.TraceListener;

import java.util.ArrayList;
import java.util.List;

/**
* Manager for natural language understanding (NLU) components in Spokestack.
*
* <p>
* Spokestack's NLU manager follows the same setup pattern as its {@link
* io.spokestack.spokestack.SpeechPipeline} and {@link io.spokestack.spokestack.tts.TTSManager}
* modules. The manager constructs the component ultimately responsible for
* classification (an {@link NLUService}) and manages the context required to
* perform these classifications and dispatch events to registered listeners.
* </p>
*/
public final class NLUManager {
private final NLUService nlu;
private final NLUContext context;

/**
* Constructs a new {@code NLUManager} with an initialized NLU service.
*
* @param builder builder with configuration parameters
* @throws Exception If there is an error constructing the service.
*/
private NLUManager(Builder builder) throws Exception {
this.context = builder.context;
this.nlu = buildService(builder);
}

private NLUService buildService(Builder builder) throws Exception {
Object constructed = Class
.forName(builder.serviceClass)
.getConstructor(SpeechConfig.class, NLUContext.class)
.newInstance(builder.config, builder.context);
return (NLUService) constructed;
}

/**
* Classify a user utterance, returning a wrapper that can either block
* until the classification is complete or call a registered callback when
* the result is ready.
*
* @param utterance The utterance to classify.
* @return An object representing the result of the asynchronous
* classification.
*/
public AsyncResult<NLUResult> classify(String utterance) {
return this.nlu.classify(utterance, this.context);
}

/**
* Add a new listener to receive trace events from the NLU subsystem.
*
* @param listener The listener to add.
*/
public void addListener(TraceListener listener) {
this.context.addTraceListener(listener);
}

/**
* Remove a trace listener, allowing it to be garbage collected.
*
* @param listener The listener to remove.
*/
public void removeListener(TraceListener listener) {
this.context.removeTraceListener(listener);
}

/**
* Fluent builder interface for initializing an NLU manager.
*/
public static class Builder {
private final List<TraceListener> traceListeners = new ArrayList<>();
private NLUContext context;
private SpeechConfig config = new SpeechConfig();
private String serviceClass;

/**
* Creates a new builder instance.
*/
public Builder() {
config.put("trace-level", EventTracer.Level.ERROR.value());
this.serviceClass = TensorflowNLU.class.getCanonicalName();
}

/**
* Sets the name of the NLU service class to be used.
*
* @param className The name of the NLU service class to be used.
* @return this
*/
public Builder setServiceClass(String className) {
this.serviceClass = className;
return this;
}

/**
* Attaches a configuration object, overwriting any existing
* configuration.
*
* @param value configuration to attach
* @return this
*/
public Builder setConfig(SpeechConfig value) {
this.config = value;
return this;
}

/**
* Sets a configuration value.
*
* @param key configuration property name
* @param value property value
* @return this
*/
public Builder setProperty(String key, Object value) {
config.put(key, value);
return this;
}

/**
* Adds a trace listener to receive events from the NLU system.
*
* @param listener the listener to register
* @return this
*/
public Builder addTraceListener(TraceListener listener) {
this.traceListeners.add(listener);
return this;
}

/**
* Create a new NLU service, automatically loading any necessary
* resources in the background. Any errors encountered during loading
* will be reported to registered {@link TraceListener}s.
*
* @return An initialized {@code NLUManager} instance
* @throws Exception If there is an error constructing the NLU service.
*/
public NLUManager build() throws Exception {
this.context = new NLUContext(this.config);
for (TraceListener listener : this.traceListeners) {
this.context.addTraceListener(listener);
}
return new NLUManager(this);
}

}
}
6 changes: 6 additions & 0 deletions src/main/java/io/spokestack/spokestack/nlu/NLUService.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
/**
* A simple interface for components that provide intent classification and slot
* recognition, either on-device or via a network request.
*
* <p>
* To participate in Spokestack's {@link NLUManager}, an NLUService must have a
* constructor that accepts instances of {@link io.spokestack.spokestack.SpeechConfig}
* and {@link NLUContext}.
* </p>
*/
public interface NLUService {

Expand Down
Loading

0 comments on commit 531d6e3

Please sign in to comment.