Skip to content

Commit

Permalink
Stop Recognizing should wait for pending service messages (#148)
Browse files Browse the repository at this point in the history 
* Commit one

* Only dialog cases failing

* Still working on the dialog connector tests

* Set turn start

* Linter fixes

* Lint errors in tests
  • Loading branch information
@rhurey
rhurey committed Feb 16, 2020
1 parent 27bb37d commit 0413695
Show file tree
Hide file tree
Showing 18 changed files with 463 additions and 545 deletions.
14 changes: 12 additions & 2 deletions src/common.browser/MicAudioSource.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ export class MicAudioSource implements IAudioSource {
public constructor(
private readonly privRecorder: IRecorder,
private readonly deviceId?: string,
audioSourceId?: string ) {
audioSourceId?: string) {

this.privOutputChunkSize = MicAudioSource.AUDIOFORMAT.avgBytesPerSec / 10;
this.privId = audioSourceId ? audioSourceId : createNoDashGuid();
Expand All @@ -85,7 +85,17 @@ export class MicAudioSource implements IAudioSource {

this.privInitializeDeferral = new Deferred<boolean>();

this.createAudioContext();
try {
this.createAudioContext();
} catch (error) {
if (error instanceof Error) {
const typedError: Error = error as Error;
this.privInitializeDeferral.reject(typedError.name + ": " + typedError.message);
} else {
this.privInitializeDeferral.reject(error);
}
return this.privInitializeDeferral.promise();
}

const nav = window.navigator as INavigator;

Expand Down
6 changes: 5 additions & 1 deletion src/common.browser/ReplayableAudioNode.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,10 @@ export class ReplayableAudioNode implements IAudioStreamNode {
// beginning of the buffer closest to the requested offset.
// A replay request will start from the last shrink point.
public shrinkBuffers(offset: number): void {
if (this.privBuffers === undefined) {
return;
}

this.privLastShrinkOffset = offset;

// Find the start point in the buffers.
Expand All @@ -115,7 +119,7 @@ export class ReplayableAudioNode implements IAudioStreamNode {

// Finds the time a buffer of audio was first seen by offset.
public findTimeAtOffset(offset: number): number {
if (offset < this.privBufferStartOffset) {
if (offset < this.privBufferStartOffset || this.privBuffers === undefined) {
return 0;
}

Expand Down
67 changes: 28 additions & 39 deletions src/common.speech/DialogServiceAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
// Do not consume directly, call fetchDialogConnection instead.
private privDialogConnectionPromise: Promise<IConnection>;

private privSuccessCallback: (e: SpeechRecognitionResult) => void;
private privConnectionLoop: Promise<IConnection>;
private terminateMessageLoop: boolean;
private agentConfigSent: boolean;
Expand Down Expand Up @@ -153,8 +152,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
this.privDialogRequestSession.requestId,
CancellationReason.Error,
CancellationErrorCode.NoError,
"Disconnecting",
undefined);
"Disconnecting");

this.terminateMessageLoop = true;
this.agentConfigSent = false;
Expand All @@ -170,10 +168,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
}
}

protected processTypeSpecificMessages(
connectionMessage: SpeechConnectionMessage,
successCallback?: (e: SpeechRecognitionResult) => void,
errorCallBack?: (e: string) => void): boolean {
protected processTypeSpecificMessages(connectionMessage: SpeechConnectionMessage): boolean {

const resultProps: PropertyCollection = new PropertyCollection();
if (connectionMessage.messageType === MessageType.Text) {
Expand Down Expand Up @@ -294,8 +289,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
requestId: string,
cancellationReason: CancellationReason,
errorCode: CancellationErrorCode,
error: string,
cancelRecoCallback: (e: SpeechRecognitionResult) => void): void {
error: string): void {

this.terminateMessageLoop = true;

Expand All @@ -319,7 +313,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
/* tslint:disable:no-empty */
} catch { }

if (!!cancelRecoCallback) {
if (!!this.privSuccessCallback) {
const result: SpeechRecognitionResult = new SpeechRecognitionResult(
undefined, // ResultId
ResultReason.Canceled,
Expand All @@ -330,7 +324,8 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
undefined, // Json
properties);
try {
cancelRecoCallback(result);
this.privSuccessCallback(result);
this.privSuccessCallback = undefined;
/* tslint:disable:no-empty */
} catch { }
}
Expand All @@ -341,9 +336,12 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
recoMode: RecognitionMode,
successCallback: (e: SpeechRecognitionResult) => void,
errorCallback: (e: string) => void
): any => {
): Promise<boolean> => {
this.privRecognizerConfig.recognitionMode = recoMode;

this.privSuccessCallback = successCallback;
this.privErrorCallback = errorCallback;

this.privDialogRequestSession.startNewRecognition();
this.privDialogRequestSession.listenForServiceTelemetry(this.privDialogAudioSource.events);

Expand All @@ -352,15 +350,13 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {

this.sendPreAudioMessages();

this.privSuccessCallback = successCallback;

return this.privDialogAudioSource
.attach(this.privDialogRequestSession.audioNodeId)
.continueWithPromise<boolean>((result: PromiseResult<IAudioStreamNode>) => {
let audioNode: ReplayableAudioNode;

if (result.isError) {
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.ConnectionFailure, result.error, successCallback);
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.ConnectionFailure, result.error);
return PromiseHelper.fromError<boolean>(result.error);
}

Expand All @@ -372,7 +368,12 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
this.privRecognizerConfig.SpeechServiceConfig.Context.audio = { source: deviceInfo };

return this.configConnection()
.on((_: IConnection) => {
.continueWithPromise<boolean>((result: PromiseResult<IConnection>): Promise<boolean> => {
if (result.isError) {
this.cancelRecognitionLocal(CancellationReason.Error, CancellationErrorCode.ConnectionFailure, result.error);
return PromiseHelper.fromError(result.error);
}

const sessionStartEventArgs: SessionEventArgs = new SessionEventArgs(this.privDialogRequestSession.sessionId);

if (!!this.privRecognizer.sessionStarted) {
Expand All @@ -383,25 +384,17 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {

// /* tslint:disable:no-empty */
audioSendPromise.on((_: boolean) => { /*add? return true;*/ }, (error: string) => {
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.RuntimeError, error, successCallback);
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.RuntimeError, error);
});

}, (error: string) => {
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.ConnectionFailure, error, successCallback);
}).continueWithPromise<boolean>((result: PromiseResult<IConnection>): Promise<boolean> => {
if (result.isError) {
return PromiseHelper.fromError(result.error);
} else {
return PromiseHelper.fromResult<boolean>(true);
}
return PromiseHelper.fromResult(true);
});
});
});
});
}

protected sendAudio = (
audioStreamNode: IAudioStreamNode): Promise<boolean> => {
protected sendAudio = (audioStreamNode: IAudioStreamNode): Promise<boolean> => {
return this.privDialogAudioSource.format.onSuccessContinueWithPromise<boolean>((audioFormat: AudioStreamFormatImpl) => {
// NOTE: Home-baked promises crash ios safari during the invocation
// of the error callback chain (looks like the recursion is way too deep, and
Expand Down Expand Up @@ -577,10 +570,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
return this.privDialogConnectionPromise;
}

private receiveDialogMessageOverride = (
successCallback?: (e: SpeechRecognitionResult) => void,
errorCallBack?: (e: string) => void
): Promise<IConnection> => {
private receiveDialogMessageOverride = (): Promise<IConnection> => {

// we won't rely on the cascading promises of the connection since we want to continually be available to receive messages
const communicationCustodian: Deferred<IConnection> = new Deferred<IConnection>();
Expand Down Expand Up @@ -611,6 +601,8 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
// turn started by the service
if (turnRequestId !== audioSessionReqId) {
this.privTurnStateManager.StartTurn(turnRequestId);
} else {
this.privDialogRequestSession.onServiceTurnStartResponse();
}
}
break;
Expand Down Expand Up @@ -673,25 +665,22 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
this.privSuccessCallback(this.privLastResult);
this.privLastResult = null;
} catch (e) {
if (!!errorCallBack) {
errorCallBack(e);
if (!!this.privErrorCallback) {
this.privErrorCallback(e);
}
}
// Only invoke the call back once.
// and if it's successful don't invoke the
// error after that.
this.privSuccessCallback = undefined;
errorCallBack = undefined;
this.privErrorCallback = undefined;
}
}
}
break;

default:
if (!this.processTypeSpecificMessages(
connectionMessage,
successCallback,
errorCallBack)) {
if (!this.processTypeSpecificMessages(connectionMessage)) {
if (!!this.serviceEvents) {
this.serviceEvents.onEvent(new ServiceEvent(connectionMessage.path.toLowerCase(), connectionMessage.textBody));
}
Expand All @@ -718,7 +707,7 @@ export class DialogServiceAdapter extends ServiceRecognizerBase {
return messageRetrievalPromise.on((r: IConnection) => {
return true;
}, (error: string) => {
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.RuntimeError, error, this.privSuccessCallback);
this.cancelRecognition(this.privDialogRequestSession.sessionId, this.privDialogRequestSession.requestId, CancellationReason.Error, CancellationErrorCode.RuntimeError, error);
});
}

Expand Down
37 changes: 17 additions & 20 deletions src/common.speech/IntentServiceRecognizer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,7 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
this.privIntentDataSent = true;
}

protected processTypeSpecificMessages(
connectionMessage: SpeechConnectionMessage,
successCallback?: (e: IntentRecognitionResult) => void,
errorCallBack?: (e: string) => void): boolean {
protected processTypeSpecificMessages(connectionMessage: SpeechConnectionMessage): boolean {

let result: IntentRecognitionResult;
let ev: IntentRecognitionEventArgs;
Expand Down Expand Up @@ -127,19 +124,19 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
}

// report result to promise.
if (!!successCallback) {
if (!!this.privSuccessCallback) {
try {
successCallback(result);
this.privSuccessCallback(result);
} catch (e) {
if (!!errorCallBack) {
errorCallBack(e);
if (!!this.privErrorCallback) {
this.privErrorCallback(e);
}
}
// Only invoke the call back once.
// and if it's successful don't invoke the
// error after that.
successCallback = undefined;
errorCallBack = undefined;
this.privSuccessCallback = undefined;
this.privErrorCallback = undefined;
}
};

Expand Down Expand Up @@ -224,19 +221,19 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
}

// report result to promise.
if (!!successCallback) {
if (!!this.privSuccessCallback) {
try {
successCallback(ev.result);
this.privSuccessCallback(ev.result);
} catch (e) {
if (!!errorCallBack) {
errorCallBack(e);
if (!!this.privErrorCallback) {
this.privErrorCallback(e);
}
}
// Only invoke the call back once.
// and if it's successful don't invoke the
// error after that.
successCallback = undefined;
errorCallBack = undefined;
this.privSuccessCallback = undefined;
this.privErrorCallback = undefined;
}
processed = true;
break;
Expand All @@ -252,8 +249,7 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
requestId: string,
cancellationReason: CancellationReason,
errorCode: CancellationErrorCode,
error: string,
cancelRecoCallback: (e: SpeechRecognitionResult) => void): void {
error: string): void {

const properties: PropertyCollection = new PropertyCollection();
properties.setProperty(CancellationErrorCodePropertyName, CancellationErrorCode[errorCode]);
Expand All @@ -273,7 +269,7 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
} catch { }
}

if (!!cancelRecoCallback) {
if (!!this.privSuccessCallback) {
const result: IntentRecognitionResult = new IntentRecognitionResult(
undefined, // Intent Id
requestId,
Expand All @@ -285,7 +281,8 @@ export class IntentServiceRecognizer extends ServiceRecognizerBase {
undefined, // Json
properties);
try {
cancelRecoCallback(result);
this.privSuccessCallback(result);
this.privSuccessCallback = undefined;
/* tslint:disable:no-empty */
} catch { }
}
Expand Down
Loading

0 comments on commit 0413695

Please sign in to comment.