voice - suspend keyword recognition when focus lost

src/vs/code/electron-main/app.ts

@@ -15,7 +15,7 @@ import { Event } from 'vs/base/common/event';
 import { stripComments } from 'vs/base/common/json';
 import { getPathLabel } from 'vs/base/common/labels';
 import { Disposable, DisposableStore } from 'vs/base/common/lifecycle';
-import { Schemas } from 'vs/base/common/network';
+import { Schemas, VSCODE_AUTHORITY } from 'vs/base/common/network';
 import { isAbsolute, join, posix } from 'vs/base/common/path';
 import { IProcessEnvironment, isLinux, isLinuxSnap, isMacintosh, isWindows, OS } from 'vs/base/common/platform';
 import { assertType } from 'vs/base/common/types';
@@ -118,7 +118,7 @@ import { ElectronPtyHostStarter } from 'vs/platform/terminal/electron-main/elect
 import { PtyHostService } from 'vs/platform/terminal/node/ptyHostService';
 import { NODE_REMOTE_RESOURCE_CHANNEL_NAME, NODE_REMOTE_RESOURCE_IPC_METHOD_NAME, NodeRemoteResourceResponse, NodeRemoteResourceRouter } from 'vs/platform/remote/common/electronRemoteResources';
 import { Lazy } from 'vs/base/common/lazy';
-import { IAuxiliaryWindowsMainService, isAuxiliaryWindow } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindows';
+import { IAuxiliaryWindowsMainService } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindows';
 import { AuxiliaryWindowsMainService } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindowsMainService';
 
 /**
@@ -392,7 +392,7 @@ export class CodeApplication extends Disposable {
 		app.on('web-contents-created', (event, contents) => {
 
 			// Auxiliary Window: delegate to `AuxiliaryWindow` class
-			if (isAuxiliaryWindow(contents)) {
+			if (contents?.opener?.url.startsWith(`${Schemas.vscodeFileResource}://${VSCODE_AUTHORITY}/`)) {
 				this.logService.trace('[aux window]  app.on("web-contents-created"): Registering auxiliary window');
 
 				this.auxiliaryWindowsMainService?.registerWindow(contents);

src/vs/platform/auxiliaryWindow/electron-main/auxiliaryWindows.ts

@@ -5,7 +5,6 @@
 
 import { BrowserWindowConstructorOptions, HandlerDetails, WebContents } from 'electron';
 import { Event } from 'vs/base/common/event';
-import { Schemas, VSCODE_AUTHORITY } from 'vs/base/common/network';
 import { IAuxiliaryWindow } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindow';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
 
@@ -30,7 +29,3 @@ export interface IAuxiliaryWindowsMainService {
 
 	getWindows(): readonly IAuxiliaryWindow[];
 }
-
-export function isAuxiliaryWindow(webContents: WebContents): boolean {
-	return webContents?.opener?.url.startsWith(`${Schemas.vscodeFileResource}://${VSCODE_AUTHORITY}/`);
-}

src/vs/platform/native/electron-main/nativeHostMainService.ts

@@ -40,7 +40,7 @@ import { VSBuffer } from 'vs/base/common/buffer';
 import { hasWSLFeatureInstalled } from 'vs/platform/remote/node/wsl';
 import { WindowProfiler } from 'vs/platform/profiling/electron-main/windowProfiling';
 import { IV8Profile } from 'vs/platform/profiling/common/profiling';
-import { IAuxiliaryWindowsMainService, isAuxiliaryWindow } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindows';
+import { IAuxiliaryWindowsMainService } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindows';
 import { IAuxiliaryWindow } from 'vs/platform/auxiliaryWindow/electron-main/auxiliaryWindow';
 import { CancellationError } from 'vs/base/common/errors';
 
@@ -105,11 +105,11 @@ export class NativeHostMainService extends Disposable implements INativeHostMain
 
 	readonly onDidBlurMainOrAuxiliaryWindow = Event.any(
 		this.onDidBlurMainWindow,
-		Event.map(Event.filter(Event.fromNodeEventEmitter(app, 'browser-window-blur', (event, window: BrowserWindow) => window), window => isAuxiliaryWindow(window.webContents)), window => window.id)
+		Event.filter(Event.fromNodeEventEmitter(app, 'browser-window-blur', (event, window: BrowserWindow) => window.id), windowId => !!this.auxiliaryWindowsMainService.getWindowById(windowId))
 	);
 	readonly onDidFocusMainOrAuxiliaryWindow = Event.any(
 		this.onDidFocusMainWindow,
-		Event.map(Event.filter(Event.fromNodeEventEmitter(app, 'browser-window-focus', (event, window: BrowserWindow) => window), window => isAuxiliaryWindow(window.webContents)), window => window.id)
+		Event.filter(Event.fromNodeEventEmitter(app, 'browser-window-focus', (event, window: BrowserWindow) => window.id), windowId => !!this.auxiliaryWindowsMainService.getWindowById(windowId))
 	);
 
 	readonly onDidResumeOS = Event.fromNodeEventEmitter(powerMonitor, 'resume');

src/vs/workbench/contrib/speech/common/speech.contribution.ts → src/vs/workbench/contrib/speech/browser/speech.contribution.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions';
-import { ISpeechService, SpeechService } from 'vs/workbench/contrib/speech/common/speechService';
+import { ISpeechService } from 'vs/workbench/contrib/speech/common/speechService';
+import { SpeechService } from 'vs/workbench/contrib/speech/browser/speechService';
 
 registerSingleton(ISpeechService, SpeechService, InstantiationType.Delayed);

src/vs/workbench/contrib/speech/browser/speechService.ts

@@ -0,0 +1,210 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { firstOrDefault } from 'vs/base/common/arrays';
+import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation';
+import { Emitter, Event } from 'vs/base/common/event';
+import { Disposable, DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
+import { IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
+import { ILogService } from 'vs/platform/log/common/log';
+import { IHostService } from 'vs/workbench/services/host/browser/host';
+import { DeferredPromise } from 'vs/base/common/async';
+import { ISpeechService, ISpeechProvider, HasSpeechProvider, ISpeechToTextSession, SpeechToTextInProgress, IKeywordRecognitionSession, KeywordRecognitionStatus, SpeechToTextStatus } from 'vs/workbench/contrib/speech/common/speechService';
+
+export class SpeechService extends Disposable implements ISpeechService {
+
+	readonly _serviceBrand: undefined;
+
+	private readonly _onDidRegisterSpeechProvider = this._register(new Emitter<ISpeechProvider>());
+	readonly onDidRegisterSpeechProvider = this._onDidRegisterSpeechProvider.event;
+
+	private readonly _onDidUnregisterSpeechProvider = this._register(new Emitter<ISpeechProvider>());
+	readonly onDidUnregisterSpeechProvider = this._onDidUnregisterSpeechProvider.event;
+
+	get hasSpeechProvider() { return this.providers.size > 0; }
+
+	private readonly providers = new Map<string, ISpeechProvider>();
+
+	private readonly hasSpeechProviderContext = HasSpeechProvider.bindTo(this.contextKeyService);
+
+	constructor(
+		@ILogService private readonly logService: ILogService,
+		@IContextKeyService private readonly contextKeyService: IContextKeyService,
+		@IHostService private readonly hostService: IHostService
+	) {
+		super();
+	}
+
+	registerSpeechProvider(identifier: string, provider: ISpeechProvider): IDisposable {
+		if (this.providers.has(identifier)) {
+			throw new Error(`Speech provider with identifier ${identifier} is already registered.`);
+		}
+
+		this.providers.set(identifier, provider);
+		this.hasSpeechProviderContext.set(true);
+
+		this._onDidRegisterSpeechProvider.fire(provider);
+
+		return toDisposable(() => {
+			this.providers.delete(identifier);
+			this._onDidUnregisterSpeechProvider.fire(provider);
+
+			if (this.providers.size === 0) {
+				this.hasSpeechProviderContext.set(false);
+			}
+		});
+	}
+
+	private readonly _onDidStartSpeechToTextSession = this._register(new Emitter<void>());
+	readonly onDidStartSpeechToTextSession = this._onDidStartSpeechToTextSession.event;
+
+	private readonly _onDidEndSpeechToTextSession = this._register(new Emitter<void>());
+	readonly onDidEndSpeechToTextSession = this._onDidEndSpeechToTextSession.event;
+
+	private _activeSpeechToTextSession: ISpeechToTextSession | undefined = undefined;
+	get hasActiveSpeechToTextSession() { return !!this._activeSpeechToTextSession; }
+
+	private readonly speechToTextInProgress = SpeechToTextInProgress.bindTo(this.contextKeyService);
+
+	createSpeechToTextSession(token: CancellationToken): ISpeechToTextSession {
+		const provider = firstOrDefault(Array.from(this.providers.values()));
+		if (!provider) {
+			throw new Error(`No Speech provider is registered.`);
+		} else if (this.providers.size > 1) {
+			this.logService.warn(`Multiple speech providers registered. Picking first one: ${provider.metadata.displayName}`);
+		}
+
+		const session = this._activeSpeechToTextSession = provider.createSpeechToTextSession(token);
+
+		const disposables = new DisposableStore();
+
+		const onSessionStoppedOrCanceled = () => {
+			if (session === this._activeSpeechToTextSession) {
+				this._activeSpeechToTextSession = undefined;
+				this.speechToTextInProgress.reset();
+				this._onDidEndSpeechToTextSession.fire();
+			}
+
+			disposables.dispose();
+		};
+
+		disposables.add(token.onCancellationRequested(() => onSessionStoppedOrCanceled()));
+		if (token.isCancellationRequested) {
+			onSessionStoppedOrCanceled();
+		}
+
+		disposables.add(session.onDidChange(e => {
+			switch (e.status) {
+				case SpeechToTextStatus.Started:
+					if (session === this._activeSpeechToTextSession) {
+						this.speechToTextInProgress.set(true);
+						this._onDidStartSpeechToTextSession.fire();
+					}
+					break;
+				case SpeechToTextStatus.Stopped:
+					onSessionStoppedOrCanceled();
+					break;
+			}
+		}));
+
+		return session;
+	}
+
+	private readonly _onDidStartKeywordRecognition = this._register(new Emitter<void>());
+	readonly onDidStartKeywordRecognition = this._onDidStartKeywordRecognition.event;
+
+	private readonly _onDidEndKeywordRecognition = this._register(new Emitter<void>());
+	readonly onDidEndKeywordRecognition = this._onDidEndKeywordRecognition.event;
+
+	private _activeKeywordRecognitionSession: IKeywordRecognitionSession | undefined = undefined;
+	get hasActiveKeywordRecognition() { return !!this._activeKeywordRecognitionSession; }
+
+	async recognizeKeyword(token: CancellationToken): Promise<KeywordRecognitionStatus> {
+		const result = new DeferredPromise<KeywordRecognitionStatus>();
+
+		const disposables = new DisposableStore();
+		disposables.add(token.onCancellationRequested(() => {
+			disposables.dispose();
+			result.complete(KeywordRecognitionStatus.Canceled);
+		}));
+
+		const recognizeKeywordDisposables = disposables.add(new DisposableStore());
+		let activeRecognizeKeywordSession: Promise<void> | undefined = undefined;
+		const recognizeKeyword = () => {
+			recognizeKeywordDisposables.clear();
+
+			const cts = new CancellationTokenSource(token);
+			recognizeKeywordDisposables.add(toDisposable(() => cts.dispose(true)));
+			const currentRecognizeKeywordSession = activeRecognizeKeywordSession = this.doRecognizeKeyword(cts.token).then(status => {
+				if (currentRecognizeKeywordSession === activeRecognizeKeywordSession) {
+					result.complete(status);
+				}
+			}, error => {
+				if (currentRecognizeKeywordSession === activeRecognizeKeywordSession) {
+					result.error(error);
+				}
+			});
+		};
+
+		disposables.add(this.hostService.onDidChangeFocus(focused => {
+			if (!focused && activeRecognizeKeywordSession) {
+				recognizeKeywordDisposables.clear();
+				activeRecognizeKeywordSession = undefined;
+			} else if (!activeRecognizeKeywordSession) {
+				recognizeKeyword();
+			}
+		}));
+
+		if (this.hostService.hasFocus) {
+			recognizeKeyword();
+		}
+
+		try {
+			return await result.p;
+		} finally {
+			disposables.dispose();
+		}
+	}
+
+	private async doRecognizeKeyword(token: CancellationToken): Promise<KeywordRecognitionStatus> {
+		const provider = firstOrDefault(Array.from(this.providers.values()));
+		if (!provider) {
+			throw new Error(`No Speech provider is registered.`);
+		} else if (this.providers.size > 1) {
+			this.logService.warn(`Multiple speech providers registered. Picking first one: ${provider.metadata.displayName}`);
+		}
+
+		const session = this._activeKeywordRecognitionSession = provider.createKeywordRecognitionSession(token);
+		this._onDidStartKeywordRecognition.fire();
+
+		const disposables = new DisposableStore();
+
+		const onSessionStoppedOrCanceled = () => {
+			if (session === this._activeKeywordRecognitionSession) {
+				this._activeKeywordRecognitionSession = undefined;
+				this._onDidEndKeywordRecognition.fire();
+			}
+
+			disposables.dispose();
+		};
+
+		disposables.add(token.onCancellationRequested(() => onSessionStoppedOrCanceled()));
+		if (token.isCancellationRequested) {
+			onSessionStoppedOrCanceled();
+		}
+
+		disposables.add(session.onDidChange(e => {
+			if (e.status === KeywordRecognitionStatus.Stopped) {
+				onSessionStoppedOrCanceled();
+			}
+		}));
+
+		try {
+			return (await Event.toPromise(session.onDidChange)).status;
+		} finally {
+			onSessionStoppedOrCanceled();
+		}
+	}
+}