use trusted URL for AudioWorklet (#732)

microsoft · Sep 19, 2023 · 6a9273d · 6a9273d
1 parent a34b157
commit 6a9273d
Show file tree

Hide file tree

Showing 36 changed files with 191 additions and 73 deletions.
diff --git a/gulpfile.js b/gulpfile.js
@@ -10,10 +10,6 @@
   var webpack = require('webpack-stream');
   var dtsBundleWebpack = require('dts-bundle-webpack');
   var tsProject = ts.createProject('tsconfig.json');
-  var tsProject2015 = ts.createProject('tsconfig.json', {
-    target: 'es2015',
-    module: 'esnext'
-  });
 
   gulp.task('build', gulp.series(function build() {
     return gulp.src([
@@ -30,29 +26,18 @@
       .pipe(tsProject())
       .pipe(sourcemaps.write('.'))
       .pipe(gulp.dest('distrib/lib'));
-  }));
-
-  gulp.task('build2015', gulp.series(function build() {
-    return gulp.src([
-      'src/**/*.ts',
-      'microsoft.cognitiveservices.speech.sdk.ts'],
-      { base: '.' })
-      .pipe(eslint({
-        formatter: 'prose',
-        configuration: 'eslint.json'
-      }))
-      .pipe(eslint.format())
-      .pipe(eslint.failAfterError())
-      .pipe(sourcemaps.init())
-      .pipe(tsProject2015())
-      .pipe(sourcemaps.write('.'))
-      .pipe(gulp.dest('distrib/es2015'));
+    }, function () {
+      return gulp.src('./src/audioworklet/speech-processor.js')
+        .pipe(gulp.dest('./distrib/lib/src/common.browser'));
   }));
 
   gulp.task('bundle', gulp.series('build', function bundle() {
     return gulp.src('bundleApp.js')
       .pipe(webpack({
-        output: { filename: 'microsoft.cognitiveservices.speech.sdk.bundle.js' },
+        entry: { 
+          'microsoft.cognitiveservices.speech.sdk.bundle': './bundleApp.js',
+        },
+        output: { filename: '[name].js' },
         devtool: 'source-map',
         module: {
           rules: [{

diff --git a/jest.config.js b/jest.config.js
@@ -8,7 +8,7 @@ module.exports = {
                 "^.+\\.ts$": "ts-jest",
             },
             testRegex: "tests/.*Tests\\.ts$",
-            testPathIgnorePatterns: ["/lib/", "/es2015/", "/node_modules/", "/src/"],
+            testPathIgnorePatterns: ["/lib/", "/node_modules/", "/src/"],
             moduleFileExtensions: ["ts", "js", "jsx", "json", "node"],
             testEnvironment: "jsdom",
             collectCoverage: false,
@@ -21,7 +21,7 @@ module.exports = {
                 "^.+\\.ts$": "ts-jest",
             },
             testRegex: "tests/.*Tests\\.ts$",
-            testPathIgnorePatterns: ["/lib/", "/es2015/", "/node_modules/", "/src/"],
+            testPathIgnorePatterns: ["/lib/", "/node_modules/", "/src/"],
             moduleFileExtensions: ["ts", "js", "jsx", "json", "node"],
             testEnvironment: "node",
             collectCoverage: false,

diff --git a/package.json b/package.json
@@ -38,11 +38,10 @@
     "net": false
   },
   "main": "distrib/lib/microsoft.cognitiveservices.speech.sdk.js",
-  "module": "distrib/es2015/microsoft.cognitiveservices.speech.sdk.js",
+  "module": "distrib/lib/microsoft.cognitiveservices.speech.sdk.js",
   "types": "distrib/lib/microsoft.cognitiveservices.speech.sdk.d.ts",
   "files": [
     "distrib/lib/**/*",
-    "distrib/es2015/**/*",
     "distrib/browser/**/*",
     "LICENSE",
     "REDIST.txt"
@@ -83,7 +82,7 @@
     "webpack-stream": "^7.0.0"
   },
   "scripts": {
-    "build": "gulp compress && gulp build2015",
+    "build": "gulp compress && gulp build",
     "test": "npm run lint && npm run jest --coverage",
     "jest": "jest",
     "lint": "eslint -c .eslintrc.js --ext .ts src",

diff --git a/src/common.browser/AudioWorkerUrl.ts b/src/common.browser/AudioWorkerUrl.ts
@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+/* webpackChunkName: 'script_processor_audioWorklet' */
+// eslint-disable-next-line @typescript-eslint/tslint/config
+export const getAudioWorkerUrl = (): string => new URL("speech-processor.js", import.meta.url).toString();
diff --git a/src/common.browser/PCMRecorder.ts b/src/common.browser/PCMRecorder.ts
@@ -2,6 +2,7 @@
 // Licensed under the MIT license.
 
 import { RiffPcmEncoder, Stream } from "../common/Exports";
+import { getAudioWorkerUrl } from "./AudioWorkerUrl";
 import { IRecorder } from "./IRecorder";
 
 export class PcmRecorder implements IRecorder {
@@ -60,59 +61,70 @@ export class PcmRecorder implements IRecorder {
             };
         };
 
+        const connectWorkletToMicInput = (context: AudioContext): void => {
+            const workletNode = new AudioWorkletNode(context, "speech-processor");
+            workletNode.port.onmessage = (ev: MessageEvent): void => {
+                const inputFrame: Float32Array = ev.data as Float32Array;
+
+                if (outputStream && !outputStream.isClosed) {
+                    const waveFrame = waveStreamEncoder.encode(inputFrame);
+                    if (!!waveFrame) {
+                        outputStream.writeStreamChunk({
+                            buffer: waveFrame,
+                            isEnd: false,
+                            timeReceived: Date.now(),
+                        });
+                    }
+                }
+            };
+            micInput.connect(workletNode);
+            workletNode.connect(context.destination);
+            this.privMediaResources = {
+                scriptProcessorNode: workletNode,
+                source: micInput,
+                stream: mediaStream,
+            };
+        };
+
         // https://webaudio.github.io/web-audio-api/#audioworklet
         // Using AudioWorklet to improve audio quality and avoid audio glitches due to blocking the UI thread
         const skipAudioWorklet = !!this.privSpeechProcessorScript && this.privSpeechProcessorScript.toLowerCase() === "ignore";
 
         if (!!context.audioWorklet && !skipAudioWorklet) {
-            if (!this.privSpeechProcessorScript) {
-                const workletScript = `class SP extends AudioWorkletProcessor {
-                    constructor(options) {
-                      super(options);
-                    }
-                    process(inputs, outputs) {
-                      const input = inputs[0];
-                      const output = [];
-                      for (let channel = 0; channel < input.length; channel += 1) {
-                        output[channel] = input[channel];
-                      }
-                      this.port.postMessage(output[0]);
-                      return true;
-                    }
-                  }
-                  registerProcessor('speech-processor', SP);`;
-                const blob = new Blob([workletScript], { type: "application/javascript; charset=utf-8" });
-                this.privSpeechProcessorScript = URL.createObjectURL(blob);
-            }
+            this.privSpeechProcessorScript = getAudioWorkerUrl();
 
             context.audioWorklet
                 .addModule(this.privSpeechProcessorScript)
                 .then((): void => {
-                    const workletNode = new AudioWorkletNode(context, "speech-processor");
-                    workletNode.port.onmessage = (ev: MessageEvent): void => {
-                        const inputFrame: Float32Array = ev.data as Float32Array;
-
-                        if (outputStream && !outputStream.isClosed) {
-                            const waveFrame = waveStreamEncoder.encode(inputFrame);
-                            if (!!waveFrame) {
-                                outputStream.writeStreamChunk({
-                                    buffer: waveFrame,
-                                    isEnd: false,
-                                    timeReceived: Date.now(),
-                                });
-                            }
-                        }
-                    };
-                    micInput.connect(workletNode);
-                    workletNode.connect(context.destination);
-                    this.privMediaResources = {
-                        scriptProcessorNode: workletNode,
-                        source: micInput,
-                        stream: mediaStream,
-                    };
+                    connectWorkletToMicInput(context);
                 })
                 .catch((): void => {
-                    attachScriptProcessor();
+                    const workletScript = `class SP extends AudioWorkletProcessor {
+                        constructor(options) {
+                        super(options);
+                        }
+                        process(inputs, outputs) {
+                        const input = inputs[0];
+                        const output = [];
+                        for (let channel = 0; channel < input.length; channel += 1) {
+                            output[channel] = input[channel];
+                        }
+                        this.port.postMessage(output[0]);
+                        return true;
+                        }
+                    }
+                    registerProcessor('speech-processor', SP);`;
+                    const blob = new Blob([workletScript], { type: "application/javascript; charset=utf-8" });
+                    this.privSpeechProcessorScript = URL.createObjectURL(blob);
+
+                    context.audioWorklet
+                        .addModule(this.privSpeechProcessorScript)
+                        .then((): void => {
+                            connectWorkletToMicInput(context);
+                        })
+                        .catch((): void => {
+                            attachScriptProcessor();
+                        });
                 });
         } else {
             try {

diff --git a/src/common.browser/WebsocketMessageAdapter.ts b/src/common.browser/WebsocketMessageAdapter.ts
@@ -3,7 +3,6 @@
 
 // Node.JS specific web socket / browser support.
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore
 import * as http from "http";
 import * as net from "net";
 import * as tls from "tls";

diff --git a/tests/AudioOutputStreamTests.ts b/tests/AudioOutputStreamTests.ts
@@ -7,6 +7,9 @@ import { Settings } from "./Settings";
 import { closeAsyncObjects } from "./Utilities";
 
 let objsToClose: any[];
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
 
 beforeAll(() => {
     // Override inputs, if necessary

diff --git a/tests/AutoSourceLangDetectionTests.ts b/tests/AutoSourceLangDetectionTests.ts
@@ -27,6 +27,10 @@ import { Settings } from "./Settings";
 import { closeAsyncObjects, WaitForCondition } from "./Utilities";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 let objsToClose: any[];
 
 beforeAll((): void => {

diff --git a/tests/ConnectionTests.ts b/tests/ConnectionTests.ts
@@ -22,6 +22,10 @@ import {
 
 import * as fs from "fs";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 let objsToClose: any[];
 
 beforeAll(() => {

diff --git a/tests/ConversationTranscriberTests.ts b/tests/ConversationTranscriberTests.ts
@@ -23,6 +23,10 @@ import { Settings } from "./Settings";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 import { closeAsyncObjects, RepeatingPullStream, WaitForCondition } from "./Utilities";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 let objsToClose: any[];
 
 beforeAll(() => {

diff --git a/tests/ConversationTranslatorTests.ts b/tests/ConversationTranslatorTests.ts
@@ -26,6 +26,10 @@ import {
 } from "./Utilities";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 // eslint-disable-next-line no-console
 const consoleInfo = console.info;
 

diff --git a/tests/DiagnosticsTests.ts b/tests/DiagnosticsTests.ts
@@ -9,6 +9,10 @@ import { closeAsyncObjects, WaitForCondition } from "./Utilities";
 
 let objsToClose: any[];
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll((): void => {
     // Override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/DialogServiceConnectorTests.ts b/tests/DialogServiceConnectorTests.ts
@@ -46,6 +46,10 @@ import {
 } from "./Utilities";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 // eslint-disable-next-line no-console
 const consoleInfo = console.info;
 const simpleMessageObj = { speak: "This is speech", text: "This is text", type: "message" };

diff --git a/tests/DynamicGrammarTests.ts b/tests/DynamicGrammarTests.ts
@@ -9,6 +9,10 @@ import {
 } from "../src/common.speech/Exports";
 import { Settings } from "./Settings";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll(() => {
     // Override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/GeneralRecognizerTests.ts b/tests/GeneralRecognizerTests.ts
@@ -5,6 +5,11 @@ import * as sdk from "../microsoft.cognitiveservices.speech.sdk";
 import { Settings } from "./Settings";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
+let bufferSize: number;
 beforeEach(() => {
     // eslint-disable-next-line no-console
     console.info("-------------------Starting test case: " + expect.getState().currentTestName + "---------------");

diff --git a/tests/IntentRecognizerTests.ts b/tests/IntentRecognizerTests.ts
@@ -18,6 +18,11 @@ import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 import { AudioStreamFormatImpl } from "../src/sdk/Audio/AudioStreamFormat";
 
 let objsToClose: any[];
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
+let bufferSize: number;
 
 beforeAll(() => {
     // override inputs, if necessary

diff --git a/tests/LanguageModelTests.ts b/tests/LanguageModelTests.ts
@@ -5,6 +5,10 @@ import * as sdk from "../microsoft.cognitiveservices.speech.sdk";
 import { LanguageUnderstandingModelImpl } from "../src/sdk/LanguageUnderstandingModel";
 import { Settings } from "./Settings";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll(() => {
     // Override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/LongRunning/SpeechRecoAuthTokenErrorMessageTests.ts b/tests/LongRunning/SpeechRecoAuthTokenErrorMessageTests.ts
@@ -9,6 +9,9 @@ import { Settings } from "../Settings";
 import { CreateRepeatingPullStream, WaitForCondition } from "../Utilities";
 
 let objsToClose: any[];
+jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
 
 beforeAll(() => {
     // override inputs, if necessary

diff --git a/tests/LongRunning/SpeechRecoAuthTokenRefreshTests.ts b/tests/LongRunning/SpeechRecoAuthTokenRefreshTests.ts
@@ -11,6 +11,10 @@ import { CreateRepeatingPullStream, WaitForCondition } from "../Utilities";
 
 let objsToClose: any[];
 
+jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll(() => {
     // override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/LongRunning/SpeechRecoReconnectTests.ts b/tests/LongRunning/SpeechRecoReconnectTests.ts
@@ -11,6 +11,10 @@ import { WaitForCondition } from "../Utilities";
 
 let objsToClose: any[];
 
+jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll((): void => {
     // override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/LongRunning/TranslationRecoReconnectTests.ts b/tests/LongRunning/TranslationRecoReconnectTests.ts
@@ -12,6 +12,10 @@ import { WaitForCondition } from "../Utilities";
 
 let objsToClose: any[];
 
+jest.mock("../../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 beforeAll((): void => {
     // override inputs, if necessary
     Settings.LoadSettings();

diff --git a/tests/MeetingTranscriberTests.ts b/tests/MeetingTranscriberTests.ts
@@ -13,6 +13,10 @@ import { Settings } from "./Settings";
 import { closeAsyncObjects } from "./Utilities";
 import { WaveFileAudioInput } from "./WaveFileAudioInputStream";
 
+jest.mock("../src/common.browser/AudioWorkerUrl", () => ({
+   getAudioWorkerUrl: (): string => "speech-processor.js"
+}));
+
 let objsToClose: any[];
 
 function sleep(milliseconds: number): Promise<any> {