Add VAD platform capability through webrtcvad

stanford-oval · May 10, 2021 · c79cf9f · c79cf9f
1 parent 6e83261
commit c79cf9f
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 5 deletions.
diff --git a/config.js b/config.js
@@ -78,6 +78,4 @@ module.exports.ENABLE_DB_ENCRYPTION = false;
 module.exports.SEMPRE_URL = process.env.THINGENGINE_NLP_URL || 'https://nlp-staging.almond.stanford.edu';
 module.exports.THINGPEDIA_URL = process.env.THINGPEDIA_URL || 'https://dev.almond.stanford.edu/thingpedia';
 module.exports.CLOUD_SYNC_URL = process.env.THINGENGINE_CLOUD_SYNC_URL || 'https://dev.almond.stanford.edu';
-module.exports.MS_SPEECH_RECOGNITION_PRIMARY_KEY = 'de1f02817356494483ba502b2ce95f6f';
-module.exports.MS_SPEECH_RECOGNITION_SECONDARY_KEY = '3dc6ce0b832940f0b0c984a1517c457e';
-
+module.exports.NL_URL = process.env.THINGENGINE_NL_URL || 'https://nlp-staging.almond.stanford.edu';
diff --git a/main.js b/main.js
@@ -63,7 +63,7 @@ async function init(platform) {
 
     if (platform.hasCapability('sound')) {
         const speech = new Genie.SpeechHandler(conversation, platform, {
-            subscriptionKey: Config.MS_SPEECH_RECOGNITION_PRIMARY_KEY
+            nlUrl: Config.NL_URL
         });
 
         let play;

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -38,7 +38,8 @@
   "optionalDependencies": {
     "canberra": "^0.1.2",
     "pulseaudio2": "^0.5.0",
-    "snowboy": "^1.3.1"
+    "snowboy": "^1.3.1",
+    "webrtcvad": "^1.0.1"
   },
   "devDependencies": {
     "coveralls": "^3.1.0",

diff --git a/service/platform/index.js b/service/platform/index.js
@@ -243,6 +243,38 @@ class SoundEffectsApi {
     }
 }
 
+let webrtcvad;
+try {
+    webrtcvad = require('webrtcvad').default;
+} catch(e) {
+    console.log("VAD not available");
+    webrtcvad = null;
+}
+
+class VAD {
+    constructor() {
+        this._instance = null;
+    }
+
+    setup(bitrate, level) {
+        if (this._instance)
+            this._instance = null;
+
+        if (webrtcvad) {
+            this._instance = new webrtcvad(bitrate, level);
+            // console.log("setup VAD bitrate", bitrate, "level", level);
+            return true;
+        }
+
+        return false;
+    }
+
+    process(chunk) {
+        if (!this._instance)
+            return false;
+        return this._instance.process(chunk);
+    }
+}
 
 class ServerPlatform extends Tp.BasePlatform {
     constructor() {
@@ -260,6 +292,7 @@ class ServerPlatform extends Tp.BasePlatform {
         safeMkdirSync(this._cacheDir);
 
         this._wakeWordDetector = null;
+        this._voiceDetector = null;
         this._soundEffects = null;
 
         this._sqliteKey = null;
@@ -317,6 +350,9 @@ class ServerPlatform extends Tp.BasePlatform {
 
             if (canberra)
                 this._soundEffects = new SoundEffectsApi();
+
+            if (webrtcvad && VAD)
+                this._voiceDetector = new VAD();
         } else {
             this._pulse = null;
         }
@@ -373,6 +409,8 @@ class ServerPlatform extends Tp.BasePlatform {
         case 'wakeword-detector':
             this._ensurePulseAudio();
             return this._wakeWordDetector !== null;
+        case 'voice-detector':
+            return this._voiceDetector !== null;
 
         case 'sound-effects':
             this._ensurePulseAudio();
@@ -400,6 +438,8 @@ class ServerPlatform extends Tp.BasePlatform {
         case 'wakeword-detector':
             this._ensurePulseAudio();
             return this._wakeWordDetector;
+        case 'voice-detector':
+            return this._voiceDetector;
         case 'sound-effects':
             this._ensurePulseAudio();
             return this._soundEffects;