From 75e9d60d685797eb50ec98d910f2bf6a0231dc5e Mon Sep 17 00:00:00 2001 From: Peter Joles Date: Tue, 9 Jun 2020 05:01:29 -0700 Subject: [PATCH] :sparkles: feat: Add ability to has TTS delivered through AWS Polly --- config/default.js | 3 + src/components/ConfigAddEditSolution.vue | 387 +++++++++++++++++++++-- src/constants/solution-config-default.js | 2 + src/store.js | 47 ++- src/utils/asr-tts.js | 51 ++- src/utils/buildConfig.js | 6 + src/utils/polly.js | 22 ++ src/utils/utils.js | 6 +- 8 files changed, 474 insertions(+), 50 deletions(-) create mode 100644 src/utils/polly.js diff --git a/config/default.js b/config/default.js index c648d3cf..f7d23137 100644 --- a/config/default.js +++ b/config/default.js @@ -94,6 +94,9 @@ const config = { sourceFile: ".env.solution.json" // relative path to your solution(s) config file - probably don't need to change } }, + tts: { + url: "" // Jaguar URL if you plan to use AWS Polly instead of default Web Speech APIs TTS + }, ui: { configArea: { shareLink: { diff --git a/src/components/ConfigAddEditSolution.vue b/src/components/ConfigAddEditSolution.vue index 01853045..15b03d24 100644 --- a/src/components/ConfigAddEditSolution.vue +++ b/src/components/ConfigAddEditSolution.vue @@ -168,7 +168,11 @@ - + - + + + + + + + + + + + + + + + + + + + Delay Teneo Responses in Milliseconds @@ -1026,24 +1076,282 @@ export default { "custom3" ], trueFalseOptions: ["true", "false"], - locales: [ - "en-us-male", - "en-us-female", - "en-uk-male", - "en-uk-female", - "fr", - "es", - "nl", - "it", - "de", - "ru", - "sv", - "no", - "da", - "jp", - "cn", - "cn(hk)", - "id" + ttsEngines: ["Web Speech API", "AWS Polly"], + webSpeechLanguages: { + Afrikaans: [["South Africa", "af-ZA"]], + Arabic: [ + ["Algeria", "ar-DZ"], + ["Bahrain", "ar-BH"], + ["Egypt", "ar-EG"], + ["Israel", "ar-IL"], + ["Iraq", "ar-IQ"], + ["Jordan", "ar-JO"], + ["Kuwait", "ar-KW"], + ["Lebanon", "ar-LB"], + ["Morocco", "ar-MA"], + ["Oman", "ar-OM"], + ["Palestinian Territory", "ar-PS"], + ["Qatar", "ar-QA"], + ["Saudi Arabia", "ar-SA"], + ["Tunisia", "ar-TN"], + ["UAE", "ar-AE"] + ], + Basque: [["Spain", "eu-ES"]], + Bulgarian: [["Bulgaria", "bg-BG"]], + Catalan: [["Spain", "ca-ES"]], + "Chinese Mandarin": [ + ["China (Simp.)", "cmn-Hans-CN"], + ["Hong Kong SAR (Trad.)", "cmn-Hans-HK"], + ["Taiwan (Trad.)", "cmn-Hant-TW"] + ], + "Chinese Cantonese": [["Hong Kong", "yue-Hant-HK"]], + Croatian: [["Croatia", "hr_HR"]], + Czech: [["Czech Republic", "cs-CZ"]], + Danish: [["Denmark", "da-DK"]], + English: [ + ["Australia", "en-AU"], + ["Canada", "en-CA"], + ["India", "en-IN"], + ["Ireland", "en-IE"], + ["New Zealand", "en-NZ"], + ["Philippines", "en-PH"], + ["South Africa", "en-ZA"], + ["United Kingdom", "en-GB"], + ["United States", "en-US"] + ], + Farsi: [["Iran", "fa-IR"]], + French: [["France", "fr-FR"]], + Filipino: [["Philippines", "fil-PH"]], + Galician: [["Spain", "gl-ES"]], + German: [["Germany", "de-DE"]], + Greek: [["Greece", "el-GR"]], + Finnish: [["Finland", "fi-FI"]], + Hebrew: [["Israel", "he-IL"]], + Hindi: [["India", "hi-IN"]], + Hungarian: [["Hungary", "hu-HU"]], + Indonesian: [["Indonesia", "id-ID"]], + Icelandic: [["Iceland", "is-IS"]], + Italian: [ + ["Italy", "it-IT"], + ["Switzerland", "it-CH"] + ], + Japanese: [["Japan", "ja-JP"]], + Korean: [["Korea", "ko-KR"]], + Lithuanian: [["Lithuania", "lt-LT"]], + Malaysian: [["Malaysia", "ms-MY"]], + Dutch: [["Netherlands", "nl-NL"]], + Norwegian: [["Norway", "nb-NO"]], + Polish: [["Poland", "pl-PL"]], + Portuguese: [ + ["Brazil", "pt-BR"], + ["Portugal", "pt-PT"] + ], + Romanian: [["Romania", "ro-RO"]], + Russian: [["Russia", "ru-RU"]], + Serbian: [["Serbia", "sr-RS"]], + Slovak: [["Slovakia", "sk-SK"]], + Slovenian: [["Slovenia", "sl-SI"]], + Spanish: [ + ["Argentina", "es-AR"], + ["Bolivia", "es-BO"], + ["Chile", "es-CL"], + ["Colombia", "es-CO"], + ["Costa Rica", "es-CR"], + ["Dominican Republic", "es-DO"], + ["Ecuador", "es-EC"], + ["El Salvador", "es-SV"], + ["Guatemala", "es-GT"], + ["Honduras", "es-HN"], + ["México", "es-MX"], + ["Nicaragua", "es-NI"], + ["Panamá", "es-PA"], + ["Paraguay", "es-PY"], + ["Perú", "es-PE"], + ["Puerto Rico", "es-PR"], + ["Spain", "es-ES"], + ["Uruguay", "es-UY"], + ["United States", "es-US"], + ["Venezuela", "es-VE"] + ], + Swedish: [["Sweden", "sv-SE"]], + Thai: [["Thailand", "th-TH"]], + Turkish: [["Turkey", "tr-TR"]], + Ukrainian: [["Ukraine", "uk-UA"]], + Vietnamese: [["Viet Nam", "vi-VN"]], + Zulu: [["South Africa", "zu-ZA"]] + }, + pollyLanguages: [ + { + lang : "English", + region : "USA", + voices : { + male: ["Matthew", "Joey"], + female: ["Joanna", "Kendra", "Kimberly", "Salli"] + } + }, + { + lang : "English", + region : "United Kingdom", + voices : { + male: ["Brian"], + female: ["Amy", "Emma"] + } + }, + { + lang : "English", + region : "Australia", + voices : { + male: ["Russell"], + female: ["Nicole"] + } + }, + { + lang : "English", + region : "Welsh", + voices : { + male: ["Geraint"], + female: [] + } + }, + { + lang : "English", + region : "India", + voices : { + male: ["Raveena"], + female: ["Aditi"] + } + }, + { + lang : "French", + region : "France", + voices : { + male: ["Mathieu"], + female: ["Celine", "Léa"] + } + }, + { + lang : "French", + region : "Canada", + voices : { + male: [], + female: ["Chantal"] + } + }, + { + lang : "Russian", + region : "Russia", + voices : { + male: ["Maxim"], + female: ["Tatyana"] + } + }, + { + lang : "Danish", + region : "Denmark", + voices : { + male: ["Mads"], + female: ["Naja"] + } + }, + { + lang : "Swedish", + region : "Sweden", + voices : { + male: [], + female: ["Astrid"] + } + }, + { + lang : "Norwegian", + region : "Norway", + voices : { + male: [], + female: ["Liv"] + } + }, + { + lang : "Norwegian", + region : "Norway", + voices : { + male: [], + female: ["Liv"] + } + }, + { + lang : "German", + region : "Germany", + voices : { + male: ["Hans"], + female: ["Marlene", "Vicki"] + } + }, + { + lang : "Italian", + region : "Italy", + voices : { + male: ["Giorgio"], + female: ["Carla", "Bianca"] + } + }, + { + lang : "Dutch", + region : "Netherlands", + voices : { + male: ["Ruben"], + female: ["Lotte"] + } + }, + { + lang : "Spanish", + region : "Spain", + voices : { + male: ["Miguel"], + female: ["Lupe", "Penelope"] + } + }, + { + lang : "Spanish", + region : "Mexico", + voices : { + male: [], + female: ["Mia"] + } + }, + { + lang : "Japanese", + region : "Japan", + voices : { + male: ["Takumi"], + female: ["Mizuki"] + } + }, + { + lang : "Chinese", + region : "China", + voices : { + male: [], + female: ["Zhiyu"] + } + } + ], + webSpeechTtsLocales: [ + {text: "English » USA » Male", value: "en-us-male"}, + {text: "English » USA » Female", value: "en-us-female"}, + {text: "English » UK » Male", value: "en-uk-male"}, + {text: "English » UK » Female", value: "en-uk-female"}, + {text: "French", value: "fr"}, + {text: "Spanish » Female", value: "es"}, + {text: "Dutch", value: "nl"}, + {text: "Italian", value: "it"}, + {text: "German", value: "de"}, + {text: "Russian", value: "ru"}, + {text: "Swedish", value: "sv"}, + {text: "Norwegian", value: "no"}, + {text: "Danish", value: "da"}, + {text: "Japanese", value: "jp"}, + {text: "Chinese", value: "cn"}, + {text: "Chinese (Hong Kong)", value: "cn(hk)"}, + {text: "Indonesian", value: "id"} ], chatIcons: [ "mdi-message-bulleted", @@ -1096,6 +1404,38 @@ export default { }, computed: { ...mapGetters(["embed", "textColor"]), + getPollyLanguages() { + let languages = []; + this.pollyLanguages.forEach(language => { + language.voices.male.forEach(voice => { + languages.push({ + text: `${language.lang} » ${language.region} » [male] » ${voice}`, + value: `${voice}` + }); + }); + language.voices.female.forEach(voice => { + languages.push({ + text: `${language.lang} » ${language.region} » [female] » ${voice}`, + value: `${voice}` + }); + }); + }); + + return languages; + }, + getAsrLangCodes() { + let languages = []; + for (let [key, values] of Object.entries(this.webSpeechLanguages)) { + console.log(`Language: ${key}`); + values.forEach(value => { + languages.push({ + text: `${key} » ${value[0]}`, + value: `${value[1]}` + }); + }); + } + return languages; + }, themeColorsFiltered() { return this.themeColors.filter(color => { return color !== "white"; @@ -1132,6 +1472,9 @@ export default { updated() {}, methods: { + isPollyConfigured() { + return window.leopardConfig.tts.url ? true : false; + }, isLight(color) { return isLight(color); }, diff --git a/src/constants/solution-config-default.js b/src/constants/solution-config-default.js index 95b65ce6..8413112c 100644 --- a/src/constants/solution-config-default.js +++ b/src/constants/solution-config-default.js @@ -40,6 +40,8 @@ okkay | ok responseIcon: "mdi-message-reply-text", sendContextParams: "login", showChatIcons: true, + ttsEngine: "Web Speech API", + enableAsrTtsOnOpen: false, lookAndFeel: { response: { iconColor: "secondary", diff --git a/src/store.js b/src/store.js index 8bc5576b..c70ef3e2 100644 --- a/src/store.js +++ b/src/store.js @@ -11,6 +11,7 @@ import Firebase from "@/utils/firebase"; import liveChatConfig from "@/utils/livechat-config"; import PostMessage from "@/utils/postMessage"; import Setup from "@/utils/setup"; +import Polly from "@/utils/polly"; // Controls Data Store and Flow for Components... import { cleanEmptyChunks, @@ -39,6 +40,8 @@ const logger = require("@/utils/logging").getLogger("store.js"); const replaceString = require("replace-string"); const TIE = require("leopard-tie-client"); +const polly = new Polly(); + const snotifyOptions = { toast: { position: SnotifyPosition.leftBottom @@ -201,6 +204,14 @@ function storeSetup(vuetify) { } }, getters: { + getPollyVoice(state) { + return state.activeSolution.pollyVoice; + }, + isPollyEnabled(state) { + return window.leopardConfig.tts.url && state.activeSolution.ttsEngine === "AWS Polly" + ? true + : false; + }, isAsrTtsOnOpenEnabled(state) { return state.tts.isAsrTtsOnOpenEnabled; }, @@ -1927,6 +1938,7 @@ function storeSetup(vuetify) { }); }, stopAudioCapture(context) { + polly.stop(); if (context.getters.tts && context.getters.tts.isSpeaking()) { logger.debug("muted TTS!"); context.getters.tts.shutUp(); @@ -1985,8 +1997,7 @@ function storeSetup(vuetify) { let now = new Date(); let currentUserInput = ""; currentUserInput = handlePromptBefore(params, now, currentUserInput, context); - - playAudioConfirmation(currentUserInput); + if (currentUserInput) playAudioConfirmation(); if (!context.getters.isLiveChat) { // normal user input - discussion with the VA @@ -2144,7 +2155,9 @@ async function handleTeneoResponse(currentUserInput, context, params, vuetify) { ) { if (context.getters.tts && context.getters.speakBackResponses) { console.log(`About to say: ${ttsText}`); - context.getters.tts.say(ttsText); + context.getters.isPollyEnabled + ? polly.say(ttsText, context.getters.getPollyVoice) + : context.getters.tts.say(ttsText); } } @@ -2279,10 +2292,7 @@ function handlePromptPollingResponse(tResp, context, params) { params.indexOf("command=prompt") !== -1 && cleanEmptyChunks(tResp.getOutputText()) !== "" ) { - try { - var audio = new Audio(require("@/assets/notification.mp3")); - audio.play(); - } catch {} + playAudioConfirmation(); } return mustStop; } @@ -2486,13 +2496,12 @@ function handleLiveChatResponse(currentUserInput, context) { context.commit("CLEAR_USER_INPUT"); } -function playAudioConfirmation(currentUserInput) { - if (currentUserInput) { - try { - var audio = new Audio(require("@/assets/notification.mp3")); - audio.play(); - } catch {} - } +function playAudioConfirmation() { + try { + var audio = new Audio(require("@/assets/notification.mp3")); + + audio.play(); + } catch {} } function handlePromptBefore(params, now, currentUserInput, context) { @@ -2568,12 +2577,16 @@ function handleLoginResponse(context, json, vuetify, resolve) { ttsText = stripHtml(tResp.getParameter("tts")); } // check if this browser supports the Web Speech API - if (context.getters.tts && context.getters.speakBackResponses && + if ( + context.getters.tts && + context.getters.speakBackResponses && Object.prototype.hasOwnProperty.call(window, "webkitSpeechRecognition") && Object.prototype.hasOwnProperty.call(window, "speechSynthesis") ) { - console.log(`About to say: ${ttsText}`); - context.getters.tts.say(ttsText); + console.log(`About to say: ${ttsText}`); + context.getters.isPollyEnabled + ? polly.say(ttsText, context.getters.getPollyVoice) + : context.getters.tts.say(ttsText); } let hasExtraData = false; diff --git a/src/utils/asr-tts.js b/src/utils/asr-tts.js index f664dd44..b1c5978f 100644 --- a/src/utils/asr-tts.js +++ b/src/utils/asr-tts.js @@ -3,37 +3,38 @@ import replaceString from "replace-string"; const logger = require("@/utils/logging").getLogger("asr-tts.js"); export function initializeTTS(locale) { - let tts = null; + let artyom = null; // Artyom Speech Recognition and TTS if ( Object.prototype.hasOwnProperty.call(window, "webkitSpeechRecognition") && Object.prototype.hasOwnProperty.call(window, "speechSynthesis") ) { - tts = new Artyom(); + artyom = new Artyom(); + // control the voices used by Web Speech API if (locale === "en-us-male") { - tts.ArtyomVoicesIdentifiers["en-US"] = [ + artyom.ArtyomVoicesIdentifiers["en-US"] = [ "Microsoft David Desktop - English (United States)", "Google US English", "en-US", "en_US" ]; } else if (locale === "en-us-female") { - tts.ArtyomVoicesIdentifiers["en-US"] = [ + artyom.ArtyomVoicesIdentifiers["en-US"] = [ "Microsoft Zira Desktop - English (United States)", "Google US English", "en-US", "en_US" ]; } else if (locale === "en-uk-male") { - tts.ArtyomVoicesIdentifiers["en-GB"] = [ + artyom.ArtyomVoicesIdentifiers["en-GB"] = [ "Google UK English Male", "Google UK English Female", "en-GB", "en_GB" ]; } else { - tts.ArtyomVoicesIdentifiers["en-GB"] = [ + artyom.ArtyomVoicesIdentifiers["en-GB"] = [ "Google UK English Female", "Google UK English Male", "en-GB", @@ -41,8 +42,36 @@ export function initializeTTS(locale) { ]; } - // artyom.ArtyomVoicesIdentifiers["en-ZA"] = ["Google US English", "en-US", "en_US"]; - tts.initialize({ + // Control the Speech Recognition for Web Speech API + // artyom.ArtyomProperties = { + // asrLang: "en-GB", + // lang: "en-GB", + // recognizing: false, + // continuous: false, + // speed: 1, + // volume: 1, + // listen: false, + // mode: "normal", + // debug: false, + // helpers: { + // redirectRecognizedTextOutput: null, + // remoteProcessorHandler: null, + // lastSay: null, + // fatalityPromiseCallback: null + // }, + // executionKeyword: null, + // obeyKeyword: null, + // speaking: false, + // obeying: true, + // soundex: true, + // name: null + // }; + + if (window.leopardConfig.asrLangCode) { + artyom.ArtyomProperties.asrLang = window.leopardConfig.asrLangCode; + } + + artyom.initialize({ soundex: true, continuous: false, listen: false, // Start recognizing @@ -75,11 +104,13 @@ export function initializeTTS(locale) { ? "id-ID" : locale.startsWith("en-us") ? "en-US" - : "en-GB", + : locale.startsWith("en-gb") + ? "en-GB" + : locale, debug: false }); } - return tts; + return artyom; } export function initializeASR(store, asrCorrections) { diff --git a/src/utils/buildConfig.js b/src/utils/buildConfig.js index c4fe984b..91b32004 100644 --- a/src/utils/buildConfig.js +++ b/src/utils/buildConfig.js @@ -44,6 +44,12 @@ let leopardConfig = { storageBucket: config.get("socialAuthentication.firebase.storageBucket"), messagingSenderId: config.get("socialAuthentication.firebase.messagingSenderId") }, + asr: { + langCode: config.get("asrLangCode", "en-GB") + }, + tts: { + url: config.get("tts.url") + }, auth: { microsoft: { tenant: config.get("socialAuthentication.firebase.microsoft.tenant"), diff --git a/src/utils/polly.js b/src/utils/polly.js new file mode 100644 index 00000000..0b3c123f --- /dev/null +++ b/src/utils/polly.js @@ -0,0 +1,22 @@ +export default class Polly { + constructor() { + this.audio = null; + } + + say(text, voice) { + if (text) { + this.audio = new Audio( + `${window.leopardConfig.tts.url}?text=${encodeURIComponent(text)}&voice=${voice}` + ); + this.audio.play(); + } + } + + stop() { + if (this.audio) { + this.audio.pause(); + this.audio.src = + "data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEAVFYAAFRWAAABAAgAZGF0YQAAAAA="; + } + } +} diff --git a/src/utils/utils.js b/src/utils/utils.js index 80d83d70..2c805d42 100644 --- a/src/utils/utils.js +++ b/src/utils/utils.js @@ -33,6 +33,10 @@ export const fixSolution = solution => { solution.id = id; } + if (!("ttsEngine" in solution)) { + solution.ttsEngine = "Web Speech API"; + } + if (!("enableAsrTtsOnOpen" in solution)) { solution.enableAsrTtsOnOpen = false; } @@ -401,7 +405,7 @@ export const scrollTo = (to, callback, duration) => { export const addTtsPauses = answerText => { return replaceAll(answerText, "||", " "); -} +}; export const cleanEmptyChunks = answerText => { let finalAnswerText = "";