Azure TTS Support

[nxturistic]

Question

Hey Louis, an excellent project by the way. Can you please help me with this Azure TTS implementation?
I'm having some issues with the synthesizer.js file. I followed the steps mentioned in Azure Text to Speech documentation.

import * as sdk from "microsoft-cognitiveservices-speech-sdk";
import { SpeechSynthesizer } from "microsoft-cognitiveservices-speech-sdk";
import Ffmpeg from "fluent-ffmpeg";
import { path as ffmpegPath } from "@ffmpeg-installer/ffmpeg";
import { path as ffprobePath } from "@ffprobe-installer/ffprobe";
import fs from "fs";

import log from "../../helpers/log.js";
import string from "../../helpers/string.js";

log.title("Azure TTS Synthesizer");

const synthesizer = {};
const voices = {
  "en-US": {
    voice: "en-US-AmberNeural",
  },
};
let client = {};
const file = `${__dirname}/../../tmp/${Date.now()}-${string.random(4)}.wav`;
const config = JSON.parse(
  fs.readFileSync(`${__dirname}/../../config/voice/azure-tts.json`, "utf8")
);
const speechConfig = sdk.SpeechConfig.fromSubscription(
  config.key,
  config.region
);
const audioConfig = sdk.AudioConfig.fromAudioFileOutput(file);
speechConfig.speechSynthesisLanguage = "en-US";
speechConfig.speechSynthesisVoiceName = voices[process.env.LEON_LANG].voice;

/**
 * Initialize Azure Text-to-Speech based on credentials in the JSON file
 */

synthesizer.init = () => {
  try {
    client = new SpeechSynthesizer(speechConfig, audioConfig);
    log.success("Synthesizer initialized");
  } catch (e) {
    log.error(`Azure TTS: ${e}`);
  }
};

/**
 * Save string to audio file
 */

synthesizer.save = (speech, em, cb) => {
  const client = new SpeechSynthesizer(speechConfig, audioConfig);
  client.speakTextAsync(speech, ({ result }) => {
    synthesizer.close();
    if (result) {
      const wStream = fs.createWriteStream(file);
      return wStream;
    }

    result.pipe(wStream);

    wStream.on("finish", () => {
      const ffmpeg = new Ffmpeg();
      ffmpeg.setFfmpegPath(ffmpegPath);
      ffmpeg.setFfprobePath(ffprobePath);

      // Get file duration thanks to ffprobe
      ffmpeg.input(file).ffprobe((err, data) => {
        if (err) log.error(err);
        else {
          const duration = data.streams[0].duration * 1000;
          em.emit("saved", duration);
          cb(file, duration);
        }
      });
    });

    wStream.on("error", (err) => {
      log.error(`Azure TTS: ${err}`);
    });
  });
};

export default synthesizer;

But whenever I pass the string input in the browser, I get the following error when leon starts talking.

---

.: CLIENT :.

✔ Connected
➡ Type: webapp
➡ Socket id: vuSABqSJmRW-WAnCAAAB

---

.: BRAIN :.

✔ New instance

---

.: TTS :.

✔ New instance
➡ Initializing TTS...

---

.: AZURE TTS SYNTHESIZER :.

✔ Synthesizer initialized

---

.: TTS :.

✔ TTS initialized

---

.: NER :.

✔ New instance

---

.: NLU :.

✔ New instance

---

.: ASR :.

✔ New instance

---

.: INITIALIZATION :.

✔ STT disabled
✔ TTS enabled

---

.: NLU :.

✔ NLP model loaded

---

.: REQUESTING :.

➡ GET /

---

.: REQUESTING :.

➡ GET /assets/index.b33e2f76.js

---

.: REQUESTING :.

➡ GET /assets/vendor.61a2236a.js

---

.: REQUESTING :.

➡ GET /assets/index.1cdd3bd6.css

---

.: REQUESTING :.

➡ GET /v1/info

---

.: GET /INFO :.

✔ Information pulled.

---

.: REQUESTING :.

➡ GET /assets/mic.d8267246.svg

---

.: CLIENT :.

✔ Connected

---

.: REQUESTING :.

➡ GET /assets/favicon.4980d64d.png
➡ Type: webapp
➡ Socket id: f-KkJFKn1-9PQ0Q2AAAD

---

.: BRAIN :.

✔ New instance

---

.: TTS :.

✔ New instance
➡ Initializing TTS...
✖ Azure TTS: Error: throwIfNotUndefined:format is already set

---

.: TTS :.

✔ TTS initialized

---

.: NER :.

✔ New instance

---

.: NLU :.

✔ New instance

---

.: ASR :.

✔ New instance

---

.: INITIALIZATION :.

✔ STT disabled
✔ TTS enabled

---

.: NLU :.

✔ NLP model loaded

---

.: SOCKET :.

➡ webapp emitted: Hello

---

.: NLU :.

➡ Processing...

---

.: NLU :.

✔ Query found

---

.: NER :.

➡ Searching for entities...
➡ No entity found

---

.: LEON PACKAGE :.

➡ {"package": "leon", "module": "greeting", "action": "run", "lang": "en", "input": "Hello", "entities": [], "output": {"type": "end", "codes": ["default"], "speech": "Hi!", "options": {}}}


---

.: LEON:.

➡ Talking...
/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Contracts.js:50
            throw new Error("throwIfNotUndefined:" + name);
            ^

Error: throwIfNotUndefined:format is already set
    at Function.Contracts.throwIfNotUndefined (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Contracts.js:50:19)
    at AudioFileWriter.set (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioFileWriter.js:38:35)
    at AudioOutputConfigImpl.set (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/Audio/AudioConfig.js:313:41)
    at SynthesisAdapterBase.set (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/common.speech/SynthesisAdapterBase.js:136:57)
    at SpeechSynthesizer.implCommonSynthesizeSetup (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/SpeechSynthesizer.js:415:44)
    at new SpeechSynthesizer (/home/admin/ai/nxturistic/node_modules/microsoft-cognitiveservices-speech-sdk/distrib/lib/src/sdk/SpeechSynthesizer.js:77:14)
    at Object.synthesizer.save (/home/admin/ai/nxturistic/server/dist/tts/azure-tts/synthesizer.js:62:18)
    at Tts.forward (/home/admin/ai/nxturistic/server/dist/tts/tts.js:70:30)
    at Tts.add (/home/admin/ai/nxturistic/server/dist/tts/tts.js:131:12)
    at Brain.talk (/home/admin/ai/nxturistic/server/dist/core/brain.js:75:18)

Can you please look into it and let me know what's the issue, I want to contribute to your project in this way 😊.

[louistiti]

Hi @nxturistic,

Thanks for willing to contribute! Sure I can check, would you mind to open a draft PR? 😃

[louistiti]

✨ [1.0.0] Azure TTS