Adding languages other than English to the text to speech block in app lab and game lab

apps/src/lib/util/audioApi.js

@@ -109,6 +109,60 @@ export const commands = {
     } else {
       Sounds.getSingleton().stopAllAudio();
     }
+  },
+  playSpeech(opts) {
+    apiValidateType(opts, 'playSpeech', 'text', opts.text, 'string');
+    apiValidateType(opts, 'playSpeech', 'gender', opts.gender, 'string');
+    apiValidateType(
+      opts,
+      'playSpeech',
+      'language',
+      opts.language,
+      'string',
+      OPTIONAL
+    );
+    const speechConfig = SpeechConfig.fromAuthorizationToken(
+      appOptions.azureSpeechServiceToken,
+      appOptions.azureSpeechServiceRegion
+    );
+    speechConfig.speechSynthesisOutputFormat =
+      SpeechSynthesisOutputFormat.Audio16Khz32KBitRateMonoMp3;
+
+    let voice = appOptions.azureSpeechServiceLanguages['English']['female'];
+    if (
+      appOptions.azureSpeechServiceLanguages[opts.language] &&
+      appOptions.azureSpeechServiceLanguages[opts.language][opts.gender]
+    ) {
+      voice =
+        appOptions.azureSpeechServiceLanguages[opts.language][opts.gender];
+    }
+    const synthesizer = new SpeechSynthesizer(speechConfig, undefined);
+    let ssml = `<speak version="1.0" xmlns="https://www.w3.org/2001/10/synthesis" xml:lang="en-US"><voice name="${voice}">${
+      opts.text
+    }</voice></speak>`;
+    synthesizer.speakSsmlAsync(
+      ssml,
+      result => {
+        // There is no way to make ajax requests from html on the filesystem.  So
+        // the only way to play sounds is using HTML5. This scenario happens when
+        // students export their apps and run them offline. At this point, their
+        // uploaded sound files are exported as well, which means varnish is not
+        // an issue.
+        const forceHTML5 = window.location.protocol === 'file:';
+        Sounds.getSingleton().playBytes(result.audioData, {
+          volume: 1.0,
+          loop: false,
+          forceHTML5: forceHTML5,
+          allowHTML5Mobile: true
+        });
+
+        synthesizer.close();
+      },
+      error => {
+        console.warn(error);
+        synthesizer.close();
+      }
+    );
   }
 };
 
@@ -119,6 +173,8 @@ export const commands = {
 export const executors = {
   playSound: (url, loop = false, callback) =>
     executeCmd(null, 'playSound', {url, loop, callback}),
-  stopSound: url => executeCmd(null, 'stopSound', {url})
+  stopSound: url => executeCmd(null, 'stopSound', {url}),
+  playSpeech: (text, gender, language = 'en-US') =>
+    executeCmd(null, 'playSpeech', {text, gender, language})
 };
 // Note to self - can we use _.zipObject to map argumentNames to arguments here?

apps/src/lib/util/audioApiDropletConfig.js

@@ -1,3 +1,4 @@
+/* globals appOptions */
 import {getStore} from '../../redux';
 import getAssetDropdown from '@cdo/apps/assetManagement/getAssetDropdown';
 import {executors} from './audioApi';
@@ -32,6 +33,19 @@ const dropletConfig = {
       0: () => getAssetDropdown('audio')
     },
     assetTooltip: {0: chooseAsset.bind(null, 'audio')}
+  },
+  playSpeech: {
+    func: 'playSpeech',
+    parent: executors,
+    paramButtons: {minArgs: 2, maxArgs: 3},
+    paletteParams: ['text', 'gender', 'language'],
+    params: ['"Hello World!"', '"female"', '"en-US"'],
+    dropdown: {
+      1: ['"female"', '"male"'],
+      2: Object.keys(appOptions.azureSpeechServiceLanguages).map(x => `"${x}"`)
+    },
+    nativeCallsBackInterpreter: true,
+    assetTooltip: {0: chooseAsset.bind(null, 'audio')}
   }
 };
 

dashboard/app/controllers/projects_controller.rb

@@ -322,6 +322,9 @@ def show
       small_footer: !iframe_embed_app_and_code && !sharing && (@game.uses_small_footer? || @level.enable_scrolling?),
       has_i18n: @game.has_i18n?,
       game_display_name: data_t("game.name", @game.name),
+      azure_speech_service_token: azure_speech_service[:azureSpeechServiceToken],
+      azure_speech_service_region: azure_speech_service[:azureSpeechServiceRegion],
+      azure_speech_service_languages: azure_speech_service[:azureSpeechServiceLanguages]
     )
 
     if params[:key] == 'artist'

dashboard/app/helpers/levels_helper.rb

@@ -5,6 +5,10 @@
 require 'firebase_token_generator'
 require 'image_size'
 require 'cdo/firehose'
+require 'cdo/languages'
+require 'net/http'
+require 'uri'
+require 'json'
 
 module LevelsHelper
   include ApplicationHelper
@@ -471,6 +475,48 @@ def firebase_options
     fb_options
   end
 
+  def azure_speech_service_options
+    speech_service_options = {}
+
+    if @level.game.use_azure_speech_service?
+      # First, get the token
+      token_uri = URI.parse("https://#{CDO.azure_speech_service_region}.api.cognitive.microsoft.com/sts/v1.0/issueToken")
+      token_header = {'Ocp-Apim-Subscription-Key': CDO.azure_speech_service_key}
+      token_http_request = Net::HTTP.new(token_uri.host, token_uri.port)
+      token_http_request.use_ssl = true
+      token_http_request.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      token_request = Net::HTTP::Post.new(token_uri.request_uri, token_header)
+      token_response = token_http_request.request(token_request)
+      speech_service_options[:azureSpeechServiceToken] = token_response.body
+      speech_service_options[:azureSpeechServiceRegion] = CDO.azure_speech_service_region
+
+      # Then, get the list of voices and languages
+      voice_uri = URI.parse("https://#{CDO.azure_speech_service_region}.tts.speech.microsoft.com/cognitiveservices/voices/list")
+      voice_header = {'Authorization': 'Bearer ' + token_response.body}
+      voice_http_request = Net::HTTP.new(voice_uri.host, voice_uri.port)
+      voice_http_request.use_ssl = true
+      voice_http_request.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      voice_request = Net::HTTP::Get.new(voice_uri.request_uri, voice_header)
+      voice_response = voice_http_request.request(voice_request)
+
+      all_voices = JSON.parse(voice_response.body)
+      language_dictionary = {}
+      all_voices.each do |voice|
+        native_locale_name = Languages.get_native_name_by_locale(voice["Locale"])
+        unless native_locale_name.empty?
+          language_dictionary[native_locale_name[0][:native_name_s]] ||= {}
+          language_dictionary[native_locale_name[0][:native_name_s]][voice["Gender"].downcase] ||= voice["ShortName"]
+        end
+      end
+
+      language_dictionary.delete_if {|_, voices| voices.length < 2}
+
+      speech_service_options[:azureSpeechServiceLanguages] = language_dictionary
+    end
+
+    speech_service_options
+  end
+
   # Options hash for Blockly
   def blockly_options
     l = @level

dashboard/app/helpers/view_options_helper.rb

@@ -34,6 +34,9 @@ module ViewOptionsHelper
     :responsive_content,
     :answerdash,
     :signed_replay_log_url,
+    :azure_speech_service_token,
+    :azure_speech_service_region,
+    :azure_speech_service_languages
   )
   # Sets custom options to be used by the view layer. The option hash is frozen once read.
   def view_options(opts = nil)

lib/cdo/languages.rb

@@ -32,6 +32,10 @@ def self.table
     table.select(:unique_language_s, :locale_s).where("locale_s = '#{locale}'").first[:unique_language_s]
   end
 
+  cached def self.get_native_name_by_locale(locale)
+    table.select(:native_name_s, :locale_s).where("locale_s = '#{locale}'").to_a
+  end
+
   cached def self.get_csf_languages
     table.select(:csf_b, :crowdin_name_s).to_a
   end