Merge pull request #456 from siata13/dev

Auto translations for localization

config/supported.localization.json

@@ -0,0 +1,3 @@
+{
+  "langs": ["bg-bg", "ca-es", "da-dk", "de-de", "el-gr", "es-es", "et-ee", "fi-fi", "fr-fr", "it-it", "ja-jp", "lt-lt", "lv-lv", "nb-no", "nl-nl", "pl-pl", "pt-pt", "ro-ro", "ru-ru", "sk-sk", "sr-latn-rs", "sv-se", "tr-tr", "vi-vn", "zh-cn", "zh-tw"]
+}

scripts/execute-translation.js

@@ -0,0 +1,201 @@
+/**
+ * Script executes the translation of the localization keys for the provided languages (config/supported.localization.json) with the usage of the Azure Cognitive services.
+ * In case the language localization file doesn't exist, it is going to be created.
+ * Only languages supported by Azure Cognitive services are supported.
+ *
+ * English is used as a main language. The translation is executed for the keys which are the same as in English or are missing.
+ */
+
+
+const path = require('path');
+const fs = require('fs');
+const _ = require('lodash');
+// Cognitive services
+const request = require('request-promise');
+const uuidv4 = require('uuid/v4');
+
+// Replace with process.env.subscriptionKey to get an access to Azure Cognitive services
+const subscriptionKey = process.env.subscriptionKey;
+const endpoint = "https://api.cognitive.microsofttranslator.com";
+
+const locHelper = require('./export-localization');
+// Load configuration for supported languages
+const languagesConfiguration = require('../config/supported.localization.json');
+
+/**
+ * Function executes the translation using cognitive services.
+ */
+async function executeTranslation(lang, inputObj) {
+  try {
+    let options = {
+      method: 'POST',
+      baseUrl: endpoint,
+      url: 'translate',
+      qs: {
+        'api-version': '3.0',
+        'to': [`${lang}`]
+      },
+      headers: {
+        'Ocp-Apim-Subscription-Key': subscriptionKey,
+        'Content-type': 'application/json',
+        'X-ClientTraceId': uuidv4().toString()
+      },
+      body: inputObj,
+      json: true,
+    };
+
+    const response = await request(options);
+    if (!response || response.length < 0) {
+      throw new Error("Somethig went wrong when obtaining translated text!");
+    }
+
+    // Go through all the results
+    const translations = response.map((item) => {
+      // Every translation is in a separate 1-element array -> make it flat
+      return item.translations[0];
+    })
+
+    return translations;
+  } catch (err) {
+    console.error(`[Exception]: Cannot execute translation for lang - ${lang}. Err=${err}`)
+    return null;
+  }
+}
+
+function compareTranslationKeys(srcObj, dstObj) {
+  // Extract all the keys and set them in alpabetyical order
+  let dstKeys = Object.keys(dstObj);
+
+  // Array<string>
+  let toTranslate = [];
+  dstKeys.forEach((locKey) => {
+    if (typeof srcObj[locKey] !== "string") {
+      // In case we have nested translation objects
+      toTranslate = toTranslate.concat(compareTranslationKeys(srcObj[locKey], dstObj[locKey]));
+    } else if (srcObj[locKey] === dstObj[locKey]) {
+      // In case the english value is the same as localized one, add it to translate
+      toTranslate.push(srcObj[locKey]);
+    }
+  });
+
+  return toTranslate;
+}
+
+let currentTranslationIndex = 0;
+function injectTranslatedKeys(srcObj, dstObj, translatedValues) {
+  const srcKeys = Object.keys(srcObj);
+  srcKeys.forEach((locKey) => {
+    if (typeof srcObj[locKey] !== "string") {
+      dstObj[locKey] = injectTranslatedKeys(srcObj[locKey], dstObj[locKey], translatedValues);
+    } else if (srcObj[locKey] === dstObj[locKey]) {
+      const translatedKey =  translatedValues[currentTranslationIndex++];
+      dstObj[locKey] = translatedKey ? translatedKey : dstObj[locKey];
+    }
+  });
+
+  return dstObj;
+}
+
+function prepareTranslationRequestMsg(wordsToTranslate) {
+  // Execute translation for every 50 words in batch
+  const result = [];
+  const chunk = 50;
+  for (let i = 0; i < wordsToTranslate.length; i += chunk) {
+    const slicedWords = wordsToTranslate.slice(i, i + chunk);
+    const slicedMessages = [];
+    // do whatever
+    slicedWords.forEach((word) => {
+      slicedMessages.push({
+        //'text': encodeURI(word)
+        'text': word
+      });
+    });
+    result.push(slicedMessages);
+  }
+
+  return result;
+}
+
+function extranctTranslatedKeys(translatedMsgs) {
+  // Flatten the result to retrieve original structure of the keys array
+  const translatedKeys = [];
+  translatedMsgs.forEach((keys) => {
+    keys.forEach((translationMsg) => {
+      // There is often a replacement of '||' to ' | | ' during the translation
+      // Replace  ' | | ' to '||'
+      const translationResult = translationMsg.text ? translationMsg.text.replace(" | | ", "||") : translationMsg.text;
+      translatedKeys.push(translationResult);
+    })
+  });
+
+  return translatedKeys;
+}
+
+async function executeLocalizationTranslation(srcObj, langObj, lang) {
+  try {
+    // Initialize result object with all english keys and localized keys
+    const dstLoc = Object.assign({}, srcObj, langObj);
+
+    // Prepare keys to translate
+    const keysToTranslate = compareTranslationKeys(srcObj, dstLoc);
+
+    if (keysToTranslate && keysToTranslate.length <= 0) {
+      console.log(`There are no keys to translate`);
+      dstLoc;
+    }
+    console.log(`There are ${keysToTranslate.length} keys to translate.`)
+
+    // Split the array to separate calls in case max limit of carachters (5000) is reached and execute translation
+    const requestMessges = prepareTranslationRequestMsg(keysToTranslate);
+    const promises = [];
+    requestMessges.forEach((msgBody) => {
+      promises.push(executeTranslation(lang, msgBody));
+    });
+
+    const translatedMsgs = await Promise.all(promises);
+    const translatedKeys = extranctTranslatedKeys(translatedMsgs);
+
+    // Inject translated keys into dstLoc object
+    // Reset the global rec counter
+    currentTranslationIndex = 0;
+    const result = injectTranslatedKeys(srcObj, dstLoc, translatedKeys);
+
+    return result;
+  } catch (err) {
+    console.log(`[Exception]: executeLocalizationTranslation : ${err.message}`);
+    return null;
+  }
+}
+
+const run = async () => {
+  // Load files in the localization directory
+  const locDirPath = path.join(__dirname, '../src/loc');
+  let locFiles = fs.readdirSync(locDirPath);
+  locFiles = locFiles.filter(f => f !== "mystrings.d.ts" && f != "en-us.ts");
+
+  // Load main localization file
+  const mainLoc = locHelper.getSPLocalizationFileAsJSON('en-us');
+
+  // Iterate over all supported languages and prepare translation request
+  for (const lang of languagesConfiguration.langs) {
+    console.log(`Processing ${lang}.`);
+
+    // If current loc file doesn't exist - copy the original one
+    let currentLoc = locHelper.getSPLocalizationFileAsJSON(lang);
+    if (!currentLoc) {
+      currentLoc = _.cloneDeep(mainLoc);
+    }
+
+    const translatedObj = await executeLocalizationTranslation(mainLoc, currentLoc, lang);
+    // Replace translated part in .ts file
+    if (translatedObj) {
+      locHelper.replaceTranslatedKeysInJSON(translatedObj, lang)
+    }
+
+    // Set delay to wait for Azure to execute the transation
+    console.log(`Finished processing ${lang}`);
+    console.log();
+  }
+};
+run();
+

scripts/export-localization.js

@@ -0,0 +1,49 @@
+/**
+ * Scripts exports english localization keys to the .csv file
+ */
+
+const path = require('path');
+const fs = require('fs');
+const ts =  require("typescript");
+require('amd-loader');
+
+const jsPlaceholder =
+`declare var define: any;
+
+define([], () => {
+  return {0};
+});`;
+
+const getSPLocalizationFileAsJSON = (locale) => {
+  const locFilePath = path.join(__dirname, `../src/loc/${locale}.ts`);
+  const tmpLocJSFilePath = path.join(__dirname, `${locale}-tmp.js`);
+
+  // Localization file doesn't exist
+  if (!fs.existsSync(locFilePath)) {
+    return null;
+  }
+
+  // Read en-us localization file and transpile if to JS
+  // Add named module to avoid amdefine excpetion
+  const enLocFileContent = fs.readFileSync(locFilePath, 'utf-8');
+  let result = ts.transpileModule(enLocFileContent, { compilerOptions: { module: ts.ModuleKind.CommonJS }});
+
+  // Create temp JS file
+  fs.writeFileSync(tmpLocJSFilePath, result.outputText);
+
+  var locResources = require(`./${locale}-tmp`);
+
+  // Remove tmp file
+  fs.unlinkSync(tmpLocJSFilePath);
+  return locResources;
+}
+
+const replaceTranslatedKeysInJSON = (translatedObj, locale) => {
+  const locFilePath = path.join(__dirname, `../src/loc/${locale}.ts`);
+  const fileContent = jsPlaceholder.replace('{0}', JSON.stringify(translatedObj, null, 2));
+
+  // Save file content
+  fs.writeFileSync(locFilePath, fileContent);
+}
+
+module.exports = { getSPLocalizationFileAsJSON , replaceTranslatedKeysInJSON }