Fetch 1l0n strings and save them locally (l10n-part 1)

Update dev deps. of package.json

bin/fetchl10nstrings.js

@@ -0,0 +1,368 @@
+#! /usr/bin/env node
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+/**
+ * Create the string bundles and then save them to disk.
+ */
+/*jshint node:true, browser:false, esnext:true*/
+"use strict";
+const async = require("marcosc-async");
+const fs = require("fs");
+const path = require("path");
+const clc = require("cli-color");
+const fetch = require("node-fetch"); // jshint ignore:line
+const repo = "https://hg.mozilla.org/releases/l10n/mozilla-aurora/";
+const newTabPath = "/raw-file/tip/browser/chrome/browser/newTab";
+const globalDirPath = "/raw-file/tip/dom/chrome/global.dtd";
+const l10nPath = path.resolve(`${__dirname}/../l10n/`);
+let defaultLocale;
+
+// CLI-Colors
+const error = clc.red.bold;
+const warn = clc.yellow;
+const notice = clc.blue;
+
+function fetchAndProcessTask(processor) {
+  return async(function*(url) {
+    let response = yield assureResponse(yield fetch(url));
+    let text = yield response.text();
+    return processor(text);
+  });
+}
+
+// Curried functions, reduce duplicate code.
+const dtdProcessorTask = fetchAndProcessTask(processDTD);
+const propProcessorsTask = fetchAndProcessTask(processProps);
+
+/**
+ * Helper object for representing string bundles.
+ *
+ * @constructor
+ * @param {String} locale The locale for the string bundle.
+ */
+function StringBundle(locale) {
+  this.locale = locale;
+}
+
+StringBundle.prototype = {
+  get properties() {
+    return `${repo}${this.locale}${newTabPath}.properties`;
+  },
+  get dtd() {
+    return `${repo}${this.locale}${newTabPath}.dtd`;
+  },
+  get dir() {
+    return `${repo}${this.locale}${globalDirPath}`;
+  },
+  /**
+   * Saves the bundle to disk.
+   *
+   * @return {Promise} Resolves once writing to disk is done is done.
+   */
+  save() {
+    return async.task(function*() {
+      // Kick-off downloads
+      let processedStrings = yield Promise.all([
+        dtdProcessorTask(this.dir),
+        dtdProcessorTask(this.dtd),
+        propProcessorsTask(this.properties)
+      ]);
+      let localeMap = processedStrings.reduce(toSingleMap, new Map());
+      checkForMissingKeys(localeMap, this.locale);
+
+      let defaultClone = new Map(Array.from(defaultLocale.entries()));
+      // Fill in gaps
+      let dirtyMap = toSingleMap(defaultClone, localeMap);
+      let trimmedMap = trimRedundantProps.call(this, dirtyMap);
+      let sortedMap = toSortedMap(trimmedMap);
+      let text = "";
+      try{
+        text = JSON.stringify(sortedMap, mapReplacer, 2);
+      }catch(err){
+        console.log("ARGHGH!!", err, this.locale, sortedMap);
+      }
+      yield writeToDisk(text, this.locale);
+    }, this);
+  }
+};
+/**
+ * Attempts to recovers from HTTP errors (404 and 503).
+ *
+ * @param  {Response} response The response to check.
+ * @return {Promise} Fulfills once an "ok" response is fetched.
+ */
+function assureResponse(response) {
+  if (response.ok) {
+    // All is good!
+    return Promise.resolve(response);
+  }
+  // Otherwise, let's try to recover.
+  return new Promise(function(resolve, reject) {
+    let msg = "";
+    switch (response.status) {
+    case 503:
+      msg = warn(`We got a 503 for: ${response.url} - trying again.`);
+      console.warn(msg);
+      setTimeout(() => {
+        fetch(response.url)
+          .then(assureResponse)
+          .then(resolve);
+      }, 2000);
+      break;
+    case 404:
+      msg = warn(`l10n File is 404 : ${response.url}`);
+      console.warn(msg);
+      // Node Response doesn't expose constructor, so fake it till you make it!
+      const fakeResponse = {
+        text() {
+          return Promise.resolve("");
+        }
+      };
+      resolve(fakeResponse);
+      break;
+    default:
+      msg = error(`Could not handle ${response.status} for: ${response.url}`);
+      reject(new Error(msg));
+    }
+  });
+}
+/**
+ * Validate the data against the canonical default object.
+ * Invalid properties are console.log()'ed.
+ *
+ * @param  {Map} results results to be validated.
+ * @param  {String} locale The corresponding locale.
+ * @return {Object[]} the validated (unmodified) results.
+ */
+function checkForMissingKeys(results, locale) {
+  let missingKeys = Array.from(defaultLocale.keys())
+    .filter(key => !results.has(key));
+  if (missingKeys.length) {
+    console.warn(`\n${warn("WARNING:")} Locale "${locale}" is missing keys. The en-US locale will fill the gaps:
+ ${notice("* " + missingKeys.join("\n * "))}`);
+  }
+  return results;
+}
+/**
+ * Writes the resulting JSON to the file system.
+ *
+ * @param  {String} data   The string to write to disk.
+ * @param  {String} locale The locale that this data is for.
+ * @return {Promise} Resolves when writing is done.
+ */
+function writeToDisk(data, locale) {
+  return new Promise((resolve, reject) => {
+    const dir = path.resolve(`${l10nPath}/${locale}/`);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir);
+    }
+    fs.writeFile(`${dir}/strings.json`, data, "utf8", (err) => {
+      if (err) {
+        return reject(err);
+      }
+      resolve();
+    });
+  });
+}
+/**
+ * Process DTD files by:
+ *   - Split on each new line,
+ *   - then filters out anything that doesn't start with "<!ENTITY"
+ *   - then removes "<" and ">" and additional quotation marks.
+ *   - then combines the remaining key/value pair into the result object.
+ *
+ * @param  {String} text Text to be processed.
+ * @return {Object} The resulting object with the key value pairs.
+ */
+function processDTD(text) {
+  return text.split("\n")
+    .filter(line => line.trim().startsWith("<!ENTITY"))
+    .map(
+      line => line.replace("<!ENTITY ", "")
+      .replace(">", "")
+      .replace(/\"/g, "")
+      .split(/\s(.+)?/)
+      .filter(item => item)
+    )
+    .map(cleanUpNameValuePairs)
+    .reduce(toMap, new Map());
+}
+/**
+ * Reduces key value pairs into an Object.
+ *
+ * @param  {Object} obj The object to reduce into.
+ * @param  {String[]} nameValue The name/value pair to convert to properties.
+ * @return {Object} The resulting object.
+ */
+function toMap(map, nameValue) {
+  let name = nameValue[0];
+  let value = nameValue[1];
+  map.set(name, value);
+  return map;
+}
+/**
+ * Trims names and replaces "." for "-"; trims values.
+ *
+ * @param  {String[]} item A name value pair
+ * @return {String[]} An new Array with the modified values
+ */
+function cleanUpNameValuePairs(item) {
+  return [
+    item[0].trim().replace(/\./g, "-"),
+    item[1].trim(),
+  ];
+}
+/**
+ * Process .properties files.
+ *  - Split on new line
+ *  - then remove comments
+ *  - then map to key value pairs
+ *  - then add key values pairs to resulting object.
+ *
+ * @param  {String} text The properties files to process.
+ * @return {Object} The resulting object.
+ */
+function processProps(text) {
+  return text.split("\n")
+    .filter(line => !line.startsWith("#") && line.trim(line))
+    .map(line => line.split(/=(.+)?/))
+    .map(cleanUpNameValuePairs)
+    .reduce(toMap, new Map());
+}
+/**
+ * Run only a few network requests at a time.
+ *
+ * @param {StringBundle[]} stringBundles the string bundles to save.
+ * @param {Number} throughPut  How many network requests to perform simultaneously.
+ */
+function* fetchRunner(stringBundles, throughPut) { // jshint ignore:line
+  // Gather N=throughPut requests, and wait until they are done before continuing.
+  for (let i = 0; i < stringBundles.length;) {
+    let bundles = [];
+    for (let j = 0; j < throughPut; j++) {
+      bundles.push(stringBundles[i++]);
+      if (i >= stringBundles.length) {
+        break;
+      }
+    }
+    yield Promise.all(
+      bundles.map(bundle => bundle.save())
+    );
+  }
+}
+
+/**
+ * Runs through the locales, downloads the data, and saves it.
+ *
+ * @param {String[]} allLocales The list of locales to download.
+ */
+function generateL10NStrings(allLocales) {
+  let stringBundles = allLocales
+    .map(locale => new StringBundle(locale));
+  let runner = fetchRunner(stringBundles, 5);
+  let fetchSequentially = () => {
+    let next = runner.next();
+    if (!next.done) {
+      return next.value.then(fetchSequentially);
+    }
+  };
+  // Fetch a few at a time, otherwise server might get upset
+  fetchSequentially();
+}
+/**
+ * Trims properties that are not used in the about:newtab page.
+ *
+ * @param  {Object} obj the object from which props will be trimmed.
+ * @return {Object} The object that got trimmed.
+ */
+function trimRedundantProps(localeMap) {
+  let redudantProps = Array.from(localeMap.keys())
+    .filter(key => !defaultLocale.has(key));
+  if (redudantProps.length) {
+    let msg = `\n${warn("WARNING:")} Redundant props in ${this.locale}:`; // jshint ignore:line
+    msg += notice(`
+ * ${redudantProps.join("\n * ")}\n`);
+    console.warn(msg);
+  }
+  redudantProps.forEach(
+    key => localeMap.delete(key)
+  );
+  return localeMap;
+}
+/**
+ * Reducer: merges entries of one map into another.
+ * @param  {[type]} targetMap [description]
+ * @param  {[type]} nextMap    [description]
+ * @return {[type]}           [description]
+ */
+function toSingleMap(prevMap, nextMap){
+  Array
+    .from(nextMap.entries())
+    .forEach(entry => prevMap.set(entry[0], entry[1]));
+  return prevMap;
+}
+
+function toSortedMap(unsortedMap){
+  return Array
+    .from(unsortedMap.keys())
+    .sort()
+    .reduce(
+      (prev, next) => {
+        prev.set(next, unsortedMap.get(next));
+        return prev;
+      }, new Map()
+    );
+}
+/**
+ * When JSON.stringify(), replaces a map for its key values.
+ * @param  {String} key The key to check.
+ * @param  {Any} value The corresponding value.
+ * @return {String} The replaced string.
+ */
+function mapReplacer(key, value){ // jshint ignore:line
+  if(!(value instanceof Map)){
+    return value;
+  }
+  let result = Array.from(value.entries())
+    .map(keyValue => {
+      let key = keyValue[0];
+      let value = String(keyValue[1]).replace(/\"/g,`\\"`).trim();
+      return `"${key}": "${value}"`;
+    })
+    .reduce(
+      // Reduce "{", then each entry, and finally "}"
+      (prev, next, i, arr) => `${prev}${next}${(i < arr.length - 1) ? "," : "}"}`  , "{"
+    );
+  return JSON.parse(result);
+}
+// Read the default locale data (en-US), get the "shipped locales",
+// and save it all to disk!
+async.task(function*() {
+  const mozCentral = "https://hg.mozilla.org/mozilla-central/raw-file/tip/";
+  const mozAurora = "https://hg.mozilla.org/releases/mozilla-aurora/raw-file/tip/";
+  const newTabDTD = `${mozCentral}browser/locales/en-US/chrome/browser/newTab.dtd`;
+  const newTabProps = `${mozCentral}browser/locales/en-US/chrome/browser/newTab.properties`;
+  const globalDTD = "https://hg.mozilla.org/releases/l10n/mozilla-aurora/an/raw-file/tip/dom/chrome/global.dtd";
+  const localeMaps = yield Promise.all([
+    dtdProcessorTask(newTabDTD),
+    propProcessorsTask(newTabProps),
+    dtdProcessorTask(globalDTD),
+  ]);
+  defaultLocale = toSortedMap(localeMaps.reduce(toSingleMap, new Map()));
+  // Save the default locale
+  let data = JSON.stringify(defaultLocale, mapReplacer, 2);
+  yield writeToDisk(data, "en-US");
+  let response = yield fetch(`${mozAurora}browser/locales/shipped-locales`);
+  let rawLocales = yield response.text();
+  //Remove default locale en-US, and discard OS specific invalid tags (e.g., "linux win32")
+  let allLocales = rawLocales.split("\n")
+    .filter(locale => locale && locale !== "en-US")
+    .map(locale => locale.split(/\s/)[0]);
+  try {
+    generateL10NStrings(allLocales);
+  } catch (err) {
+    console.error(error(err));
+  }
+}).catch(err => console.log(err));