🏗 build and copy story localization strings to dist on amp build an…

…d `amp dist` (#37623)

* add compile localization code

* build and copy story localization strings to dist on `amp build` and `amp dist`

We need to start deploying these json files on the Google AMP cache as
we will soon lazy load amp-story strings to reduce the size of the story
JavaScript binaries.

* fix lint

* make sure to ensure fallback codes

* Update build-system/tasks/build-story-localization.js

Co-authored-by: Daniel Rozenberg <me@danielrozenberg.com>

* Update build-system/tasks/build-story-localization.js

Co-authored-by: Daniel Rozenberg <me@danielrozenberg.com>

* fix refactor bug

* add watch functionality

* re-use the code found in the json importer babel transformer

* remove comment

* rename function

* rename unsed param to _ to appease typecheck

Co-authored-by: Daniel Rozenberg <me@danielrozenberg.com>

build-system/babel-plugins/babel-plugin-transform-json-import/index.js

@@ -1,5 +1,5 @@
 const {dirname, join, relative, resolve} = require('path');
-const {readFileSync} = require('fs');
+const {readJson} = require('../../json-locales');
 
 /**
  * Transforms JSON imports into a `JSON.parse` call:
@@ -19,32 +19,6 @@ const {readFileSync} = require('fs');
  */
 module.exports = function (babel) {
   const {template, types: t} = babel;
-
-  /**
-   * JSON reviver that converts {"string": "foo", ...} to just "foo".
-   * This minifies the format of locale files.
-   * @param {string} _
-   * @param {*} value
-   * @return {*}
-   */
-  function minifyLocalesJsonReviver(_, value) {
-    // Always default to original `value` since this reviver is called for any
-    // property pair, including the higher-level containing object.
-    return value?.string || value;
-  }
-
-  /**
-   * @param {string} filename
-   * @return {*}
-   */
-  function readJson(filename) {
-    // Treat files under /_locales/ specially in order to minify their format.
-    const reviver = filename.includes('/_locales/')
-      ? minifyLocalesJsonReviver
-      : undefined;
-    return JSON.parse(readFileSync(filename, 'utf8'), reviver);
-  }
-
   return {
     manipulateOptions(_opts, parserOpts) {
       parserOpts.plugins.push('importAssertions');

build-system/json-locales/index.js

@@ -0,0 +1,27 @@
+const {readFileSync} = require('fs');
+/**
+ * JSON reviver that converts {"string": "foo", ...} to just "foo".
+ * This minifies the format of locale files.
+ * @param {string} _
+ * @param {*} value
+ * @return {*}
+ */
+function minifyLocalesJsonReviver(_, value) {
+  // Always default to original `value` since this reviver is called for any
+  // property pair, including the higher-level containing object.
+  return value?.string || value;
+}
+
+/**
+ * @param {string} filename
+ * @return {*}
+ */
+function readJson(filename) {
+  // Treat files under /_locales/ specially in order to minify their format.
+  const reviver = filename.includes('/_locales/')
+    ? minifyLocalesJsonReviver
+    : undefined;
+  return JSON.parse(readFileSync(filename, 'utf8'), reviver);
+}
+
+module.exports = {readJson, minifyLocalesJsonReviver};

build-system/tasks/build-story-localization.js

@@ -0,0 +1,105 @@
+const fs = require('fs-extra');
+const fastGlob = require('fast-glob');
+const pathMod = require('path');
+const debounce = require('../common/debounce');
+const {endBuildStep, watchDebounceDelay} = require('./helpers');
+const {watch} = require('chokidar');
+const {readJson} = require('../json-locales');
+
+const dest = 'dist/v0';
+
+const LOCALES_DIR = 'extensions/amp-story/1.0/_locales/*.json';
+
+const FALLBACK_LANGUAGE_CODE = 'en';
+
+const LANGUAGE_CODE_CHUNK_REGEX = /\w+/gi;
+
+/**
+ * Finds fallback language codes for the current language code.
+ * @param {string} languageCode the language code to get fallbacks for
+ * @return {string[]}
+ */
+function getLanguageCodeFallbacks(languageCode) {
+  if (!languageCode) {
+    return [FALLBACK_LANGUAGE_CODE];
+  }
+  const matches = languageCode.match(LANGUAGE_CODE_CHUNK_REGEX) || [];
+  return matches.reduce(
+    (fallbackLanguageCodeList, _, index) => {
+      const fallbackLanguageCode = matches
+        .slice(0, index + 1)
+        .join('-')
+        .toLowerCase();
+      fallbackLanguageCodeList.push(fallbackLanguageCode);
+      return fallbackLanguageCodeList;
+    },
+    [FALLBACK_LANGUAGE_CODE]
+  );
+}
+
+/**
+ * Reads the language files found in amp-story and stores it in a
+ * Object.
+ * @return {Promise<Object>}
+ */
+async function getLanguageStrings() {
+  const langs = Object.create(null);
+  const jsonFiles = await fastGlob(LOCALES_DIR);
+  for (const jsonFile of jsonFiles) {
+    const langKey = pathMod.basename(jsonFile, '.json');
+    langs[langKey] = readJson(jsonFile);
+  }
+  return langs;
+}
+
+/**
+ * Retrieves the fallback language codes for each current locale
+ * and merges any strings from the fallback language.
+ * @param {Object} languages
+ */
+function mergeFallbacks(languages) {
+  for (const langKey in languages) {
+    languages[langKey] = getLanguageCodeFallbacks(langKey)
+      .map((x) => languages[x])
+      .reduce((prev, cur) => Object.assign(prev, cur), Object.create(null));
+  }
+}
+
+/**
+ * Write the localization files to dist/v0/
+ * @return {Promise<void>}
+ */
+async function writeStoryLocalizationFiles() {
+  const startTime = Date.now();
+  await fs.ensureDir(dest);
+  const languages = await getLanguageStrings();
+  mergeFallbacks(languages);
+  // Write out each individual lang file.
+  for (const langKey in languages) {
+    await fs.writeJson(`${dest}/amp-story.${langKey}.json`, languages[langKey]);
+  }
+  // Write out all the languages into one file.
+  await fs.writeJson(`${dest}/amp-story.all-lang.json`, languages);
+  endBuildStep('Generated Story JSON Localization files into', dest, startTime);
+}
+
+/**
+ * Flattens the structure of the json locale strings and writes them out to the
+ * dist directory.
+ * @param {Object=} options
+ * @return {Promise<void>}
+ */
+async function buildStoryLocalization(options = {}) {
+  if (options.watch) {
+    watch(LOCALES_DIR).on(
+      'change',
+      debounce(writeStoryLocalizationFiles, watchDebounceDelay)
+    );
+  }
+  await writeStoryLocalizationFiles();
+}
+
+module.exports = {buildStoryLocalization};
+
+buildStoryLocalization.description =
+  'Flattens the structure of the json locale strings and writes them out to the dist directory';

build-system/tasks/build.js

@@ -15,6 +15,7 @@ const {buildExtensions} = require('./extension-helpers');
 const {buildVendorConfigs} = require('./3p-vendor-helpers');
 const {compileCss} = require('./css');
 const {parseExtensionFlags} = require('./extension-helpers');
+const {buildStoryLocalization} = require('./build-story-localization');
 
 const argv = require('minimist')(process.argv.slice(2));
 
@@ -26,7 +27,11 @@ const argv = require('minimist')(process.argv.slice(2));
  * @return {Promise}
  */
 async function runPreBuildSteps(options) {
-  return Promise.all([compileCss(options), bootstrapThirdPartyFrames(options)]);
+  return Promise.all([
+    buildStoryLocalization(options),
+    compileCss(options),
+    bootstrapThirdPartyFrames(options),
+  ]);
 }
 
 /**

build-system/tasks/dist.js

@@ -32,6 +32,7 @@ const {compileJison} = require('./compile-jison');
 const {formatExtractedMessages} = require('../compile/log-messages');
 const {log} = require('../common/logging');
 const {VERSION} = require('../compile/internal-version');
+const {buildStoryLocalization} = require('./build-story-localization');
 
 const {cyan, green} = colors;
 const argv = require('minimist')(process.argv.slice(2));
@@ -93,6 +94,7 @@ async function runPreDistSteps(options) {
   await compileJison();
   await copyParsers();
   await bootstrapThirdPartyFrames(options);
+  await buildStoryLocalization(options);
   displayLifecycleDebugging();
 }