Split up localization data for lazy loading

This changes the format of the localization data to enable apps to
trivially lazy-load translations.  It also adds --locales to the build
scripts to allow app developers to choose the compiled-in locales.
The generated output now goes into dist/ and is not checked into
revision control.  Finally, it adds "description" and "meaning" fields
to the source messages to allow us to more easily integrate with a
context-aware human translation service.  The "description" field
provides application context for the translator, while the "meaning"
field provides linguistic disambiguation for words with multiple
meanings or parts of speech in the original English.

Because the translation service wants to collapse messages with
identical text, we had to merge several messages together.  To this
end, we have removed the prefixes "ARIA_LABEL_" and "LABEL_" from the
messages themselves and collapsed what remained.

Issue #1688

Change-Id: I24c17e71c73f6663cf123cfdba118c486fa80ecc

build/all.py

@@ -24,6 +24,7 @@
 import compiler
 import docs
 import gendeps
+import generateLocalizations
 import shakaBuildHelpers
 
 import os
@@ -34,6 +35,13 @@ def main(args):
       description='User facing build script for building the Shaka'
                   ' Player Project.')
 
+  parser.add_argument(
+      '--locales',
+      type=str,
+      nargs='+',
+      default=generateLocalizations.DEFAULT_LOCALES,
+      help='The list of locales to compile in (default %(default)r)')
+
   parser.add_argument(
       '--fix',
       help='Automatically fix style violations.',
@@ -59,6 +67,20 @@ def main(args):
 
   parsed_args = parser.parse_args(args)
 
+  # Make the dist/ folder, ignore errors.
+  base = shakaBuildHelpers.get_source_base()
+  try:
+    os.mkdir(os.path.join(base, 'dist'))
+  except OSError:
+    pass
+
+  # Generate localizations before running gendeps, so the output is available
+  # to the deps system.
+  # TODO(#1858): It might be time to look at a third-party build system.
+  localizations = compiler.GenerateLocalizations(parsed_args.locales)
+  if not localizations.generate(parsed_args.force):
+    return 1
+
   if gendeps.main([]) != 0:
     return 1
 
@@ -77,7 +99,6 @@ def main(args):
     return 1
 
   match = re.compile(r'.*\.less$')
-  base = shakaBuildHelpers.get_source_base()
   main_less_src = os.path.join(base, 'ui', 'controls.less')
   all_less_srcs = shakaBuildHelpers.get_all_files(
       os.path.join(base, 'ui'), match)
@@ -88,6 +109,7 @@ def main(args):
     return 1
 
   build_args_with_ui = ['--name', 'ui', '+@complete']
+  build_args_with_ui += ['--locales'] + parsed_args.locales
   build_args_without_ui = ['--name', 'compiled', '+@complete', '-@ui']
 
   if parsed_args.force:

build/build.py

@@ -47,6 +47,7 @@
 import re
 
 import compiler
+import generateLocalizations
 import shakaBuildHelpers
 
 
@@ -170,6 +171,18 @@ def add_core(self):
       logging.error('Cannot exclude files from core')
     self.include |= core_files
 
+  def has_ui(self):
+    """Returns True if the UI library is in the build."""
+    for path in self.include:
+      if '/ui/' in path:
+        return True
+    return False
+
+  def generate_localizations(self, locales, force):
+    localizations = compiler.GenerateLocalizations(locales)
+    localizations.generate(force)
+    self.include.add(os.path.abspath(localizations.output))
+
   def parse_build(self, lines, root):
     """Parses a Build object from the given lines of commands.
 
@@ -238,11 +251,12 @@ def parse_build(self, lines, root):
 
     return True
 
-  def build_library(self, name, force, is_debug):
+  def build_library(self, name, locales, force, is_debug):
     """Builds Shaka Player using the files in |self.include|.
 
     Args:
       name: The name of the build.
+      locales: A list of strings of locale identifiers.
       force: True to rebuild, False to ignore if no changes are detected.
       is_debug: True to compile for debugging, false for release.
 
@@ -251,6 +265,8 @@ def build_library(self, name, force, is_debug):
     """
     self.add_closure()
     self.add_core()
+    if self.has_ui():
+      self.generate_localizations(locales, force)
 
     if is_debug:
       name += '.debug'
@@ -281,6 +297,13 @@ def main(args):
       description=__doc__,
       formatter_class=argparse.RawDescriptionHelpFormatter)
 
+  parser.add_argument(
+      '--locales',
+      type=str,
+      nargs='+',
+      default=generateLocalizations.DEFAULT_LOCALES,
+      help='The list of locales to compile in (requires UI, default %(default)r)')
+
   parser.add_argument(
       '--force',
       '-f',
@@ -325,10 +348,11 @@ def main(args):
     return 1
 
   name = parsed_args.name
+  locales = parsed_args.locales
   force = parsed_args.force
   is_debug = parsed_args.mode == 'debug'
 
-  if not custom_build.build_library(name, force, is_debug):
+  if not custom_build.build_library(name, locales, force, is_debug):
     return 1
 
   return 0

build/check.py

@@ -120,6 +120,10 @@ def get(*path_components):
               get('third_party', 'language-mapping-list'))
   files.add(os.path.join(base, 'demo', 'common', 'assets.js'))
 
+  localizations = compiler.GenerateLocalizations(None)
+  localizations.generate(args.force)
+  files.add(localizations.output)
+
   closure_opts = build.common_closure_opts + build.common_closure_defines
   closure_opts += build.debug_closure_opts + build.debug_closure_defines
 

build/compiler.py

@@ -22,6 +22,7 @@
 import shutil
 import subprocess
 
+import generateLocalizations
 import shakaBuildHelpers
 
 
@@ -354,3 +355,51 @@ def build(self, force=False):
         return False
 
     return True
+
+
+class GenerateLocalizations(object):
+  def __init__(self, locales):
+    self.locales = locales
+    self.source_files = shakaBuildHelpers.get_all_files(
+        _get_source_path('ui/locales'))
+    self.output = _get_source_path('dist/locales.js')
+
+  def _locales_changed(self):
+    # If locales is None, it means we are being called by a caller who doesn't
+    # care what locales are in use.  This is true, for example, when we are
+    # running a compiler pass over the tests.
+    if self.locales is None:
+      return False
+
+    # Find out what locales we used before.  If they have changed, we must
+    # regenerate the output.
+    last_locales = None
+    try:
+      prefix = '// LOCALES: '
+      with open(self.output, 'r') as f:
+        for line in f:
+          if line.startswith(prefix):
+            last_locales = line.replace(prefix, '').strip().split(', ')
+    except IOError:
+      # The file wasn't found or couldn't be read, so it needs to be redone.
+      return True
+
+    return set(last_locales) != set(self.locales)
+
+  def generate(self, force=False):
+    """Generate runtime localizations.
+
+    Args:
+      force: Generate the localizations even if the inputs and locales have not
+             changed.
+
+    Returns:
+      True on success; False on failure.
+    """
+
+    if (not force and not _must_build(self.output, self.source_files) and
+        not self._locales_changed()):
+      return True
+
+    generateLocalizations.main(['--locales'] + self.locales)
+    return True

build/gendeps.py

@@ -30,6 +30,7 @@
     '--root_with_prefix=third_party/closure ../../../third_party/closure',
     '--root_with_prefix=third_party/language-mapping-list ' +
         '../../../third_party/language-mapping-list',
+    '--root_with_prefix=dist ../../../dist',
 ]