locale_tool.py

#!/usr/bin/env vpython
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Helper script used to manage locale-related files in Chromium.

This script is used to check, and potentially fix, many locale-related files
in your Chromium workspace, such as:

  - GRIT input files (.grd) and the corresponding translations (.xtb).

  - BUILD.gn files listing Android localized resource string resource .xml
    generated by GRIT for all supported Chrome locales. These correspond to
    <output> elements that use the type="android" attribute.

The --scan-dir <dir> option can be used to check for all files under a specific
directory, and the --fix-inplace option can be used to try fixing any file
that doesn't pass the check.

This can be very handy to avoid tedious and repetitive work when adding new
translations / locales to the Chrome code base, since this script can update
said input files for you.

Important note: checks and fix may fail on some input files. For example
remoting/resources/remoting_strings.grd contains an in-line comment element
inside its <outputs> section that breaks the script. The check will fail, and
trying to fix it too, but at least the file will not be modified.
"""

from __future__ import print_function

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import unittest

# Assume this script is under build/
_SCRIPT_DIR = os.path.dirname(__file__)
_SCRIPT_NAME = os.path.join(_SCRIPT_DIR, os.path.basename(__file__))
_TOP_SRC_DIR = os.path.join(_SCRIPT_DIR, '..')

# Need to import android/gyp/util/resource_utils.py here.
sys.path.insert(0, os.path.join(_SCRIPT_DIR, 'android/gyp'))

from util import build_utils
from util import resource_utils


# This locale is the default and doesn't have translations.
_DEFAULT_LOCALE = 'en-US'

# Misc terminal codes to provide human friendly progress output.
_CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 = '\x1b[0G'
_CONSOLE_CODE_ERASE_LINE = '\x1b[K'
_CONSOLE_START_LINE = (
    _CONSOLE_CODE_MOVE_CURSOR_TO_COLUMN_0 + _CONSOLE_CODE_ERASE_LINE)

##########################################################################
##########################################################################
#####
#####    G E N E R I C   H E L P E R   F U N C T I O N S
#####
##########################################################################
##########################################################################

def _FixChromiumLangAttribute(lang):
  """Map XML "lang" attribute values to Chromium locale names."""
  _CHROMIUM_LANG_FIXES = {
      'en': 'en-US',  # For now, Chromium doesn't have an 'en' locale.
      'iw': 'he',  # 'iw' is the obsolete form of ISO 639-1 for Hebrew
      'no': 'nb',  # 'no' is used by the Translation Console for Norwegian (nb).
  }
  return _CHROMIUM_LANG_FIXES.get(lang, lang)


def _FixTranslationConsoleLocaleName(locale):
  _FIXES = {
      'nb': 'no',  # Norwegian.
      'he': 'iw',  # Hebrew
  }
  return _FIXES.get(locale, locale)


def _CompareLocaleLists(list_a, list_expected, list_name):
  """Compare two lists of locale names. Print errors if they differ.

  Args:
    list_a: First list of locales.
    list_expected: Second list of locales, as expected.
    list_name: Name of list printed in error messages.
  Returns:
    On success, return False. On error, print error messages and return True.
  """
  errors = []
  missing_locales = sorted(set(list_a) - set(list_expected))
  if missing_locales:
    errors.append('Missing locales: %s' % missing_locales)

  extra_locales = sorted(set(list_expected) - set(list_a))
  if extra_locales:
    errors.append('Unexpected locales: %s' % extra_locales)

  if errors:
    print('Errors in %s definition:' % list_name)
    for error in errors:
      print('  %s\n' % error)
    return True

  return False


def _BuildIntervalList(input_list, predicate):
  """Find ranges of contiguous list items that pass a given predicate.

  Args:
    input_list: An input list of items of any type.
    predicate: A function that takes a list item and return True if it
      passes a given test.
  Returns:
    A list of (start_pos, end_pos) tuples, where all items in
    [start_pos, end_pos) pass the predicate.
  """
  result = []
  size = len(input_list)
  start = 0
  while True:
    # Find first item in list that passes the predicate.
    while start < size and not predicate(input_list[start]):
      start += 1

    if start >= size:
      return result

    # Find first item in the rest of the list that does not pass the
    # predicate.
    end = start + 1
    while end < size and predicate(input_list[end]):
      end += 1

    result.append((start, end))
    start = end + 1


def _SortListSubRange(input_list, start, end, key_func):
  """Sort an input list's sub-range according to a specific key function.

  Args:
    input_list: An input list.
    start: Sub-range starting position in list.
    end: Sub-range limit position in list.
    key_func: A function that extracts a sort key from a line.
  Returns:
    A copy of |input_list|, with all items in [|start|, |end|) sorted
    according to |key_func|.
  """
  result = input_list[:start]
  inputs = []
  for pos in xrange(start, end):
    line = input_list[pos]
    key = key_func(line)
    inputs.append((key, line))

  for _, line in sorted(inputs):
    result.append(line)

  result += input_list[end:]
  return result


def _SortElementsRanges(lines, element_predicate, element_key):
  """Sort all elements of a given type in a list of lines by a given key.

  Args:
    lines: input lines.
    element_predicate: predicate function to select elements to sort.
    element_key: lambda returning a comparison key for each element that
      passes the predicate.
  Returns:
    A new list of input lines, with lines [start..end) sorted.
  """
  intervals = _BuildIntervalList(lines, element_predicate)
  for start, end in intervals:
    lines = _SortListSubRange(lines, start, end, element_key)

  return lines


def _ProcessFile(input_file, locales, check_func, fix_func):
  """Process a given input file, potentially fixing it.

  Args:
    input_file: Input file path.
    locales: List of Chrome locales to consider / expect.
    check_func: A lambda called to check the input file lines with
      (input_lines, locales) argument. It must return an list of error
      messages, or None on success.
    fix_func: None, or a lambda called to fix the input file lines with
      (input_lines, locales). It must return the new list of lines for
      the input file, and may raise an Exception in case of error.
  Returns:
    True at the moment.
  """
  print('%sProcessing %s...' % (_CONSOLE_START_LINE, input_file), end=' ')
  sys.stdout.flush()
  with open(input_file) as f:
    input_lines = f.readlines()
  errors = check_func(input_file, input_lines, locales)
  if errors:
    print('\n%s%s' % (_CONSOLE_START_LINE, '\n'.join(errors)))
    if fix_func:
      try:
        input_lines = fix_func(input_file, input_lines, locales)
        output = ''.join(input_lines)
        with open(input_file, 'wt') as f:
          f.write(output)
        print('Fixed %s.' % input_file)
      except Exception as e:  # pylint: disable=broad-except
        print('Skipped %s: %s' % (input_file, e))

  return True


def _ScanDirectoriesForFiles(scan_dirs, file_predicate):
  """Scan a directory for files that match a given predicate.

  Args:
    scan_dir: A list of top-level directories to start scan in.
    file_predicate: lambda function which is passed the file's base name
      and returns True if its full path, relative to |scan_dir|, should be
      passed in the result.
  Returns:
    A list of file full paths.
  """
  result = []
  for src_dir in scan_dirs:
    for root, _, files in os.walk(src_dir):
      result.extend(os.path.join(root, f) for f in files if file_predicate(f))
  return result


def _WriteFile(file_path, file_data):
  """Write |file_data| to |file_path|."""
  with open(file_path, 'w') as f:
    f.write(file_data)


def _FindGnExecutable():
  """Locate the real GN executable used by this Chromium checkout.

  This is needed because the depot_tools 'gn' wrapper script will look
  for .gclient and other things we really don't need here.

  Returns:
    Path of real host GN executable from current Chromium src/ checkout.
  """
  # Simply scan buildtools/*/gn and return the first one found so we don't
  # have to guess the platform-specific sub-directory name (e.g. 'linux64'
  # for 64-bit Linux machines).
  buildtools_dir = os.path.join(_TOP_SRC_DIR, 'buildtools')
  for subdir in os.listdir(buildtools_dir):
    subdir_path = os.path.join(buildtools_dir, subdir)
    if not os.path.isdir(subdir_path):
      continue
    gn_path = os.path.join(subdir_path, 'gn')
    if os.path.exists(gn_path):
      return gn_path
  return None


def _PrettyPrintListAsLines(input_list, available_width, trailing_comma=False):
  result = []
  input_str = ', '.join(input_list)
  while len(input_str) > available_width:
    pos = input_str.rfind(',', 0, available_width)
    result.append(input_str[:pos + 1])
    input_str = input_str[pos + 1:].lstrip()
  if trailing_comma and input_str:
    input_str += ','
  result.append(input_str)
  return result


class _PrettyPrintListAsLinesTest(unittest.TestCase):

  def test_empty_list(self):
    self.assertListEqual([''], _PrettyPrintListAsLines([], 10))

  def test_wrapping(self):
    input_list = ['foo', 'bar', 'zoo', 'tool']
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 8),
        ['foo,', 'bar,', 'zoo,', 'tool'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 12), ['foo, bar,', 'zoo, tool'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 79), ['foo, bar, zoo, tool'])

  def test_trailing_comma(self):
    input_list = ['foo', 'bar', 'zoo', 'tool']
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 8, trailing_comma=True),
        ['foo,', 'bar,', 'zoo,', 'tool,'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 12, trailing_comma=True),
        ['foo, bar,', 'zoo, tool,'])
    self.assertListEqual(
        _PrettyPrintListAsLines(input_list, 79, trailing_comma=True),
        ['foo, bar, zoo, tool,'])


##########################################################################
##########################################################################
#####
#####    L O C A L E S   L I S T S
#####
##########################################################################
##########################################################################

# Various list of locales that will be extracted from build/config/locales.gni
# Do not use these directly, use ChromeLocales(), AndroidAPKOmittedLocales() and
# IosUnsupportedLocales() instead to access these lists.
_INTERNAL_CHROME_LOCALES = []
_INTERNAL_ANDROID_APK_OMITTED_LOCALES = []
_INTERNAL_IOS_UNSUPPORTED_LOCALES = []


def ChromeLocales():
  """Return the list of all locales supported by Chrome."""
  if not _INTERNAL_CHROME_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_CHROME_LOCALES


def AndroidAPKOmittedLocales():
  """Return the list of locales omitted from Android APKs."""
  if not _INTERNAL_ANDROID_APK_OMITTED_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_ANDROID_APK_OMITTED_LOCALES


def IosUnsupportedLocales():
  """Return the list of locales that are unsupported on iOS."""
  if not _INTERNAL_IOS_UNSUPPORTED_LOCALES:
    _ExtractAllChromeLocalesLists()
  return _INTERNAL_IOS_UNSUPPORTED_LOCALES


def _PrepareTinyGnWorkspace(work_dir, out_subdir_name='out'):
  """Populate an empty directory with a tiny set of working GN config files.

  This allows us to run 'gn gen <out> --root <work_dir>' as fast as possible
  to generate files containing the locales list. This takes about 300ms on
  a decent machine, instead of more than 5 seconds when running the equivalent
  commands from a real Chromium workspace, which requires regenerating more
  than 23k targets.

  Args:
    work_dir: target working directory.
    out_subdir_name: Name of output sub-directory.
  Returns:
    Full path of output directory created inside |work_dir|.
  """
  # Create top-level .gn file that must point to the BUILDCONFIG.gn.
  _WriteFile(os.path.join(work_dir, '.gn'),
             'buildconfig = "//BUILDCONFIG.gn"\n')
  # Create BUILDCONFIG.gn which must set a default toolchain. Also add
  # all variables that may be used in locales.gni in a declare_args() block.
  _WriteFile(
      os.path.join(work_dir, 'BUILDCONFIG.gn'),
      r'''set_default_toolchain("toolchain")
declare_args () {
  is_ios = false
  is_android = true
}
''')

  # Create fake toolchain required by BUILDCONFIG.gn.
  os.mkdir(os.path.join(work_dir, 'toolchain'))
  _WriteFile(os.path.join(work_dir, 'toolchain', 'BUILD.gn'),
             r'''toolchain("toolchain") {
  tool("stamp") {
    command = "touch {{output}}"  # Required by action()
  }
}
''')

  # Create top-level BUILD.gn, GN requires at least one target to build so do
  # that with a fake action which will never be invoked. Also write the locales
  # to misc files in the output directory.
  _WriteFile(
      os.path.join(work_dir, 'BUILD.gn'), r'''import("//locales.gni")

action("create_foo") {   # fake action to avoid GN complaints.
  script = "//build/create_foo.py"
  inputs = []
  outputs = [ "$target_out_dir/$target_name" ]
}

# Write the locales lists to files in the output directory.
_filename = root_build_dir + "/foo"
write_file(_filename + ".locales", locales, "json")
write_file(_filename + ".android_apk_omitted_locales",
            android_apk_omitted_locales,
            "json")
write_file(_filename + ".ios_unsupported_locales",
            ios_unsupported_locales,
            "json")
''')

  # Copy build/config/locales.gni to the workspace, as required by BUILD.gn.
  shutil.copyfile(os.path.join(_TOP_SRC_DIR, 'build', 'config', 'locales.gni'),
                  os.path.join(work_dir, 'locales.gni'))

  # Create output directory.
  out_path = os.path.join(work_dir, out_subdir_name)
  os.mkdir(out_path)

  # And ... we're good.
  return out_path


# Set this global variable to the path of a given temporary directory
# before calling _ExtractAllChromeLocalesLists() if you want to debug
# the locales list extraction process.
_DEBUG_LOCALES_WORK_DIR = None


def _ReadJsonList(file_path):
  """Read a JSON file that must contain a list, and return it."""
  with open(file_path) as f:
    data = json.load(f)
    assert isinstance(data, list), "JSON file %s is not a list!" % file_path
  return [item.encode('utf8') for item in data]


def _ExtractAllChromeLocalesLists():
  with build_utils.TempDir() as tmp_path:
    if _DEBUG_LOCALES_WORK_DIR:
      tmp_path = _DEBUG_LOCALES_WORK_DIR
      build_utils.DeleteDirectory(tmp_path)
      build_utils.MakeDirectory(tmp_path)

    out_path = _PrepareTinyGnWorkspace(tmp_path, 'out')

    # NOTE: The file suffixes used here should be kept in sync with
    # build/config/locales.gni
    gn_executable = _FindGnExecutable()
    try:
      subprocess.check_output(
          [gn_executable, 'gen', out_path, '--root=' + tmp_path])
    except subprocess.CalledProcessError as e:
      print(e.output)
      raise e

    global _INTERNAL_CHROME_LOCALES
    _INTERNAL_CHROME_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.locales'))

    global _INTERNAL_ANDROID_APK_OMITTED_LOCALES
    _INTERNAL_ANDROID_APK_OMITTED_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.android_apk_omitted_locales'))

    global _INTERNAL_IOS_UNSUPPORTED_LOCALES
    _INTERNAL_IOS_UNSUPPORTED_LOCALES = _ReadJsonList(
        os.path.join(out_path, 'foo.ios_unsupported_locales'))


##########################################################################
##########################################################################
#####
#####    G R D   H E L P E R   F U N C T I O N S
#####
##########################################################################
##########################################################################

# Technical note:
#
# Even though .grd files are XML, an xml parser library is not used in order
# to preserve the original file's structure after modification. ElementTree
# tends to re-order attributes in each element when re-writing an XML
# document tree, which is undesirable here.
#
# Thus simple line-based regular expression matching is used instead.
#

# Misc regular expressions used to match elements and their attributes.
_RE_OUTPUT_ELEMENT = re.compile(r'<output (.*)\s*/>')
_RE_TRANSLATION_ELEMENT = re.compile(r'<file( | .* )path="(.*\.xtb)".*/>')
_RE_FILENAME_ATTRIBUTE = re.compile(r'filename="([^"]*)"')
_RE_LANG_ATTRIBUTE = re.compile(r'lang="([^"]*)"')
_RE_PATH_ATTRIBUTE = re.compile(r'path="([^"]*)"')
_RE_TYPE_ANDROID_ATTRIBUTE = re.compile(r'type="android"')



def _IsGritInputFile(input_file):
  """Returns True iff this is a GRIT input file."""
  return input_file.endswith('.grd')


def _GetXmlLangAttribute(xml_line):
  """Extract the lang attribute value from an XML input line."""
  m = _RE_LANG_ATTRIBUTE.search(xml_line)
  if not m:
    return None
  return m.group(1)


class _GetXmlLangAttributeTest(unittest.TestCase):
  TEST_DATA = {
      '': None,
      'foo': None,
      'lang=foo': None,
      'lang="foo"': 'foo',
      '<something lang="foo bar" />': 'foo bar',
      '<file lang="fr-CA" path="path/to/strings_fr-CA.xtb" />': 'fr-CA',
  }

  def test_GetXmlLangAttribute(self):
    for test_line, expected in self.TEST_DATA.iteritems():
      self.assertEquals(_GetXmlLangAttribute(test_line), expected)


def _SortGrdElementsRanges(grd_lines, element_predicate):
  """Sort all .grd elements of a given type by their lang attribute."""
  return _SortElementsRanges(grd_lines, element_predicate, _GetXmlLangAttribute)


def _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales):
  """Check the element 'lang' attributes in specific .grd lines range.

  This really checks the following:
    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  locales = set()
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      errors.append('%d: Missing "lang" attribute in <output> element' % pos +
                    1)
      continue
    cr_locale = _FixChromiumLangAttribute(lang)
    if cr_locale in locales:
      errors.append(
          '%d: Redefinition of <output> for "%s" locale' % (pos + 1, lang))
    locales.add(cr_locale)

  extra_locales = locales.difference(wanted_locales)
  if extra_locales:
    errors.append('%d-%d: Extra locales found: %s' % (start + 1, end + 1,
                                                      sorted(extra_locales)))

  missing_locales = wanted_locales.difference(locales)
  if missing_locales:
    errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
                                                  sorted(missing_locales)))

  return errors


##########################################################################
##########################################################################
#####
#####    G R D   A N D R O I D   O U T P U T S
#####
##########################################################################
##########################################################################

def _IsGrdAndroidOutputLine(line):
  """Returns True iff this is an Android-specific <output> line."""
  m = _RE_OUTPUT_ELEMENT.search(line)
  if m:
    return 'type="android"' in m.group(1)
  return False

assert _IsGrdAndroidOutputLine('  <output type="android"/>')

# Many of the functions below have unused arguments due to genericity.
# pylint: disable=unused-argument

def _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
                                               wanted_locales):
  """Check all <output> elements in specific input .grd lines range.

  This really checks the following:
    - Filenames exist for each listed locale.
    - Filenames are well-formed.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      continue
    cr_locale = _FixChromiumLangAttribute(lang)

    m = _RE_FILENAME_ATTRIBUTE.search(line)
    if not m:
      errors.append('%d: Missing filename attribute in <output> element' % pos +
                    1)
    else:
      filename = m.group(1)
      if not filename.endswith('.xml'):
        errors.append(
            '%d: Filename should end with ".xml": %s' % (pos + 1, filename))

      dirname = os.path.basename(os.path.dirname(filename))
      prefix = ('values-%s' % resource_utils.ToAndroidLocaleName(cr_locale)
                if cr_locale != _DEFAULT_LOCALE else 'values')
      if dirname != prefix:
        errors.append(
            '%s: Directory name should be %s: %s' % (pos + 1, prefix, filename))

  return errors


def _CheckGrdAndroidOutputElements(grd_file, grd_lines, wanted_locales):
  """Check all <output> elements related to Android.

  Args:
    grd_file: Input .grd file path.
    grd_lines: List of input .grd lines.
    wanted_locales: set of wanted Chromium locale names.
  Returns:
    List of error message strings. Empty on success.
  """
  intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
    errors += _CheckGrdElementRangeAndroidOutputFilename(grd_lines, start, end,
                                                         wanted_locales)
  return errors


def _AddMissingLocalesInGrdAndroidOutputs(grd_file, grd_lines, wanted_locales):
  """Fix an input .grd line by adding missing Android outputs.

  Args:
    grd_file: Input .grd file path.
    grd_lines: Input .grd line list.
    wanted_locales: set of Chromium locale names.
  Returns:
    A new list of .grd lines, containing new <output> elements when needed
    for locales from |wanted_locales| that were not part of the input.
  """
  intervals = _BuildIntervalList(grd_lines, _IsGrdAndroidOutputLine)
  for start, end in reversed(intervals):
    locales = set()
    for pos in xrange(start, end):
      lang = _GetXmlLangAttribute(grd_lines[pos])
      locale = _FixChromiumLangAttribute(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'bg'
    src_lang_attribute = 'lang="%s"' % src_locale
    src_line = None
    for pos in xrange(start, end):
      if src_lang_attribute in grd_lines[pos]:
        src_line = grd_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find <output> element with "%s" lang attribute' % src_locale)

    line_count = end - 1
    for locale in missing_locales:
      android_locale = resource_utils.ToAndroidLocaleName(locale)
      dst_line = src_line.replace(
          'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
              'values-%s/' % src_locale, 'values-%s/' % android_locale)
      grd_lines.insert(line_count, dst_line)
      line_count += 1

  # Sort the new <output> elements.
  return _SortGrdElementsRanges(grd_lines, _IsGrdAndroidOutputLine)


##########################################################################
##########################################################################
#####
#####    G R D   T R A N S L A T I O N S
#####
##########################################################################
##########################################################################


def _IsTranslationGrdOutputLine(line):
  """Returns True iff this is an output .xtb <file> element."""
  m = _RE_TRANSLATION_ELEMENT.search(line)
  return m is not None


class _IsTranslationGrdOutputLineTest(unittest.TestCase):

  def test_GrdTranslationOutputLines(self):
    _VALID_INPUT_LINES = [
        '<file path="foo/bar.xtb" />',
        '<file path="foo/bar.xtb"/>',
        '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb"/>',
        '<file lang="fr-CA" path="translations/aw_strings_fr-CA.xtb" />',
        '  <file path="translations/aw_strings_ar.xtb" lang="ar" />',
    ]
    _INVALID_INPUT_LINES = ['<file path="foo/bar.xml" />']

    for line in _VALID_INPUT_LINES:
      self.assertTrue(
          _IsTranslationGrdOutputLine(line),
          '_IsTranslationGrdOutputLine() returned False for [%s]' % line)

    for line in _INVALID_INPUT_LINES:
      self.assertFalse(
          _IsTranslationGrdOutputLine(line),
          '_IsTranslationGrdOutputLine() returned True for [%s]' % line)


def _CheckGrdTranslationElementRange(grd_lines, start, end,
                                     wanted_locales):
  """Check all <translations> sub-elements in specific input .grd lines range.

  This really checks the following:
    - Each item has a 'path' attribute.
    - Each such path value ends up with '.xtb'.

  Args:
    grd_lines: Input .grd lines.
    start: Sub-range start position in input line list.
    end: Sub-range limit position in input line list.
    wanted_locales: Set of wanted Chromium locale names.
  Returns:
    List of error message strings for this input. Empty on success.
  """
  errors = []
  for pos in xrange(start, end):
    line = grd_lines[pos]
    lang = _GetXmlLangAttribute(line)
    if not lang:
      continue
    m = _RE_PATH_ATTRIBUTE.search(line)
    if not m:
      errors.append('%d: Missing path attribute in <file> element' % pos +
                    1)
    else:
      filename = m.group(1)
      if not filename.endswith('.xtb'):
        errors.append(
            '%d: Path should end with ".xtb": %s' % (pos + 1, filename))

  return errors


def _CheckGrdTranslations(grd_file, grd_lines, wanted_locales):
  """Check all <file> elements that correspond to an .xtb output file.

  Args:
    grd_file: Input .grd file path.
    grd_lines: List of input .grd lines.
    wanted_locales: set of wanted Chromium locale names.
  Returns:
    List of error message strings. Empty on success.
  """
  wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGrdElementRangeLang(grd_lines, start, end, wanted_locales)
    errors += _CheckGrdTranslationElementRange(grd_lines, start, end,
                                              wanted_locales)
  return errors


# Regular expression used to replace the lang attribute inside .xtb files.
_RE_TRANSLATIONBUNDLE = re.compile('<translationbundle lang="(.*)">')


def _CreateFakeXtbFileFrom(src_xtb_path, dst_xtb_path, dst_locale):
  """Create a fake .xtb file.

  Args:
    src_xtb_path: Path to source .xtb file to copy from.
    dst_xtb_path: Path to destination .xtb file to write to.
    dst_locale: Destination locale, the lang attribute in the source file
      will be substituted with this value before its lines are written
      to the destination file.
  """
  with open(src_xtb_path) as f:
    src_xtb_lines = f.readlines()

  def replace_xtb_lang_attribute(line):
    m = _RE_TRANSLATIONBUNDLE.search(line)
    if not m:
      return line
    return line[:m.start(1)] + dst_locale + line[m.end(1):]

  dst_xtb_lines = [replace_xtb_lang_attribute(line) for line in src_xtb_lines]
  with build_utils.AtomicOutput(dst_xtb_path) as tmp:
    tmp.writelines(dst_xtb_lines)


def _AddMissingLocalesInGrdTranslations(grd_file, grd_lines, wanted_locales):
  """Fix an input .grd line by adding missing Android outputs.

  This also creates fake .xtb files from the one provided for 'en-GB'.

  Args:
    grd_file: Input .grd file path.
    grd_lines: Input .grd line list.
    wanted_locales: set of Chromium locale names.
  Returns:
    A new list of .grd lines, containing new <output> elements when needed
    for locales from |wanted_locales| that were not part of the input.
  """
  wanted_locales = wanted_locales - set([_DEFAULT_LOCALE])
  intervals = _BuildIntervalList(grd_lines, _IsTranslationGrdOutputLine)
  for start, end in reversed(intervals):
    locales = set()
    for pos in xrange(start, end):
      lang = _GetXmlLangAttribute(grd_lines[pos])
      locale = _FixChromiumLangAttribute(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'en-GB'
    src_lang_attribute = 'lang="%s"' % src_locale
    src_line = None
    for pos in xrange(start, end):
      if src_lang_attribute in grd_lines[pos]:
        src_line = grd_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find <file> element with "%s" lang attribute' % src_locale)

    src_path = os.path.join(
        os.path.dirname(grd_file),
        _RE_PATH_ATTRIBUTE.search(src_line).group(1))

    line_count = end - 1
    for locale in missing_locales:
      dst_line = src_line.replace(
          'lang="%s"' % src_locale, 'lang="%s"' % locale).replace(
              '_%s.xtb' % src_locale, '_%s.xtb' % locale)
      grd_lines.insert(line_count, dst_line)
      line_count += 1

      dst_path = src_path.replace('_%s.xtb' % src_locale, '_%s.xtb' % locale)
      _CreateFakeXtbFileFrom(src_path, dst_path, locale)


  # Sort the new <output> elements.
  return _SortGrdElementsRanges(grd_lines, _IsTranslationGrdOutputLine)


##########################################################################
##########################################################################
#####
#####    G N   A N D R O I D   O U T P U T S
#####
##########################################################################
##########################################################################

_RE_GN_VALUES_LIST_LINE = re.compile(
    r'^\s*".*values(\-([A-Za-z0-9-]+))?/.*\.xml",\s*$')

def _IsBuildGnInputFile(input_file):
  """Returns True iff this is a BUILD.gn file."""
  return os.path.basename(input_file) == 'BUILD.gn'


def _GetAndroidGnOutputLocale(line):
  """Check a GN list, and return its Android locale if it is an output .xml"""
  m = _RE_GN_VALUES_LIST_LINE.match(line)
  if not m:
    return None

  if m.group(1):  # First group is optional and contains group 2.
    return m.group(2)

  return resource_utils.ToAndroidLocaleName(_DEFAULT_LOCALE)


def _IsAndroidGnOutputLine(line):
  """Returns True iff this is an Android-specific localized .xml output."""
  return _GetAndroidGnOutputLocale(line) != None


def _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
  """Check that a range of GN lines corresponds to localized strings.

  Special case: Some BUILD.gn files list several non-localized .xml files
  that should be ignored by this function, e.g. in
  components/cronet/android/BUILD.gn, the following appears:

    inputs = [
      ...
      "sample/res/layout/activity_main.xml",
      "sample/res/layout/dialog_url.xml",
      "sample/res/values/dimens.xml",
      "sample/res/values/strings.xml",
      ...
    ]

  These are non-localized strings, and should be ignored. This function is
  used to detect them quickly.
  """
  for pos in xrange(start, end):
    if not 'values/' in gn_lines[pos]:
      return True
  return False


def _CheckGnOutputsRange(gn_lines, start, end, wanted_locales):
  if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
    return []

  errors = []
  locales = set()
  for pos in xrange(start, end):
    line = gn_lines[pos]
    android_locale = _GetAndroidGnOutputLocale(line)
    assert android_locale != None
    cr_locale = resource_utils.ToChromiumLocaleName(android_locale)
    if cr_locale in locales:
      errors.append('%s: Redefinition of output for "%s" locale' %
                    (pos + 1, android_locale))
    locales.add(cr_locale)

  extra_locales = locales.difference(wanted_locales)
  if extra_locales:
    errors.append('%d-%d: Extra locales: %s' % (start + 1, end + 1,
                                                sorted(extra_locales)))

  missing_locales = wanted_locales.difference(locales)
  if missing_locales:
    errors.append('%d-%d: Missing locales: %s' % (start + 1, end + 1,
                                                  sorted(missing_locales)))

  return errors


def _CheckGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  errors = []
  for start, end in intervals:
    errors += _CheckGnOutputsRange(gn_lines, start, end, wanted_locales)
  return errors


def _AddMissingLocalesInGnAndroidOutputs(gn_file, gn_lines, wanted_locales):
  intervals = _BuildIntervalList(gn_lines, _IsAndroidGnOutputLine)
  # NOTE: Since this may insert new lines to each interval, process the
  # list in reverse order to maintain valid (start,end) positions during
  # the iteration.
  for start, end in reversed(intervals):
    if not _CheckGnOutputsRangeForLocalizedStrings(gn_lines, start, end):
      continue

    locales = set()
    for pos in xrange(start, end):
      lang = _GetAndroidGnOutputLocale(gn_lines[pos])
      locale = resource_utils.ToChromiumLocaleName(lang)
      locales.add(locale)

    missing_locales = wanted_locales.difference(locales)
    if not missing_locales:
      continue

    src_locale = 'bg'
    src_values = 'values-%s/' % resource_utils.ToAndroidLocaleName(src_locale)
    src_line = None
    for pos in xrange(start, end):
      if src_values in gn_lines[pos]:
        src_line = gn_lines[pos]
        break

    if not src_line:
      raise Exception(
          'Cannot find output list item with "%s" locale' % src_locale)

    line_count = end - 1
    for locale in missing_locales:
      if locale == _DEFAULT_LOCALE:
        dst_line = src_line.replace('values-%s/' % src_locale, 'values/')
      else:
        dst_line = src_line.replace(
            'values-%s/' % src_locale,
            'values-%s/' % resource_utils.ToAndroidLocaleName(locale))
      gn_lines.insert(line_count, dst_line)
      line_count += 1

    gn_lines = _SortListSubRange(
        gn_lines, start, line_count,
        lambda line: _RE_GN_VALUES_LIST_LINE.match(line).group(1))

  return gn_lines


##########################################################################
##########################################################################
#####
#####    T R A N S L A T I O N   E X P E C T A T I O N S
#####
##########################################################################
##########################################################################

_EXPECTATIONS_FILENAME = 'translation_expectations.pyl'

# Technical note: the format of translation_expectations.pyl
# is a 'Python literal', which defines a python dictionary, so should
# be easy to parse. However, when modifying it, care should be taken
# to respect the line comments and the order of keys within the text
# file.


def _ReadPythonLiteralFile(pyl_path):
  """Read a .pyl file into a Python data structure."""
  with open(pyl_path) as f:
    pyl_content = f.read()
  # Evaluate as a Python data structure, use an empty global
  # and local dictionary.
  return eval(pyl_content, dict(), dict())


def _UpdateLocalesInExpectationLines(pyl_lines,
                                     wanted_locales,
                                     available_width=79):
  """Update the locales list(s) found in an expectations file.

  Args:
    pyl_lines: Iterable of input lines from the file.
    wanted_locales: Set or list of new locale names.
    available_width: Optional, number of character colums used
      to word-wrap the new list items.
  Returns:
    New list of updated lines.
  """
  locales_list = ['"%s"' % loc for loc in sorted(wanted_locales)]
  result = []
  line_count = len(pyl_lines)
  line_num = 0
  DICT_START = '"languages": ['
  while line_num < line_count:
    line = pyl_lines[line_num]
    line_num += 1
    result.append(line)
    # Look for start of "languages" dictionary.
    pos = line.find(DICT_START)
    if pos < 0:
      continue

    start_margin = pos
    start_line = line_num
    # Skip over all lines from the list.
    while (line_num < line_count and
           not pyl_lines[line_num].rstrip().endswith('],')):
      line_num += 1
      continue

    if line_num == line_count:
      raise Exception('%d: Missing list termination!' % start_line)

    # Format the new list according to the new margin.
    locale_width = available_width - (start_margin + 2)
    locale_lines = _PrettyPrintListAsLines(
        locales_list, locale_width, trailing_comma=True)
    for locale_line in locale_lines:
      result.append(' ' * (start_margin + 2) + locale_line)
    result.append(' ' * start_margin + '],')
    line_num += 1

  return result


class _UpdateLocalesInExpectationLinesTest(unittest.TestCase):

  def test_simple(self):
    self.maxDiff = 1000
    input_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
  "android_grd": {
    "languages": [
      "aa", "bb", "cc", "dd", "ee",
      "ff", "gg", "hh", "ii", "jj",
      "kk"],
  },
  # Example with bad indentation in input.
  "another_grd": {
         "languages": [
  "aa", "bb", "cc", "dd", "ee", "ff", "gg", "hh", "ii", "jj", "kk",
      ],
  },
}
'''
    expected_text = r'''
# This comment should be preserved
# 23456789012345678901234567890123456789
{
  "android_grd": {
    "languages": [
      "A2", "AA", "BB", "CC", "DD",
      "E2", "EE", "FF", "GG", "HH",
      "I2", "II", "JJ", "KK",
    ],
  },
  # Example with bad indentation in input.
  "another_grd": {
         "languages": [
           "A2", "AA", "BB", "CC", "DD",
           "E2", "EE", "FF", "GG", "HH",
           "I2", "II", "JJ", "KK",
         ],
  },
}
'''
    input_lines = input_text.splitlines()
    test_locales = ([
        'AA', 'BB', 'CC', 'DD', 'EE', 'FF', 'GG', 'HH', 'II', 'JJ', 'KK', 'A2',
        'E2', 'I2'
    ])
    expected_lines = expected_text.splitlines()
    self.assertListEqual(
        _UpdateLocalesInExpectationLines(input_lines, test_locales, 40),
        expected_lines)

  def test_missing_list_termination(self):
    input_lines = r'''
  "languages": ['
    "aa", "bb", "cc", "dd"
'''.splitlines()
    with self.assertRaises(Exception) as cm:
      _UpdateLocalesInExpectationLines(input_lines, ['a', 'b'], 40)

    self.assertEqual(str(cm.exception), '2: Missing list termination!')


def _UpdateLocalesInExpectationFile(pyl_path, wanted_locales):
  """Update all locales listed in a given expectations file.

  Args:
    pyl_path: Path to .pyl file to update.
    wanted_locales: List of locales that need to be written to
      the file.
  """
  tc_locales = {
      _FixTranslationConsoleLocaleName(locale)
      for locale in set(wanted_locales) - set([_DEFAULT_LOCALE])
  }

  with open(pyl_path) as f:
    input_lines = [l.rstrip() for l in f.readlines()]

  updated_lines = _UpdateLocalesInExpectationLines(input_lines, tc_locales)
  with build_utils.AtomicOutput(pyl_path) as f:
    f.writelines('\n'.join(updated_lines) + '\n')


##########################################################################
##########################################################################
#####
#####    C H E C K   E V E R Y T H I N G
#####
##########################################################################
##########################################################################

# pylint: enable=unused-argument


def _IsAllInputFile(input_file):
  return _IsGritInputFile(input_file) or _IsBuildGnInputFile(input_file)


def _CheckAllFiles(input_file, input_lines, wanted_locales):
  errors = []
  if _IsGritInputFile(input_file):
    errors += _CheckGrdTranslations(input_file, input_lines, wanted_locales)
    errors += _CheckGrdAndroidOutputElements(
        input_file, input_lines, wanted_locales)
  elif _IsBuildGnInputFile(input_file):
    errors += _CheckGnAndroidOutputs(input_file, input_lines, wanted_locales)
  return errors


def _AddMissingLocalesInAllFiles(input_file, input_lines, wanted_locales):
  if _IsGritInputFile(input_file):
    lines = _AddMissingLocalesInGrdTranslations(
        input_file, input_lines, wanted_locales)
    lines = _AddMissingLocalesInGrdAndroidOutputs(
        input_file, lines, wanted_locales)
  elif _IsBuildGnInputFile(input_file):
    lines = _AddMissingLocalesInGnAndroidOutputs(
        input_file, input_lines, wanted_locales)
  return lines


##########################################################################
##########################################################################
#####
#####    C O M M A N D   H A N D L I N G
#####
##########################################################################
##########################################################################

class _Command(object):
  """A base class for all commands recognized by this script.

  Usage is the following:
    1) Derived classes must re-define the following class-based fields:
       - name: Command name (e.g. 'list-locales')
       - description: Command short description.
       - long_description: Optional. Command long description.
         NOTE: As a convenience, if the first character is a newline,
         it will be omitted in the help output.

    2) Derived classes for commands that take arguments should override
       RegisterExtraArgs(), which receives a corresponding argparse
       sub-parser as argument.

    3) Derived classes should implement a Run() command, which can read
       the current arguments from self.args.
  """
  name = None
  description = None
  long_description = None

  def __init__(self):
    self._parser = None
    self.args = None

  def RegisterExtraArgs(self, subparser):
    pass

  def RegisterArgs(self, parser):
    subp = parser.add_parser(
        self.name, help=self.description,
        description=self.long_description or self.description,
        formatter_class=argparse.RawDescriptionHelpFormatter)
    self._parser = subp
    subp.set_defaults(command=self)
    group = subp.add_argument_group('%s arguments' % self.name)
    self.RegisterExtraArgs(group)

  def ProcessArgs(self, args):
    self.args = args


class _ListLocalesCommand(_Command):
  """Implement the 'list-locales' command to list locale lists of interest."""
  name = 'list-locales'
  description = 'List supported Chrome locales'
  long_description = r'''
List locales of interest, by default this prints all locales supported by
Chrome, but `--type=android_apk_omitted` can be used to print the list of
locales omitted from Android APKs (but not app bundles), and
`--type=ios_unsupported` for the list of locales unsupported on iOS.

These values are extracted directly from build/config/locales.gni.

Additionally, use the --as-json argument to print the list as a JSON list,
instead of the default format (which is a space-separated list of locale names).
'''

  # Maps type argument to a function returning the corresponding locales list.
  TYPE_MAP = {
      'all': ChromeLocales,
      'android_apk_omitted': AndroidAPKOmittedLocales,
      'ios_unsupported': IosUnsupportedLocales,
  }

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '--as-json',
        action='store_true',
        help='Output as JSON list.')
    group.add_argument(
        '--type',
        choices=tuple(self.TYPE_MAP.viewkeys()),
        default='all',
        help='Select type of locale list to print.')

  def Run(self):
    locale_list = self.TYPE_MAP[self.args.type]()
    if self.args.as_json:
      print('[%s]' % ", ".join("'%s'" % loc for loc in locale_list))
    else:
      print(' '.join(locale_list))


class _CheckInputFileBaseCommand(_Command):
  """Used as a base for other _Command subclasses that check input files.

  Subclasses should also define the following class-level variables:

  - select_file_func:
      A predicate that receives a file name (not path) and return True if it
      should be selected for inspection. Used when scanning directories with
      '--scan-dir <dir>'.

  - check_func:
  - fix_func:
      Two functions passed as parameters to _ProcessFile(), see relevant
      documentation in this function's definition.
  """
  select_file_func = None
  check_func = None
  fix_func = None

  def RegisterExtraArgs(self, group):
    group.add_argument(
      '--scan-dir',
      action='append',
      help='Optional directory to scan for input files recursively.')
    group.add_argument(
      'input',
      nargs='*',
      help='Input file(s) to check.')
    group.add_argument(
      '--fix-inplace',
      action='store_true',
      help='Try to fix the files in-place too.')
    group.add_argument(
      '--add-locales',
      help='Space-separated list of additional locales to use')

  def Run(self):
    args = self.args
    input_files = []
    if args.input:
      input_files = args.input
    if args.scan_dir:
      input_files.extend(_ScanDirectoriesForFiles(
          args.scan_dir, self.select_file_func.__func__))
    locales = ChromeLocales()
    if args.add_locales:
      locales.extend(args.add_locales.split(' '))

    locales = set(locales)

    for input_file in input_files:
      _ProcessFile(input_file,
                   locales,
                   self.check_func.__func__,
                   self.fix_func.__func__ if args.fix_inplace else None)
    print('%sDone.' % (_CONSOLE_START_LINE))


class _CheckGrdAndroidOutputsCommand(_CheckInputFileBaseCommand):
  name = 'check-grd-android-outputs'
  description = (
      'Check the Android resource (.xml) files outputs in GRIT input files.')
  long_description = r'''
Check the Android .xml files outputs in one or more input GRIT (.grd) files
for the following conditions:

    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.
    - Filenames exist for each listed locale.
    - Filenames are well-formed.
'''
  select_file_func = _IsGritInputFile
  check_func = _CheckGrdAndroidOutputElements
  fix_func = _AddMissingLocalesInGrdAndroidOutputs


class _CheckGrdTranslationsCommand(_CheckInputFileBaseCommand):
  name = 'check-grd-translations'
  description = (
      'Check the translation (.xtb) files outputted by .grd input files.')
  long_description = r'''
Check the translation (.xtb) file outputs in one or more input GRIT (.grd) files
for the following conditions:

    - Each item has a correct 'lang' attribute.
    - There are no duplicated lines for the same 'lang' attribute.
    - That there are no extra locales that Chromium doesn't want.
    - That no wanted locale is missing.
    - Each item has a 'path' attribute.
    - Each such path value ends up with '.xtb'.
'''
  select_file_func = _IsGritInputFile
  check_func = _CheckGrdTranslations
  fix_func = _AddMissingLocalesInGrdTranslations


class _CheckGnAndroidOutputsCommand(_CheckInputFileBaseCommand):
  name = 'check-gn-android-outputs'
  description = 'Check the Android .xml file lists in GN build files.'
  long_description = r'''
Check one or more BUILD.gn file, looking for lists of Android resource .xml
files, and checking that:

  - There are no duplicated output files in the list.
  - Each output file belongs to a wanted Chromium locale.
  - There are no output files for unwanted Chromium locales.
'''
  select_file_func = _IsBuildGnInputFile
  check_func = _CheckGnAndroidOutputs
  fix_func = _AddMissingLocalesInGnAndroidOutputs


class _CheckAllCommand(_CheckInputFileBaseCommand):
  name = 'check-all'
  description = 'Check everything.'
  long_description = 'Equivalent to calling all other check-xxx commands.'
  select_file_func = _IsAllInputFile
  check_func = _CheckAllFiles
  fix_func = _AddMissingLocalesInAllFiles


class _UpdateExpectationsCommand(_Command):
  name = 'update-expectations'
  description = 'Update translation expectations file.'
  long_description = r'''
Update %s files to match the current list of locales supported by Chromium.
This is especially useful to add new locales before updating any GRIT or GN
input file with the --add-locales option.
''' % _EXPECTATIONS_FILENAME

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '--add-locales',
        help='Space-separated list of additional locales to use.')

  def Run(self):
    locales = ChromeLocales()
    add_locales = self.args.add_locales
    if add_locales:
      locales.extend(add_locales.split(' '))

    expectation_paths = [
        'tools/gritsettings/translation_expectations.pyl',
        'clank/tools/translation_expectations.pyl',
    ]
    missing_expectation_files = []
    for path in enumerate(expectation_paths):
      file_path = os.path.join(_TOP_SRC_DIR, path)
      if not os.path.exists(file_path):
        missing_expectation_files.append(file_path)
        continue
      _UpdateLocalesInExpectationFile(file_path, locales)

    if missing_expectation_files:
      sys.stderr.write('WARNING: Missing file(s): %s\n' %
                       (', '.join(missing_expectation_files)))


class _UnitTestsCommand(_Command):
  name = 'unit-tests'
  description = 'Run internal unit-tests for this script'

  def RegisterExtraArgs(self, group):
    group.add_argument(
        '-v', '--verbose', action='count', help='Increase test verbosity.')
    group.add_argument('args', nargs=argparse.REMAINDER)

  def Run(self):
    argv = [_SCRIPT_NAME] + self.args.args
    unittest.main(argv=argv, verbosity=self.args.verbose)


# List of all commands supported by this script.
_COMMANDS = [
    _ListLocalesCommand,
    _CheckGrdAndroidOutputsCommand,
    _CheckGrdTranslationsCommand,
    _CheckGnAndroidOutputsCommand,
    _CheckAllCommand,
    _UpdateExpectationsCommand,
    _UnitTestsCommand,
]


def main(argv):
  parser = argparse.ArgumentParser(
      description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)

  subparsers = parser.add_subparsers()
  commands = [clazz() for clazz in _COMMANDS]
  for command in commands:
    command.RegisterArgs(subparsers)

  if not argv:
    argv = ['--help']

  args = parser.parse_args(argv)
  args.command.ProcessArgs(args)
  args.command.Run()


if __name__ == "__main__":
  main(sys.argv[1:])