diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index 236196c88c6..af02212efc8 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -4,9 +4,14 @@ import click -from ckan.cli import click_config_option, db, load_config, search_index, server +from ckan.cli import ( + click_config_option, db, load_config, search_index, server, + translation, +) + from ckan.config.middleware import make_app + log = logging.getLogger(__name__) @@ -28,3 +33,4 @@ def ckan(ctx, config, *args, **kwargs): ckan.add_command(server.run) ckan.add_command(db.db) ckan.add_command(search_index.search_index) +ckan.add_command(translation.translation) diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py new file mode 100644 index 00000000000..edf3d27cdc3 --- /dev/null +++ b/ckan/cli/translation.py @@ -0,0 +1,165 @@ +# encoding: utf-8 + +import polib +import re +import logging +import os + +import click + +from ckan.cli import error_shout +from ckan.common import config +from ckan.lib.i18n import build_js_translations + +ckan_path = os.path.join(os.path.dirname(__file__), u'..') + +log = logging.getLogger(__name__) + + +@click.group(name=u'translation', short_help=u'Translation management') +def translation(): + pass + + +@translation.command( + u'js', short_help=u'Generate the javascript translations.' +) +def js(): + build_js_translations() + click.secho(u'JS translation build: SUCCESS', fg=u'green', bold=True) + + +@translation.command( + u'mangle', short_help=u'Mangle the zh_TW translations for testing.' +) +def mangle(): + u'''This will mangle the zh_TW translations for translation coverage testing. + + NOTE: This will destroy the current translations fot zh_TW + ''' + i18n_path = get_i18n_path() + pot_path = os.path.join(i18n_path, u'ckan.pot') + po = polib.pofile(pot_path) + # we don't want to mangle the following items in strings + # %(...)s %s %0.3f %1$s %2$0.3f [1:...] {...} etc + + # sprintf bit after % + spf_reg_ex = r"\+?(0|'.)?-?\d*(.\d*)?[\%bcdeufosxX]" + + extract_reg_ex = r'(\%\([^\)]*\)' + spf_reg_ex + \ + r'|\[\d*\:[^\]]*\]' + \ + r'|\{[^\}]*\}' + \ + r'|<[^>}]*>' + \ + r'|\%((\d)*\$)?' + spf_reg_ex + r')' + + for entry in po: + msg = entry.msgid.encode(u'utf-8') + matches = re.finditer(extract_reg_ex, msg) + length = len(msg) + position = 0 + translation = u'' + for match in matches: + translation += '-' * (match.start() - position) + position = match.end() + translation += match.group(0) + translation += '-' * (length - position) + entry.msgstr = translation + out_dir = os.path.join(i18n_path, u'zh_TW', u'LC_MESSAGES') + try: + os.makedirs(out_dir) + except OSError: + pass + po.metadata[u'Plural-Forms'] = u"nplurals=1; plural=0\n" + out_po = os.path.join(out_dir, u'ckan.po') + out_mo = os.path.join(out_dir, u'ckan.mo') + po.save(out_po) + po.save_as_mofile(out_mo) + click.secho(u'zh_TW has been mangled', fg=u'green', bold=True) + + +@translation.command( + u'check-po', short_help=u'Check po files for common mistakes' +) +@click.argument(u'files', nargs=-1, type=click.Path(exists=True)) +def check_po(files): + for file in files: + errors = check_po_file(file) + for msgid, msgstr in errors: + click.echo(u"Format specifiers don't match:") + click.echo( + u'\t{} -> {}'.format( + msgid, msgstr.encode(u'ascii', u'replace') + ) + ) + + +def get_i18n_path(): + return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n')) + + +def simple_conv_specs(s): + '''Return the simple Python string conversion specifiers in the string s. + + e.g. ['%s', '%i'] + + See http://docs.python.org/library/stdtypes.html#string-formatting + ''' + simple_conv_specs_re = re.compile(r'\%\w') + return simple_conv_specs_re.findall(s) + + +def mapping_keys(s): + '''Return a sorted list of the mapping keys in the string s. + + e.g. ['%(name)s', '%(age)i'] + + See http://docs.python.org/library/stdtypes.html#string-formatting + ''' + mapping_keys_re = re.compile(r'\%\([^\)]*\)\w') + return sorted(mapping_keys_re.findall(s)) + + +def replacement_fields(s): + '''Return a sorted list of the Python replacement fields in the string s. + + e.g. ['{}', '{2}', '{object}', '{target}'] + + See http://docs.python.org/library/string.html#formatstrings + ''' + repl_fields_re = re.compile(r'\{[^\}]*\}') + return sorted(repl_fields_re.findall(s)) + + +def check_translation(validator, msgid, msgstr): + if not validator(msgid) == validator(msgstr): + return msgid, msgstr + + +def check_po_file(path): + errors = [] + + po = polib.pofile(path) + for entry in po.translated_entries(): + if entry.msgid_plural and entry.msgstr_plural: + for function in ( + simple_conv_specs, mapping_keys, replacement_fields + ): + for key, msgstr in entry.msgstr_plural.iteritems(): + if key == u'0': + error = check_translation( + function, entry.msgid, entry.msgstr_plural[key] + ) + else: + error = check_translation( + function, entry.msgid_plural, + entry.msgstr_plural[key] + ) + if error: + errors.append(error) + + elif entry.msgstr: + for function in ( + simple_conv_specs, mapping_keys, replacement_fields + ): + check_translation(function, entry.msgid, entry.msgstr) + return errors