From 04ce589e5ce35821290660e5e1730a678e897a5f Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 26 Mar 2019 13:34:49 +0200 Subject: [PATCH 1/3] CLI. Implement `translation` command --- ckan/cli/cli.py | 8 +- ckan/cli/translation.py | 165 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 ckan/cli/translation.py diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py index 236196c88c6..af02212efc8 100644 --- a/ckan/cli/cli.py +++ b/ckan/cli/cli.py @@ -4,9 +4,14 @@ import click -from ckan.cli import click_config_option, db, load_config, search_index, server +from ckan.cli import ( + click_config_option, db, load_config, search_index, server, + translation, +) + from ckan.config.middleware import make_app + log = logging.getLogger(__name__) @@ -28,3 +33,4 @@ def ckan(ctx, config, *args, **kwargs): ckan.add_command(server.run) ckan.add_command(db.db) ckan.add_command(search_index.search_index) +ckan.add_command(translation.translation) diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py new file mode 100644 index 00000000000..edf3d27cdc3 --- /dev/null +++ b/ckan/cli/translation.py @@ -0,0 +1,165 @@ +# encoding: utf-8 + +import polib +import re +import logging +import os + +import click + +from ckan.cli import error_shout +from ckan.common import config +from ckan.lib.i18n import build_js_translations + +ckan_path = os.path.join(os.path.dirname(__file__), u'..') + +log = logging.getLogger(__name__) + + +@click.group(name=u'translation', short_help=u'Translation management') +def translation(): + pass + + +@translation.command( + u'js', short_help=u'Generate the javascript translations.' +) +def js(): + build_js_translations() + click.secho(u'JS translation build: SUCCESS', fg=u'green', bold=True) + + +@translation.command( + u'mangle', short_help=u'Mangle the zh_TW translations for testing.' +) +def mangle(): + u'''This will mangle the zh_TW translations for translation coverage testing. + + NOTE: This will destroy the current translations fot zh_TW + ''' + i18n_path = get_i18n_path() + pot_path = os.path.join(i18n_path, u'ckan.pot') + po = polib.pofile(pot_path) + # we don't want to mangle the following items in strings + # %(...)s %s %0.3f %1$s %2$0.3f [1:...] {...} etc + + # sprintf bit after % + spf_reg_ex = r"\+?(0|'.)?-?\d*(.\d*)?[\%bcdeufosxX]" + + extract_reg_ex = r'(\%\([^\)]*\)' + spf_reg_ex + \ + r'|\[\d*\:[^\]]*\]' + \ + r'|\{[^\}]*\}' + \ + r'|<[^>}]*>' + \ + r'|\%((\d)*\$)?' + spf_reg_ex + r')' + + for entry in po: + msg = entry.msgid.encode(u'utf-8') + matches = re.finditer(extract_reg_ex, msg) + length = len(msg) + position = 0 + translation = u'' + for match in matches: + translation += '-' * (match.start() - position) + position = match.end() + translation += match.group(0) + translation += '-' * (length - position) + entry.msgstr = translation + out_dir = os.path.join(i18n_path, u'zh_TW', u'LC_MESSAGES') + try: + os.makedirs(out_dir) + except OSError: + pass + po.metadata[u'Plural-Forms'] = u"nplurals=1; plural=0\n" + out_po = os.path.join(out_dir, u'ckan.po') + out_mo = os.path.join(out_dir, u'ckan.mo') + po.save(out_po) + po.save_as_mofile(out_mo) + click.secho(u'zh_TW has been mangled', fg=u'green', bold=True) + + +@translation.command( + u'check-po', short_help=u'Check po files for common mistakes' +) +@click.argument(u'files', nargs=-1, type=click.Path(exists=True)) +def check_po(files): + for file in files: + errors = check_po_file(file) + for msgid, msgstr in errors: + click.echo(u"Format specifiers don't match:") + click.echo( + u'\t{} -> {}'.format( + msgid, msgstr.encode(u'ascii', u'replace') + ) + ) + + +def get_i18n_path(): + return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n')) + + +def simple_conv_specs(s): + '''Return the simple Python string conversion specifiers in the string s. + + e.g. ['%s', '%i'] + + See http://docs.python.org/library/stdtypes.html#string-formatting + ''' + simple_conv_specs_re = re.compile(r'\%\w') + return simple_conv_specs_re.findall(s) + + +def mapping_keys(s): + '''Return a sorted list of the mapping keys in the string s. + + e.g. ['%(name)s', '%(age)i'] + + See http://docs.python.org/library/stdtypes.html#string-formatting + ''' + mapping_keys_re = re.compile(r'\%\([^\)]*\)\w') + return sorted(mapping_keys_re.findall(s)) + + +def replacement_fields(s): + '''Return a sorted list of the Python replacement fields in the string s. + + e.g. ['{}', '{2}', '{object}', '{target}'] + + See http://docs.python.org/library/string.html#formatstrings + ''' + repl_fields_re = re.compile(r'\{[^\}]*\}') + return sorted(repl_fields_re.findall(s)) + + +def check_translation(validator, msgid, msgstr): + if not validator(msgid) == validator(msgstr): + return msgid, msgstr + + +def check_po_file(path): + errors = [] + + po = polib.pofile(path) + for entry in po.translated_entries(): + if entry.msgid_plural and entry.msgstr_plural: + for function in ( + simple_conv_specs, mapping_keys, replacement_fields + ): + for key, msgstr in entry.msgstr_plural.iteritems(): + if key == u'0': + error = check_translation( + function, entry.msgid, entry.msgstr_plural[key] + ) + else: + error = check_translation( + function, entry.msgid_plural, + entry.msgstr_plural[key] + ) + if error: + errors.append(error) + + elif entry.msgstr: + for function in ( + simple_conv_specs, mapping_keys, replacement_fields + ): + check_translation(function, entry.msgid, entry.msgstr) + return errors From c993f7d99ab61a4af67c7f7cb7d0f4c8fcac3c78 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 26 Mar 2019 14:01:29 +0200 Subject: [PATCH 2/3] prefix literals --- ckan/cli/translation.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py index edf3d27cdc3..e138dd76404 100644 --- a/ckan/cli/translation.py +++ b/ckan/cli/translation.py @@ -44,13 +44,13 @@ def mangle(): # %(...)s %s %0.3f %1$s %2$0.3f [1:...] {...} etc # sprintf bit after % - spf_reg_ex = r"\+?(0|'.)?-?\d*(.\d*)?[\%bcdeufosxX]" + spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\%bcdeufosxX]" - extract_reg_ex = r'(\%\([^\)]*\)' + spf_reg_ex + \ - r'|\[\d*\:[^\]]*\]' + \ - r'|\{[^\}]*\}' + \ - r'|<[^>}]*>' + \ - r'|\%((\d)*\$)?' + spf_reg_ex + r')' + extract_reg_ex = u'(\\%\\([^\\)]*\\)' + spf_reg_ex + \ + u'|\\[\\d*\\:[^\\]]*\\]' + \ + u'|\\{[^\\}]*\\}' + \ + u'|<[^>}]*>' + \ + u'|\\%((\\d)*\\$)?' + spf_reg_ex + u')' for entry in po: msg = entry.msgid.encode(u'utf-8') @@ -59,10 +59,10 @@ def mangle(): position = 0 translation = u'' for match in matches: - translation += '-' * (match.start() - position) + translation += u'-' * (match.start() - position) position = match.end() translation += match.group(0) - translation += '-' * (length - position) + translation += u'-' * (length - position) entry.msgstr = translation out_dir = os.path.join(i18n_path, u'zh_TW', u'LC_MESSAGES') try: @@ -104,7 +104,7 @@ def simple_conv_specs(s): See http://docs.python.org/library/stdtypes.html#string-formatting ''' - simple_conv_specs_re = re.compile(r'\%\w') + simple_conv_specs_re = re.compile(u'\\%\\w') return simple_conv_specs_re.findall(s) @@ -115,7 +115,7 @@ def mapping_keys(s): See http://docs.python.org/library/stdtypes.html#string-formatting ''' - mapping_keys_re = re.compile(r'\%\([^\)]*\)\w') + mapping_keys_re = re.compile(u'\\%\\([^\\)]*\\)\\w') return sorted(mapping_keys_re.findall(s)) @@ -126,7 +126,7 @@ def replacement_fields(s): See http://docs.python.org/library/string.html#formatstrings ''' - repl_fields_re = re.compile(r'\{[^\}]*\}') + repl_fields_re = re.compile(u'\\{[^\\}]*\\}') return sorted(repl_fields_re.findall(s)) From 0ebc75e8163cab11fcf11e643d7c19ecdec0bfde Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 28 Mar 2019 17:39:40 +0200 Subject: [PATCH 3/3] Remove mention of `color` command from docs --- doc/maintaining/paster.rst | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst index 4362eb14b53..c14a57e486b 100644 --- a/doc/maintaining/paster.rst +++ b/doc/maintaining/paster.rst @@ -172,7 +172,6 @@ The following paster commands are supported by CKAN: ================= ============================================================ check-po-files Check po files for common mistakes -color Create or remove a color scheme. create-test-data Create test data in the database. dataset Manage datasets. datastore Perform commands to set up the datastore. @@ -202,20 +201,6 @@ Usage:: check-po-files [options] [FILE] ... -color: Create or remove a color scheme -====================================== - -After running this command, you'll need to regenerate the css files. See :ref:`less` for details. - -Usage:: - - color - creates a random color scheme - color clear - clears any color scheme - color <'HEX'> - uses as base color eg '#ff00ff' must be quoted. - color - a float between 0.0 and 1.0 used as base hue - color - html color name used for base color eg lightblue - - create-test-data: Create test data ==================================