merge upstream

ckan/cli/cli.py

@@ -6,10 +6,13 @@
 
 from ckan.cli import (
     click_config_option, db, load_config, search_index, server,
+    translation,
     dataset,
 )
+
 from ckan.config.middleware import make_app
 
+
 log = logging.getLogger(__name__)
 
 
@@ -31,4 +34,5 @@ def ckan(ctx, config, *args, **kwargs):
 ckan.add_command(server.run)
 ckan.add_command(db.db)
 ckan.add_command(search_index.search_index)
+ckan.add_command(translation.translation)
 ckan.add_command(dataset.dataset)

ckan/cli/translation.py

@@ -0,0 +1,165 @@
+# encoding: utf-8
+
+import polib
+import re
+import logging
+import os
+
+import click
+
+from ckan.cli import error_shout
+from ckan.common import config
+from ckan.lib.i18n import build_js_translations
+
+ckan_path = os.path.join(os.path.dirname(__file__), u'..')
+
+log = logging.getLogger(__name__)
+
+
+@click.group(name=u'translation', short_help=u'Translation management')
+def translation():
+    pass
+
+
+@translation.command(
+    u'js', short_help=u'Generate the javascript translations.'
+)
+def js():
+    build_js_translations()
+    click.secho(u'JS translation build: SUCCESS', fg=u'green', bold=True)
+
+
+@translation.command(
+    u'mangle', short_help=u'Mangle the zh_TW translations for testing.'
+)
+def mangle():
+    u'''This will mangle the zh_TW translations for translation coverage testing.
+
+    NOTE: This will destroy the current translations fot zh_TW
+    '''
+    i18n_path = get_i18n_path()
+    pot_path = os.path.join(i18n_path, u'ckan.pot')
+    po = polib.pofile(pot_path)
+    # we don't want to mangle the following items in strings
+    # %(...)s  %s %0.3f %1$s %2$0.3f [1:...] {...} etc
+
+    # sprintf bit after %
+    spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\%bcdeufosxX]"
+
+    extract_reg_ex = u'(\\%\\([^\\)]*\\)' + spf_reg_ex + \
+                     u'|\\[\\d*\\:[^\\]]*\\]' + \
+                     u'|\\{[^\\}]*\\}' + \
+                     u'|<[^>}]*>' + \
+                     u'|\\%((\\d)*\\$)?' + spf_reg_ex + u')'
+
+    for entry in po:
+        msg = entry.msgid.encode(u'utf-8')
+        matches = re.finditer(extract_reg_ex, msg)
+        length = len(msg)
+        position = 0
+        translation = u''
+        for match in matches:
+            translation += u'-' * (match.start() - position)
+            position = match.end()
+            translation += match.group(0)
+        translation += u'-' * (length - position)
+        entry.msgstr = translation
+    out_dir = os.path.join(i18n_path, u'zh_TW', u'LC_MESSAGES')
+    try:
+        os.makedirs(out_dir)
+    except OSError:
+        pass
+    po.metadata[u'Plural-Forms'] = u"nplurals=1; plural=0\n"
+    out_po = os.path.join(out_dir, u'ckan.po')
+    out_mo = os.path.join(out_dir, u'ckan.mo')
+    po.save(out_po)
+    po.save_as_mofile(out_mo)
+    click.secho(u'zh_TW has been mangled', fg=u'green', bold=True)
+
+
+@translation.command(
+    u'check-po', short_help=u'Check po files for common mistakes'
+)
+@click.argument(u'files', nargs=-1, type=click.Path(exists=True))
+def check_po(files):
+    for file in files:
+        errors = check_po_file(file)
+        for msgid, msgstr in errors:
+            click.echo(u"Format specifiers don't match:")
+            click.echo(
+                u'\t{} -> {}'.format(
+                    msgid, msgstr.encode(u'ascii', u'replace')
+                )
+            )
+
+
+def get_i18n_path():
+    return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n'))
+
+
+def simple_conv_specs(s):
+    '''Return the simple Python string conversion specifiers in the string s.
+
+    e.g. ['%s', '%i']
+
+    See http://docs.python.org/library/stdtypes.html#string-formatting
+    '''
+    simple_conv_specs_re = re.compile(u'\\%\\w')
+    return simple_conv_specs_re.findall(s)
+
+
+def mapping_keys(s):
+    '''Return a sorted list of the mapping keys in the string s.
+
+    e.g. ['%(name)s', '%(age)i']
+
+    See http://docs.python.org/library/stdtypes.html#string-formatting
+    '''
+    mapping_keys_re = re.compile(u'\\%\\([^\\)]*\\)\\w')
+    return sorted(mapping_keys_re.findall(s))
+
+
+def replacement_fields(s):
+    '''Return a sorted list of the Python replacement fields in the string s.
+
+    e.g. ['{}', '{2}', '{object}', '{target}']
+
+    See http://docs.python.org/library/string.html#formatstrings
+    '''
+    repl_fields_re = re.compile(u'\\{[^\\}]*\\}')
+    return sorted(repl_fields_re.findall(s))
+
+
+def check_translation(validator, msgid, msgstr):
+    if not validator(msgid) == validator(msgstr):
+        return msgid, msgstr
+
+
+def check_po_file(path):
+    errors = []
+
+    po = polib.pofile(path)
+    for entry in po.translated_entries():
+        if entry.msgid_plural and entry.msgstr_plural:
+            for function in (
+                simple_conv_specs, mapping_keys, replacement_fields
+            ):
+                for key, msgstr in entry.msgstr_plural.iteritems():
+                    if key == u'0':
+                        error = check_translation(
+                            function, entry.msgid, entry.msgstr_plural[key]
+                        )
+                    else:
+                        error = check_translation(
+                            function, entry.msgid_plural,
+                            entry.msgstr_plural[key]
+                        )
+                    if error:
+                        errors.append(error)
+
+        elif entry.msgstr:
+            for function in (
+                simple_conv_specs, mapping_keys, replacement_fields
+            ):
+                check_translation(function, entry.msgid, entry.msgstr)
+    return errors

ckan/logic/__init__.py

@@ -430,11 +430,13 @@ def get_action(action):
                 auth_function.auth_audit_exempt = True
                 fetched_actions[name] = auth_function
     for name, func_list in chained_actions.iteritems():
-        if name not in fetched_actions:
+        if name not in fetched_actions and name not in _actions:
+            # nothing to override from plugins or core
             raise NotFound('The action %r is not found for chained action' % (
                 name))
         for func in reversed(func_list):
-            prev_func = fetched_actions[name]
+            # try other plugins first, fall back to core
+            prev_func = fetched_actions.get(name, _actions.get(name))
             fetched_actions[name] = functools.partial(func, prev_func)
 
     # Use the updated ones in preference to the originals.

ckanext/datastore/tests/test_chained_action.py

@@ -4,12 +4,19 @@
 import ckan.plugins as p
 import ckan.tests.helpers as helpers
 import ckan.tests.factories as factories
+from ckan.logic.action.get import package_list as core_package_list
 
 from ckanext.datastore.tests.helpers import DatastoreFunctionalTestBase
 
 assert_equals = nose.tools.assert_equals
 assert_raises = nose.tools.assert_raises
 
+package_list_message = u'The content of this message is largely irrelevant'
+
+
+class TestActionException(Exception):
+    pass
+
 
 @p.toolkit.chained_action
 def datastore_delete(up_func, context, data_dict):
@@ -22,11 +29,19 @@ def datastore_delete(up_func, context, data_dict):
     return result
 
 
+@p.toolkit.chained_action
+def package_list(next_func, context, data_dict):
+    # check it's received the core function as the first arg
+    assert_equals(next_func, core_package_list)
+    raise TestActionException(package_list_message)
+
+
 class ExampleDataStoreDeletedWithCountPlugin(p.SingletonPlugin):
     p.implements(p.IActions)
 
     def get_actions(self):
-        return ({u'datastore_delete': datastore_delete})
+        return ({u'datastore_delete': datastore_delete,
+                 u'package_list': package_list})
 
 
 class TestChainedAction(DatastoreFunctionalTestBase):
@@ -67,3 +82,8 @@ def _create_datastore_resource(self, records):
         helpers.call_action(u'datastore_create', **data)
 
         return resource
+
+    def test_chain_core_action(self):
+        with assert_raises(TestActionException) as raise_context:
+            helpers.call_action(u'package_list', {})
+        assert_equals(raise_context.exception.message, package_list_message)

doc/maintaining/paster.rst

@@ -172,7 +172,6 @@ The following paster commands are supported by CKAN:
 
 ================= ============================================================
 check-po-files    Check po files for common mistakes
-color             Create or remove a color scheme.
 create-test-data  Create test data in the database.
 dataset           Manage datasets.
 datastore         Perform commands to set up the datastore.
@@ -202,20 +201,6 @@ Usage::
     check-po-files [options] [FILE] ...
 
 
-color: Create or remove a color scheme
-======================================
-
-After running this command, you'll need to regenerate the css files. See :ref:`less` for details.
-
-Usage::
-
-    color               - creates a random color scheme
-    color clear         - clears any color scheme
-    color <'HEX'>       - uses as base color eg '#ff00ff' must be quoted.
-    color <VALUE>       - a float between 0.0 and 1.0 used as base hue
-    color <COLOR_NAME>  - html color name used for base color eg lightblue
-
-
 create-test-data: Create test data
 ==================================