Merge branch 'master' into 3484_revision_ui_removal2

ckan/authz.py

@@ -102,13 +102,19 @@ def _build(self):
                     fetched_auth_functions[name] = auth_function
 
         for name, func_list in chained_auth_functions.iteritems():
-            if name not in fetched_auth_functions:
+            if (name not in fetched_auth_functions and
+                    name not in self._functions):
                 raise Exception('The auth %r is not found for chained auth' % (
                     name))
             # create the chain of functions in the correct order
             for func in reversed(func_list):
-                prev_func = fetched_auth_functions[name]
-                fetched_auth_functions[name] = functools.partial(func, prev_func)
+                if name in fetched_auth_functions:
+                    prev_func = fetched_auth_functions[name]
+                else:
+                    # fallback to chaining off the builtin auth function
+                    prev_func = self._functions[name]
+                fetched_auth_functions[name] = (
+                    functools.partial(func, prev_func))
 
         # Use the updated ones in preference to the originals.
         self._functions.update(fetched_auth_functions)
@@ -200,7 +206,7 @@ def is_authorized(action, context, data_dict=None):
             return {
                 'success': False,
                 'msg': 'Action {0} requires an authenticated user'.format(
-                    auth_function.__name__)
+                    action)
             }
 
         return auth_function(context, data_dict)

ckan/cli/cli.py

@@ -3,10 +3,15 @@
 import logging
 
 import click
+from ckan.cli import config_tool
+from ckan.cli import (
+    click_config_option, db, load_config, search_index, server,
+    translation,
+)
 
-from ckan.cli import click_config_option, db, load_config, search_index, server
 from ckan.config.middleware import make_app
 
+
 log = logging.getLogger(__name__)
 
 
@@ -25,6 +30,8 @@ def ckan(ctx, config, *args, **kwargs):
     ctx.obj = CkanCommand(config)
 
 
+ckan.add_command(config_tool.config_tool)
 ckan.add_command(server.run)
 ckan.add_command(db.db)
 ckan.add_command(search_index.search_index)
+ckan.add_command(translation.translation)

ckan/cli/config_tool.py

@@ -0,0 +1,77 @@
+# encoding: utf-8
+
+import logging
+
+import click
+
+from ckan.cli import error_shout
+import ckan.lib.config_tool as ct
+
+log = logging.getLogger(__name__)
+
+
+class ConfigOption(click.ParamType):
+    name = u'config-option'
+
+    def convert(self, value, param, ctx):
+        if u'=' not in value:
+            self.fail(
+                u'An option does not have an equals sign. '
+                u'It should be \'key=value\'. If there are spaces '
+                u'you\'ll need to quote the option.\n'
+            )
+        return value
+
+
+@click.command(
+    name=u'config-tool',
+    short_help=u'Tool for editing options in a CKAN config file.'
+)
+@click.option(
+    u'--section',
+    u'-s',
+    default=u'app:main',
+    help=u'Section of the config file'
+)
+@click.option(
+    u'--edit',
+    u'-e',
+    is_flag=True,
+    help=u'Checks the option already exists in the config file.'
+)
+@click.option(
+    u'--file',
+    u'-f',
+    u'merge_filepath',
+    help=u'Supply an options file to merge in.'
+)
+@click.argument(u'config_filepath', type=click.Path(exists=True))
+@click.argument(u'options', nargs=-1, type=ConfigOption())
+def config_tool(config_filepath, options, section, edit, merge_filepath):
+    u'''Tool for editing options in a CKAN config file
+
+    paster config-tool <default.ini> <key>=<value> [<key>=<value> ...]
+
+    paster config-tool <default.ini> -f <custom_options.ini>
+
+    Examples:
+
+      paster config-tool default.ini sqlalchemy.url=123 'ckan.site_title=ABC'
+
+      paster config-tool default.ini -s server:main -e port=8080
+
+      paster config-tool default.ini -f custom_options.ini
+    '''
+
+    if merge_filepath:
+        ct.config_edit_using_merge_file(
+            config_filepath, merge_filepath
+        )
+    if not (options or merge_filepath):
+        return error_shout(u'No options provided')
+    try:
+        ct.config_edit_using_option_strings(
+            config_filepath, options, section, edit=edit
+        )
+    except ct.ConfigToolError as e:
+        error_shout(e)

ckan/cli/translation.py

@@ -0,0 +1,165 @@
+# encoding: utf-8
+
+import polib
+import re
+import logging
+import os
+
+import click
+
+from ckan.cli import error_shout
+from ckan.common import config
+from ckan.lib.i18n import build_js_translations
+
+ckan_path = os.path.join(os.path.dirname(__file__), u'..')
+
+log = logging.getLogger(__name__)
+
+
+@click.group(name=u'translation', short_help=u'Translation management')
+def translation():
+    pass
+
+
+@translation.command(
+    u'js', short_help=u'Generate the javascript translations.'
+)
+def js():
+    build_js_translations()
+    click.secho(u'JS translation build: SUCCESS', fg=u'green', bold=True)
+
+
+@translation.command(
+    u'mangle', short_help=u'Mangle the zh_TW translations for testing.'
+)
+def mangle():
+    u'''This will mangle the zh_TW translations for translation coverage testing.
+
+    NOTE: This will destroy the current translations fot zh_TW
+    '''
+    i18n_path = get_i18n_path()
+    pot_path = os.path.join(i18n_path, u'ckan.pot')
+    po = polib.pofile(pot_path)
+    # we don't want to mangle the following items in strings
+    # %(...)s  %s %0.3f %1$s %2$0.3f [1:...] {...} etc
+
+    # sprintf bit after %
+    spf_reg_ex = u"\\+?(0|'.)?-?\\d*(.\\d*)?[\%bcdeufosxX]"
+
+    extract_reg_ex = u'(\\%\\([^\\)]*\\)' + spf_reg_ex + \
+                     u'|\\[\\d*\\:[^\\]]*\\]' + \
+                     u'|\\{[^\\}]*\\}' + \
+                     u'|<[^>}]*>' + \
+                     u'|\\%((\\d)*\\$)?' + spf_reg_ex + u')'
+
+    for entry in po:
+        msg = entry.msgid.encode(u'utf-8')
+        matches = re.finditer(extract_reg_ex, msg)
+        length = len(msg)
+        position = 0
+        translation = u''
+        for match in matches:
+            translation += u'-' * (match.start() - position)
+            position = match.end()
+            translation += match.group(0)
+        translation += u'-' * (length - position)
+        entry.msgstr = translation
+    out_dir = os.path.join(i18n_path, u'zh_TW', u'LC_MESSAGES')
+    try:
+        os.makedirs(out_dir)
+    except OSError:
+        pass
+    po.metadata[u'Plural-Forms'] = u"nplurals=1; plural=0\n"
+    out_po = os.path.join(out_dir, u'ckan.po')
+    out_mo = os.path.join(out_dir, u'ckan.mo')
+    po.save(out_po)
+    po.save_as_mofile(out_mo)
+    click.secho(u'zh_TW has been mangled', fg=u'green', bold=True)
+
+
+@translation.command(
+    u'check-po', short_help=u'Check po files for common mistakes'
+)
+@click.argument(u'files', nargs=-1, type=click.Path(exists=True))
+def check_po(files):
+    for file in files:
+        errors = check_po_file(file)
+        for msgid, msgstr in errors:
+            click.echo(u"Format specifiers don't match:")
+            click.echo(
+                u'\t{} -> {}'.format(
+                    msgid, msgstr.encode(u'ascii', u'replace')
+                )
+            )
+
+
+def get_i18n_path():
+    return config.get(u'ckan.i18n_directory', os.path.join(ckan_path, u'i18n'))
+
+
+def simple_conv_specs(s):
+    '''Return the simple Python string conversion specifiers in the string s.
+
+    e.g. ['%s', '%i']
+
+    See http://docs.python.org/library/stdtypes.html#string-formatting
+    '''
+    simple_conv_specs_re = re.compile(u'\\%\\w')
+    return simple_conv_specs_re.findall(s)
+
+
+def mapping_keys(s):
+    '''Return a sorted list of the mapping keys in the string s.
+
+    e.g. ['%(name)s', '%(age)i']
+
+    See http://docs.python.org/library/stdtypes.html#string-formatting
+    '''
+    mapping_keys_re = re.compile(u'\\%\\([^\\)]*\\)\\w')
+    return sorted(mapping_keys_re.findall(s))
+
+
+def replacement_fields(s):
+    '''Return a sorted list of the Python replacement fields in the string s.
+
+    e.g. ['{}', '{2}', '{object}', '{target}']
+
+    See http://docs.python.org/library/string.html#formatstrings
+    '''
+    repl_fields_re = re.compile(u'\\{[^\\}]*\\}')
+    return sorted(repl_fields_re.findall(s))
+
+
+def check_translation(validator, msgid, msgstr):
+    if not validator(msgid) == validator(msgstr):
+        return msgid, msgstr
+
+
+def check_po_file(path):
+    errors = []
+
+    po = polib.pofile(path)
+    for entry in po.translated_entries():
+        if entry.msgid_plural and entry.msgstr_plural:
+            for function in (
+                simple_conv_specs, mapping_keys, replacement_fields
+            ):
+                for key, msgstr in entry.msgstr_plural.iteritems():
+                    if key == u'0':
+                        error = check_translation(
+                            function, entry.msgid, entry.msgstr_plural[key]
+                        )
+                    else:
+                        error = check_translation(
+                            function, entry.msgid_plural,
+                            entry.msgstr_plural[key]
+                        )
+                    if error:
+                        errors.append(error)
+
+        elif entry.msgstr:
+            for function in (
+                simple_conv_specs, mapping_keys, replacement_fields
+            ):
+                check_translation(function, entry.msgid, entry.msgstr)
+    return errors

ckan/config/middleware/pylons_app.py

@@ -138,10 +138,7 @@ def make_pylons_stack(conf, full_stack=True, static_files=True,
     )
 
     # Establish the Registry for this application
-    # The RegistryManager includes code to pop
-    # registry values after the stream has completed,
-    # so we need to prevent this with `streaming` set to True.
-    app = RegistryManager(app, streaming=True)
+    app = RegistryManager(app, streaming=False)
 
     if asbool(static_files):
         # Serve static files

ckan/controllers/group.py

@@ -220,6 +220,10 @@ def read(self, id, limit=20):
             # Do not query for the group datasets when dictizing, as they will
             # be ignored and get requested on the controller anyway
             data_dict['include_datasets'] = False
+
+            # Do not query group members as they aren't used in the view
+            data_dict['include_users'] = False
+
             c.group_dict = self._action('group_show')(context, data_dict)
             c.group = context['group']
         except (NotFound, NotAuthorized):

ckan/lib/dictization/model_dictize.py

@@ -177,10 +177,10 @@ def package_dictize(pkg, context):
         assert 'display_name' not in tag
         tag['display_name'] = tag['name']
 
-    # extras
+    # extras - no longer revisioned, so always provide latest
     extra = model.package_extra_table
     q = select([extra]).where(extra.c.package_id == pkg.id)
-    result = execute(q, extra, context)
+    result = _execute(q, extra, context)
     result_dict["extras"] = extras_list_dictize(result, context)
 
     # groups

ckan/lib/dictization/model_save.py

@@ -96,16 +96,14 @@ def package_resource_list_save(res_dicts, package, context):
         resource_list.append(resource)
 
 
-def package_extras_save(extra_dicts, obj, context):
+def package_extras_save(extra_dicts, pkg, context):
     allow_partial_update = context.get("allow_partial_update", False)
     if extra_dicts is None and allow_partial_update:
         return
 
-    model = context["model"]
     session = context["session"]
 
-    extras_list = obj.extras_list
-    old_extras = dict((extra.key, extra) for extra in extras_list)
+    old_extras = pkg._extras
 
     new_extras = {}
     for extra_dict in extra_dicts or []:
@@ -116,28 +114,22 @@ def package_extras_save(extra_dicts, obj, context):
             pass
         else:
             new_extras[extra_dict["key"]] = extra_dict["value"]
+
     #new
     for key in set(new_extras.keys()) - set(old_extras.keys()):
-        state = 'active'
-        extra = model.PackageExtra(state=state, key=key, value=new_extras[key])
-        session.add(extra)
-        extras_list.append(extra)
+        pkg.extras[key] = new_extras[key]
     #changed
     for key in set(new_extras.keys()) & set(old_extras.keys()):
         extra = old_extras[key]
-        if new_extras[key] == extra.value and extra.state != 'deleted':
+        if new_extras[key] == extra.value:
             continue
-        state = 'active'
         extra.value = new_extras[key]
-        extra.state = state
         session.add(extra)
     #deleted
     for key in set(old_extras.keys()) - set(new_extras.keys()):
         extra = old_extras[key]
-        if extra.state == 'deleted':
-            continue
-        state = 'deleted'
-        extra.state = state
+        extra.delete()
+
 
 def package_tag_list_save(tag_dicts, package, context):
     allow_partial_update = context.get("allow_partial_update", False)
@@ -305,7 +297,7 @@ def package_dict_save(pkg_dict, context):
         objects = pkg_dict.get('relationships_as_object')
         relationship_list_save(objects, pkg, 'relationships_as_object', context)
 
-    extras = package_extras_save(pkg_dict.get("extras"), pkg, context)
+    package_extras_save(pkg_dict.get("extras"), pkg, context)
 
     return pkg