Merge remote-tracking branch 'origin/master' into circle-v2

Dockerfile

@@ -1,5 +1,5 @@
 # See CKAN docs on installation from Docker Compose on usage
-FROM debian:jessie
+FROM debian:stretch
 MAINTAINER Open Knowledge
 
 # Install required system packages

ckan/config/environment.py

@@ -226,9 +226,9 @@ def update_config():
     routes_map = routing.make_map()
 
     lib_plugins.reset_package_plugins()
-    lib_plugins.set_default_package_plugin()
+    lib_plugins.register_package_plugins()
     lib_plugins.reset_group_plugins()
-    lib_plugins.set_default_group_plugin()
+    lib_plugins.register_group_plugins()
 
     config['routes.map'] = routes_map
     # The RoutesMiddleware needs its mapper updating if it exists

ckan/config/middleware/flask_app.py

@@ -182,8 +182,8 @@ def hello_world_post():
         if hasattr(plugin, 'get_blueprint'):
             app.register_extension_blueprint(plugin.get_blueprint())
 
-    lib_plugins.register_group_plugins(app)
-    lib_plugins.register_package_plugins(app)
+    lib_plugins.register_package_blueprints(app)
+    lib_plugins.register_group_blueprints(app)
 
     # Set flask routes in named_routes
     for rule in app.url_map.iter_rules():
@@ -308,7 +308,8 @@ def ckan_after_request(response):
 
 def helper_functions():
     u'''Make helper functions (`h`) available to Flask templates'''
-    helpers.load_plugin_helpers()
+    if not helpers.helper_functions:
+        helpers.load_plugin_helpers()
     return dict(h=helpers.helper_functions)
 
 

ckan/lib/plugins.py

@@ -3,7 +3,6 @@
 import logging
 import os
 import sys
-from importlib import import_module
 
 from flask import Blueprint
 
@@ -13,7 +12,6 @@
 from ckan import plugins
 import ckan.authz
 import ckan.plugins.toolkit as toolkit
-from flask import Blueprint
 
 log = logging.getLogger(__name__)
 
@@ -92,39 +90,58 @@ def lookup_group_blueprints(group_type=None):
     return _group_blueprints.get(group_type)
 
 
-def register_package_plugins(app):
+def register_package_plugins():
     """
     Register the various IDatasetForm instances.
 
     This method will setup the mappings between package types and the
-    registered IDatasetForm instances. If it's called more than once an
-    exception will be raised.
+    registered IDatasetForm instances.
     """
     global _default_package_plugin
 
-    from ckan.views.dataset import dataset, register_dataset_plugin_rules
-    from ckan.views.resource import resource, register_dataset_plugin_rules as dataset_resource_rules
-
     # Create the mappings and register the fallback behaviour if one is found.
     for plugin in plugins.PluginImplementations(plugins.IDatasetForm):
         if plugin.is_fallback():
             if _default_package_plugin is not None and not isinstance(_default_package_plugin, DefaultDatasetForm):
                 raise ValueError("More than one fallback "
                                  "IDatasetForm has been registered")
             _default_package_plugin = plugin
+        for package_type in plugin.package_types():
+            if package_type in _package_plugins:
+                raise ValueError("An existing IDatasetForm is "
+                                 "already associated with the package type "
+                                 "'%s'" % package_type)
+
+            _package_plugins[package_type] = plugin
+
+    # Setup the fallback behaviour if one hasn't been defined.
+    set_default_package_plugin()
+
+
+def register_package_blueprints(app):
+    """
+    Register a Flask blueprint for the various IDatasetForm instances.
+
+    Actually two blueprints per IDatasetForm instance, one for the dataset routes
+    and one for the resources one.
+    """
+
+    from ckan.views.dataset import dataset, register_dataset_plugin_rules
+    from ckan.views.resource import resource, register_dataset_plugin_rules as dataset_resource_rules
+
+    # Create the mappings and register the fallback behaviour if one is found.
+    for plugin in plugins.PluginImplementations(plugins.IDatasetForm):
         for package_type in plugin.package_types():
 
             if package_type == u'dataset':
                 # The default routes are registered with the core
                 # 'dataset' blueprint
                 continue
 
-            elif package_type in _package_plugins:
-                raise ValueError("An existing IDatasetForm is "
-                                 "already associated with the package type "
-                                 "'%s'" % package_type)
-
-            _package_plugins[package_type] = plugin
+            elif package_type in app.blueprints:
+                raise ValueError(
+                    'A blueprint for has already been associated for the '
+                    'package type {}'.format(package_type))
 
             dataset_blueprint = Blueprint(
                 package_type,
@@ -141,9 +158,9 @@ def register_package_plugins(app):
                 url_defaults={u'package_type': package_type})
             dataset_resource_rules(resource_blueprint)
             app.register_blueprint(resource_blueprint)
-
-    # Setup the fallback behaviour if one hasn't been defined.
-    set_default_package_plugin()
+            log.debug(
+                'Registered blueprints for custom dataset type \'{}\''.format(
+                    package_type))
 
 
 def set_default_package_plugin():
@@ -152,21 +169,18 @@ def set_default_package_plugin():
         _default_package_plugin = DefaultDatasetForm()
 
 
-def register_group_plugins(app):
+def register_group_plugins():
     """
     Register the various IGroupForm instances.
 
     This method will setup the mappings between group types and the
-    registered IGroupForm instances. If it's called more than once an
-    exception will be raised.
+    registered IGroupForm instances.
 
     It will register IGroupForm instances for both groups and organizations
     """
     global _default_group_plugin
     global _default_organization_plugin
 
-    from ckan.views.group import group, register_group_plugin_rules
-    # Create the mappings and register the fallback behaviour if one is found.
     for plugin in plugins.PluginImplementations(plugins.IGroupForm):
 
         # Get group_controller from plugin if there is one,
@@ -199,18 +213,51 @@ def register_group_plugins(app):
 
         for group_type in plugin.group_types():
 
-            if group_type in (u'group', u'organization'):
-                # The default routes are registered with the core
-                # 'group' or 'organization' blueprint
-                _group_plugins[group_type] = plugin
-                continue
-            elif group_type in _group_plugins:
+            if group_type in _group_plugins:
                 raise ValueError("An existing IGroupForm is "
                                  "already associated with the group type "
                                  "'%s'" % group_type)
             _group_plugins[group_type] = plugin
             _group_controllers[group_type] = group_controller
 
+    # Setup the fallback behaviour if one hasn't been defined.
+    set_default_group_plugin()
+
+
+def register_group_blueprints(app):
+    """
+    Register a Flask blueprint for the various IGroupForm instances.
+
+    It will register blueprints for both groups and organizations
+    """
+
+    from ckan.views.group import group, register_group_plugin_rules
+
+    for plugin in plugins.PluginImplementations(plugins.IGroupForm):
+
+        # Get group_controller from plugin if there is one,
+        # otherwise use 'group'
+        try:
+            group_controller = plugin.group_controller()
+        except AttributeError:
+            group_controller = 'group'
+
+        if hasattr(plugin, 'is_organization'):
+            is_organization = plugin.is_organization
+        else:
+            is_organization = group_controller == 'organization'
+
+        for group_type in plugin.group_types():
+
+            if group_type in (u'group', u'organization'):
+                # The default routes are registered with the core
+                # 'group' or 'organization' blueprint
+                continue
+            elif group_type in app.blueprints:
+                raise ValueError(
+                    'A blueprint for has already been associated for the '
+                    'group type {}'.format(group_type))
+
             blueprint = Blueprint(group_type,
                                   group.import_name,
                                   url_prefix='/{}'.format(group_type),
@@ -219,8 +266,6 @@ def register_group_plugins(app):
             register_group_plugin_rules(blueprint)
             app.register_blueprint(blueprint)
 
-        set_default_group_plugin()
-
 
 def set_default_group_plugin():
     global _default_group_plugin

ckan/lib/search/query.py

@@ -20,7 +20,7 @@
 VALID_SOLR_PARAMETERS = set([
     'q', 'fl', 'fq', 'rows', 'sort', 'start', 'wt', 'qf', 'bf', 'boost',
     'facet', 'facet.mincount', 'facet.limit', 'facet.field',
-    'extras', 'fq_list', 'tie', 'defType', 'mm'
+    'extras', 'fq_list', 'tie', 'defType', 'mm', 'df'
 ])
 
 # for (solr) package searches, this specifies the fields that are searched

ckan/logic/action/get.py

@@ -1814,6 +1814,9 @@ def package_search(context, data_dict):
     for key in [key for key in data_dict.keys() if key.startswith('ext_')]:
         data_dict['extras'][key] = data_dict.pop(key)
 
+    # set default search field
+    data_dict['df'] = 'text'
+
     # check if some extension needs to modify the search params
     for item in plugins.PluginImplementations(plugins.IPackageController):
         data_dict = item.before_search(data_dict)

ckan/templates/snippets/language_selector.html

@@ -3,7 +3,7 @@
   <label for="field-lang-select">{{ _('Language') }}</label>
   <select id="field-lang-select" name="url" data-module="autocomplete" data-module-dropdown-class="lang-dropdown" data-module-container-class="lang-container">
     {% for locale in h.get_available_locales() %}
-      <option value="{% url_for h.current_url(), locale=locale.short_name %}" {% if locale.identifier == current_lang %}selected="selected"{% endif %}>
+      <option value="{% url_for h.current_url(), locale=locale.short_name %}" {% if locale.short_name == current_lang %}selected="selected"{% endif %}>
         {{ locale.display_name or locale.english_name }}
       </option>
     {% endfor %}

ckanext/datastore/backend/postgres.py

@@ -1230,9 +1230,9 @@ def search_data(context, data_dict):
         ).replace('%', '%%')
         sql_fmt = u'''
             SELECT '[' || array_to_string(array_agg(j.v), ',') || ']' FROM (
-                SELECT '[' || {select} || ']' v
+                SELECT {distinct} '[' || {select} || ']' v
                 FROM (
-                    SELECT {distinct} * FROM "{resource}" {ts_query}
+                    SELECT * FROM "{resource}" {ts_query}
                     {where} {sort} LIMIT {limit} OFFSET {offset}) as z
             ) AS j'''
     elif records_format == u'csv':
@@ -1287,9 +1287,11 @@ def search_data(context, data_dict):
     _insert_links(data_dict, limit, offset)
 
     if data_dict.get('include_total', True):
-        count_sql_string = u'''SELECT {distinct} count(*)
-            FROM "{resource}" {ts_query} {where};'''.format(
+        count_sql_string = u'''SELECT count(*) FROM (
+            SELECT {distinct} {select}
+            FROM "{resource}" {ts_query} {where}) as t;'''.format(
             distinct=distinct,
+            select=select_columns,
             resource=resource_id,
             ts_query=ts_query,
             where=where_clause)
@@ -1412,7 +1414,10 @@ def upsert(context, data_dict):
         context['connection'].execute(
             u'SET LOCAL statement_timeout TO {0}'.format(timeout))
         upsert_data(context, data_dict)
-        trans.commit()
+        if data_dict.get(u'dry_run', False):
+            trans.rollback()
+        else:
+            trans.commit()
         return _unrename_json_field(data_dict)
     except IntegrityError as e:
         if e.orig.pgcode == _PG_ERR_CODE['unique_violation']:

ckanext/datastore/logic/action.py

@@ -229,6 +229,9 @@ def datastore_upsert(context, data_dict):
                    Possible options are: upsert, insert, update
                    (optional, default: upsert)
     :type method: string
+    :param dry_run: set to True to abort transaction instead of committing,
+                    e.g. to check for validation or type errors.
+    :type dry_run: bool (optional, default: False)
 
     **Results:**
 

ckanext/datastore/logic/schema.py

@@ -135,6 +135,7 @@ def datastore_upsert_schema():
         'id': [ignore_missing],
         'method': [ignore_missing, text_type, OneOf(
             ['upsert', 'insert', 'update'])],
+        'dry_run': [ignore_missing, boolean_validator],
         '__junk': [empty],
         '__before': [rename('id', 'resource_id')]
     }

ckanext/datastore/tests/test_search.py

@@ -284,7 +284,7 @@ def test_search_distinct(self):
         res_dict = json.loads(res.body)
         assert res_dict['success'] is True
         result = res_dict['result']
-        assert result['total'] == 2
+        assert result['total'] == 1
         assert result['records'] == [{u'author': 'tolstoy'}], result['records']
 
     def test_search_filters(self):