Merge remote-tracking branch 'upstream/master' into flask-helpers-update

CHANGELOG.rst

@@ -47,8 +47,7 @@ General notes:
    configuration option.
  * This version requires a requirements upgrade on source installations
  * This version requires a database upgrade
- * This version does not require a Solr schema upgrade (You may want to
-   upgrade the schema if you want to target Solr>=5, see #2914)
+ * This version requires a Solr schema upgrade
  * There are several old features being officially deprecated starting from
    this version. Check the *Deprecations* section to be prepared.
 

ckan/config/environment.py

@@ -251,7 +251,7 @@ def update_config():
     if extra_template_paths:
         # must be first for them to override defaults
         template_paths = extra_template_paths.split(',') + template_paths
-    config['pylons.app_globals'].template_paths = template_paths
+    config['computed_template_paths'] = template_paths
 
     # Set the default language for validation messages from formencode
     # to what is set as the default locale in the config

ckan/config/middleware/flask_app.py

@@ -6,6 +6,8 @@
 import itertools
 import pkgutil
 
+from jinja2 import ChoiceLoader
+
 from flask import Flask, Blueprint
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
@@ -24,6 +26,7 @@
 import ckan.model as model
 from ckan.lib import helpers
 from ckan.lib import jinja_extensions
+from ckan.lib.render import CkanextTemplateLoader
 from ckan.common import config, g, request, ungettext
 import ckan.lib.app_globals as app_globals
 from ckan.plugins import PluginImplementations
@@ -54,6 +57,10 @@ def make_flask_stack(conf, **app_conf):
     app.template_folder = os.path.join(root, 'templates')
     app.app_ctx_globals_class = CKAN_AppCtxGlobals
     app.url_rule_class = CKAN_Rule
+    app.jinja_loader = ChoiceLoader([
+        app.jinja_loader,
+        CkanextTemplateLoader()
+    ])
 
     # Update Flask config with the CKAN values. We use the common config
     # object as values might have been modified on `load_environment`
@@ -370,7 +377,9 @@ def _register_core_blueprints(app):
     def is_blueprint(mm):
         return isinstance(mm, Blueprint)
 
-    for loader, name, _ in pkgutil.iter_modules(['ckan/views'], 'ckan.views.'):
+    path = os.path.join(os.path.dirname(__file__), '..', '..', 'views')
+
+    for loader, name, _ in pkgutil.iter_modules([path], 'ckan.views.'):
         module = loader.find_module(name).load_module(name)
         for blueprint in inspect.getmembers(module, is_blueprint):
             app.register_blueprint(blueprint[1])

ckan/config/routing.py

@@ -183,7 +183,7 @@ def make_map():
     with SubMapper(map, controller='package') as m:
         m.connect('search', '/dataset', action='search',
                   highlight_actions='index search')
-        m.connect('add dataset', '/dataset/new', action='new')
+        m.connect('dataset_new', '/dataset/new', action='new')
         m.connect('/dataset/{action}',
                   requirements=dict(action='|'.join([
                       'list',

ckan/config/solr/schema.xml

@@ -24,7 +24,7 @@
 <!-- We update the version when there is a backward-incompatible change to this
 schema. In this case the version should be set to the next CKAN version number.
 (x.y but not x.y.z since it needs to be a float) -->
-<schema name="ckan" version="2.7">
+<schema name="ckan" version="2.8">
 
 <types>
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
@@ -112,7 +112,7 @@ schema. In this case the version should be set to the next CKAN version number.
     <field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
 
     <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
-    <field name="permission_labels" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
 
     <field name="res_name" type="textgen" indexed="true" stored="true" multiValued="true" />
     <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>

ckan/controllers/package.py

@@ -162,7 +162,8 @@ def drill_down_url(alternative_url=None, **by):
 
         def remove_field(key, value=None, replace=None):
             return h.remove_url_param(key, value=value, replace=replace,
-                                      controller='package', action='search')
+                                      controller='package', action='search',
+                                      alternative_url=package_type)
 
         c.remove_field = remove_field
 

ckan/lib/render.py

@@ -4,6 +4,8 @@
 import re
 import logging
 
+from jinja2 import FileSystemLoader
+
 from ckan.common import config
 
 log = logging.getLogger(__name__)
@@ -17,7 +19,7 @@ def reset_template_info_cache():
 def find_template(template_name):
     ''' looks through the possible template paths to find a template
     returns the full path is it exists. '''
-    template_paths = config['pylons.app_globals'].template_paths
+    template_paths = config['computed_template_paths']
     for path in template_paths:
         if os.path.exists(os.path.join(path, template_name.encode('utf-8'))):
             return os.path.join(path, template_name)
@@ -47,3 +49,16 @@ def template_info(template_name):
                   'template_type' : t_type,}
         _template_info_cache[template_name] = t_data
     return template_path, t_type
+
+
+class CkanextTemplateLoader(FileSystemLoader):
+    def __init__(self):
+        super(CkanextTemplateLoader, self).__init__([])
+
+    @property
+    def searchpath(self):
+        return config['computed_template_paths']
+
+    @searchpath.setter
+    def searchpath(self, _):
+        pass

ckan/lib/search/__init__.py

@@ -31,7 +31,7 @@ def text_traceback():
     return res
 
 
-SUPPORTED_SCHEMA_VERSIONS = ['2.7']
+SUPPORTED_SCHEMA_VERSIONS = ['2.8']
 
 DEFAULT_OPTIONS = {
     'limit': 20,

ckan/templates/package/search.html

@@ -4,7 +4,7 @@
 {% block subtitle %}{{ _("Datasets") }}{% endblock %}
 
 {% block breadcrumb_content %}
-  <li class="active">{{ h.nav_link(_('Datasets'), controller='package', action='search', highlight_actions = 'new index') }}</li>
+  <li class="active">{{ h.nav_link(_(dataset_type.title() + 's'), controller='package', action='search', named_route=dataset_type + '_search', highlight_actions = 'new index') }}</li>
 {% endblock %}
 
 {% block primary_content %}
@@ -13,7 +13,7 @@
       {% block page_primary_action %}
         {% if h.check_access('package_create') %}
           <div class="page_primary_action">
-            {% snippet 'snippets/add_dataset.html' %}
+            {{ h.snippet ('snippets/add_dataset.html', dataset_type=dataset_type) }}
           </div>
         {% endif %}
       {% endblock %}
@@ -32,7 +32,7 @@
           (_('Last Modified'), 'metadata_modified desc'),
           (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ]
         %}
-        {% snippet 'snippets/search_form.html', form_id='dataset-search-form', type='dataset', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.params, error=c.query_error, fields=c.fields %}
+        {% snippet 'snippets/search_form.html', form_id='dataset-search-form', type=dataset_type, query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search ' + dataset_type + 's') + '...', facets=facets, show_empty=request.params, error=c.query_error, fields=c.fields %}
       {% endblock %}
       {% block package_search_results_list %}
         {{ h.snippet('snippets/package_list.html', packages=c.page.items) }}

ckan/templates/snippets/add_dataset.html

@@ -1,7 +1,9 @@
 {# Adds 'Add Dataset' button #}
 
+{% set dataset_type = dataset_type if dataset_type else 'dataset' %}
+
 {% if group %}
     {% link_for _('Add Dataset'), controller='package', action='new', group=group, class_='btn btn-primary', icon='plus-square' %}
 {% else %}
-    {% link_for _('Add Dataset'), controller='package', action='new', class_="btn btn-primary", icon="plus-square" %}
+    {% link_for _('Add ' + dataset_type.title()), controller='package', action='new', named_route=dataset_type + '_new', class_='btn btn-primary', icon='plus-square' %}
 {% endif %}

ckan/templates/snippets/package_item.html

@@ -33,7 +33,7 @@ <h3 class="dataset-heading">
               {% endif %}
             {% endblock %}
             {% block heading_title %}
-              {{ h.link_to(h.truncate(title, truncate_title), h.url_for(controller='package', action='read', id=package.name)) }}
+              {{ h.link_to(h.truncate(title, truncate_title), h.url_for(package.type + '_read', controller='package', action='read', id=package.name)) }}
             {% endblock %}
             {% block heading_meta %}
               {% if package.get('state', '').startswith('draft') %}

ckan/templates/snippets/search_result_text.html

@@ -28,18 +28,31 @@
   {% set text_query_none = _('No organizations found for "{query}"') %}
   {% set text_no_query = ungettext('{number} organization found', '{number} organizations found', count) %}
   {% set text_no_query_none = _('No organizations found') %}
+
+{% else %}
+  {% set text_query_singular = '{number} ' + type + ' found for "{query}"' %}
+  {% set text_query_plural = '{number} ' + type + 's found for "{query}"' %}
+  {% set text_query_none_plural = 'No ' + type + 's found for "{query}"' %}
+  {% set text_no_query_singular = '{number} ' + type + ' found' %}        
+  {% set text_no_query_plural = '{number} ' + type + 's found' %}
+  {% set text_no_query_none_plural = 'No ' + type + 's found' %}      
+
+  {% set text_query = ungettext(text_query_singular, text_query_plural, count) %}
+  {% set text_query_none = _(text_query_none_plural) %}
+  {% set text_no_query = ungettext(text_no_query_singular, text_no_query_plural, count) %}
+  {% set text_no_query_none = _(text_no_query_none_plural) %}
 {%- endif -%}
 
 {% if query %}
   {%- if count -%}
-    {{ text_query.format(number=h.localised_number(count), query=query) }}
+    {{ text_query.format(number=h.localised_number(count), query=query, type=type) }}
   {%- else -%}
-    {{ text_query_none.format(query=query) }}
+    {{ text_query_none.format(query=query, type=type) }}
   {%- endif -%}
 {%- else -%}
   {%- if count -%}
-    {{ text_no_query.format(number=h.localised_number(count)) }}
+    {{ text_no_query.format(number=h.localised_number(count), type=type) }}
   {%- else -%}
-    {{ text_no_query_none }}
+    {{ text_no_query_none.format(type=type) }}
   {%- endif -%}
 {%- endif -%}

ckan/tests/controllers/test_api.py

@@ -266,6 +266,7 @@ def test_api_info(self):
 
         if not p.plugin_loaded('datastore'):
             p.load('datastore')
+
         app = self._get_test_app()
         page = app.get(url, status=200)
         p.unload('datastore')
@@ -277,7 +278,7 @@ def test_api_info(self):
             '<code>http://test.ckan.net/api/3/action/datastore_search',
             'http://test.ckan.net/api/3/action/datastore_search_sql',
             'http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&amp;limit=5',
-            'http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&amp;q=jones',
+            'http://test.ckan.net/api/3/action/datastore_search?q=jones&amp;resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b',
             'http://test.ckan.net/api/3/action/datastore_search_sql?sql=SELECT * from &#34;588dfa82-760c-45a2-b78a-e3bc314a4a9b&#34; WHERE title LIKE &#39;jones&#39;',
             "url: 'http://test.ckan.net/api/3/action/datastore_search'",
             "http://test.ckan.net/api/3/action/datastore_search?resource_id=588dfa82-760c-45a2-b78a-e3bc314a4a9b&amp;limit=5&amp;q=title:jones",

ckan/views/__init__.py

@@ -62,12 +62,12 @@ def set_cors_headers_for_response(response):
             cors_origin_allowed = request.headers.get(u'Origin')
 
         if cors_origin_allowed is not None:
-            response.headers[u'Access-Control-Allow-Origin'] = \
+            response.headers[b'Access-Control-Allow-Origin'] = \
                 cors_origin_allowed
-            response.headers[u'Access-Control-Allow-Methods'] = \
-                u'POST, PUT, GET, DELETE, OPTIONS'
-            response.headers[u'Access-Control-Allow-Headers'] = \
-                u'X-CKAN-API-KEY, Authorization, Content-Type'
+            response.headers[b'Access-Control-Allow-Methods'] = \
+                b'POST, PUT, GET, DELETE, OPTIONS'
+            response.headers[b'Access-Control-Allow-Headers'] = \
+                b'X-CKAN-API-KEY, Authorization, Content-Type'
 
     return response
 

ckan/views/api.py

@@ -445,7 +445,12 @@ def snippet(snippet_path, ver=API_REST_DEFAULT_VERSION):
         We only allow snippets in templates/ajax_snippets and its subdirs
     '''
     snippet_path = u'ajax_snippets/' + snippet_path
-    return render(snippet_path, extra_vars=dict(request.args))
+    # werkzeug.datastructures.ImmutableMultiDict.to_dict
+    # by default returns flattened dict with first occurences of each key.
+    # For retrieving multiple values per key, use named argument `flat`
+    # set to `False`
+    extra_vars = request.args.to_dict()
+    return render(snippet_path, extra_vars=extra_vars)
 
 
 def i18n_js_translations(lang, ver=API_REST_DEFAULT_VERSION):

doc/maintaining/authorization.rst

@@ -44,12 +44,18 @@ dataset searches but are shown in dataset searches within the organization.
 When a user joins an organization, an organization admin gives them one of
 three roles: member, editor or admin.
 
-An organization **admin** can:
+A **member** can:
+
+* View the organization's private datasets.
+
+An **editor** can do everything as **member** plus:
 
-* View the organization's private datasets
 * Add new datasets to the organization
 * Edit or delete any of the organization's datasets
-* Make  datasets public or private.
+* Make datasets public or private.
+
+An organization **admin** can do everything as **editor** plus:
+
 * Add users to the organization, and choose whether to make the new user a
   member, editor or admin
 * Change the role of any user in the organization, including other admin users
@@ -58,16 +64,6 @@ An organization **admin** can:
   description or image)
 * Delete the organization
 
-An **editor** can:
-
-* View the organization's private datasets
-* Add new datasets to the organization
-* Edit or delete any of the organization's datasets
-
-A **member** can:
-
-* View the organization's private datasets.
-
 When a user creates a new organization, they automatically become the first
 admin of that organization.
 

doc/maintaining/configuration.rst

@@ -547,7 +547,7 @@ Example::
 Default value: ``False``
 
 
-Allow new user accounts to be created via the API.
+Allow new user accounts to be created via the API by anyone. When ``False`` only sysadmins are authorised.
 
 .. _ckan.auth.create_user_via_web:
 

requirements.in

@@ -10,7 +10,6 @@ Flask-Babel==0.11.2
 Jinja2==2.8
 Markdown==2.6.7
 ofs==0.4.2
-ordereddict==1.1
 Pairtree==0.7.1-T
 passlib==1.6.5
 paste==1.7.5.1

requirements.txt

@@ -23,7 +23,6 @@ Markdown==2.6.7
 MarkupSafe==0.23          # via jinja2, mako, webhelpers
 nose==1.3.7               # via pylons
 ofs==0.4.2
-ordereddict==1.1
 Pairtree==0.7.1-T
 passlib==1.6.5
 paste==1.7.5.1