Merge branch 'canada-v2.5' into canada-v2.6

Conflicts:
	.travis.yml
	ckan/authz.py
	ckan/config/environment.py
	ckan/config/middleware/common_middleware.py
	ckan/controllers/group.py
	ckan/controllers/package.py
	ckan/controllers/user.py
	ckan/lib/activity_streams.py
	ckan/lib/base.py
	ckan/lib/helpers.py
	ckan/lib/plugins.py
	ckan/lib/search/query.py
	ckan/logic/__init__.py
	ckan/logic/action/create.py
	ckan/logic/action/get.py
	ckan/logic/auth/get.py
	ckan/plugins/toolkit.py
	ckan/public/base/javascript/modules/image-upload.js
	ckan/templates/package/base.html
	ckan/templates/package/confirm_delete.html
	ckan/templates/package/snippets/resource_view.html
	ckan/templates/snippets/home_breadcrumb_item.html
	ckan/templates/snippets/related.html
	ckan/templates/user/edit_user_form.html
	ckanext/datapusher/tests/test.py
	ckanext/datastore/db.py
	ckanext/datastore/logic/action.py
	ckanext/datastore/tests/helpers.py
	ckanext/datastore/tests/test_create.py
	ckanext/datastore/tests/test_delete.py
	ckanext/datastore/tests/test_helpers.py
	ckanext/datastore/tests/test_search.py
	ckanext/datastore/tests/test_unit.py
	ckanext/datastore/tests/test_upsert.py
	requirements.txt

ckan/authz.py

@@ -1,7 +1,6 @@
 # encoding: utf-8
 
 import sys
-import re
 from logging import getLogger
 
 from ckan.common import config
@@ -89,7 +88,7 @@ def _build(self):
         self._functions.update(fetched_auth_functions)
 
 _AuthFunctions = AuthFunctions()
-#remove the class
+# remove the class
 del AuthFunctions
 
 
@@ -111,7 +110,8 @@ def is_sysadmin(username):
 
 
 def _get_user(username):
-    ''' Try to get the user from c, if possible, and fallback to using the DB '''
+    '''Try to get the user from c, if possible, and fallback to using the DB
+    '''
     if not username:
         return None
     # See if we can get the user without touching the DB
@@ -148,11 +148,22 @@ def is_authorized(action, context, data_dict=None):
     if context.get('ignore_auth'):
         return {'success': True}
 
+    read_only = asbool(config.get('ckan.read_only', 'false'))
+
     auth_function = _AuthFunctions.get(action)
     if auth_function:
         username = context.get('user')
-        user = _get_user(username)
 
+        if read_only:
+            # If the auth function has not been wrapped with a auth_read_safe
+            # decorator, we should deny it if read-only mode is on.
+            if not getattr(auth_function, 'auth_read_safe', False):
+                return {
+                    'success': False,
+                    'msg': 'Read-only mode is enabled.'
+                }
+
+        user = _get_user(username)
         if user:
             # deleted users are always unauthorized
             if user.is_deleted():
@@ -169,10 +180,12 @@ def is_authorized(action, context, data_dict=None):
         # access straight away
         if not getattr(auth_function, 'auth_allow_anonymous_access', False) \
            and not context.get('auth_user_obj'):
-            return {'success': False,
-                    'msg': '{0} requires an authenticated user'
-                            .format(auth_function)
-                   }
+            return {
+                'success': False,
+                'msg': '{0} requires an authenticated user'.format(
+                    auth_function
+                )
+            }
 
         return auth_function(context, data_dict)
     else:
@@ -182,7 +195,13 @@ def is_authorized(action, context, data_dict=None):
 # these are the permissions that roles have
 ROLE_PERMISSIONS = OrderedDict([
     ('admin', ['admin']),
-    ('editor', ['read', 'delete_dataset', 'create_dataset', 'update_dataset', 'manage_group']),
+    ('editor', [
+        'read',
+        'delete_dataset',
+        'create_dataset',
+        'update_dataset',
+        'manage_group'
+    ]),
     ('member', ['read', 'manage_group']),
 ])
 
@@ -253,7 +272,8 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission):
         return True
     # Handle when permissions cascade. Check the user's roles on groups higher
     # in the group hierarchy for permission.
-    for capacity in check_config_permission('roles_that_cascade_to_sub_groups'):
+    for capacity in check_config_permission(
+            'roles_that_cascade_to_sub_groups'):
         parent_groups = group.get_parent_group_hierarchy(type=group.type)
         group_ids = [group_.id for group_ in parent_groups]
         if _has_user_permission_for_groups(user_id, permission, group_ids,
@@ -335,7 +355,7 @@ def has_user_permission_for_some_org(user_name, permission):
 
     # see if any of the groups are orgs
     q = model.Session.query(model.Group) \
-        .filter(model.Group.is_organization == True) \
+        .filter(model.Group.is_organization.is_(True)) \
         .filter(model.Group.state == 'active') \
         .filter(model.Group.id.in_(group_ids))
 
@@ -417,6 +437,7 @@ def auth_is_registered_user():
     '''
     return auth_is_loggedin_user()
 
+
 def auth_is_loggedin_user():
     ''' Do we have a logged in user '''
     try:
@@ -425,6 +446,7 @@ def auth_is_loggedin_user():
         context_user = None
     return bool(context_user)
 
+
 def auth_is_anon_user(context):
     ''' Is this an anonymous user?
         eg Not logged in if a web request and not user defined in context

ckan/config/environment.py

@@ -2,11 +2,16 @@
 
 '''CKAN environment configuration'''
 import os
+import tempfile
+import atexit
+import shutil
+from functools import partial
 import logging
 import warnings
 from urlparse import urlparse
 import pytz
 
+import jinja2
 import sqlalchemy
 from pylons import config as pylons_config
 import formencode
@@ -230,17 +235,32 @@ def update_config():
     logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
 
     # Create Jinja2 environment
+    cache_dir = config.get('jinja2_cache_dir', None)
+    if not cache_dir:
+        cache_dir = tempfile.mkdtemp()
+        config['jinja2_cache_dir'] = cache_dir
+        atexit.register(partial(shutil.rmtree, cache_dir))
+    elif not os.path.exists(cache_dir):
+        os.makedirs(cache_dir)
     env = jinja_extensions.Environment(
         loader=jinja_extensions.CkanFileSystemLoader(template_paths),
         autoescape=True,
-        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
-                    jinja_extensions.SnippetExtension,
-                    jinja_extensions.CkanExtend,
-                    jinja_extensions.CkanInternationalizationExtension,
-                    jinja_extensions.LinkForExtension,
-                    jinja_extensions.ResourceExtension,
-                    jinja_extensions.UrlForStaticExtension,
-                    jinja_extensions.UrlForExtension]
+        auto_reload=False,
+        extensions=[
+            'jinja2.ext.do',
+            'jinja2.ext.with_',
+            'jinja2.ext.InternationalizationExtension',
+            jinja_extensions.SnippetExtension,
+            jinja_extensions.CkanExtend,
+            jinja_extensions.LinkForExtension,
+            jinja_extensions.ResourceExtension,
+            jinja_extensions.UrlForStaticExtension,
+            jinja_extensions.UrlForExtension
+        ],
+        # The pre-2.8 default was only 50, the post-2.8 default is 400.
+        cache_size=400,
+
+        bytecode_cache=jinja2.FileSystemBytecodeCache(cache_dir)
     )
     env.install_gettext_callables(_, ungettext, newstyle=True)
     # custom filters

ckan/config/routing.py

@@ -240,24 +240,24 @@ def make_map():
                       'api_data',
                   ])))
         m.connect('dataset_edit', '/dataset/edit/{id}', action='edit',
-                  ckan_icon='edit')
+                  ckan_icon='pencil-square-o')
         m.connect('dataset_followers', '/dataset/followers/{id}',
-                  action='followers', ckan_icon='group')
+                  action='followers', ckan_icon='users')
         m.connect('dataset_activity', '/dataset/activity/{id}',
-                  action='activity', ckan_icon='time')
+                  action='activity', ckan_icon='clock-o')
         m.connect('/dataset/activity/{id}/{offset}', action='activity')
         m.connect('dataset_groups', '/dataset/groups/{id}',
-                  action='groups', ckan_icon='group')
+                  action='groups', ckan_icon='users')
         m.connect('dataset_resources', '/dataset/resources/{id}',
-                  action='resources', ckan_icon='reorder')
+                  action='resources', ckan_icon='bars')
         m.connect('dataset_read', '/dataset/{id}', action='read',
                   ckan_icon='sitemap')
         m.connect('/dataset/{id}/resource/{resource_id}',
                   action='resource_read')
         m.connect('/dataset/{id}/resource_delete/{resource_id}',
                   action='resource_delete')
         m.connect('resource_edit', '/dataset/{id}/resource_edit/{resource_id}',
-                  action='resource_edit', ckan_icon='edit')
+                  action='resource_edit', ckan_icon='pencil-square-o')
         m.connect('/dataset/{id}/resource/{resource_id}/download',
                   action='resource_download')
         m.connect('/dataset/{id}/resource/{resource_id}/download/{filename}',
@@ -270,12 +270,12 @@ def make_map():
         m.connect('/dataset/{id}/resource/{resource_id}/preview',
                   action='resource_datapreview')
         m.connect('views', '/dataset/{id}/resource/{resource_id}/views',
-                  action='resource_views', ckan_icon='reorder')
+                  action='resource_views', ckan_icon='bars')
         m.connect('new_view', '/dataset/{id}/resource/{resource_id}/new_view',
-                  action='edit_view', ckan_icon='edit')
+                  action='edit_view', ckan_icon='pencil-square-o')
         m.connect('edit_view',
                   '/dataset/{id}/resource/{resource_id}/edit_view/{view_id}',
-                  action='edit_view', ckan_icon='edit')
+                  action='edit_view', ckan_icon='pencil-square-o')
         m.connect('resource_view',
                   '/dataset/{id}/resource/{resource_id}/view/{view_id}',
                   action='resource_view')
@@ -307,13 +307,13 @@ def make_map():
                       'activity',
                   ])))
         m.connect('group_about', '/group/about/{id}', action='about',
-                  ckan_icon='info-sign'),
+                  ckan_icon='info-circle'),
         m.connect('group_edit', '/group/edit/{id}', action='edit',
-                  ckan_icon='edit')
+                  ckan_icon='pencil-square-o')
         m.connect('group_members', '/group/members/{id}', action='members',
-                  ckan_icon='group'),
+                  ckan_icon='users'),
         m.connect('group_activity', '/group/activity/{id}/{offset}',
-                  action='activity', ckan_icon='time'),
+                  action='activity', ckan_icon='clock-o'),
         m.connect('group_read', '/group/{id}', action='read',
                   ckan_icon='sitemap')
 
@@ -331,16 +331,16 @@ def make_map():
                       'history'
                   ])))
         m.connect('organization_activity', '/organization/activity/{id}/{offset}',
-                  action='activity', ckan_icon='time')
+                  action='activity', ckan_icon='clock-o')
         m.connect('organization_read', '/organization/{id}', action='read')
         m.connect('organization_about', '/organization/about/{id}',
-                  action='about', ckan_icon='info-sign')
+                  action='about', ckan_icon='info-circle')
         m.connect('organization_read', '/organization/{id}', action='read',
                   ckan_icon='sitemap')
         m.connect('organization_edit', '/organization/edit/{id}',
-                  action='edit', ckan_icon='edit')
+                  action='edit', ckan_icon='pencil-square-o')
         m.connect('organization_members', '/organization/members/{id}',
-                  action='members', ckan_icon='group')
+                  action='members', ckan_icon='users')
         m.connect('organization_bulk_process',
                   '/organization/bulk_process/{id}',
                   action='bulk_process', ckan_icon='sitemap')
@@ -364,20 +364,20 @@ def make_map():
         m.connect('user_generate_apikey', '/user/generate_key/{id}', action='generate_apikey')
         m.connect('/user/activity/{id}/{offset}', action='activity')
         m.connect('user_activity_stream', '/user/activity/{id}',
-                  action='activity', ckan_icon='time')
+                  action='activity', ckan_icon='clock-o')
         m.connect('user_dashboard', '/dashboard', action='dashboard',
                   ckan_icon='list')
         m.connect('user_dashboard_datasets', '/dashboard/datasets',
                   action='dashboard_datasets', ckan_icon='sitemap')
         m.connect('user_dashboard_groups', '/dashboard/groups',
-                  action='dashboard_groups', ckan_icon='group')
+                  action='dashboard_groups', ckan_icon='users')
         m.connect('user_dashboard_organizations', '/dashboard/organizations',
-                  action='dashboard_organizations', ckan_icon='building')
+                  action='dashboard_organizations', ckan_icon='building-o')
         m.connect('/dashboard/{offset}', action='dashboard')
         m.connect('user_follow', '/user/follow/{id}', action='follow')
         m.connect('/user/unfollow/{id}', action='unfollow')
         m.connect('user_followers', '/user/followers/{id:.*}',
-                  action='followers', ckan_icon='group')
+                  action='followers', ckan_icon='users')
         m.connect('user_edit', '/user/edit/{id:.*}', action='edit',
                   ckan_icon='cog')
         m.connect('user_delete', '/user/delete/{id}', action='delete')
@@ -395,13 +395,6 @@ def make_map():
                   ckan_icon='sitemap')
         m.connect('user_index', '/user', action='index')
 
-    with SubMapper(map, controller='revision') as m:
-        m.connect('/revision', action='index')
-        m.connect('/revision/edit/{id}', action='edit')
-        m.connect('/revision/diff/{id}', action='diff')
-        m.connect('/revision/list', action='list')
-        m.connect('/revision/{id}', action='read')
-
     # feeds
     with SubMapper(map, controller='feed') as m:
         m.connect('/feeds/group/{id}.atom', action='group')
@@ -411,11 +404,11 @@ def make_map():
         m.connect('/feeds/custom.atom', action='custom')
 
     map.connect('ckanadmin_index', '/ckan-admin', controller='admin',
-                action='index', ckan_icon='legal')
+                action='index', ckan_icon='gavel')
     map.connect('ckanadmin_config', '/ckan-admin/config', controller='admin',
-                action='config', ckan_icon='check')
+                action='config', ckan_icon='check-square-o')
     map.connect('ckanadmin_trash', '/ckan-admin/trash', controller='admin',
-                action='trash', ckan_icon='trash')
+                action='trash', ckan_icon='trash-o')
     map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')
 
     with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m:

ckan/config/solr/schema.xml

@@ -118,6 +118,10 @@ schema. In this case the version should be set to the next CKAN version number.
     <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="keywords" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="keywords_fra" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="subject" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="subject_fra" type="string" indexed="true" stored="true" multiValued="true"/>
 
     <!-- catchall field, containing all other searchable text fields (implemented
          via copyField further on in this schema  -->