Merge branch 'master' into postgres-9.3

CHANGELOG.rst

@@ -21,6 +21,70 @@ Note: This version requires re-running the 'datastore set-permissions' command
    CKAN developers should also re-run set-permissions on the test database:
    :ref:`datastore-test-set-permissions`
 
+Changes and deprecations:
+ * The old Celery based background jobs have been removed in CKAN 2.8 in favour of the new RQ based
+   jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
+   of course use Celery but they will need to handle the management themselves.
+
+v2.7.3 2018-03-15
+=================
+
+General notes:
+ * As with all patch releases this one does not include requirement changes.
+   However in some scenarios you might encounter the following error while
+   installing or upgrading this version of CKAN::
+
+     Error: could not determine PostgreSQL version from '10.2'
+
+   This is due to a bug in the psycopg2 version pinned to the release. To solve
+   it, upgrade psycopg2 with the following command::
+
+     pip install --upgrade psycopg2==2.7.3.2
+
+ * This release does not require a Solr schema upgrade, but if you are having the
+   issues described in #3863 (datasets wrongly indexed in multilingual setups),
+   you can upgrade the Solr schema and reindex to solve them.
+
+ * #3422 (implemented in #3425) introduced a major bug where if a resource was
+   deleted and the DataStore was active extras from all resources on the site where
+   changed. This is now fixed as part of this release but if your database is already
+   affected you will need to run a script to restore the extras to their
+   previous state. Remember, you only need to run the script if all the following are
+   true:
+
+   1. You are currently running CKAN 2.7.0 or 2.7.2, and
+   2. You have enabled the DataStore, and
+   3. One or more resources with data on the DataStore have been deleted (or you
+      suspect they might have been)
+
+   If all these are true you can run the following script to restore the extras to
+   their previous state:
+
+   https://github.com/ckan/ckan/blob/dev-v2.7/scripts/4042_fix_resource_extras.py
+
+   This issue is described in #4042
+
+Fixes:
+ * Fix toggle bars header icon (#3880)
+ * Change CORS header keys and values to string instead of unicode (#3855)
+ * Fix cors header when all origins are allowed (#3898)
+ * Update SOLR schema.xml reference in Dockerfile
+ * Build local SOLR container by default
+ * Create datastore indexes only if they are not exist
+ * Properly close file responses
+ * Use javascript content-type for jsonp responses (#4022)
+ * Add Data Dictionary documentation (#3989)
+ * Fix SOLR index delete_package implementation
+ * Add second half of DataStore set-permissions command(Docs)
+ * Fix extras overriding for removed resources (#4042)
+ * Return a 403 if not authorized on the search page (#4081)
+ * Add support for user/pass for Solr as ENV var
+ * Change permission_labels type to string in schema.xml (#3863)
+ * Disallow solr local parameters
+ * Improve text view rendering
+ * Update Orgs/Groups logic for custom fields delete and update (#4094)
+ * Upgrade Solr Docker image
+
 v2.7.2 2017-09-28
 =================
 
@@ -139,6 +203,28 @@ Deprecations:
    jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
    of course use Celery but they will need to handle the management themselves.
 
+v2.6.5 2018-03-15
+=================
+
+Note: This version requires a database upgrade
+
+* Activity Time stored in UTC (#2882)
+* Migration script to adjust current activity timestamps to UTC
+* Change CORS header keys and values to string instead of unicode (#3855)
+* Fix cors header when all origins are allowed (#3898)
+* Update SOLR schema.xml reference in Dockerfile
+* Build local SOLR container by default
+* Create datastore indexes only if they don't exist
+* Properly close file responses
+* Use javascript content-type for jsonp responses (#4022)
+* Fix SOLR index delete_package implementation
+* Add second half of DataStore set-permissions command (Docs)
+* Return a 403 if not authorized on the search page (#4081)
+* Add support for user/pass for Solr as ENV var
+* Disallow solr local parameters
+* Improve text view rendering
+* Update Orgs/Groups logic for custom fields delete and update (#4094)
+
 v2.6.4 2017-09-27
 =================
 
@@ -288,6 +374,27 @@ Bug fixes:
 API changes and deprecations:
  * Replace `c.__version__` with new helper `h.ckan_version()` (`#3103 <https://github.com/ckan/ckan/pull/3103>`_)
 
+v2.5.8 2018-03-15
+=================
+
+Note: This version requires a database upgrade
+
+* Fix language switcher
+* Activity Time stored in UTC (#2882)
+* Migration script to adjust current activity timestamps to UTC
+* Change CORS header keys and values to string instead of unicode (#3855)
+* Fix cors header when all origins are allowed (#3898)
+* Create datastore indexes only if they are not exist
+* Use javascript content-type for jsonp responses (#4022)
+* Fix SOLR index delete_package implementation
+* Add second half of DataStore set-permissions command(Docs)
+* Update SOLR client (pysolr -> solrpy)
+* Return a 403 if not authorized on the search page (#4081)
+* Add support for user/pass for Solr as ENV var
+* Disallow solr local parameters
+* Improve text view rendering
+* Update Orgs/Groups logic for custom fields delete and update (#4094)
+
 v2.5.7 2017-09-27
 =================
 

LICENSE.txt

@@ -2,7 +2,7 @@ License
 +++++++
 
 CKAN - Data Catalogue Software
-Copyright (c) 2006-2017 Open Knowledge International and contributors
+Copyright (c) 2006-2018 Open Knowledge International and contributors
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as
@@ -23,7 +23,7 @@ Note
 
 CKAN is sometimes packaged directly with other software (listed in
 requirements.txt and dev-requirements.txt).
-In these cases, we are required to list the licenses of the packaged softare
+In these cases, we are required to list the licenses of the packaged software
 too. They are all AGPL compatible and read as follows in the next sections.
 
 WebHelpers
@@ -59,35 +59,6 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-
-Migrate
--------
-
-Open Source Initiative OSI - The MIT License:Licensing
-[OSI Approved License]
-
-The MIT License
-
-Copyright (c) <year> <copyright holders>
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-
 SQLAlchemy
 ----------
 
@@ -125,4 +96,3 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-

README.rst

@@ -77,7 +77,7 @@ ckan-dev mailing list.
 Copying and License
 -------------------
 
-This material is copyright (c) 2006-2017 Open Knowledge International and contributors.
+This material is copyright (c) 2006-2018 Open Knowledge International and contributors.
 
 It is open and licensed under the GNU Affero General Public License (AGPL) v3.0
 whose full text may be found at:

ckan/authz.py

@@ -1,7 +1,10 @@
 # encoding: utf-8
 
+import functools
 import sys
 import re
+
+from collections import defaultdict
 from logging import getLogger
 
 from ckan.common import config
@@ -37,6 +40,14 @@ def get(self, function):
             self._build()
         return self._functions.get(function)
 
+    @staticmethod
+    def _is_chained_auth_function(func):
+        '''
+        Helper function to check if a function is a chained auth function, i.e.
+        it has been decorated with the chain auth function decorator.
+        '''
+        return getattr(func, 'chained_auth_function', False)
+
     def _build(self):
         ''' Gather the auth functions.
 
@@ -74,17 +85,31 @@ def _build(self):
         # Then overwrite them with any specific ones in the plugins:
         resolved_auth_function_plugins = {}
         fetched_auth_functions = {}
+        chained_auth_functions = defaultdict(list)
         for plugin in p.PluginImplementations(p.IAuthFunctions):
             for name, auth_function in plugin.get_auth_functions().items():
-                if name in resolved_auth_function_plugins:
+                if self._is_chained_auth_function(auth_function):
+                    chained_auth_functions[name].append(auth_function)
+                elif name in resolved_auth_function_plugins:
                     raise Exception(
                         'The auth function %r is already implemented in %r' % (
                             name,
                             resolved_auth_function_plugins[name]
                         )
                     )
-                resolved_auth_function_plugins[name] = plugin.name
-                fetched_auth_functions[name] = auth_function
+                else:
+                    resolved_auth_function_plugins[name] = plugin.name
+                    fetched_auth_functions[name] = auth_function
+
+        for name, func_list in chained_auth_functions.iteritems():
+            if name not in fetched_auth_functions:
+                raise Exception('The auth %r is not found for chained auth' % (
+                    name))
+            # create the chain of functions in the correct order
+            for func in reversed(func_list):
+                prev_func = fetched_auth_functions[name]
+                fetched_auth_functions[name] = functools.partial(func, prev_func)
+
         # Use the updated ones in preference to the originals.
         self._functions.update(fetched_auth_functions)
 

ckan/ckan_nose_plugin.py

@@ -94,7 +94,7 @@ def wantFunction(self, fn):
 
     def finalize(self, report):
         if self.segments:
-            print 'Segments: %s' % self.segments
+            print('Segments: %s' % self.segments)
 
     def configure(self, settings, config):
         CkanNose.settings = settings

ckan/common.py

@@ -46,6 +46,24 @@ def is_flask_request():
              not pylons_request_available))
 
 
+def streaming_response(
+        data, mimetype=u'application/octet-stream', with_context=False):
+    iter_data = iter(data)
+    if is_flask_request():
+        # Removal of context variables for pylon's app is prevented
+        # inside `pylons_app.py`. It would be better to decide on the fly
+        # whether we need to preserve context, but it won't affect performance
+        # in any visible way and we are going to get rid of pylons anyway.
+        # Flask allows to do this in easy way.
+        if with_context:
+            iter_data = flask.stream_with_context(iter_data)
+        resp = flask.Response(iter_data, mimetype=mimetype)
+    else:
+        response.app_iter = iter_data
+        resp = response.headers['Content-type'] = mimetype
+    return resp
+
+
 def ugettext(*args, **kwargs):
     if is_flask_request():
         return flask_ugettext(*args, **kwargs)

ckan/config/environment.py

@@ -135,6 +135,8 @@ def find_controller(self, controller):
     'ckan.datastore.read_url': 'CKAN_DATASTORE_READ_URL',
     'ckan.redis.url': 'CKAN_REDIS_URL',
     'solr_url': 'CKAN_SOLR_URL',
+    'solr_user': 'CKAN_SOLR_USER',
+    'solr_password': 'CKAN_SOLR_PASSWORD',
     'ckan.site_id': 'CKAN_SITE_ID',
     'ckan.site_url': 'CKAN_SITE_URL',
     'ckan.storage_path': 'CKAN_STORAGE_PATH',
@@ -143,7 +145,8 @@ def find_controller(self, controller):
     'smtp.starttls': 'CKAN_SMTP_STARTTLS',
     'smtp.user': 'CKAN_SMTP_USER',
     'smtp.password': 'CKAN_SMTP_PASSWORD',
-    'smtp.mail_from': 'CKAN_SMTP_MAIL_FROM'
+    'smtp.mail_from': 'CKAN_SMTP_MAIL_FROM',
+    'ckan.max_resource_size': 'CKAN_MAX_UPLOAD_SIZE_MB'
 }
 # End CONFIG_FROM_ENV_VARS
 
@@ -266,17 +269,7 @@ def update_config():
 
     # Create Jinja2 environment
     env = jinja_extensions.Environment(
-        loader=jinja_extensions.CkanFileSystemLoader(template_paths),
-        autoescape=True,
-        extensions=['jinja2.ext.do', 'jinja2.ext.with_',
-                    jinja_extensions.SnippetExtension,
-                    jinja_extensions.CkanExtend,
-                    jinja_extensions.CkanInternationalizationExtension,
-                    jinja_extensions.LinkForExtension,
-                    jinja_extensions.ResourceExtension,
-                    jinja_extensions.UrlForStaticExtension,
-                    jinja_extensions.UrlForExtension]
-    )
+        **jinja_extensions.get_jinja_env_options())
     env.install_gettext_callables(_, ungettext, newstyle=True)
     # custom filters
     env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape