Merge branch 'master' of github.com:okfn/ckan into 1155-document-upgr…

…ading-dependencies

.travis.yml

@@ -5,7 +5,8 @@ python:
 env:
   - PGVERSION=9.1
   - PGVERSION=8.4
-script: ./bin/travis-build
+install: ./bin/travis-install-dependencies
+script: ./bin/travis-run-tests
 notifications:
   irc:
     channels:

CHANGELOG.rst

@@ -12,10 +12,28 @@ v2.2
 
 API changes and deprecations:
 
+
+* The `ckan.api_url` has been completely removed and it can no longer be used
 * The edit() and after_update() methods of IPackageController plugins are now
   called when updating a resource using the web frontend or the
   resource_update API action [#1052]
 
+v2.1.1 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Fix place-holder images on non-root locations (#1309)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Wrong auth checks for sysadmins on package_create (#1184)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+ * Fix organization not set when editing a dataset (#1199)
+ * Fix PDF previews (#1194)
+ * Fix preview failing on private datastore resources (#1221)
 
 v2.1 2013-08-13
 ===============
@@ -96,6 +114,18 @@ Known issues:
  * Under certain authorization setups the frontend for the groups functionality
    may not work as expected (See #1176 #1175).
 
+v2.0.3 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+
 
 v2.0.2 2013-08-13
 =================

README.rst

@@ -5,8 +5,8 @@ CKAN: The Open Source Data Portal Software
     :target: http://travis-ci.org/okfn/ckan
     :alt: Build Status
 
-.. image:: https://coveralls.io/repos/okfn/ckan/badge.png?branch=coveralls
-	:target: https://coveralls.io/r/okfn/ckan?branch=coveralls
+.. image:: https://coveralls.io/repos/okfn/ckan/badge.png
+	:target: https://coveralls.io/r/okfn/ckan
 	:alt: Test coverage
 
 **CKAN is the world’s leading open-source data portal platform**.

bin/travis-build → bin/travis-install-dependencies

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Exit immediately if any command fails
+set -e
 
 # Drop Travis' postgres cluster if we're building using a different pg version
 TRAVIS_PGVERSION='9.1'
@@ -32,43 +35,15 @@ python setup.py develop
 # Install npm dpes for mocha
 npm install -g mocha-phantomjs phantomjs
 
-# Configure Solr
-echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
-sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
-
 paster db init -c test-core.ini
 
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
-  psql -c 'CREATE USER datastore_default;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default@\/datastore_test/' test-core.ini
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini
   paster datastore set-permissions postgres -c test-core.ini
 else
   sed -i -e 's/.*datastore.read_url.*//' test-core.ini
 fi
 
 cat test-core.ini
-
-# Run mocha front-end tests
-# We need ckan to be running for some tests
-paster serve test-core.ini &
-sleep 5 # Make sure the server has fully started
-mocha-phantomjs http://localhost:5000/base/test/index.html
-# Did an error occur?
-MOCHA_ERROR=$?
-# We are done so kill ckan
-killall paster
-
-# And finally, run the nosetests
-nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext  --with-coverage --cover-package=ckanext --cover-package=ckan
-# Did an error occur?
-NOSE_ERROR=$?
-
-[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
-[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
-
-# If an error occurred in our tests make sure travis knows
-exit `expr $MOCHA_ERROR + $NOSE_ERROR`

bin/travis-run-tests

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Configure Solr
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
+sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+
+# Run mocha front-end tests
+# We need ckan to be running for some tests
+paster serve test-core.ini &
+sleep 5 # Make sure the server has fully started
+mocha-phantomjs http://localhost:5000/base/test/index.html
+# Did an error occur?
+MOCHA_ERROR=$?
+# We are done so kill ckan
+killall paster
+
+# And finally, run the nosetests
+nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext
+# Did an error occur?
+NOSE_ERROR=$?
+
+[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
+[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
+
+# If an error occurred in our tests make sure travis knows
+exit `expr $MOCHA_ERROR + $NOSE_ERROR`

ckan/config/deployment.ini_tmpl

@@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true
 ckan.auth.user_delete_groups = true
 ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
 
 
 ## Search Settings
@@ -79,6 +80,7 @@ ckan.site_id = default
 ## Plugins Settings
 
 # Note: Add ``datastore`` to enable the CKAN DataStore
+#       Add ``datapusher`` to enable DataPusher
 #       Add ``pdf_preview`` to enable the resource preview for PDFs
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
@@ -146,8 +148,8 @@ ckan.feeds.author_link =
 
 # Make sure you have set up the DataStore
 
-datapusher.formats = csv
-datapusher.url = http://datapusher.ckan.org/
+ckan.datapusher.formats = csv
+ckan.datapusher.url = http://datapusher.ckan.org/
 
 ## Activity Streams Settings
 

ckan/config/middleware.py

@@ -4,6 +4,7 @@
 import logging
 import json
 import hashlib
+import os
 
 import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
@@ -22,6 +23,7 @@
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales_from_config
+import ckan.lib.uploader as uploader
 
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
@@ -147,6 +149,20 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
                 cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
+        storage_directory = uploader.get_storage_path()
+        if storage_directory:
+            path = os.path.join(storage_directory, 'storage')
+            try:
+                os.makedirs(path)
+            except OSError, e:
+                ## errno 17 is file already exists
+                if e.errno != 17:
+                    raise
+
+            storage_app = StaticURLParser(path,
+                cache_max_age=static_max_age)
+            static_parsers.insert(0, storage_app)
+
         # Configurable extra static file paths
         extra_static_parsers = []
         for public_path in config.get('extra_public_paths', '').split(','):

ckan/controllers/api.py

@@ -114,8 +114,10 @@ def _finish_ok(self, response_data=None,
 
         return self._finish(status_int, response_data, content_type)
 
-    def _finish_not_authz(self):
+    def _finish_not_authz(self, extra_msg=None):
         response_data = _('Access denied')
+        if extra_msg:
+            response_data = '%s - %s' % (response_data, extra_msg)
         return self._finish(403, response_data, 'json')
 
     def _finish_not_found(self, extra_msg=None):
@@ -194,10 +196,14 @@ def action(self, logic_function, ver=None):
                                     'data': request_data}
             return_dict['success'] = False
             return self._finish(400, return_dict, content_type='json')
-        except NotAuthorized:
+        except NotAuthorized, e:
             return_dict['error'] = {'__type': 'Authorization Error',
                                     'message': _('Access denied')}
             return_dict['success'] = False
+
+            if e.extra_msg:
+                return_dict['error']['message'] += ': %s' % e.extra_msg
+
             return self._finish(403, return_dict, content_type='json')
         except NotFound, e:
             return_dict['error'] = {'__type': 'Not Found Error',
@@ -277,8 +283,9 @@ def list(self, ver=None, register=None, subregister=None, id=None):
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
 
     def show(self, ver=None, register=None, subregister=None,
              id=None, id2=None):
@@ -308,8 +315,9 @@ def show(self, ver=None, register=None, subregister=None,
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
 
     def _represent_package(self, package):
         return package.as_dict(ref_package_by=self.ref_package_by,
@@ -354,8 +362,9 @@ def create(self, ver=None, register=None, subregister=None,
                                           data_dict.get("id")))
             return self._finish_ok(response_data,
                                    resource_location=location)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
@@ -410,8 +419,9 @@ def update(self, ver=None, register=None, subregister=None,
         try:
             response_data = action(context, data_dict)
             return self._finish_ok(response_data)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
@@ -458,8 +468,9 @@ def delete(self, ver=None, register=None, subregister=None,
         try:
             response_data = action(context, data_dict)
             return self._finish_ok(response_data)
-        except NotAuthorized:
-            return self._finish_not_authz()
+        except NotAuthorized, e:
+            extra_msg = e.extra_msg
+            return self._finish_not_authz(extra_msg)
         except NotFound, e:
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)

ckan/controllers/group.py

@@ -1,6 +1,8 @@
 import re
+import os
 import logging
 import genshi
+import cgi
 import datetime
 from urllib import urlencode
 
@@ -278,7 +280,8 @@ def pager_url(q=None, page=None):
 
             facets = OrderedDict()
 
-            default_facet_titles = {'groups': _('Groups'),
+            default_facet_titles = {'organization': _('Organizations'),
+                                    'groups': _('Groups'),
                                     'tags': _('Tags'),
                                     'res_format': _('Formats'),
                                     'license_id': _('License')}
@@ -344,6 +347,9 @@ def pager_url(q=None, page=None):
             c.facets = {}
             c.page = h.Page(collection=[])
 
+        self._setup_template_variables(context, {'id':id},
+            group_type=group_type)
+
     def bulk_process(self, id):
         ''' Allow bulk processing of datasets for an organization.  Make
         private/public or delete. For organization admins.'''
@@ -421,6 +427,9 @@ def new(self, data=None, errors=None, error_summary=None):
             return self._save_new(context, group_type)
 
         data = data or {}
+        if not data.get('image_url', '').startswith('http'):
+            data.pop('image_url', None)
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors,
@@ -520,7 +529,6 @@ def _save_edit(self, id, context):
             data_dict['id'] = id
             context['allow_partial_update'] = True
             group = self._action('group_update')(context, data_dict)
-
             if id != group['name']:
                 self._force_reindex(group)
 
@@ -828,7 +836,9 @@ def admins(self, id):
 
     def about(self, id):
         c.group_dict = self._get_group_dict(id)
-        return render(self._about_template(c.group_dict['type']))
+        group_type = c.group_dict['type']
+        self._setup_template_variables({}, {'id': id}, group_type=group_type)
+        return render(self._about_template(group_type))
 
     def _get_group_dict(self, id):
         ''' returns the result of group_show action or aborts if there is a

ckan/controllers/home.py

@@ -67,6 +67,7 @@ def index(self):
             c.search_facets = query['search_facets']
 
             c.facet_titles = {
+                'organization': _('Organizations'),
                 'groups': _('Groups'),
                 'tags': _('Tags'),
                 'res_format': _('Formats'),