Merge branch 'master' into 877-nginx-instructions

.travis.yml

@@ -5,7 +5,8 @@ python:
 env:
   - PGVERSION=9.1
   - PGVERSION=8.4
-script: ./bin/travis-build
+install: ./bin/travis-install-dependencies
+script: ./bin/travis-run-tests
 notifications:
   irc:
     channels:

CHANGELOG.rst

@@ -12,10 +12,28 @@ v2.2
 
 API changes and deprecations:
 
+
+* The `ckan.api_url` has been completely removed and it can no longer be used
 * The edit() and after_update() methods of IPackageController plugins are now
   called when updating a resource using the web frontend or the
   resource_update API action [#1052]
 
+v2.1.1 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Fix place-holder images on non-root locations (#1309)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Wrong auth checks for sysadmins on package_create (#1184)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+ * Fix organization not set when editing a dataset (#1199)
+ * Fix PDF previews (#1194)
+ * Fix preview failing on private datastore resources (#1221)
 
 v2.1 2013-08-13
 ===============
@@ -96,6 +114,18 @@ Known issues:
  * Under certain authorization setups the frontend for the groups functionality
    may not work as expected (See #1176 #1175).
 
+v2.0.3 2013-11-8
+================
+
+Bug fixes:
+ * Fix errors on preview on non-root locations (#960)
+ * Don't accept invalid URLs in resource proxy (#1106)
+ * Make sure came_from url is local (#1039)
+ * Fix logout redirect in non-root locations (#1025)
+ * Don't return private datasets on package_list (#1295)
+ * Stop tracking failing when no lang/encoding headers (#1192)
+ * Fix for paster db clean command getting frozen
+
 
 v2.0.2 2013-08-13
 =================

CONTRIBUTING.rst

@@ -61,13 +61,15 @@ When writing code for CKAN, try to respect our coding standards:
    css-coding-standards
    javascript-coding-standards
    testing-coding-standards
+   upgrading-dependencies
 
 * `CKAN coding standards <http://docs.ckan.org/en/latest/ckan-coding-standards.html>`_
 * `Python coding standards <http://docs.ckan.org/en/latest/python-coding-standards.html>`_
 * `HTML coding standards <http://docs.ckan.org/en/latest/html-coding-standards.html>`_
 * `CSS coding standards <http://docs.ckan.org/en/latest/css-coding-standards.html>`_
 * `JavaScript coding standards <http://docs.ckan.org/en/latest/javascript-coding-standards.html>`_
 * `Testing coding standards <http://docs.ckan.org/en/latest/testing-coding-standards.html>`_
+* `Upgrading CKAN's dependencies <http://docs.ckan.org/en/latest/upgrading-dependencies.html>`_
 
 
 ---------------

bin/travis-build → bin/travis-install-dependencies

@@ -1,4 +1,7 @@
-#!/bin/sh
+#!/bin/bash
+
+# Exit immediately if any command fails
+set -e
 
 # Drop Travis' postgres cluster if we're building using a different pg version
 TRAVIS_PGVERSION='9.1'
@@ -13,7 +16,7 @@ fi
 # We need this ppa so we can install postgres-8.4
 sudo add-apt-repository -yy ppa:pitti/postgresql
 sudo apt-get update -qq
-sudo apt-get install solr-jetty postgresql-$PGVERSION
+sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1
 
 sudo service postgresql reload
 
@@ -32,43 +35,15 @@ python setup.py develop
 # Install npm dpes for mocha
 npm install -g mocha-phantomjs phantomjs
 
-# Configure Solr
-echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
-sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
-sudo service jetty restart
-
 paster db init -c test-core.ini
 
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
-  psql -c 'CREATE USER datastore_default;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default@\/datastore_test/' test-core.ini
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini
   paster datastore set-permissions postgres -c test-core.ini
 else
   sed -i -e 's/.*datastore.read_url.*//' test-core.ini
 fi
 
 cat test-core.ini
-
-# Run mocha front-end tests
-# We need ckan to be running for some tests
-paster serve test-core.ini &
-sleep 5 # Make sure the server has fully started
-mocha-phantomjs http://localhost:5000/base/test/index.html
-# Did an error occur?
-MOCHA_ERROR=$?
-# We are done so kill ckan
-killall paster
-
-# And finally, run the nosetests
-nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext  --with-coverage --cover-package=ckanext --cover-package=ckan
-# Did an error occur?
-NOSE_ERROR=$?
-
-[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
-[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
-
-# If an error occurred in our tests make sure travis knows
-exit `expr $MOCHA_ERROR + $NOSE_ERROR`

bin/travis-run-tests

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+# Configure Solr
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
+sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+
+# Run mocha front-end tests
+# We need ckan to be running for some tests
+paster serve test-core.ini &
+sleep 5 # Make sure the server has fully started
+mocha-phantomjs http://localhost:5000/base/test/index.html
+# Did an error occur?
+MOCHA_ERROR=$?
+# We are done so kill ckan
+killall paster
+
+# And finally, run the nosetests
+nosetests --ckan --reset-db --with-pylons=test-core.ini --nologcapture ckan ckanext
+# Did an error occur?
+NOSE_ERROR=$?
+
+[ "0" -ne "$MOCHA_ERROR" ] && echo MOCKA tests have failed
+[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
+
+# If an error occurred in our tests make sure travis knows
+exit `expr $MOCHA_ERROR + $NOSE_ERROR`

ckan/ckan_nose_plugin.py

@@ -19,21 +19,26 @@ def startContext(self, ctx):
 
         if 'new_tests' in repr(ctx):
             # We don't want to do the stuff below for new-style tests.
+            if not CkanNose.settings.reset_database:
+                model.repo.tables_created_and_initialised = True
             return
 
         if isclass(ctx):
             if hasattr(ctx, "no_db") and ctx.no_db:
                 return
-            if self.is_first_test or CkanNose.settings.ckan_migration:
+            if (not CkanNose.settings.reset_database
+                    and not CkanNose.settings.ckan_migration):
+                model.Session.close_all()
+                model.repo.tables_created_and_initialised = True
+                model.repo.rebuild_db()
+                self.is_first_test = False
+            elif self.is_first_test or CkanNose.settings.ckan_migration:
                 model.Session.close_all()
                 model.repo.clean_db()
                 self.is_first_test = False
                 if CkanNose.settings.ckan_migration:
                     model.Session.close_all()
                     model.repo.upgrade_db()
-            # init_db is run at the start of every class because
-            # when you use an in-memory sqlite db, it appears that
-            # the db is destroyed after every test when you Session.Remove().
 
             ## This is to make sure the configuration is run again.
             ## Plugins use configure to make their own tables and they
@@ -43,6 +48,9 @@ def startContext(self, ctx):
             for plugin in PluginImplementations(IConfigurable):
                 plugin.configure(config)
 
+            # init_db is run at the start of every class because
+            # when you use an in-memory sqlite db, it appears that
+            # the db is destroyed after every test when you Session.Remove().
             model.repo.init_db()
 
     def options(self, parser, env):
@@ -66,6 +74,11 @@ def options(self, parser, env):
             dest='segments',
             help='A string containing a hex digits that represent which of'
                  'the 16 test segments to run. i.e 15af will run segments 1,5,a,f')
+        parser.add_option(
+            '--reset-db',
+            action='store_true',
+            dest='reset_database',
+            help='drop database and reinitialize before tests are run')
 
     def wantClass(self, cls):
         name = cls.__name__

ckan/config/deployment.ini_tmpl

@@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true
 ckan.auth.user_delete_groups = true
 ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
+ckan.auth.create_user_via_web = true
 
 
 ## Search Settings

ckan/config/middleware.py

@@ -4,6 +4,7 @@
 import logging
 import json
 import hashlib
+import os
 
 import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
@@ -22,6 +23,7 @@
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales_from_config
+import ckan.lib.uploader as uploader
 
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
@@ -147,6 +149,20 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
                 cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
+        storage_directory = uploader.get_storage_path()
+        if storage_directory:
+            path = os.path.join(storage_directory, 'storage')
+            try:
+                os.makedirs(path)
+            except OSError, e:
+                ## errno 17 is file already exists
+                if e.errno != 17:
+                    raise
+
+            storage_app = StaticURLParser(path,
+                cache_max_age=static_max_age)
+            static_parsers.insert(0, storage_app)
+
         # Configurable extra static file paths
         extra_static_parsers = []
         for public_path in config.get('extra_public_paths', '').split(','):

ckan/controllers/group.py

@@ -1,6 +1,8 @@
 import re
+import os
 import logging
 import genshi
+import cgi
 import datetime
 from urllib import urlencode
 
@@ -425,6 +427,9 @@ def new(self, data=None, errors=None, error_summary=None):
             return self._save_new(context, group_type)
 
         data = data or {}
+        if not data.get('image_url', '').startswith('http'):
+            data.pop('image_url', None)
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors,
@@ -524,7 +529,6 @@ def _save_edit(self, id, context):
             data_dict['id'] = id
             context['allow_partial_update'] = True
             group = self._action('group_update')(context, data_dict)
-
             if id != group['name']:
                 self._force_reindex(group)
 

ckan/controllers/user.py

@@ -68,7 +68,7 @@ def _setup_template_variables(self, context, data_dict):
         try:
             user_dict = get_action('user_show')(context, data_dict)
         except NotFound:
-            h.redirect_to(controller='user', action='login', id=None)
+            abort(404, _('User not found'))
         except NotAuthorized:
             abort(401, _('Not authorized to see this page'))
         c.user_dict = user_dict
@@ -117,10 +117,6 @@ def read(self, id=None):
                    'for_view': True}
         data_dict = {'id': id,
                      'user_obj': c.userobj}
-        try:
-            check_access('user_show', context, data_dict)
-        except NotAuthorized:
-            abort(401, _('Not authorized to see this page'))
 
         context['with_related'] = True
 

ckan/lib/app_globals.py

@@ -44,7 +44,6 @@
     'ckan.template_footer_end': {},
     'ckan.dumps_url': {},
     'ckan.dumps_format': {},
-    'ckan.api_url': {},
     'ofs.impl': {'name': 'ofs_impl'},
     'ckan.homepage_style': {'default': '1'},
 

ckan/lib/dictization/model_dictize.py

@@ -10,6 +10,7 @@
 import ckan.lib.dictization as d
 import ckan.new_authz as new_authz
 import ckan.lib.search as search
+import ckan.lib.munge as munge
 
 ## package save
 
@@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context,
 
         group_dict['display_name'] = obj.display_name
 
+        image_url = group_dict.get('image_url')
+        group_dict['image_display_url'] = image_url
+        if image_url and not image_url.startswith('http'):
+            #munge here should not have an effect only doing it incase
+            #of potential vulnerability of dodgy api input
+            image_url = munge.munge_filename(image_url)
+            group_dict['image_display_url'] = h.url_for_static(
+                'uploads/group/%s' % group_dict.get('image_url')
+            )
+
         if obj.is_organization:
             group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0)
         else:
@@ -357,6 +368,16 @@ def group_dictize(group, context):
         for item in plugins.PluginImplementations(plugin):
             result_dict = item.before_view(result_dict)
 
+    image_url = result_dict.get('image_url')
+    result_dict['image_display_url'] = image_url
+    if image_url and not image_url.startswith('http'):
+        #munge here should not have an effect only doing it incase
+        #of potential vulnerability of dodgy api input
+        image_url = munge.munge_filename(image_url)
+        result_dict['image_display_url'] = h.url_for_static(
+            'uploads/group/%s' % result_dict.get('image_url'),
+            qualified = True
+        )
     return result_dict
 
 def tag_list_dictize(tag_list, context):

ckan/lib/helpers.py

@@ -37,6 +37,7 @@
 import ckan.lib.maintain as maintain
 import ckan.lib.datapreview as datapreview
 import ckan.logic as logic
+import ckan.lib.uploader as uploader
 
 from ckan.common import (
     _, ungettext, g, c, request, session, json, OrderedDict
@@ -1664,6 +1665,10 @@ def new_activities():
     action = logic.get_action('dashboard_new_activities_count')
     return action({}, {})
 
+def uploads_enabled():
+    if uploader.get_storage_path():
+        return True
+    return False
 
 def get_featured_organizations(count=1):
     '''Returns a list of favourite organization in the form
@@ -1836,6 +1841,7 @@ def get_site_statistics():
     'radio',
     'submit',
     'asbool',
+    'uploads_enabled',
     'get_featured_organizations',
     'get_featured_groups',
     'get_site_statistics',