Merge branch 'master' of github.com:okfn/ckan into 408-remove-legacy-…

…org-code

.travis.yml

@@ -1,34 +1,16 @@
 language: python
 python:
+  - "2.6"
   - "2.7"
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq solr-jetty
-install:
-  - "pip install -r pip-requirements.txt --use-mirrors"
-  - "pip install -r pip-requirements-test.txt --use-mirrors"
-before_script:
-  - psql -c 'CREATE DATABASE ckantest;' -U postgres
-  - psql -c 'CREATE DATABASE datastore;' -U postgres
-  - psql -c 'CREATE USER readonlyuser;' -U postgres
-  - python setup.py develop
-  - paster make-config ckan development.ini --no-interactive
-  - sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' development.ini
-  - sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' development.ini
-  - sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@localhost\/ckantest/' development.ini
-  - sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@localhost\/datastore/' development.ini
-  - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@localhost\/datastore/' development.ini
-  - cat development.ini
-  - echo -e "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
-  - sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
-  - sudo service jetty restart
-  - paster --plugin=ckan db init
-  - sed -i -e 's/ -W//g' ckanext/datastore/bin/datastore_setup.py
-  - paster datastore set-permissions postgres
-script: "nosetests --ckan ckan && nosetests --ckan --with-pylons=test-core.ini --nologcapture --cover-package=ckanext.datastore ckanext/datastore/tests -x"
+env:
+  - PGVERSION=9.1
+  - PGVERSION=8.4
+script: ./bin/travis-build
 notifications:
   irc:
     channels:
       - "irc.freenode.org#ckan"
+    on_success: never
+    on_failure: change
     template:
-      - "%{repository} (%{branch} - %{commit}) [%{build_url}] %{author}: %{message}"
+      - "%{repository} %{branch} %{commit} %{build_url} %{author}: %{message}"

bin/travis-build

@@ -0,0 +1,55 @@
+#!/bin/sh
+
+# Drop Travis' postgres cluster if we're building using a different pg version
+TRAVIS_PGVERSION='9.1'
+if [ $PGVERSION != $TRAVIS_PGVERSION ]
+then
+  sudo -u postgres pg_dropcluster --stop $TRAVIS_PGVERSION main
+  # Make psql use $PGVERSION
+  export PGCLUSTER=$PGVERSION/main
+fi
+
+# Install postgres and solr
+# We need this ppa so we can install postgres-8.4
+sudo add-apt-repository -yy ppa:pitti/postgresql
+sudo apt-get update -qq
+sudo apt-get install solr-jetty postgresql-$PGVERSION
+
+# Don't require a password to access DB
+sudo sed -i -e 's/ident/trust/g' /etc/postgresql/$PGVERSION/main/pg_hba.conf 
+
+sudo service postgresql reload
+
+pip install -r pip-requirements.txt --use-mirrors
+pip install -r pip-requirements-test.txt --use-mirrors
+
+psql -c 'CREATE DATABASE ckantest;' -U postgres
+psql -c 'CREATE DATABASE datastore;' -U postgres
+
+python setup.py develop
+
+# Configure CKAN's configuration file
+paster make-config ckan development.ini --no-interactive
+sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' development.ini
+sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' development.ini
+sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@\/ckantest/' development.ini
+sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@\/datastore/' development.ini
+
+# Configure Solr
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+
+paster --plugin=ckan db init
+
+# If Postgres >= 9.0, we don't need to use datastore's legacy mode.
+if [ $PGVERSION != '8.4' ]
+then
+  psql -c 'CREATE USER readonlyuser;' -U postgres
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/datastore/' development.ini
+  paster datastore set-permissions postgres
+fi
+
+
+# And finally, run the tests
+nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext

ckan/common.py

@@ -0,0 +1,17 @@
+# This file contains commonly used parts of external libraries. The idea is
+# to help in removing helpers from being used as a dependency by many files
+# but at the same time making it easy to change for example the json lib
+# used.
+#
+# NOTE:  This file is specificaly created for
+# from ckan.common import x, y, z to be allowed
+
+
+from pylons.i18n import _, ungettext
+from pylons import g, c, request, session, response
+import simplejson as json
+
+try:
+    from collections import OrderedDict  # from python 2.7
+except ImportError:
+    from sqlalchemy.util import OrderedDict

ckan/config/middleware.py

@@ -27,10 +27,10 @@
 import ckan.lib.app_globals as app_globals
 
 
-def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
+def make_app(conf, full_stack=True, static_files=True, **app_conf):
     """Create a Pylons WSGI application and return it
 
-    ``global_conf``
+    ``conf``
         The inherited configuration for this application. Normally from
         the [DEFAULT] section of the Paste ini file.
 
@@ -51,7 +51,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     """
     # Configure the Pylons environment
-    load_environment(global_conf, app_conf)
+    load_environment(conf, app_conf)
 
     # The Pylons WSGI app
     app = PylonsApp()
@@ -90,7 +90,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
 
     if asbool(full_stack):
         # Handle Python exceptions
-        app = ErrorHandler(app, global_conf, **config['pylons.errorware'])
+        app = ErrorHandler(app, conf, **config['pylons.errorware'])
 
         # Display error documents for 401, 403, 404 status codes (and
         # 500 when debug is disabled)
@@ -100,7 +100,7 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
             app = StatusCodeRedirect(app, [400, 404, 500])
 
     # Initialize repoze.who
-    who_parser = WhoConfig(global_conf['here'])
+    who_parser = WhoConfig(conf['here'])
     who_parser.parse(open(app_conf['who.config_file']))
 
     if asbool(config.get('openid_enabled', 'true')):

ckan/config/routing.py

@@ -184,21 +184,6 @@ def make_map():
     ## /END API
     ###########
 
-
-    ## Webstore
-    if config.get('ckan.datastore.enabled', False):
-        with SubMapper(map, controller='datastore') as m:
-            m.connect('datastore_read', '/api/data/{id}{url:(/.*)?}',
-                      action='read', url='', conditions=GET)
-            m.connect('datastore_write', '/api/data/{id}{url:(/.*)?}',
-                      action='write', url='', conditions=PUT_POST_DELETE)
-            m.connect('datastore_read_shortcut',
-                      '/dataset/{dataset}/resource/{id}/api{url:(/.*)?}',
-                      action='read', url='', conditions=GET)
-            m.connect('datastore_write_shortcut',
-                      '/dataset/{dataset}/resource/{id}/api{url:(/.*)?}',
-                      action='write', url='', conditions=PUT_POST_DELETE)
-
     map.redirect('/packages', '/dataset')
     map.redirect('/packages/{url:.*}', '/dataset/{url}')
     map.redirect('/package', '/dataset')
@@ -212,8 +197,8 @@ def make_map():
                   action='delete')
         m.connect('related_list', '/dataset/{id}/related', action='list',
                   ckan_icon='picture')
-        m.connect('related_read', '/apps/{id}', action='read')
-        m.connect('related_dashboard', '/apps', action='dashboard')
+        m.connect('related_read', '/related/{id}', action='read')
+        m.connect('related_dashboard', '/related', action='dashboard')
 
     with SubMapper(map, controller='package') as m:
         m.connect('search', '/dataset', action='search',
@@ -299,12 +284,15 @@ def make_map():
           'follow',
           'unfollow',
           'admins',
-          'about',
           'activity',
           ]))
           )
-        m.connect('group_activity', '/group/activity/{id}/{offset}', action='activity'),
-        m.connect('group_read', '/group/{id}', action='read')
+        m.connect('group_about', '/group/about/{id}', action='about',
+                  ckan_icon='info-sign'),
+        m.connect('group_activity', '/group/activity/{id}/{offset}',
+                  action='activity', ckan_icon='time'),
+        m.connect('group_read', '/group/{id}', action='read',
+                  ckan_icon='sitemap')
 
     # organizations these basically end up being the same as groups
     with SubMapper(map, controller='organization') as m:
@@ -313,17 +301,26 @@ def make_map():
         m.connect('/organization/new', action='new')
         m.connect('/organization/{action}/{id}',
           requirements=dict(action='|'.join([
-          'edit',
           'delete',
           'admins',
-          'members',
           'member_new',
           'member_delete',
-          'history',
-          'about'
+          'history'
           ]))
           )
+        m.connect('organization_activity', '/organization/activity/{id}',
+                  action='activity', ckan_icon='time')
         m.connect('organization_read', '/organization/{id}', action='read')
+        m.connect('organization_about', '/organization/about/{id}',
+                  action='about', ckan_icon='info-sign')
+        m.connect('organization_read', '/organization/{id}', action='read',
+                  ckan_icon='sitemap')
+        m.connect('organization_edit', '/organization/edit/{id}',
+                  action='edit', ckan_icon='edit')
+        m.connect('organization_members', '/organization/members/{id}',
+                  action='members', ckan_icon='group')
+        m.connect('organization_bulk_process', '/organization/bulk_process/{id}',
+                  action='bulk_process', ckan_icon='sitemap')
     register_package_plugins(map)
     register_group_plugins(map)
 
@@ -344,12 +341,14 @@ def make_map():
         m.connect('user_activity_stream', '/user/activity/{id}',
                   action='activity', ckan_icon='time')
         m.connect('/dashboard/{offset}', action='dashboard')
-        m.connect('/dashboard', action='dashboard')
+        m.connect('user_dashboard', '/dashboard', action='dashboard',
+                  ckan_icon='list')
         m.connect('user_follow', '/user/follow/{id}', action='follow')
         m.connect('/user/unfollow/{id}', action='unfollow')
         m.connect('user_followers', '/user/followers/{id:.*}',
                   action='followers', ckan_icon='group')
-        m.connect('/user/edit/{id:.*}', action='edit')
+        m.connect('user_edit', '/user/edit/{id:.*}', action='edit',
+                  ckan_icon='cog')
         m.connect('/user/reset/{id:.*}', action='perform_reset')
         m.connect('register', '/user/register', action='register')
         m.connect('login', '/user/login', action='login')
@@ -378,7 +377,10 @@ def make_map():
         m.connect('/feeds/dataset.atom', action='general')
         m.connect('/feeds/custom.atom', action='custom')
 
-    map.connect('ckanadmin_index', '/ckan-admin', controller='admin', action='index')
+    map.connect('ckanadmin_index', '/ckan-admin', controller='admin',
+                action='index', ckan_icon='legal')
+    map.connect('ckanadmin_config', '/ckan-admin/config', controller='admin',
+                action='config', ckan_icon='check')
     map.connect('ckanadmin', '/ckan-admin/{action}', controller='admin')
 
     # Storage routes