Merge pull request #1 from data-govt-nz/ckan-2.7-support-spatial

Ckan 2.7 support spatial

.travis.yml

@@ -1,11 +1,20 @@
 language: python
+dist: trusty
 python:
     - "2.7"
+cache: pip
 env:
-    - CKANVERSION=master POSTGISVERSION=2
-    - CKANVERSION=2.2 POSTGISVERSION=2
-    - CKANVERSION=2.3 POSTGISVERSION=2
-    - CKANVERSION=2.4 POSTGISVERSION=2
+    - CKANVERSION=master
+    - CKANVERSION=release-v2.5-latest
+    - CKANVERSION=release-v2.6-latest
+    - CKANVERSION=2.7
+    - CKANVERSION=2.8
+sudo: required
+addons:
+  postgresql: 9.6
+  apt:
+    packages:
+    - postgresql-9.6-postgis-2.3
 services:
     - redis-server
 install:

README.rst

@@ -5,7 +5,6 @@ ckanext-spatial - Geo related plugins for CKAN
 .. image:: https://travis-ci.org/ckan/ckanext-spatial.svg?branch=master
     :target: https://travis-ci.org/ckan/ckanext-spatial
 
-
 This extension contains plugins that add geospatial capabilities to CKAN_,
 including:
 

bin/travis-build.bash

@@ -5,35 +5,23 @@ echo "This is travis-build.bash..."
 
 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
-sudo apt-get install postgresql-9.1 solr-jetty libcommons-fileupload-java:amd64=1.2.2-1
-
-echo "Installing PostGIS..."
-if [ $POSTGISVERSION == '1' ]
-then
-    sudo apt-get install postgresql-9.1-postgis=1.5.3-2
-fi
-# PostGIS 2.1 already installed on Travis
-
-echo "Patching lxml..."
-wget ftp://xmlsoft.org/libxml2/libxml2-2.9.0.tar.gz
-tar zxf libxml2-2.9.0.tar.gz
-cd libxml2-2.9.0/
-./configure --quiet --libdir=/usr/lib/x86_64-linux-gnu
-make --silent
-sudo make --silent install
-xmllint --version
-cd -
+sudo apt-get install solr-jetty
 
 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
 cd ckan
 if [ $CKANVERSION != 'master' ]
 then
-    git checkout release-v$CKANVERSION-latest
+    git checkout $CKANVERSION
 fi
+
+# Unpin CKAN's psycopg2 dependency get an important bugfix
+# https://stackoverflow.com/questions/47044854/error-installing-psycopg2-2-6-2
+sed -i '/psycopg2/c\psycopg2' requirements.txt
+
 python setup.py develop
-pip install -r requirements.txt --allow-all-external
-pip install -r dev-requirements.txt --allow-all-external
+pip install -r requirements.txt
+pip install -r dev-requirements.txt
 cd -
 
 echo "Setting up Solr..."
@@ -50,18 +38,9 @@ sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
 sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
 
 echo "Setting up PostGIS on the database..."
-if [ $POSTGISVERSION == '1' ]
-then
-    sudo -u postgres psql -d ckan_test -f /usr/share/postgresql/9.1/contrib/postgis-1.5/postgis.sql
-    sudo -u postgres psql -d ckan_test -f /usr/share/postgresql/9.1/contrib/postgis-1.5/spatial_ref_sys.sql
-    sudo -u postgres psql -d ckan_test -c 'ALTER TABLE geometry_columns OWNER TO ckan_default;'
-    sudo -u postgres psql -d ckan_test -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;'
-elif [ $POSTGISVERSION == '2' ]
-then
-    sudo -u postgres psql -d ckan_test -c 'CREATE EXTENSION postgis;'
-    sudo -u postgres psql -d ckan_test -c 'ALTER VIEW geometry_columns OWNER TO ckan_default;'
-    sudo -u postgres psql -d ckan_test -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;'
-fi
+sudo -u postgres psql -d ckan_test -c 'CREATE EXTENSION postgis;'
+sudo -u postgres psql -d ckan_test -c 'ALTER VIEW geometry_columns OWNER TO ckan_default;'
+sudo -u postgres psql -d ckan_test -c 'ALTER TABLE spatial_ref_sys OWNER TO ckan_default;'
 
 echo "Install other libraries required..."
 sudo apt-get install python-dev libxml2-dev libxslt1-dev libgeos-c1
@@ -75,13 +54,13 @@ echo "Installing ckanext-harvest and its requirements..."
 git clone https://github.com/ckan/ckanext-harvest
 cd ckanext-harvest
 python setup.py develop
-pip install -r pip-requirements.txt --allow-all-external
+pip install -r pip-requirements.txt
 
 paster harvester initdb -c ../ckan/test-core.ini
 cd -
 
 echo "Installing ckanext-spatial and its requirements..."
-pip install -r pip-requirements.txt --allow-all-external
+pip install -r pip-requirements.txt
 python setup.py develop
 
 

ckanext/spatial/controllers/api.py

@@ -1,7 +1,9 @@
 import logging
 
-try: from cStringIO import StringIO
-except ImportError: from StringIO import StringIO
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
 
 from pylons import response
 from pkg_resources import resource_stream
@@ -16,63 +18,69 @@
 
 log = logging.getLogger(__name__)
 
+
 class ApiController(BaseApiController):
 
     def spatial_query(self):
 
-        error_400_msg = 'Please provide a suitable bbox parameter [minx,miny,maxx,maxy]'
+        error_400_msg = \
+            'Please provide a suitable bbox parameter [minx,miny,maxx,maxy]'
 
         if not 'bbox' in request.params:
-            abort(400,error_400_msg)
+            abort(400, error_400_msg)
 
         bbox = validate_bbox(request.params['bbox'])
 
         if not bbox:
-            abort(400,error_400_msg)
+            abort(400, error_400_msg)
 
-        srid = get_srid(request.params.get('crs')) if 'crs' in request.params else None
+        srid = get_srid(request.params.get('crs')) if 'crs' in \
+            request.params else None
 
-        extents = bbox_query(bbox,srid)
+        extents = bbox_query(bbox, srid)
 
-        format = request.params.get('format','')
+        format = request.params.get('format', '')
 
-        return self._output_results(extents,format)
+        return self._output_results(extents, format)
 
-    def _output_results(self,extents,format=None):
+    def _output_results(self, extents, format=None):
 
         ids = [extent.package_id for extent in extents]
-
-        output = dict(count=len(ids),results=ids)
+        output = dict(count=len(ids), results=ids)
 
         return self._finish_ok(output)
 
+
 class HarvestMetadataApiController(BaseApiController):
 
     def _get_content(self, id):
 
         obj = Session.query(HarvestObject) \
-                        .filter(HarvestObject.id==id).first()
+                        .filter(HarvestObject.id == id).first()
         if obj:
             return obj.content
         else:
             return None
 
     def _get_original_content(self, id):
         extra = Session.query(HarvestObjectExtra).join(HarvestObject) \
-                        .filter(HarvestObject.id==id) \
-                        .filter(HarvestObjectExtra.key=='original_document').first()
+                        .filter(HarvestObject.id == id) \
+                        .filter(
+                            HarvestObjectExtra.key == 'original_document'
+                        ).first()
         if extra:
             return extra.value
         else:
             return None
 
     def _transform_to_html(self, content, xslt_package=None, xslt_path=None):
 
-        xslt_package = xslt_package or 'ckanext.spatial'
-        xslt_path = xslt_path or 'templates/ckanext/spatial/gemini2-html-stylesheet.xsl'
+        xslt_package = xslt_package or __name__
+        xslt_path = xslt_path or \
+            '../templates/ckanext/spatial/gemini2-html-stylesheet.xsl'
 
-        ## optimise -- read transform only once and compile rather
-        ## than at each request
+        # optimise -- read transform only once and compile rather
+        # than at each request
         with resource_stream(xslt_package, xslt_path) as style:
             style_xml = etree.parse(style)
             transformer = etree.XSLT(style_xml)
@@ -90,7 +98,8 @@ def _transform_to_html(self, content, xslt_package=None, xslt_path=None):
     def _get_xslt(self, original=False):
 
         if original:
-            config_option = 'ckanext.spatial.harvest.xslt_html_content_original'
+            config_option = \
+                'ckanext.spatial.harvest.xslt_html_content_original'
         else:
             config_option = 'ckanext.spatial.harvest.xslt_html_content'
 
@@ -103,8 +112,9 @@ def _get_xslt(self, original=False):
                 xslt_package = xslt[0]
                 xslt_path = xslt[1]
             else:
-                log.error('XSLT should be defined in the form <package>:<path>' +
-                        ', eg ckanext.myext:templates/my.xslt')
+                log.error(
+                    'XSLT should be defined in the form <package>:<path>' +
+                    ', eg ckanext.myext:templates/my.xslt')
 
         return xslt_package, xslt_path
 
@@ -121,7 +131,7 @@ def display_xml_original(self, id):
             content = u'<?xml version="1.0" encoding="UTF-8"?>\n' + content
         return content.encode('utf-8')
 
-    def display_html(self,id):
+    def display_html(self, id):
         content = self._get_content(id)
 
         if not content:

ckanext/spatial/harvesters/base.py

@@ -24,6 +24,7 @@
 from ckan import logic
 from ckan.lib.navl.validators import not_empty
 from ckan.lib.search.index import PackageSearchIndex
+from ckanext.harvest.harvesters.base import munge_tag
 
 from ckanext.harvest.harvesters.base import HarvesterBase
 from ckanext.harvest.model import HarvestObject
@@ -144,12 +145,21 @@ def validate_config(self, source_config):
             if 'default_tags' in source_config_obj:
                 if not isinstance(source_config_obj['default_tags'],list):
                     raise ValueError('default_tags must be a list')
+                elif not isinstance(source_config_obj['default_tags'][0], dict):
+                    # This check is taken from ckanext-harvest CKANHarvester to ensure consistency
+                    # between harversters.
+                    raise ValueError('default_tags must be a list of dicts like {"name": "name-of-tag"}')
 
             if 'default_extras' in source_config_obj:
                 if not isinstance(source_config_obj['default_extras'],dict):
                     raise ValueError('default_extras must be a dictionary')
 
-            for key in ('override_extras'):
+            if 'default_groups' in source_config_obj:
+                # Also taken from CKANHarvester to ensure consistency.
+                if not isinstance(source_config_obj['default_groups'], list):
+                    raise ValueError('default_groups must be a *list* of group names/ids')
+
+            for key in ('override_extras', 'clean_tags'):
                 if key in source_config_obj:
                     if not isinstance(source_config_obj[key],bool):
                         raise ValueError('%s must be boolean' % key)
@@ -202,24 +212,37 @@ def get_package_dict(self, context, data_dict):
         :returns: A dataset dictionary (package_dict)
         :rtype: dict
         '''
-
+        
         tags = []
+
         if 'tags' in iso_values:
-            for tag in iso_values['tags']:
-                tag = tag[:50] if len(tag) > 50 else tag
-                tags.append({'name': tag})
+            do_clean = self.source_config.get('clean_tags')
+            tags_val = [munge_tag(tag) if do_clean else tag[:100] for tag in iso_values['tags']]
+            tags = [{'name': tag} for tag in tags_val]
 
-        # Add default_tags from config
-        default_tags = self.source_config.get('default_tags',[])
-        if default_tags:
-           for tag in default_tags:
-              tags.append({'name': tag})
+
+        # Add default_tags from config.
+        # For consistency with CKANHarvester `default_tags` is now a list of
+        # dicts, which we can add to tags without further parsing.
+        tags.extend(self.source_config.get('default_tags', []))
+
+        # Adding default_groups from config. This was previously not supported
+        # by ckanext-spatial.
+        context = {'model': model, 'user': p.toolkit.c.user}
+        groups = []
+        for group_name_or_id in self.source_config.get('default_groups', []):
+            try:
+                group = p.toolkit.get_action('group_show')(context, {'id': group_name_or_id})
+                groups.append({'id': group['id'], 'name': group['name']})
+            except p.toolkit.ObjectNotFound, e:
+                logging.error('Default group %s not found, proceeding without.' % group_name_or_id)
 
         package_dict = {
             'title': iso_values['title'],
             'notes': iso_values['abstract'],
-            'tags': tags,
+            'tags': dict((tag['name'], tag) for tag in tags).values(),
             'resources': [],
+            'groups': dict((group['id'], group) for group in groups).values(),
         }
 
         # We need to get the owner organization (if any) from the harvest

ckanext/spatial/helpers.py

@@ -6,6 +6,7 @@
 
 log = logging.getLogger(__name__)
 
+
 def get_reference_date(date_str):
     '''
         Gets a reference date extra created by the harvesters and formats it
@@ -30,6 +31,7 @@ def get_reference_date(date_str):
     except (ValueError, TypeError):
         return date_str
 
+
 def get_responsible_party(value):
     '''
         Gets a responsible party extra created by the harvesters and formats it
@@ -59,6 +61,7 @@ def get_responsible_party(value):
     except (ValueError, TypeError):
         return value
 
+
 def get_common_map_config():
     '''
         Returns a dict with all configuration options related to the common

ckanext/spatial/plugin.py

@@ -5,8 +5,6 @@
 
 from pylons import config
 
-
-
 from ckan import plugins as p
 
 from ckan.lib.helpers import json
@@ -51,7 +49,8 @@ def prettify(field_name):
     summary = {}
     for key, error in error_dict.iteritems():
         if key == 'resources':
-            summary[p.toolkit._('Resources')] = p.toolkit._('Package resource(s) invalid')
+            summary[p.toolkit._('Resources')] = p.toolkit._(
+                'Package resource(s) invalid')
         elif key == 'extras':
             summary[p.toolkit._('Extras')] = p.toolkit._('Missing Value')
         elif key == 'extras_validation':
@@ -176,6 +175,7 @@ def before_map(self, map):
 
     def before_index(self, pkg_dict):
         import shapely
+        import shapely.geometry
 
         if pkg_dict.get('extras_spatial', None) and self.search_backend in ('solr', 'solr-spatial-field'):
             try:
@@ -317,6 +317,9 @@ def _params_for_solr_spatial_field_search(self, bbox, search_params):
             +spatial_geom:"Intersects(ENVELOPE({minx}, {miny}, {maxx}, {maxy}))
 
         '''
+        if bbox['maxx'] > 180:
+            bbox['maxx'] = -180 + (bbox['maxx'] - 180)
+
         search_params['fq_list'] = search_params.get('fq_list', [])
         search_params['fq_list'].append('+spatial_geom:"Intersects(ENVELOPE({minx}, {maxx}, {maxy}, {miny}))"'
                                         .format(minx=bbox['minx'], miny=bbox['miny'], maxx=bbox['maxx'], maxy=bbox['maxy']))