Merge pull request #1 from NaturalHistoryMuseum/josh/ckan-upgrade-2.9

CKAN 2.9.x upgrade

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+relative_files = True

.gitignore

@@ -40,3 +40,5 @@ coverage.xml
 
 # Sphinx documentation
 docs/_build/
+
+.idea

.travis.yml

@@ -0,0 +1,16 @@
+sudo: required
+
+language: python
+
+services:
+  - docker
+
+# we need coveralls and this also prevents travis from running pip install -r requirements.txt
+install: pip install coveralls
+
+script:
+  - docker-compose build
+  - docker-compose run ckan
+
+after_success: coveralls
+

ckanext/iiif/__init__.py

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/iiif/lib/manifest.py

@@ -8,90 +8,94 @@
 
 class IIIFRecordManifestBuilder(object):
     # the group names here must match the parameters for the get_builder function below
-    regex = re.compile(u'resource/(?P<resource_id>.+?)/record/(?P<record_id>.+)$')
+    regex = re.compile('resource/(?P<resource_id>.+?)/record/(?P<record_id>.+)$')
 
     @staticmethod
     def get_builder(resource_id, record_id):
         # this will throw an error if the resource can't be found
-        resource = toolkit.get_action(u'resource_show')({}, {u'id': resource_id})
+        resource = toolkit.get_action('resource_show')({}, {'id': resource_id})
         # this will throw an error if the record can't be found
-        record = toolkit.get_action(u'record_show')({}, {u'resource_id': resource_id,
-                                                         u'record_id': record_id})
-        return IIIFRecordManifestBuilder(resource, record[u'data'])
+        record = toolkit.get_action('record_show')({}, {'resource_id': resource_id,
+                                                        'record_id': record_id})
+        return IIIFRecordManifestBuilder(resource, record['data'])
 
     def __init__(self, resource, record):
         self.resource = resource
         self.record = record
-        self.resource_id = resource[u'id']
-        self.record_id = record[u'_id']
+        self.resource_id = resource['id']
+        self.record_id = record['_id']
 
     @property
     def manifest_id(self):
-        return u'resource/{}/record/{}'.format(self.resource_id, self.record_id)
+        return f'resource/{self.resource_id}/record/{self.record_id}'
 
     @property
     def label(self):
-        return wrap_language(self.record[self.resource[u'_title_field']])
+        return wrap_language(self.record[self.resource['_title_field']])
 
     @property
     def images(self):
-        value = self.record[self.resource[u'_image_field']]
-        delimeter = self.resource[u'_image_delimiter']
-        return value.split(delimeter) if delimeter else [value]
+        value = self.record[self.resource['_image_field']]
+        image_delimiter = self.resource.get('_image_delimiter', None)
+        if image_delimiter:
+            return value.split(image_delimiter)
+        else:
+            return value if isinstance(value, list) else [value]
 
     @property
     def rights(self):
-        license_id = self.resource.get(u'_image_licence', None)
+        license_id = self.resource.get('_image_licence', None)
         # if the license is '' or None we override it
         if not license_id:
             # default the license to cc-by
-            license_id = u'cc-by'
+            license_id = 'cc-by'
         license = model.Package.get_license_register()[license_id]
         return license.url
 
     @property
     def metadata(self):
+        # TODO: this function does not handle lists of values well, nor nested dicts...
         return [
-            {u'label': wrap_language(field), u'value': wrap_language(unicode(value))}
+            {'label': wrap_language(field), 'value': wrap_language(str(value))}
             for field, value in self.record.items()
         ]
 
     def build_canvas(self, image):
-        canvas_id = create_id_url(u'{}/canvas/{}'.format(self.manifest_id, image))
+        canvas_id = create_id_url(f'{self.manifest_id}/canvas/{image}')
         # TODO: need to pass the type based on some logic (user setting/custom code)
         image_id = create_image_server_url(image)
 
         return {
-            u'id': canvas_id,
-            u'type': u'Canvas',
+            'id': canvas_id,
+            'type': 'Canvas',
             # we don't have access to the image data or metadata here so just use 1000x1000
-            u'width': 1000,
-            u'height': 1000,
+            'width': 1000,
+            'height': 1000,
             # TODO: label needs to be using a field defined by the user
-            u'label': wrap_language(image),
-            u'items': [
+            'label': wrap_language(image),
+            'items': [
                 {
                     # TODO: do we need an id here?
-                    u'type': u'AnnotationPage',
-                    u'items': [
+                    'type': 'AnnotationPage',
+                    'items': [
                         {
                             # TODO: do we need an id here?
-                            u'type': u'Annotation',
-                            u'motivation': u'painting',
-                            u'body': {
-                                u'id': u'{}/{}'.format(image_id, u'info.json'),
-                                u'type': u'Image',
-                                u'format': u'image/jpeg',
-                                u'service': [
+                            'type': 'Annotation',
+                            'motivation': 'painting',
+                            'body': {
+                                'id': f'{image_id}/info.json',
+                                'type': 'Image',
+                                'format': 'image/jpeg',
+                                'service': [
                                     {
-                                        u'@context': u'http://iiif.io/api/image/3/context.json',
-                                        u'id': image_id,
-                                        u'type': u'ImageService3',
-                                        u'profile': u'level0'
+                                        '@context': 'http://iiif.io/api/image/3/context.json',
+                                        'id': image_id,
+                                        'type': 'ImageService3',
+                                        'profile': 'level0'
                                     }
                                 ],
                             },
-                            u'target': canvas_id,
+                            'target': canvas_id,
                         },
                     ]
                 }
@@ -101,20 +105,20 @@ def build_canvas(self, image):
     def build(self):
         # TODO: add more properties
         return {
-            u'@context': u'http://iiif.io/api/presentation/3/context.json',
-            u'id': create_id_url(self.manifest_id),
-            u'type': u'Manifest',
-            u'label': self.label,
-            u'metadata': self.metadata,
-            u'rights': self.rights,
-            u'items': [self.build_canvas(image) for image in self.images],
-            u'logo': [
+            '@context': 'http://iiif.io/api/presentation/3/context.json',
+            'id': create_id_url(self.manifest_id),
+            'type': 'Manifest',
+            'label': self.label,
+            'metadata': self.metadata,
+            'rights': self.rights,
+            'items': [self.build_canvas(image) for image in self.images],
+            'logo': [
                 {
-                    u'id': u'{}/images/logo.png'.format(config.get(u'ckan.site_url')),
-                    u'type': u'Image',
-                    u'format': u'image/png',
-                    u'width': 120,
-                    u'height': 56,
+                    'id': f'{config.get("ckan.site_url")}/images/logo.png',
+                    'type': 'Image',
+                    'format': 'image/png',
+                    'width': 120,
+                    'height': 56,
                 }
             ],
         }

ckanext/iiif/lib/utils.py

@@ -3,15 +3,15 @@
 
 
 def create_id_url(identifier):
-    return toolkit.url_for(u'iiif.resource', identifier=identifier, _external=True)
+    return toolkit.url_for('iiif.resource', identifier=identifier, _external=True)
 
 
-def create_image_server_url(identifier_name, identifier_type=u'vfactor'):
-    image_server_url = config.get(u'ckanext.iiif.image_server_url')
-    return u'{}/{}:{}'.format(image_server_url, identifier_type, identifier_name)
+def create_image_server_url(identifier_name, identifier_type='vfactor'):
+    image_server_url = config.get('ckanext.iiif.image_server_url')
+    return f'{image_server_url}/{identifier_type}:{identifier_name}'
 
 
-def wrap_language(value, language=u'en'):
+def wrap_language(value, language='en'):
     if not isinstance(value, list):
         value = [value]
     return {language: value}

ckanext/iiif/plugin.py

@@ -1,8 +1,7 @@
-import logging
-
 import ckan.plugins as plugins
+import logging
 from ckan.plugins import toolkit
-from contextlib2 import suppress
+from contextlib import suppress
 
 from . import interfaces
 from . import routes
@@ -35,7 +34,7 @@ def configure(self, ckan_config):
         for plugin in plugins.PluginImplementations(interfaces.IIIIF):
             for builder in plugin.get_builders():
                 if builder.regex in iiif.builders:
-                    log.warning(u'Duplicate IIIF builder regex: {}'.format(builder.regex.pattern))
+                    log.warning('Duplicate IIIF builder regex: {}'.format(builder.regex.pattern))
                 iiif.builders[builder.regex] = builder.get_builder
 
     # IBlueprint
@@ -45,17 +44,16 @@ def get_blueprint(self):
     # IVersionedDatastore
     def datastore_multisearch_modify_response(self, response):
         resource_cache = {}
-        resource_show = toolkit.get_action(u'resource_show')
-        vfactor_resource_id = toolkit.config.get(u'ckanext.nhm.vfactor_resource_id')
+        resource_show = toolkit.get_action('resource_show')
+        vfactor_resource_id = toolkit.config.get('ckanext.nhm.vfactor_resource_id')
 
-        for record in response[u'records']:
-            resource_id = record[u'resource']
+        for record in response['records']:
+            resource_id = record['resource']
             if resource_id == vfactor_resource_id:
                 if resource_id not in resource_cache:
-                    resource_cache[resource_id] = resource_show({}, {u'id': resource_id})
+                    resource_cache[resource_id] = resource_show({}, {'id': resource_id})
                 with suppress(Exception):
-                    builder = IIIFRecordManifestBuilder(resource_cache[resource_id],
-                                                        record[u'data'])
-                    record[u'iiif'] = builder.build()
+                    builder = IIIFRecordManifestBuilder(resource_cache[resource_id], record['data'])
+                    record['iiif'] = builder.build()
 
         return response

ckanext/iiif/routes/iiif.py

@@ -1,19 +1,18 @@
-from collections import OrderedDict
-
 from ckan.plugins import toolkit
+from collections import OrderedDict
 from flask import Blueprint, jsonify
 
-blueprint = Blueprint(name=u'iiif', import_name=__name__, url_prefix=u'/iiif')
+blueprint = Blueprint(name='iiif', import_name=__name__, url_prefix='/iiif')
 
 builders = OrderedDict()
 
 
-@blueprint.route(u'/<path:identifier>')
+@blueprint.route('/<path:identifier>')
 def resource(identifier):
     for regex, get_builder_function in builders.items():
         match = regex.match(identifier)
         if match:
             builder = get_builder_function(**match.groupdict())
             return jsonify(builder.build())
 
-    return toolkit.abort(status_code=404, detail=u'Unknown IIIF identifier')
+    return toolkit.abort(status_code=404, detail='Unknown IIIF identifier')

dev_requirements.txt

@@ -0,0 +1,2 @@
+pytest>=4.6.5
+pytest-cov>=2.7.1

docker-compose.yml

@@ -0,0 +1,39 @@
+version: "3"
+
+services:
+  ckan:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    environment:
+      PYTHONUNBUFFERED: 1
+      PYTHONDONTWRITEBYTECODE: 1
+    depends_on:
+      - db
+      - solr
+      - redis
+    volumes:
+      - .:/srv/app/src/ckanext-iiif
+
+  solr:
+    build:
+      context: https://github.com/okfn/docker-ckan.git#:solr
+    logging:
+      driver: none
+
+  db:
+    build:
+      context: https://github.com/okfn/docker-ckan.git#:postgresql
+      args:
+        - DATASTORE_READONLY_PASSWORD=password
+        - POSTGRES_PASSWORD=password
+    environment:
+      - DATASTORE_READONLY_PASSWORD=password
+      - POSTGRES_PASSWORD=password
+    logging:
+      driver: none
+
+  redis:
+    image: redis:latest
+    logging:
+      driver: none

docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM openknowledge/ckan-dev:2.9
+
+# ckan is installed in /srv/app/src/ckan in the ckan-dev image we're basing this image on
+WORKDIR /srv/app/src/ckanext-iiif
+
+# copy over the ckanext-iiif source
+COPY . .
+
+# might as well update pip while we're here!
+RUN pip3 install --upgrade pip
+
+# fixes this https://github.com/ckan/ckan/issues/5570
+RUN pip3 install pytest-ckan
+
+# install the dependencies
+RUN python3 setup.py develop && \
+    pip3 install -r requirements.txt && \
+    pip3 install -r dev_requirements.txt
+
+# this entrypoint ensures our service dependencies (postgresql, solr and redis) are running before
+# running the cmd
+ENTRYPOINT ["/bin/bash", "docker/entrypoint.sh"]
+
+# run the tests with coverage output
+CMD ["pytest", "--cov=ckanext.iiif", "--ckan-ini=test.ini", "tests"]

docker/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+echo "Wait for PostgreSQL to start..."
+while ! pg_isready -h db -U ckan; do
+    sleep 1;
+done
+echo "PostgreSQL started"
+
+echo "Wait for Solr to start..."
+while ! curl -s "http://solr:8983/solr/ckan/admin/ping" | grep -q OK; do
+  sleep 1;
+done
+echo "Solr started"
+
+echo "Wait for Redis to start..."
+while ! echo -e "PING" | nc -w 1 redis 6379 | grep -q "+PONG"; do
+  sleep 1;
+done
+echo "Redis started"
+
+echo "All services up, running command"
+
+exec "$@"