Switch tests over to docker-compose, bump version

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+relative_files = True

.gitignore

@@ -1,3 +1,5 @@
 *.pyc
 *.egg-info/
 *.db
+.idea
+.coverage

.travis.yml

@@ -1,25 +1,16 @@
-dist: trusty
-language: python
-
-python:
-  - "2.7"
-
-before_install:
-  - sudo chown -R elasticsearch:elasticsearch /etc/elasticsearch && sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
-  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.7.2.deb && sudo dpkg -i --force-confnew elasticsearch-6.7.2.deb && sudo service elasticsearch restart
+sudo: required
 
-install:
-  - sh ckanext/statistics/tests/bin/build.sh
+language: python
 
 services:
-  - redis-server
-  - postgresql
-  - elasticsearch
-  - mongodb
+  - docker
 
-addons:
-  postgresql: "9.4"
+# we need coveralls and this also prevents travis from running pip install -r requirements.txt
+install: pip install coveralls
 
-script: coverage run --source=ckanext.statistics setup.py nosetests --ckan --with-pylons=ckanext/statistics/tests/bin/test.ini --nologcapture --debug=ckantest,ckanext.statistics --rednose
+script:
+  - docker-compose build
+  - docker-compose run ckan
 
 after_success: coveralls
+

ckanext/statistics/model/__init__.py

@@ -3,8 +3,3 @@
 #
 # This file is part of ckanext-statistics
 # Created by the Natural History Museum in London, UK
-
-
-from sqlalchemy.ext.declarative import declarative_base
-
-Base = declarative_base()

dev_requirements.txt

@@ -1,6 +1,3 @@
-nose
-rednose
 mock
-httpretty
-coveralls
--e git+https://github.com/NaturalHistoryMuseum/ckantest.git#egg=ckantest
+pytest>=4.6.5
+pytest-cov>=2.7.1

docker-compose.yml

@@ -0,0 +1,53 @@
+version: "3"
+
+services:
+  ckan:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    environment:
+      PYTHONUNBUFFERED: 1
+      PYTHONDONTWRITEBYTECODE: 1
+    depends_on:
+      - db
+      - solr
+      - redis
+      - elasticsearch
+      - mongodb
+    volumes:
+      - .:/srv/app/src/ckanext-statistics
+
+  solr:
+    build:
+      context: https://github.com/okfn/docker-ckan.git#:solr
+    logging:
+      driver: none
+
+  db:
+    build:
+      context: https://github.com/okfn/docker-ckan.git#:postgresql
+      args:
+        - DATASTORE_READONLY_PASSWORD=password
+        - POSTGRES_PASSWORD=password
+    environment:
+      - DATASTORE_READONLY_PASSWORD=password
+      - POSTGRES_PASSWORD=password
+    logging:
+      driver: none
+
+  redis:
+    image: redis:latest
+    logging:
+      driver: none
+
+  elasticsearch:
+    image: elasticsearch:6.8.13
+    environment:
+      discovery.type: single-node
+    logging:
+      driver: none
+
+  mongodb:
+    image: mongo:4.2
+    logging:
+      driver: none

docker/Dockerfile

@@ -0,0 +1,25 @@
+FROM openknowledge/ckan-dev:2.9-py2
+
+# ckan is installed in /srv/app/src/ckan in the ckan-dev image we're basing this image on
+WORKDIR /srv/app/src/ckanext-statistics
+
+# copy over the ckanext-statistics source
+COPY . .
+
+# might as well update pip while we're here!
+RUN pip2 install --upgrade pip
+
+# fixes this https://github.com/ckan/ckan/issues/5570
+RUN pip2 install pytest-ckan
+
+# install the dependencies
+RUN python setup.py develop && \
+    pip2 install -r requirements.txt && \
+    pip2 install -r dev_requirements.txt
+
+# this entrypoint ensures our service dependencies (postgresql, solr and redis) are running before
+# running the cmd
+ENTRYPOINT ["/bin/bash", "docker/entrypoint.sh"]
+
+# run the tests with coverage output
+CMD ["pytest", "--cov=ckanext.statistics", "--ckan-ini=test.ini", "tests"]

docker/entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+
+echo "Wait for PostgreSQL to start..."
+while ! pg_isready -h db -U ckan; do
+    sleep 1;
+done
+echo "PostgreSQL started"
+
+echo "Wait for Solr to start..."
+while ! curl -s "http://solr:8983/solr/ckan/admin/ping" | grep -q OK; do
+  sleep 1;
+done
+echo "Solr started"
+
+echo "Wait for Redis to start..."
+while ! echo -e "PING" | nc -w 1 redis 6379 | grep -q "+PONG"; do
+  sleep 1;
+done
+echo "Redis started"
+
+echo "Wait for Elasticsearch to start..."
+while ! curl -s "http://elasticsearch:9200/_cluster/health" | grep -q green; do
+  sleep 1;
+done
+echo "Elasticsearch started"
+
+echo "Wait for MongoDB to start..."
+while ! nc -z mongodb 27017; do
+  sleep 1;
+done
+echo "MongoDB started"
+
+
+echo "All services up, running command"
+
+exec "$@"

setup.py

@@ -6,15 +6,21 @@
 
 from setuptools import find_packages, setup
 
-__version__ = u'1.0.0-alpha'
+__version__ = u'1.0.1'
 
 with open(u'README.md', u'r') as f:
     __long_description__ = f.read()
 
-dependencies = {
-    u'ckanext-ckanpackager': u'git+https://github.com/NaturalHistoryMuseum/ckanext-ckanpackager.git#egg=ckanext-ckanpackager',
-    u'ckanext-versioned-datastore': u'git+https://github.com/NaturalHistoryMuseum/ckanext-versioned-datastore.git#egg=ckanext-versioned-datastore',
-}
+
+def nhm_github(name, tag):
+    return name, u'git+https://github.com/NaturalHistoryMuseum/{name}.git@{tag}#egg={name}'.format(
+        name=name, tag=tag)
+
+
+dependencies = dict([
+    nhm_github(u'ckanext-ckanpackager', u'319bd63158757a9287336034122cae66c2991a41'),
+    nhm_github(u'ckanext-versioned-datastore', u'd88e167a838e95af2448b60c7df67f2e2fe86eed'),
+])
 
 setup(
     name=u'ckanext-statistics',
@@ -36,13 +42,11 @@
     include_package_data=True,
     zip_safe=False,
     install_requires=[
-        'requests',
-        'pysolr==3.6.0',
-        'tqdm>=4.55.1',
-        ] + [u'{0} @ {1}'.format(k, v) for k, v in dependencies.items()],
-    dependency_links=dependencies.values(),
-    entry_points= \
-        u'''
+        u'requests',
+        u'tqdm>=4.55.1',
+    ] + [u'{0} @ {1}'.format(k, v) for k, v in dependencies.items()],
+    dependency_links=list(dependencies.values()),
+    entry_points=u'''
         [ckan.plugins]
             statistics=ckanext.statistics.plugin:StatisticsPlugin
         ''',

test.ini

@@ -0,0 +1,58 @@
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+# use a full path to ensure the config is picked up wherever the tests are run from (this covers off
+# both the default setup, where the tests are run from /srv/app/src/ckanext-statistics/, and running
+# them from an IDE like PyCharm which will copy the code into /opt/project/ and run it from there
+use = config:/srv/app/src/ckan/test-core.ini
+
+# the hosts referenced here resolve to the other docker containers configured in docker-compose.yml
+sqlalchemy.url = postgresql://ckan:password@db/ckan
+ckan.datastore.write_url = postgresql://ckan:password@db/datastore
+ckan.datastore.read_url = postgresql://datastore_ro:password@db/datastore
+ckan.redis.url = redis://redis:6379/1
+solr_url = http://solr:8983/solr/ckan
+
+ckanext.versioned_datastore.elasticsearch_hosts = elasticsearch
+ckanext.versioned_datastore.elasticsearch_port = 9200
+ckanext.versioned_datastore.elasticsearch_index_prefix = vdstest-
+ckanext.versioned_datastore.mongo_host = mongodb
+ckanext.versioned_datastore.mongo_port = 27017
+ckanext.versioned_datastore.mongo_database = vdstest
+
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s