Add integration tests

The integration tests require a running PostGIS instance and spatial
database. This adds two new tox environments to allow for running the
unit tests without needing to have PostGIS set up. The full suite of
tests will be run during CI.

.travis.yml

@@ -1,17 +1,23 @@
 notifications:
   email: false
 language: python
+services:
+    - postgresql
+env:
+  global:
+    POSTGIS_DB="postgresql://postgres@localhost/slingshot_test"
 matrix:
   include:
-    - env: TOX_ENV=py27
+    - env: TOX_ENV=py27-integration
     - python: 3.5
-      env: TOX_ENV=py35
-    - python: 3.5
-      env: TOX_ENV=setup
+      env: TOX_ENV=py35-integration
     - python: 3.5
       env: TOX_ENV=flake8
     - python: 3.5
       env: TOX_ENV=coveralls
+before_script:
+    - psql -U postgres -c "CREATE DATABASE slingshot_test;"
+    - psql -U postgres -d slingshot_test -c "CREATE EXTENSION postgis;"
 install:
   - pip install tox
 script:

slingshot/app.py

@@ -67,7 +67,7 @@ def make_uuid(value, namespace='mit.edu'):
     try:
         ns = uuid.uuid5(uuid.NAMESPACE_DNS, namespace)
         uid = uuid.uuid5(ns, value)
-    except UnicodeDecodeError:
+    except UnicodeDecodeError:  # pragma: no cover
         # Python 2 requires a byte string for the second argument.
         # Python 3 requires a unicode string for the second argument.
         value, namespace = [bytearray(s, 'utf-8') for s in (value, namespace)]

slingshot/cli.py

@@ -16,7 +16,7 @@
     unpack_zip,
 )
 from slingshot.app import load_layer
-from slingshot.db import engine, metadata
+from slingshot.db import engine
 
 
 @click.group()
@@ -52,7 +52,6 @@ def bag(layers, bags, db_uri, workspace, public, secure):
     PostGIS. If a Bag already exists the layer will be skipped.
     """
     engine.configure(db_uri)
-    metadata.bind = engine()
     if os.path.isdir(layers):
         zips = [os.path.join(layers, l) for l in os.listdir(layers)
                 if l.endswith('.zip')]

slingshot/db.py

@@ -21,6 +21,7 @@ def __call__(self):
 
     def configure(self, url):
         self._engine = self._engine or create_engine(url)
+        metadata.bind = self._engine
 
 
 engine = Engine()

tests/integration/test_command_line.py

@@ -0,0 +1,81 @@
+import os
+import shutil
+import tempfile
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
+
+from click.testing import CliRunner
+from dotenv import load_dotenv, find_dotenv
+import pytest
+
+from slingshot.cli import main
+from slingshot.db import engine, metadata
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+@pytest.fixture
+def db():
+    load_dotenv(find_dotenv())
+    uri = os.environ.get('POSTGIS_DB')
+    engine.configure(uri)
+    return engine
+
+
+@pytest.fixture(autouse=True)
+def db_setup(db):
+    metadata.drop_all()
+    metadata.clear()
+
+
+def test_bag_loads_shapefile(db, runner, shapefile):
+    store = tempfile.mkdtemp()
+    layers = tempfile.mkdtemp()
+    shutil.copy2(shapefile, layers)
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', layers, store])
+    assert res.exit_code == 0
+    with db().connect() as conn:
+        r = conn.execute('SELECT COUNT(*) FROM bermuda').scalar()
+        assert r == 713
+
+
+def test_bag_creates_bag(runner, shapefile):
+    store = tempfile.mkdtemp()
+    layers = tempfile.mkdtemp()
+    shutil.copy2(shapefile, layers)
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', layers, store])
+    assert res.exit_code == 0
+    assert 'Loaded layer bermuda' in res.output
+
+
+def test_bag_skips_existing_layers(runner, shapefile, bags_dir):
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', shapefile, bags_dir])
+    assert res.exit_code == 0
+    assert 'Skipping existing layer bermuda' in res.output
+
+
+def test_bag_removes_failed_bag(runner, shapefile):
+    store = tempfile.mkdtemp()
+    uri = os.environ.get('POSTGIS_DB')
+    with patch('slingshot.cli.load_layer') as m:
+        m.side_effect = Exception
+        res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                                   'mock://example.com', '--secure',
+                                   'mock://example.com', shapefile, store])
+        assert res.exit_code == 0
+        assert 'Failed creating bag bermuda' in res.output
+        assert not os.listdir(store)

tests/unit/test_cli.py

@@ -0,0 +1,38 @@
+from click.testing import CliRunner
+import pytest
+import requests_mock
+
+from slingshot.cli import main
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+def test_publish_publishes_layer(runner, bags_dir):
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/public/rest/workspaces/mit/datastores'
+               '/data/featuretypes')
+        m.post('mock://example.com/solr/update')
+        m.post('mock://example.com/solr/update/json/docs')
+        res = runner.invoke(main, ['publish', '--public',
+                                   'mock://example.com/public', '--secure',
+                                   'mock://example.com/secure', '--solr',
+                                   'mock://example.com/solr', bags_dir])
+        assert res.exit_code == 0
+        assert 'Loaded bermuda' in res.output
+
+
+def test_reindex_deletes_and_reloads(runner, bags_dir):
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/solr/update')
+        m.post('mock://example.com/solr/update/json/docs')
+        res = runner.invoke(main, ['reindex', '--solr',
+                                   'mock://example.com/solr', bags_dir])
+        assert res.exit_code == 0
+        assert m.request_history[0].json() == \
+            {'delete': {'query':
+                        'dct_provenance_s:MIT AND dc_format_s:Shapefile'}}
+        assert 'Indexed bermuda' in res.output
+        assert m.request_history[2].json() == {'commit': {}}

tests/test_db.py → tests/unit/test_db.py

@@ -1,9 +1,9 @@
 import re
 
 import pytest
-from shapefile import Reader
 from sqlalchemy import Boolean, Date, Float, Integer, Text
 
+from slingshot.app import ShapeReader
 from slingshot.db import (
     metadata,
     multiply,
@@ -94,46 +94,46 @@ def test_multiply_does_not_modify_points():
 
 
 def test_pg_reader_read_returns_size(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    assert pg.read(17) == '1\t45683\t58443\t32.'
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        assert pg.read(17) == '1\t45683\t58443\t32.'
 
 
 def test_pg_reader_reads_to_end(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = ''
-    while True:
-        chunk = pg.read(1024)
-        if not chunk:
-            break
-        buf += chunk
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = ''
+        while True:
+            chunk = pg.read(1024)
+            if not chunk:
+                break
+            buf += chunk
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
 
 
 def test_pg_reader_reads_all(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = pg.read()
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = pg.read()
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
 
 
 def test_pg_reader_reads_line(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    assert pg.readline().startswith('1\t45683\t58443')
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        assert pg.readline().startswith('1\t45683\t58443')
 
 
 def test_pg_reader_readline_reads_to_end(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = ''
-    while True:
-        line = pg.readline()
-        if not line:
-            break
-        buf += line
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = ''
+        while True:
+            line = pg.readline()
+            if not line:
+                break
+            buf += line
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)