Merge pull request #42 from MITLibraries/int-tests

Add integration tests

.travis.yml

@@ -1,17 +1,23 @@
 notifications:
   email: false
 language: python
+services:
+    - postgresql
+env:
+  global:
+    POSTGIS_DB="postgresql://postgres@localhost/slingshot_test"
 matrix:
   include:
-    - env: TOX_ENV=py27
+    - env: TOX_ENV=py27-integration
     - python: 3.5
-      env: TOX_ENV=py35
-    - python: 3.5
-      env: TOX_ENV=setup
+      env: TOX_ENV=py35-integration
     - python: 3.5
       env: TOX_ENV=flake8
     - python: 3.5
       env: TOX_ENV=coveralls
+before_script:
+    - psql -U postgres -c "CREATE DATABASE slingshot_test;"
+    - psql -U postgres -d slingshot_test -c "CREATE EXTENSION postgis;"
 install:
   - pip install tox
 script:

slingshot/app.py

@@ -67,7 +67,7 @@ def make_uuid(value, namespace='mit.edu'):
     try:
         ns = uuid.uuid5(uuid.NAMESPACE_DNS, namespace)
         uid = uuid.uuid5(ns, value)
-    except UnicodeDecodeError:
+    except UnicodeDecodeError:  # pragma: no cover
         # Python 2 requires a byte string for the second argument.
         # Python 3 requires a unicode string for the second argument.
         value, namespace = [bytearray(s, 'utf-8') for s in (value, namespace)]

slingshot/cli.py

@@ -16,7 +16,7 @@
     unpack_zip,
 )
 from slingshot.app import load_layer
-from slingshot.db import engine, metadata
+from slingshot.db import engine
 
 
 @click.group()
@@ -52,7 +52,6 @@ def bag(layers, bags, db_uri, workspace, public, secure):
     PostGIS. If a Bag already exists the layer will be skipped.
     """
     engine.configure(db_uri)
-    metadata.bind = engine()
     if os.path.isdir(layers):
         zips = [os.path.join(layers, l) for l in os.listdir(layers)
                 if l.endswith('.zip')]

slingshot/db.py

@@ -21,6 +21,7 @@ def __call__(self):
 
     def configure(self, url):
         self._engine = self._engine or create_engine(url)
+        metadata.bind = self._engine
 
 
 engine = Engine()

tests/integration/test_command_line.py

@@ -0,0 +1,81 @@
+import os
+import shutil
+import tempfile
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
+
+from click.testing import CliRunner
+from dotenv import load_dotenv, find_dotenv
+import pytest
+
+from slingshot.cli import main
+from slingshot.db import engine, metadata
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+@pytest.fixture
+def db():
+    load_dotenv(find_dotenv())
+    uri = os.environ.get('POSTGIS_DB')
+    engine.configure(uri)
+    return engine
+
+
+@pytest.fixture(autouse=True)
+def db_setup(db):
+    metadata.drop_all()
+    metadata.clear()
+
+
+def test_bag_loads_shapefile(db, runner, shapefile):
+    store = tempfile.mkdtemp()
+    layers = tempfile.mkdtemp()
+    shutil.copy2(shapefile, layers)
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', layers, store])
+    assert res.exit_code == 0
+    with db().connect() as conn:
+        r = conn.execute('SELECT COUNT(*) FROM bermuda').scalar()
+        assert r == 713
+
+
+def test_bag_creates_bag(runner, shapefile):
+    store = tempfile.mkdtemp()
+    layers = tempfile.mkdtemp()
+    shutil.copy2(shapefile, layers)
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', layers, store])
+    assert res.exit_code == 0
+    assert 'Loaded layer bermuda' in res.output
+
+
+def test_bag_skips_existing_layers(runner, shapefile, bags_dir):
+    uri = os.environ.get('POSTGIS_DB')
+    res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                               'mock://example.com', '--secure',
+                               'mock://example.com', shapefile, bags_dir])
+    assert res.exit_code == 0
+    assert 'Skipping existing layer bermuda' in res.output
+
+
+def test_bag_removes_failed_bag(runner, shapefile):
+    store = tempfile.mkdtemp()
+    uri = os.environ.get('POSTGIS_DB')
+    with patch('slingshot.cli.load_layer') as m:
+        m.side_effect = Exception
+        res = runner.invoke(main, ['bag', '--db-uri', uri, '--public',
+                                   'mock://example.com', '--secure',
+                                   'mock://example.com', shapefile, store])
+        assert res.exit_code == 0
+        assert 'Failed creating bag bermuda' in res.output
+        assert not os.listdir(store)

tests/unit/test_cli.py

@@ -0,0 +1,38 @@
+from click.testing import CliRunner
+import pytest
+import requests_mock
+
+from slingshot.cli import main
+
+
+@pytest.fixture
+def runner():
+    return CliRunner()
+
+
+def test_publish_publishes_layer(runner, bags_dir):
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/public/rest/workspaces/mit/datastores'
+               '/data/featuretypes')
+        m.post('mock://example.com/solr/update')
+        m.post('mock://example.com/solr/update/json/docs')
+        res = runner.invoke(main, ['publish', '--public',
+                                   'mock://example.com/public', '--secure',
+                                   'mock://example.com/secure', '--solr',
+                                   'mock://example.com/solr', bags_dir])
+        assert res.exit_code == 0
+        assert 'Loaded bermuda' in res.output
+
+
+def test_reindex_deletes_and_reloads(runner, bags_dir):
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/solr/update')
+        m.post('mock://example.com/solr/update/json/docs')
+        res = runner.invoke(main, ['reindex', '--solr',
+                                   'mock://example.com/solr', bags_dir])
+        assert res.exit_code == 0
+        assert m.request_history[0].json() == \
+            {'delete': {'query':
+                        'dct_provenance_s:MIT AND dc_format_s:Shapefile'}}
+        assert 'Indexed bermuda' in res.output
+        assert m.request_history[2].json() == {'commit': {}}

tests/test_db.py → tests/unit/test_db.py

@@ -1,9 +1,9 @@
 import re
 
 import pytest
-from shapefile import Reader
 from sqlalchemy import Boolean, Date, Float, Integer, Text
 
+from slingshot.app import ShapeReader
 from slingshot.db import (
     metadata,
     multiply,
@@ -94,46 +94,46 @@ def test_multiply_does_not_modify_points():
 
 
 def test_pg_reader_read_returns_size(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    assert pg.read(17) == '1\t45683\t58443\t32.'
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        assert pg.read(17) == '1\t45683\t58443\t32.'
 
 
 def test_pg_reader_reads_to_end(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = ''
-    while True:
-        chunk = pg.read(1024)
-        if not chunk:
-            break
-        buf += chunk
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = ''
+        while True:
+            chunk = pg.read(1024)
+            if not chunk:
+                break
+            buf += chunk
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
 
 
 def test_pg_reader_reads_all(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = pg.read()
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = pg.read()
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
 
 
 def test_pg_reader_reads_line(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    assert pg.readline().startswith('1\t45683\t58443')
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        assert pg.readline().startswith('1\t45683\t58443')
 
 
 def test_pg_reader_readline_reads_to_end(shapefile_unpacked):
-    shp = Reader(shapefile_unpacked + '/bermuda.shp')
-    pg = PGShapeReader(shp, 4326)
-    buf = ''
-    while True:
-        line = pg.readline()
-        if not line:
-            break
-        buf += line
-    assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
-                     '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)
+    with ShapeReader(shapefile_unpacked + '/bermuda.shp') as shp:
+        pg = PGShapeReader(shp, 4326)
+        buf = ''
+        while True:
+            line = pg.readline()
+            if not line:
+                break
+            buf += line
+        assert re.search('Zeta Island\t1995-08-16\tSRID=4326;POINT '
+                         '\(-64\.[0-9]+ 32\.[0-9]+\)\n$', buf)