Merge pull request #32 from MITLibraries/reindex

Add reindex function

slingshot/app.py

@@ -65,12 +65,6 @@ def register_layer(layer_name, geoserver, workspace, datastore, auth=None):
     r.raise_for_status()
 
 
-def index_layer(record, solr, auth=None):
-    url = '{}/update/json/docs'.format(solr.rstrip('/'))
-    r = requests.post(url, json=record, auth=auth)
-    r.raise_for_status()
-
-
 def make_uuid(value, namespace='mit.edu'):
     try:
         ns = uuid.uuid5(uuid.NAMESPACE_DNS, namespace)
@@ -90,6 +84,28 @@ def make_slug(name):
     return 'mit-' + b32.decode('ascii').rstrip('=').lower()
 
 
+class Solr(object):
+    def __init__(self, url, auth=None):
+        self.url = url.rstrip('/')
+        self.session = requests.Session()
+        self.session.auth = auth
+
+    def add(self, record):
+        url = self.url + '/update/json/docs'
+        r = self.session.post(url, json=record)
+        r.raise_for_status()
+
+    def delete(self, query='dct_provenance_s:MIT'):
+        url = self.url + '/update'
+        r = self.session.post(url, json={'delete': {'query': query}})
+        r.raise_for_status()
+
+    def commit(self):
+        url = self.url + '/update'
+        r = self.session.post(url, json={'commit': {}})
+        r.raise_for_status()
+
+
 class GeoBag(object):
     def __init__(self, bag):
         self.bag = bagit.Bag(bag)

slingshot/cli.py

@@ -10,9 +10,9 @@
 from slingshot.app import (
     create_record,
     GeoBag,
-    index_layer,
     make_slug,
     register_layer,
+    Solr,
     unpack_zip,
 )
 from slingshot.app import load_layer
@@ -91,31 +91,46 @@ def bag(layers, bags, db_uri, workspace, public, secure):
 @click.option('--workspace', default='mit')
 @click.option('--datastore', default='data')
 @click.option('--public', envvar='PUBLIC_GEOSERVER')
-@click.option('--public-user', envvar='PUBLIC_GEOSERVER_USER')
-@click.option('--public-password', envvar='PUBLIC_GEOSERVER_PASSWORD')
 @click.option('--secure', envvar='SECURE_GEOSERVER')
-@click.option('--secure-user', envvar='SECURE_GEOSERVER_USER')
-@click.option('--secure-password', envvar='SECURE_GEOSERVER_PASSWORD')
+@click.option('--geoserver-user', envvar='GEOSERVER_USER')
+@click.option('--geoserver-password', envvar='GEOSERVER_PASSWORD')
 @click.option('--solr', envvar='SOLR')
 @click.option('--solr-user', envvar='SOLR_USER')
 @click.option('--solr-password', envvar='SOLR_PASSWORD')
-def publish(bags, workspace, datastore, public, public_user, public_password,
-            secure, secure_user, secure_password, solr, solr_user,
-            solr_password):
-    public_auth = (public_user, public_password) if public_user and \
-        public_password else ()
-    secure_auth = (secure_user, secure_password) if secure_user and \
-        secure_password else ()
+def publish(bags, workspace, datastore, public, secure, geoserver_user,
+            geoserver_password, solr, solr_user, solr_password):
+    gs_auth = (geoserver_user, geoserver_password) if geoserver_user and \
+        geoserver_password else None
     solr_auth = (solr_user, solr_password) if solr_user and solr_password \
-        else ()
+        else None
+    s = Solr(solr, solr_auth)
     for b in os.listdir(bags):
         try:
             bag = GeoBag(os.path.join(bags, b))
             geoserver = public if bag.is_public() else secure
-            geoserver_auth = public_auth if bag.is_public() else secure_auth
             register_layer(bag.name, geoserver, workspace, datastore,
-                           auth=geoserver_auth)
-            index_layer(bag.record, solr, auth=solr_auth)
+                           auth=gs_auth)
+            s.add(bag.record)
             click.echo('Loaded {}'.format(bag.name))
         except Exception as e:
             click.echo('Failed loading {}: {}'.format(b, e))
+
+
+@main.command()
+@click.argument('bags')
+@click.option('--solr', envvar='SOLR')
+@click.option('--solr-user', envvar='SOLR_USER')
+@click.option('--solr-password', envvar='SOLR_PASSWORD')
+def reindex(bags, solr, solr_user, solr_password):
+    solr_auth = (solr_user, solr_password) if solr_user and solr_password \
+        else None
+    s = Solr(solr, solr_auth)
+    s.delete('dct_provenance_s:MIT AND dc_format_s:Shapefile')
+    for b in os.listdir(bags):
+        try:
+            bag = GeoBag(os.path.join(bags, b))
+            s.add(bag.record)
+            click.echo('Indexed {}'.format(bag.name))
+        except Exception as e:
+            click.echo('Failed indexing {}: {}'.format(b, e))
+    s.commit()

tests/test_app.py

@@ -11,10 +11,10 @@
     create_record,
     GeoBag,
     get_srid,
-    index_layer,
     make_slug,
     make_uuid,
     register_layer,
+    Solr,
     unpack_zip,
 )
 
@@ -57,13 +57,31 @@ def test_register_layer_adds_layer_to_geoserver():
             '<featureType><name>bermuda</name></featureType>'
 
 
-def test_index_layer_adds_layer_to_solr():
+def test_solr_adds_layer_to_solr():
     with requests_mock.Mocker() as m:
         m.post('mock://example.com/update/json/docs')
-        index_layer({'foo': 'bar'}, 'mock://example.com/')
+        s = Solr('mock://example.com/')
+        s.add({'foo': 'bar'})
         assert m.request_history[0].json() == {'foo': 'bar'}
 
 
+def test_solr_deletes_by_query():
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/update')
+        s = Solr('mock://example.com/')
+        s.delete()
+        assert m.request_history[0].json() == \
+            {'delete': {'query': 'dct_provenance_s:MIT'}}
+
+
+def test_solr_commits_changes():
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/update')
+        s = Solr('mock://example.com/')
+        s.commit()
+        assert m.request_history[0].json() == {'commit': {}}
+
+
 def test_make_uuid_creates_uuid_string():
     assert make_uuid('bermuda', 'mit.edu') == \
         uuid.UUID('df04b29c-0e51-58a8-8a37-557e4f4917df')

tests/test_cli.py

@@ -62,3 +62,17 @@ def test_publish_publishes_layer(runner, bags_dir):
                                    'mock://example.com/solr', bags_dir])
         assert res.exit_code == 0
         assert 'Loaded bermuda' in res.output
+
+
+def test_reindex_deletes_and_reloads(runner, bags_dir):
+    with requests_mock.Mocker() as m:
+        m.post('mock://example.com/solr/update')
+        m.post('mock://example.com/solr/update/json/docs')
+        res = runner.invoke(main, ['reindex', '--solr',
+                                   'mock://example.com/solr', bags_dir])
+        assert res.exit_code == 0
+        assert m.request_history[0].json() == \
+            {'delete': {'query':
+                        'dct_provenance_s:MIT AND dc_format_s:Shapefile'}}
+        assert 'Indexed bermuda' in res.output
+        assert m.request_history[2].json() == {'commit': {}}