Merge pull request #24 from MITLibraries/skip-existing

Skip existing layers

slingshot/app.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
 import base64
 import json
 import os
-import shutil
 import tempfile
 import uuid
 from zipfile import ZipFile
@@ -35,18 +35,6 @@ def unpack_zip(source, destination):
                 fp.write(zf.read(f))
 
 
-def make_bag_dir(destination, overwrite=False):
-    try:
-        os.mkdir(destination)
-    except OSError:
-        if overwrite:
-            shutil.rmtree(destination)
-            os.mkdir(destination)
-        else:
-            raise
-    return destination
-
-
 def create_record(bag, public, secure, **kwargs):
     r = parse(bag.fgdc, FGDCParser)
     r.update(**kwargs)

slingshot/cli.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+
 import os
 import shutil
 import traceback
@@ -10,7 +11,6 @@
     create_record,
     GeoBag,
     index_layer,
-    make_bag_dir,
     make_slug,
     register_layer,
     unpack_zip,
@@ -60,7 +60,12 @@ def bag(layers, bags, db_uri, workspace, public, secure):
         zips = [layers]
     for layer in zips:
         name = os.path.splitext(os.path.basename(layer))[0]
-        dest = make_bag_dir(os.path.join(bags, name))
+        dest = os.path.join(bags, name)
+        try:
+            os.mkdir(dest)
+        except OSError:
+            click.echo('Skipping existing layer {}'.format(name))
+            continue
         try:
             unpack_zip(layer, dest)
             bag = GeoBag.create(dest)

tests/test_app.py

@@ -12,7 +12,6 @@
     GeoBag,
     get_srid,
     index_layer,
-    make_bag_dir,
     make_slug,
     make_uuid,
     register_layer,
@@ -30,33 +29,6 @@ def test_unpack_zip_extracts_to_top_of_dir(shapefile, temp_dir):
     assert os.path.isfile(os.path.join(temp_dir, 'bermuda.shp'))
 
 
-def test_make_bag_dir_creates_directory(temp_dir):
-    target = os.path.join(temp_dir, 'TEST_BAG')
-    make_bag_dir(target)
-    assert os.path.isdir(target)
-
-
-def test_make_bag_dir_returns_dir_name(temp_dir):
-    target = os.path.join(temp_dir, 'TEST_BAG')
-    assert make_bag_dir(target) == target
-
-
-def test_make_bag_dir_raises_error_when_dir_exists(temp_dir):
-    bag = os.path.join(temp_dir, 'TEST_BAG')
-    os.mkdir(bag)
-    with pytest.raises(OSError):
-        make_bag_dir(bag)
-
-
-def test_make_bag_dir_overwrites_existing_dir(temp_dir):
-    bag = os.path.join(temp_dir, 'TEST_BAG')
-    os.mkdir(bag)
-    with open(os.path.join(bag, 'foobar'), 'w') as fp:
-        fp.write("I shouldn't be here.")
-    make_bag_dir(bag, overwrite=True)
-    assert not os.path.isfile(os.path.join(bag, 'foobar'))
-
-
 def test_create_record_creates_mit_record(bag):
     record = create_record(GeoBag(bag), public='mock://example.com',
                            secure='mock://example.com')

tests/test_cli.py

@@ -1,3 +1,4 @@
+import os
 import shutil
 import tempfile
 try:
@@ -29,6 +30,27 @@ def test_bag_creates_bag(runner, shapefile):
     assert 'Loaded layer bermuda' in res.output
 
 
+def test_bag_skips_existing_layers(runner, shapefile, bags_dir):
+    with patch('slingshot.cli.load_layer'):
+        res = runner.invoke(main, ['bag', '--db-uri', 'sqlite://', '--public',
+                                   'mock://example.com', '--secure',
+                                   'mock://example.com', shapefile, bags_dir])
+    assert res.exit_code == 0
+    assert 'Skipping existing layer bermuda' in res.output
+
+
+def test_bag_removes_failed_bag(runner, shapefile):
+    store = tempfile.mkdtemp()
+    with patch('slingshot.cli.load_layer') as m:
+        m.side_effect = Exception
+        res = runner.invoke(main, ['bag', '--db-uri', 'sqlite://', '--public',
+                                   'mock://example.com', '--secure',
+                                   'mock://example.com', shapefile, store])
+        assert res.exit_code == 0
+        assert 'Failed creating bag bermuda' in res.output
+        assert not os.listdir(store)
+
+
 def test_publish_publishes_layer(runner, bags_dir):
     with requests_mock.Mocker() as m:
         m.post('mock://example.com/public/rest/workspaces/mit/datastores'