Stop execution after specified number of failures

Closes #9. Provides an option to set the number of consecutive
failures allowed before the script stops execution.

slingshot/cli.py

@@ -28,7 +28,9 @@ def main():
 @click.option('--username', help="Username for kepler submission.")
 @click.option('--password',
               help="Password for kepler submission. Omit for prompt.")
-def run(layers, store, url, namespace, username, password):
+@click.option('--fail-after', default=5,
+              help="Stop after number of consecutive failures. Default is 5.")
+def run(layers, store, url, namespace, username, password, fail_after):
     """Create and upload bags to the specified endpoint.
 
     This script will create bags from all the layers in the LAYERS
@@ -50,17 +52,23 @@ def run(layers, store, url, namespace, username, password):
     auth = username, password
     if not all(auth):
         auth = None
+    failures = 0
     for data_layer in uploadable(layers, store):
         bag = prep_bag(os.path.join(layers, data_layer), store)
         try:
             bagit.make_bag(bag)
             bag_name = make_uuid(os.path.basename(bag), namespace)
             with temp_archive(bag, bag_name) as zf:
                 submit(zf, url, auth)
+            click.echo("%sZ: %s uploaded" % (datetime.utcnow().isoformat(),
+                                             data_layer))
+            failures = 0
         except Exception as e:
             shutil.rmtree(bag, ignore_errors=True)
+            failures += 1
             click.echo("%sZ: %s failed with %r" %
                        (datetime.utcnow().isoformat(), data_layer, e))
-            raise e
-        click.echo("%sZ: %s uploaded" % (datetime.utcnow().isoformat(),
-                                         data_layer))
+            if failures >= fail_after:
+                click.echo("%sZ: Maximum number of consecutive failures (%d)" %
+                           (datetime.utcnow().isoformat(), failures))
+                raise e

tests/test_cli.py

@@ -77,3 +77,12 @@ def test_run_logs_failed_layers_to_stdout(runner, layers_dir):
         res = runner.invoke(main, ['run', layers_dir, store,
                                    'http://localhost'])
         assert 'SDE_DATA_BD_A8GNS_2003.zip failed' in res.output
+
+
+def test_run_fails_after_consecutive_failures(runner, layers_dir):
+    with requests_mock.Mocker() as m:
+        store = tempfile.mkdtemp()
+        m.post('http://localhost', status_code=500)
+        res = runner.invoke(main, ['run', layers_dir, store,
+                                   'http://localhost', '--fail-after', 1])
+        assert 'Maximum number of consecutive failures' in res.output