Merge remote-tracking branch 'remotes/origin/feature/configurable_sle…

…ep_time' into feature/datatable_errors_in_report

goodtables/pipeline/batch.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import os
+import time
 from . import pipeline
 from ..utilities import helpers
 from .. import datatable
@@ -19,14 +20,15 @@ class Batch(object):
     * `source_type`: 'csv' or 'datapackage'
     * `data_key`: The key for the data (only used when source_type is 'csv')
     * `schema_key`: The key for the schema (only used when source_type is 'csv').
+    * `sleep`:  The time (in seconds) to wait beteween pipelines
     * `pipeline_options`: Options to pass to each pipeline instance.
     * `post_task`: Task handler to run after all pipelines have run.
     * `pipeline_post_task`: Task handler passed to each pipeline instance.
 
     """
 
     def __init__(self, source, source_type='csv', data_key='data',
-                 schema_key='schema', pipeline_options=None,
+                 schema_key='schema', sleep=1, pipeline_options=None,
                  post_task=None, pipeline_post_task=None):
 
         self.source = source
@@ -44,6 +46,10 @@ def __init__(self, source, source_type='csv', data_key='data',
         self.post_task = post_task
         self.pipeline_post_task = pipeline_post_task
 
+        if not isinstance(sleep, (int, float)):
+            raise ValueError('Received non int or float for the \'sleep\' argument.')
+        self.sleep = sleep
+
     def get_dataset(self):
         """Get the dataset for this batch process."""
 
@@ -127,6 +133,8 @@ def run(self):
             result, report = self.current_pipeline.run()
             self.reports.append(report)
 
+            time.sleep(self.sleep)
+
         if self.post_task:
             self.post_task(self)
 

tests/test_batch.py

@@ -5,6 +5,8 @@
 from __future__ import unicode_literals
 
 import os
+from timeit import default_timer as timer
+
 from goodtables.pipeline import Batch
 from goodtables import exceptions
 from tests import base
@@ -61,3 +63,21 @@ def test_bad_pipeline_post_task_raises(self):
         say_hi = 'Say Hi!'
         self.assertRaises(exceptions.InvalidHandlerError, Batch,
                           self.batch_csv, pipeline_post_task=say_hi)
+
+    def test_batch_with_batch_sleep_time(self):
+        def normal_time():
+            batch = Batch(self.batch_csv, sleep=0)
+            start = timer(); batch.run(); end = timer()
+            return end - start
+
+        def default_time():
+            batch = Batch(self.batch_csv)
+            start = timer(); batch.run(); end = timer()
+            return end - start
+
+        def greater_time():
+            batch = Batch(self.batch_csv, sleep=3)
+            start = timer(); batch.run(); end = timer()
+            return end - start
+
+        self.assertTrue(normal_time() < default_time() < greater_time())