Issue #407: Added capabilty to monitor progress with a logger

petl/test/util/test_timing.py

@@ -2,7 +2,7 @@
 
 
 from petl.util.counting import nrows
-from petl.util.timing import progress
+from petl.util.timing import progress, log_progress
 
 
 def test_progress():
@@ -13,3 +13,10 @@ def test_progress():
              ('b', 3))
     nrows(progress(table))
 
+def test_log_progress():
+    # make sure log_progress doesn't raise exception
+    table = (('foo', 'bar', 'baz'),
+             ('a', 1, True),
+             ('b', 2, True),
+             ('b', 3))
+    nrows(log_progress(table))

petl/util/__init__.py

@@ -21,7 +21,7 @@
 from petl.util.materialise import listoflists, listoftuples, tupleoflists, \
     tupleoftuples, columns, facetcolumns
 
-from petl.util.timing import progress, clock
+from petl.util.timing import progress, log_progress, clock
 
 from petl.util.statistics import limits, stats
 

petl/util/timing.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import, print_function, division
 
 
+import abc
+import logging
 import sys
 import time
 
@@ -9,9 +11,9 @@
 from petl.util.statistics import onlinestats
 
 
-def progress(table, batchsize=1000, prefix="", out=sys.stderr):
+def progress(table, batchsize=1000, prefix="", out=None):
     """
-    Report progress on rows passing through. E.g.::
+    Report progress on rows passing through to a log. E.g.::
 
         >>> import petl as etl
         >>> table = etl.dummytable(100000)
@@ -35,16 +37,49 @@ def progress(table, batchsize=1000, prefix="", out=sys.stderr):
     return ProgressView(table, batchsize, prefix, out)
 
 
+def log_progress(table, batchsize=1000, prefix="", out=None, level=logging.INFO):
+    """
+    Report progress on rows passing through. E.g.::
+
+        >>> import petl as etl
+        >>> table = etl.dummytable(100000)
+        >>> table.progress(10000).tocsv('example.csv')
+        10000 rows in 0.13s (78363 row/s); batch in 0.13s (78363 row/s)
+        20000 rows in 0.22s (91679 row/s); batch in 0.09s (110448 row/s)
+        30000 rows in 0.31s (96573 row/s); batch in 0.09s (108114 row/s)
+        40000 rows in 0.40s (99535 row/s); batch in 0.09s (109625 row/s)
+        50000 rows in 0.49s (101396 row/s); batch in 0.09s (109591 row/s)
+        60000 rows in 0.59s (102245 row/s); batch in 0.09s (106709 row/s)
+        70000 rows in 0.68s (103221 row/s); batch in 0.09s (109498 row/s)
+        80000 rows in 0.77s (103810 row/s); batch in 0.09s (108126 row/s)
+        90000 rows in 0.90s (99465 row/s); batch in 0.13s (74516 row/s)
+        100000 rows in 1.02s (98409 row/s); batch in 0.11s (89821 row/s)
+        100000 rows in 1.02s (98402 row/s); batches in 0.10 +/- 0.02s [0.09-0.13] (100481 +/- 13340 rows/s [74516-110448])
+
+    See also :func:`petl.util.timing.clock`.
+
+    """
+
+    return LoggingProgressView(table, batchsize, prefix, out, level=level)
+
+
 Table.progress = progress
+Table.log_progress = log_progress
 
 
-class ProgressView(Table):
+class ProgressViewBase(Table):
+    """
+    Abstract base class for reporting on proecessing status
+    """
 
-    def __init__(self, inner, batchsize, prefix, out):
+    def __init__(self, inner, batchsize, prefix):
         self.inner = inner
         self.batchsize = batchsize
         self.prefix = prefix
-        self.out = out
+
+    @abc.abstractmethod
+    def print_message(self, message):
+        pass
 
     def __iter__(self):
         start = time.time()
@@ -76,9 +111,7 @@ def __iter__(self):
                 message = self.prefix + \
                     '%s rows in %.2fs (%s row/s); ' \
                     'batch in %.2fs (%s row/s)' % v
-                print(message, file=self.out)
-                if hasattr(self.out, 'flush'):
-                    self.out.flush()
+                self.print_message(message)
                 batchstart = batchend
                 batchtimemean, batchtimevar = \
                     onlinestats(batchtime, batchn, mean=batchtimemean,
@@ -120,9 +153,43 @@ def __iter__(self):
             v = (n, elapsedtime, rate)
             message = self.prefix + '%s rows in %.2fs (%s row/s)' % v
 
-        print(message, file=self.out)
-        if hasattr(self.out, 'flush'):
-            self.out.flush()
+        self.print_message(message)
+
+
+class ProgressView(ProgressViewBase):
+    """
+    Reports progress to a file_object like sys.stdout or a file handler
+    """
+
+    def __init__(self, inner, batchsize, prefix, out):
+        if out is None:
+            self.file_object = sys.stderr
+        else:
+            self.file_object = out
+        super(ProgressView, self).__init__(inner, batchsize, prefix)
+
+    def print_message(self, message):
+        print(message, file=self.file_object)
+        if hasattr(self.file_object, 'flush'):
+            self.file_object.flush()
+
+
+class LoggingProgressView(ProgressViewBase):
+    """
+    Reports progress to a logger, log handler, or log adapter
+    """
+
+    def __init__(self, inner, batchsize, prefix, out, level=logging.INFO):
+        if out is None:
+            self.logger = logging.getLogger(__name__)
+            self.logger.setLevel(level)
+        else:
+            self.logger = out
+        self.level = level
+        super(LoggingProgressView, self).__init__(inner, batchsize, prefix)
+
+    def print_message(self, message):
+        self.logger.log(self.level, message)
 
 
 def clock(table):