Merge 12b0181 into b226acf

docs/config.rst

@@ -0,0 +1,6 @@
+Configuration
+===============================
+
+.. automodule:: petl.config
+    :members:
+

docs/index.rst

@@ -36,6 +36,7 @@ see the :ref:`genindex`.
    io
    transform
    util
+   config
    changes
    contributing
    acknowledgments

petl/config.py

@@ -16,3 +16,15 @@
 display_index_header = False
 display_vrepr = text_type
 sort_buffersize = 100000
+failonerror=False # False, True, 'inline'
+"""
+Controls what happens when unhandled exceptions are raised in a
+transformation:
+
+    - If `False`, exceptions are suppressed.  If present, the value
+      provided in the `errorvalue` argument is returned.
+
+    - If `True`, the first unhandled exception is raised.
+
+    - If `'inline'`, unhandled exceptions are returned.
+"""

petl/test/failonerror.py

@@ -0,0 +1,118 @@
+from petl.test.helpers import ieq, eq_
+import petl.config as config
+
+from nose.tools import nottest
+
+
+@nottest
+def test_failonerror(input_fn, expected_output):
+    """In the input rows, the first row should process through the
+    transformation cleanly.  The second row should generate an
+    exception.  There are no requirements for any other rows."""
+    #=========================================================
+    # Test function parameters with default config settings
+    #=========================================================
+    # test the default config setting: failonerror == False
+    eq_(config.failonerror, False)
+
+    # By default, a bad conversion does not raise an exception, and
+    # values for the failed conversion are returned as None
+    table2 = input_fn()
+    ieq(expected_output, table2)
+    ieq(expected_output, table2)
+
+    # When called with failonerror is False or None, a bad conversion
+    # does not raise an exception, and values for the failed conversion
+    # are returned as None
+    table3 = input_fn(failonerror=False)
+    ieq(expected_output, table3)
+    ieq(expected_output, table3)
+    table3 = input_fn(failonerror=None)
+    ieq(expected_output, table3)
+    ieq(expected_output, table3)
+
+    # When called with failonerror=True, a bad conversion raises an
+    # exception
+    try:
+        table4 = input_fn(failonerror=True)
+        table4.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When called with failonerror='inline', a bad conversion
+    # does not raise an exception, and an Exception for the failed
+    # conversion is returned in the result.
+    expect5 = expected_output[0], expected_output[1]
+    table5 = input_fn(failonerror='inline')
+    ieq(expect5, table5.head(1))
+    ieq(expect5, table5.head(1))
+    excp = table5[2][0]
+    assert isinstance(excp, Exception)
+
+    #=========================================================
+    # Test config settings
+    #=========================================================
+    # Save config setting
+    saved_config_failonerror = config.failonerror
+
+    # When config.failonerror == True, a bad conversion raises an
+    # exception
+    config.failonerror = True
+    try:
+        table6 = input_fn()
+        table6.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When config.failonerror == 'inline', a bad conversion
+    # does not raise an exception, and an Exception for the failed
+    # conversion is returned in the result.
+    expect7 = expected_output[0], expected_output[1]
+    config.failonerror = 'inline'
+    table7 = input_fn()
+    ieq(expect7, table7.head(1))
+    ieq(expect7, table7.head(1))
+    excp = table7[2][0]
+    assert isinstance(excp, Exception)
+
+    # When config.failonerror is an invalid value, but still truthy, it
+    # behaves the same as if == True
+    config.failonerror = 'invalid'
+    try:
+        table8 = input_fn()
+        table8.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # When config.failonerror is None, it behaves the same as if
+    # config.failonerror is False
+    config.failonerror = None
+    table9 = input_fn()
+    ieq(expected_output, table9)
+    ieq(expected_output, table9)
+
+    # A False keyword parameter overrides config.failonerror == True
+    config.failonerror = True
+    table10 = input_fn(failonerror=False)
+    ieq(expected_output, table10)
+    ieq(expected_output, table10)
+
+    # A None keyword parameter uses config.failonerror == True
+    config.failonerror = True
+    try:
+        table11 = input_fn(failonerror=None)
+        table11.nrows()
+    except Exception:
+        pass
+    else:
+        raise Exception('expected exception not raised')
+
+    # restore config setting
+    config.failonerror = saved_config_failonerror
+

petl/test/transform/test_conversions.py

@@ -1,10 +1,13 @@
 from __future__ import absolute_import, print_function, division
 
 
+from petl.test.failonerror import test_failonerror
 from petl.test.helpers import ieq
 from petl.transform.conversions import convert, convertall, convertnumbers, \
     replace, update, format, interpolate
 
+from functools import partial
+
 
 def test_convert():
 
@@ -289,6 +292,16 @@ def test_convert_where():
     ieq(expect, actual)
 
 
+def test_convert_failonerror():
+    input_  = (('foo',), ('A',), (1,))
+    cvt_    = {'foo': 'lower'}
+    expect_ = (('foo',), ('a',), (None,))
+
+    test_failonerror(
+            input_fn=partial(convert, input_, cvt_),
+            expected_output=expect_)
+
+
 def test_replace_where():
 
     tbl1 = (('foo', 'bar'),
@@ -363,3 +376,4 @@ def test_interpolate():
     actual = interpolate(table, 'bar', '%02d')
     ieq(expect, actual)
     ieq(expect, actual)
+

petl/test/transform/test_maps.py

@@ -1,10 +1,13 @@
 from __future__ import absolute_import, print_function, division
 
-
 from collections import OrderedDict
+
+from petl.test.failonerror import test_failonerror
 from petl.test.helpers import ieq
 from petl.transform.maps import fieldmap, rowmap, rowmapmany
 
+from functools import partial
+
 
 def test_fieldmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
@@ -88,6 +91,16 @@ def test_fieldmap_empty():
     ieq(expect, actual)
 
 
+def test_fieldmap_failonerror():
+    input_  = (('foo',), ('A',), (1,))
+    mapper_ = {'bar': ('foo', lambda v: v.lower())}
+    expect_ = (('bar',), ('a',), (None,))
+
+    test_failonerror(
+            input_fn=partial(fieldmap, input_, mapper_),
+            expected_output=expect_)
+
+
 def test_rowmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -147,6 +160,17 @@ def rowmapper(row):
     ieq(expect, actual)
 
 
+def test_rowmap_failonerror():
+    input_  = (('foo',), ('A',), (1,), ('B',))
+    mapper  = lambda r: [r[0].lower()]
+    # exceptions in rowmappers do not generate an output row
+    expect_ = (('foo',), ('a',), ('b',))
+
+    test_failonerror(
+            input_fn=partial(rowmap, input_, mapper, header=('foo',)),
+            expected_output=expect_)
+
+
 def test_recordmap():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -222,6 +246,16 @@ def rowgenerator(row):
     ieq(expect, actual)  # can iteratate twice?
 
 
+def test_rowmapmany_failonerror():
+    input_  = (('foo',), ('A',), (1,), ('B',))
+    mapper  = lambda r: [r[0].lower()]
+    expect_ = (('foo',), ('a',), ('b',),)
+
+    test_failonerror(
+            input_fn=partial(rowmapmany, input_, mapper, header=('foo',)),
+            expected_output=expect_)
+
+
 def test_recordmapmany():
     table = (('id', 'sex', 'age', 'height', 'weight'),
              (1, 'male', 16, 1.45, 62.0),
@@ -252,3 +286,4 @@ def rowgenerator(rec):
               (4, 'age_months', 21 * 12))
     ieq(expect, actual)
     ieq(expect, actual)  # can iteratate twice?
+

petl/transform/conversions.py

@@ -4,6 +4,7 @@
 from petl.compat import next, integer_types, string_types, text_type
 
 
+import petl.config as config
 from petl.errors import ArgumentError, FieldSelectionError
 from petl.util.base import Table, expr, header, Record
 from petl.util.parsers import numparser
@@ -165,6 +166,9 @@ def convert(table, *args, **kwargs):
     arguments to the conversion function (so, i.e., the conversion function
     should accept two arguments).
 
+    Also accepts `failonerror` and `errorvalue` keyword arguments,
+    documented under :func:`petl.config.failonerror`
+
     """
 
     converters = None
@@ -286,7 +290,7 @@ def convertnumbers(table, strict=False, **kwargs):
 
 class FieldConvertView(Table):
 
-    def __init__(self, source, converters=None, failonerror=False,
+    def __init__(self, source, converters=None, failonerror=None,
                  errorvalue=None, where=None, pass_row=False):
         self.source = source
         if converters is None:
@@ -297,7 +301,8 @@ def __init__(self, source, converters=None, failonerror=False,
             self.converters = dict([(i, v) for i, v in enumerate(converters)])
         else:
             raise ArgumentError('unexpected converters: %r' % converters)
-        self.failonerror = failonerror
+        self.failonerror = (config.failonerror if failonerror is None
+                                else failonerror)
         self.errorvalue = errorvalue
         self.where = where
         self.pass_row = pass_row
@@ -366,7 +371,9 @@ def transform_value(i, v, *args):
             try:
                 return converter_functions[i](v, *args)
             except Exception as e:
-                if failonerror:
+                if failonerror == 'inline':
+                    return e
+                elif failonerror:
                     raise e
                 else:
                     return errorvalue