Catch and log serialization exceptions (#39)

CHANGES.rst

@@ -4,6 +4,9 @@ Changelog
 master
 ------
 
+* Catch and log serialization exceptions instead of crashing. Fixes #38, merge
+  of #39.
+
 * Fix bug in default code filter when Python lib paths are symlinked. Merge of
   #40. Thanks Simon Gomizelj.
 

doc/stores.rst

@@ -41,10 +41,7 @@ interface.
     Store one or more :class:`~monkeytype.typing.CallTrace` instances.
 
     Implementations of this method will probably find the
-    :class:`~monkeytype.encoding.CallTraceRow` class and its
-    :meth:`~monkeytype.encoding.CallTraceRow.from_trace` classmethod useful; it
-    implements a standardized serialization of
-    :class:`~monkeytype.typing.CallTrace` instances to JSON.
+    :func:`~monkeytype.encoding.serialize_traces` function useful.
 
   .. method:: filter(module: str, qualname_prefix: Optional[str] = None, limit: int = 2000) -> List[CallTraceThunk]
 
@@ -97,6 +94,15 @@ call traces in a SQLite database in a local file.
 
 .. module:: monkeytype.encoding
 
+serialize_traces
+~~~~~~~~~~~~~~~~
+
+.. function:: serialize_traces(traces: Iterable[CallTrace]) -> Iterable[CallTraceRow]
+
+  Serialize an iterable of :class:`~monkeytype.tracing.CallTrace` to an iterable
+  of :class:`CallTraceRow` (via :meth:`CallTraceRow.from_trace`). If any trace
+  fails to serialize, the exception is logged and serialization continues.
+
 CallTraceRow
 ~~~~~~~~~~~~
 

monkeytype/db/sqlite.py

@@ -19,7 +19,10 @@
     CallTraceStore,
     CallTraceThunk,
 )
-from monkeytype.encoding import CallTraceRow
+from monkeytype.encoding import (
+    CallTraceRow,
+    serialize_traces,
+)
 from monkeytype.tracing import CallTrace
 
 logger = logging.getLogger(__name__)
@@ -81,8 +84,7 @@ def make_store(cls, connection_string: str) -> 'CallTraceStore':
 
     def add(self, traces: Iterable[CallTrace]) -> None:
         values = []
-        for trace in traces:
-            row = CallTraceRow.from_trace(trace)
+        for row in serialize_traces(traces):
             values.append((datetime.datetime.now(), row.module, row.qualname,
                            row.arg_types, row.return_type, row.yield_type))
         with self.conn:

monkeytype/encoding.py

@@ -4,13 +4,15 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 import json
+import logging
 
 # _Any and _Union aren't visible from stubs
 from typing import (  # type: ignore
     Any,
     Callable,
     Dict,
     GenericMeta,
+    Iterable,
     Optional,
     Type,
     TypeVar,
@@ -28,6 +30,9 @@
 )
 
 
+logger = logging.getLogger(__name__)
+
+
 # Types are converted to dictionaries of the following form before
 # being JSON encoded and sent to storage:
 #
@@ -180,3 +185,34 @@ def to_trace(self) -> CallTrace:
         return_type = maybe_decode_type(type_from_json, self.return_type)
         yield_type = maybe_decode_type(type_from_json, self.yield_type)
         return CallTrace(function, arg_types, return_type, yield_type)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, CallTraceRow):
+            return (
+                self.module,
+                self.qualname,
+                self.arg_types,
+                self.return_type,
+                self.yield_type,
+            ) == (
+                other.module,
+                other.qualname,
+                other.arg_types,
+                other.return_type,
+                other.yield_type,
+            )
+        return NotImplemented
+
+
+def serialize_traces(traces: Iterable[CallTrace]) -> Iterable[CallTraceRow]:
+    """Serialize an iterable of CallTraces to an iterable of CallTraceRow.
+
+    Catches and logs exceptions, so a failure to serialize one CallTrace doesn't
+    lose all traces.
+
+    """
+    for trace in traces:
+        try:
+            yield CallTraceRow.from_trace(trace)
+        except Exception:
+            logger.exception("Failed to serialize trace")

tests/test_encoding.py

@@ -24,6 +24,7 @@
     type_from_json,
     type_to_dict,
     type_to_json,
+    serialize_traces,
 )
 from monkeytype.exceptions import InvalidTypeError
 from monkeytype.tracing import CallTrace
@@ -96,7 +97,6 @@ def test_maybe_encode_type(self, encoder, typ, expected, should_call_encoder):
         ret = maybe_encode_type(encoder, typ)
         if should_call_encoder:
             encoder.assert_called_with(typ)
-
         else:
             encoder.assert_not_called()
         assert ret == expected
@@ -117,3 +117,19 @@ def test_maybe_decode_type(self, encoder, typ, expected, should_call_encoder):
         else:
             encoder.assert_not_called()
         assert ret == expected
+
+
+class TestSerializeTraces:
+    def test_log_failure_and_continue(self, caplog):
+        traces = [
+            CallTrace(dummy_func, {'a': int, 'b': int}, int),
+            CallTrace(object(), {}),  # object() will fail to serialize
+            CallTrace(dummy_func, {'a': str, 'b': str}, str),
+        ]
+        rows = list(serialize_traces(traces))
+        expected = [
+            CallTraceRow.from_trace(traces[0]),
+            CallTraceRow.from_trace(traces[2]),
+        ]
+        assert rows == expected
+        assert [r.msg for r in caplog.records] == ["Failed to serialize trace"]