diff --git a/rollbar/__init__.py b/rollbar/__init__.py index ce1552ad..1ba37f88 100644 --- a/rollbar/__init__.py +++ b/rollbar/__init__.py @@ -327,6 +327,7 @@ def _get_fastapi_request(): 'request_pool_connections': None, 'request_pool_maxsize': None, 'request_max_retries': None, + 'batch_transforms': False, } _CURRENT_LAMBDA_CONTEXT = None @@ -341,11 +342,13 @@ def _get_fastapi_request(): from rollbar.lib.transforms.scrub_redact import REDACT_REF from rollbar.lib import transforms +from rollbar.lib import type_info from rollbar.lib.transforms.scrub import ScrubTransform from rollbar.lib.transforms.scruburl import ScrubUrlTransform from rollbar.lib.transforms.scrub_redact import ScrubRedactTransform from rollbar.lib.transforms.serializable import SerializableTransform from rollbar.lib.transforms.shortener import ShortenerTransform +from rollbar.lib.transforms.batched import BatchedTransform ## public api @@ -1082,10 +1085,11 @@ def _add_locals_data(trace_data, exc_info): def _serialize_frame_data(data): - for transform in (ScrubRedactTransform(), _serialize_transform): - data = transforms.transform(data, transform) - - return data + return transforms.transform( + data, + [ScrubRedactTransform(), _serialize_transform], + batch_transforms=SETTINGS['batch_transforms'] + ) def _add_lambda_context_data(data): @@ -1477,10 +1481,12 @@ def _build_server_data(): def _transform(obj, key=None): - for transform in _transforms: - obj = transforms.transform(obj, transform, key=key) - - return obj + return transforms.transform( + obj, + _transforms, + key=key, + batch_transforms=SETTINGS['batch_transforms'] + ) def _build_payload(data): diff --git a/rollbar/lib/transform.py b/rollbar/lib/transform.py new file mode 100644 index 00000000..fa08e5cd --- /dev/null +++ b/rollbar/lib/transform.py @@ -0,0 +1,39 @@ +class Transform(object): + def default(self, o, key=None): + return o + + def transform_circular_reference(self, o, key=None, ref_key=None): + # By default, we just perform a no-op for circular references. + # Subclasses should implement this method to return whatever representation + # for the circular reference they need. + return self.default(o, key=key) + + def transform_tuple(self, o, key=None): + return self.default(o, key=key) + + def transform_namedtuple(self, o, key=None): + return self.default(o, key=key) + + def transform_list(self, o, key=None): + return self.default(o, key=key) + + def transform_dict(self, o, key=None): + return self.default(o, key=key) + + def transform_number(self, o, key=None): + return self.default(o, key=key) + + def transform_py2_str(self, o, key=None): + return self.default(o, key=key) + + def transform_py3_bytes(self, o, key=None): + return self.default(o, key=key) + + def transform_unicode(self, o, key=None): + return self.default(o, key=key) + + def transform_boolean(self, o, key=None): + return self.default(o, key=key) + + def transform_custom(self, o, key=None): + return self.default(o, key=key) diff --git a/rollbar/lib/transforms/__init__.py b/rollbar/lib/transforms/__init__.py index 3b995055..2869f2d4 100644 --- a/rollbar/lib/transforms/__init__.py +++ b/rollbar/lib/transforms/__init__.py @@ -1,6 +1,23 @@ +try: + # Python 3 + from collections.abc import Iterable +except ImportError: + # Python 2.7 + from collections import Iterable + from rollbar.lib import ( - python_major_version, binary_type, string_types, integer_types, - number_types, traverse) + python_major_version, + binary_type, + string_types, + integer_types, + number_types, + traverse, + type_info, +) +# NOTE: Don't remove this import, it would cause a breaking change to the library's API. +# The `Transform` class was moved out of this file to prevent a cyclical dependency issue. +from rollbar.lib.transform import Transform +from rollbar.lib.transforms.batched import BatchedTransform _ALLOWED_CIRCULAR_REFERENCE_TYPES = [binary_type, bool, type(None)] @@ -17,72 +34,47 @@ _ALLOWED_CIRCULAR_REFERENCE_TYPES = tuple(_ALLOWED_CIRCULAR_REFERENCE_TYPES) -class Transform(object): - def default(self, o, key=None): - return o - - def transform_circular_reference(self, o, key=None, ref_key=None): - # By default, we just perform a no-op for circular references. - # Subclasses should implement this method to return whatever representation - # for the circular reference they need. - return self.default(o, key=key) - - def transform_tuple(self, o, key=None): - return self.default(o, key=key) - - def transform_namedtuple(self, o, key=None): - return self.default(o, key=key) - - def transform_list(self, o, key=None): - return self.default(o, key=key) - - def transform_dict(self, o, key=None): - return self.default(o, key=key) +def transform(obj, transforms, key=None, batch_transforms=False): + if isinstance(transforms, Transform): + transforms = [transforms] - def transform_number(self, o, key=None): - return self.default(o, key=key) + if batch_transforms: + transforms = [BatchedTransform(transforms)] - def transform_py2_str(self, o, key=None): - return self.default(o, key=key) + for transform in transforms: + obj = _transform(obj, transform, key=key) - def transform_py3_bytes(self, o, key=None): - return self.default(o, key=key) + return obj - def transform_unicode(self, o, key=None): - return self.default(o, key=key) - def transform_boolean(self, o, key=None): - return self.default(o, key=key) - - def transform_custom(self, o, key=None): - return self.default(o, key=key) - - -def transform(obj, transform, key=None): +def _transform(obj, transform, key=None): key = key or () def do_transform(type_name, val, key=None, **kw): - fn = getattr(transform, 'transform_%s' % type_name, transform.transform_custom) + fn = getattr(transform, "transform_%s" % type_name, transform.transform_custom) val = fn(val, key=key, **kw) return val if python_major_version() < 3: + def string_handler(s, key=None): if isinstance(s, str): - return do_transform('py2_str', s, key=key) + return do_transform("py2_str", s, key=key) elif isinstance(s, unicode): - return do_transform('unicode', s, key=key) + return do_transform("unicode", s, key=key) + else: + def string_handler(s, key=None): if isinstance(s, bytes): - return do_transform('py3_bytes', s, key=key) + return do_transform("py3_bytes", s, key=key) elif isinstance(s, str): - return do_transform('unicode', s, key=key) + return do_transform("unicode", s, key=key) def default_handler(o, key=None): if isinstance(o, bool): - return do_transform('boolean', o, key=key) + return do_transform("boolean", o, key=key) # There is a quirk in the current version (1.1.6) of the enum # backport enum34 which causes it to not have the same @@ -90,26 +82,29 @@ def default_handler(o, key=None): # they are instances of numbers but not number types. if isinstance(o, number_types): if type(o) not in number_types: - return do_transform('custom', o, key=key) + return do_transform("custom", o, key=key) else: - return do_transform('number', o, key=key) + return do_transform("number", o, key=key) - return do_transform('custom', o, key=key) + return do_transform("custom", o, key=key) handlers = { - 'string_handler': string_handler, - 'tuple_handler': lambda o, key=None: do_transform('tuple', o, key=key), - 'namedtuple_handler': lambda o, key=None: do_transform('namedtuple', o, key=key), - 'list_handler': lambda o, key=None: do_transform('list', o, key=key), - 'set_handler': lambda o, key=None: do_transform('set', o, key=key), - 'mapping_handler': lambda o, key=None: do_transform('dict', o, key=key), - 'circular_reference_handler': lambda o, key=None, ref_key=None: - do_transform('circular_reference', o, key=key, ref_key=ref_key), - 'default_handler': default_handler, - 'allowed_circular_reference_types': _ALLOWED_CIRCULAR_REFERENCE_TYPES + "string_handler": string_handler, + "tuple_handler": lambda o, key=None: do_transform("tuple", o, key=key), + "namedtuple_handler": lambda o, key=None: do_transform( + "namedtuple", o, key=key + ), + "list_handler": lambda o, key=None: do_transform("list", o, key=key), + "set_handler": lambda o, key=None: do_transform("set", o, key=key), + "mapping_handler": lambda o, key=None: do_transform("dict", o, key=key), + "circular_reference_handler": lambda o, key=None, ref_key=None: do_transform( + "circular_reference", o, key=key, ref_key=ref_key + ), + "default_handler": default_handler, + "allowed_circular_reference_types": _ALLOWED_CIRCULAR_REFERENCE_TYPES, } return traverse.traverse(obj, key=key, **handlers) -__all__ = ['transform', 'Transform'] +__all__ = ["transform", "Transform"] diff --git a/rollbar/lib/transforms/batched.py b/rollbar/lib/transforms/batched.py new file mode 100644 index 00000000..5ba9eadd --- /dev/null +++ b/rollbar/lib/transforms/batched.py @@ -0,0 +1,88 @@ +from rollbar.lib.transform import Transform +from rollbar.lib import ( + python_major_version, + number_types, + type_info, +) + + +def do_transform(transform, type_name, val, key=None, **kw): + fn = getattr(transform, "transform_%s" % type_name, transform.transform_custom) + val = fn(val, key=key, **kw) + + return val + + +if python_major_version() < 3: + + def string_handler(transform, s, key=None): + if isinstance(s, str): + return do_transform(transform, "py2_str", s, key=key) + elif isinstance(s, unicode): + return do_transform(transform, "unicode", s, key=key) + +else: + + def string_handler(transform, s, key=None): + if isinstance(s, bytes): + return do_transform(transform, "py3_bytes", s, key=key) + elif isinstance(s, str): + return do_transform(transform, "unicode", s, key=key) + + +def default_handler(transform, o, key=None): + if isinstance(o, bool): + return do_transform(transform, "boolean", o, key=key) + + # There is a quirk in the current version (1.1.6) of the enum + # backport enum34 which causes it to not have the same + # behavior as Python 3.4+. One way to identify IntEnums is that + # they are instances of numbers but not number types. + if isinstance(o, number_types): + if type(o) not in number_types: + return do_transform(transform, "custom", o, key=key) + else: + return do_transform(transform, "number", o, key=key) + + return do_transform(transform, "custom", o, key=key) + + +handlers = { + type_info.STRING: string_handler, + type_info.TUPLE: lambda transform, o, key=None: do_transform( + transform, "tuple", o, key=key + ), + type_info.NAMEDTUPLE: lambda transform, o, key=None: do_transform( + transform, "namedtuple", o, key=key + ), + type_info.LIST: lambda transform, o, key=None: do_transform( + transform, "list", o, key=key + ), + type_info.SET: lambda transform, o, key=None: do_transform( + transform, "set", o, key=key + ), + type_info.MAPPING: lambda transform, o, key=None: do_transform( + transform, "dict", o, key=key + ), + type_info.CIRCULAR: lambda transform, o, key=None, ref_key=None: do_transform( + transform, "circular_reference", o, key=key, ref_key=ref_key + ), + type_info.DEFAULT: default_handler, +} + + +class BatchedTransform(Transform): + def __init__(self, transforms): + super(BatchedTransform, self).__init__() + self._transforms = transforms + + def default(self, o, key=None): + for transform in self._transforms: + node_type = type_info.get_type(o) + handler = handlers.get(node_type, handlers.get(type_info.DEFAULT)) + o = handler(transform, o, key=key) + + return o + + +__all__ = ["BatchedTransform"] diff --git a/rollbar/lib/transforms/scrub.py b/rollbar/lib/transforms/scrub.py index 8032f648..96158d85 100644 --- a/rollbar/lib/transforms/scrub.py +++ b/rollbar/lib/transforms/scrub.py @@ -1,7 +1,7 @@ import random from rollbar.lib import build_key_matcher, text -from rollbar.lib.transforms import Transform +from rollbar.lib.transform import Transform class ScrubTransform(Transform): diff --git a/rollbar/lib/transforms/serializable.py b/rollbar/lib/transforms/serializable.py index 20102896..756b9a58 100644 --- a/rollbar/lib/transforms/serializable.py +++ b/rollbar/lib/transforms/serializable.py @@ -6,7 +6,7 @@ undecodable_object_label, unencodable_object_label) from rollbar.lib import iteritems, python_major_version, text -from rollbar.lib.transforms import Transform +from rollbar.lib.transform import Transform class SerializableTransform(Transform): diff --git a/rollbar/lib/transforms/shortener.py b/rollbar/lib/transforms/shortener.py index 68574d9d..44508e0b 100644 --- a/rollbar/lib/transforms/shortener.py +++ b/rollbar/lib/transforms/shortener.py @@ -12,7 +12,7 @@ from rollbar.lib import ( integer_types, iteritems, key_in, number_types, reprlib, sequence_types, string_types, text) -from rollbar.lib.transforms import Transform +from rollbar.lib.transform import Transform _type_name_mapping = { diff --git a/rollbar/lib/traverse.py b/rollbar/lib/traverse.py index dcfd0fc0..f732771f 100644 --- a/rollbar/lib/traverse.py +++ b/rollbar/lib/traverse.py @@ -1,30 +1,29 @@ import logging -try: - # Python 3 - from collections.abc import Mapping - from collections.abc import Sequence -except ImportError: - # Python 2.7 - from collections import Mapping - from collections import Sequence from rollbar.lib import binary_type, iteritems, string_types, circular_reference_label -CIRCULAR = -1 -DEFAULT = 0 -MAPPING = 1 -TUPLE = 2 -NAMEDTUPLE = 3 -LIST = 4 -SET = 5 -STRING = 6 +# NOTE: Don't remove this line of code as it would cause a breaking change +# to the library's API. The items imported here were originally in this file +# but were moved to a new file for easier use elsewhere. +from rollbar.lib.type_info import ( + get_type, + CIRCULAR, + DEFAULT, + MAPPING, + TUPLE, + NAMEDTUPLE, + LIST, + SET, + STRING, +) + log = logging.getLogger(__name__) def _noop_circular(a, **kw): - return circular_reference_label(a, ref_key=kw.get('ref_key')) + return circular_reference_label(a, ref_key=kw.get("ref_key")) def _noop(a, **_): @@ -63,64 +62,45 @@ def _noop_mapping(a, **_): } -def get_type(obj): - if isinstance(obj, (string_types, binary_type)): - return STRING - - if isinstance(obj, Mapping): - return MAPPING - - if isinstance(obj, tuple): - if hasattr(obj, '_fields'): - return NAMEDTUPLE - - return TUPLE - - if isinstance(obj, set): - return SET - - if isinstance(obj, Sequence): - return LIST - - return DEFAULT - - -def traverse(obj, - key=(), - string_handler=_default_handlers[STRING], - tuple_handler=_default_handlers[TUPLE], - namedtuple_handler=_default_handlers[NAMEDTUPLE], - list_handler=_default_handlers[LIST], - set_handler=_default_handlers[SET], - mapping_handler=_default_handlers[MAPPING], - default_handler=_default_handlers[DEFAULT], - circular_reference_handler=_default_handlers[CIRCULAR], - allowed_circular_reference_types=None, - memo=None, - **custom_handlers): - +def traverse( + obj, + key=(), + string_handler=_default_handlers[STRING], + tuple_handler=_default_handlers[TUPLE], + namedtuple_handler=_default_handlers[NAMEDTUPLE], + list_handler=_default_handlers[LIST], + set_handler=_default_handlers[SET], + mapping_handler=_default_handlers[MAPPING], + default_handler=_default_handlers[DEFAULT], + circular_reference_handler=_default_handlers[CIRCULAR], + allowed_circular_reference_types=None, + memo=None, + **custom_handlers +): memo = memo or {} obj_id = id(obj) obj_type = get_type(obj) ref_key = memo.get(obj_id) if ref_key: - if not allowed_circular_reference_types or not isinstance(obj, allowed_circular_reference_types): + if not allowed_circular_reference_types or not isinstance( + obj, allowed_circular_reference_types + ): return circular_reference_handler(obj, key=key, ref_key=ref_key) memo[obj_id] = key kw = { - 'string_handler': string_handler, - 'tuple_handler': tuple_handler, - 'namedtuple_handler': namedtuple_handler, - 'list_handler': list_handler, - 'set_handler': set_handler, - 'mapping_handler': mapping_handler, - 'default_handler': default_handler, - 'circular_reference_handler': circular_reference_handler, - 'allowed_circular_reference_types': allowed_circular_reference_types, - 'memo': memo + "string_handler": string_handler, + "tuple_handler": tuple_handler, + "namedtuple_handler": namedtuple_handler, + "list_handler": list_handler, + "set_handler": set_handler, + "mapping_handler": mapping_handler, + "default_handler": default_handler, + "circular_reference_handler": circular_reference_handler, + "allowed_circular_reference_types": allowed_circular_reference_types, + "memo": memo, } kw.update(custom_handlers) @@ -128,25 +108,50 @@ def traverse(obj, if obj_type is STRING: return string_handler(obj, key=key) elif obj_type is TUPLE: - return tuple_handler(tuple(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key) + return tuple_handler( + tuple( + traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj) + ), + key=key, + ) elif obj_type is NAMEDTUPLE: - return namedtuple_handler(obj._make(traverse(v, key=key + (k,), **kw) for k, v in iteritems(obj._asdict())), key=key) + return namedtuple_handler( + obj._make( + traverse(v, key=key + (k,), **kw) + for k, v in iteritems(obj._asdict()) + ), + key=key, + ) elif obj_type is LIST: - return list_handler(list(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key) + return list_handler( + list( + traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj) + ), + key=key, + ) elif obj_type is SET: - return set_handler(set(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), key=key) + return set_handler( + set(traverse(elem, key=key + (i,), **kw) for i, elem in enumerate(obj)), + key=key, + ) elif obj_type is MAPPING: - return mapping_handler(dict((k, traverse(v, key=key + (k,), **kw)) for k, v in iteritems(obj)), key=key) + return mapping_handler( + dict((k, traverse(v, key=key + (k,), **kw)) for k, v in iteritems(obj)), + key=key, + ) elif obj_type is DEFAULT: for handler_type, handler in iteritems(custom_handlers): if isinstance(obj, handler_type): return handler(obj, key=key) except: # use the default handler for unknown object types - log.debug("Exception while traversing object using type-specific " - "handler. Switching to default handler.", exc_info=True) + log.debug( + "Exception while traversing object using type-specific " + "handler. Switching to default handler.", + exc_info=True, + ) return default_handler(obj, key=key) -__all__ = ['traverse'] +__all__ = ["traverse"] diff --git a/rollbar/lib/type_info.py b/rollbar/lib/type_info.py new file mode 100644 index 00000000..a15b44bf --- /dev/null +++ b/rollbar/lib/type_info.py @@ -0,0 +1,58 @@ +from rollbar.lib import binary_type, string_types + + +try: + # Python 3 + from collections.abc import Mapping + from collections.abc import Sequence + from collections.abc import Set +except ImportError: + # Python 2.7 + from collections import Mapping + from collections import Sequence + from collections import Set + + +CIRCULAR = -1 +DEFAULT = 0 +MAPPING = 1 +TUPLE = 2 +NAMEDTUPLE = 3 +LIST = 4 +SET = 5 +STRING = 6 + + +def get_type(obj): + if isinstance(obj, (string_types, binary_type)): + return STRING + + if isinstance(obj, Mapping): + return MAPPING + + if isinstance(obj, tuple): + if hasattr(obj, "_fields"): + return NAMEDTUPLE + + return TUPLE + + if isinstance(obj, set): + return SET + + if isinstance(obj, Sequence): + return LIST + + return DEFAULT + + +__all__ = [ + "CIRCULAR", + "DEFAULT", + "MAPPING", + "TUPLE", + "NAMEDTUPLE", + "LIST", + "SET", + "STRING", + "get_type", +] diff --git a/rollbar/test/test_batched_transform.py b/rollbar/test/test_batched_transform.py new file mode 100644 index 00000000..96c7d04c --- /dev/null +++ b/rollbar/test/test_batched_transform.py @@ -0,0 +1,59 @@ +from rollbar.lib.transforms import transform +from rollbar.lib.transform import Transform +from rollbar.lib.traverse import traverse + +from rollbar.test import BaseTest + + +class TrackingTransformer(Transform): + def __init__(self): + self.got = [] + + def default(self, o, key=None): + self.got.append((o, key)) + return o + + +class BatchedTransformTest(BaseTest): + def assertTrackingTransform(self, input): + tracking_transformer = TrackingTransformer() + + transforms = [ + tracking_transformer, + tracking_transformer, + ] + + transform(input, transforms, batch_transforms=True) + + want = [] + + def dup_watch_handler(o, key=None): + want.append((o, key)) + want.append((o, key)) + return o + + traverse( + input, + string_handler=dup_watch_handler, + tuple_handler=dup_watch_handler, + namedtuple_handler=dup_watch_handler, + list_handler=dup_watch_handler, + set_handler=dup_watch_handler, + mapping_handler=dup_watch_handler, + default_handler=dup_watch_handler, + circular_reference_handler=dup_watch_handler, + ) + + self.assertEqual(want, tracking_transformer.got) + + def test_number(self): + self.assertTrackingTransform(1) + + def test_flat_list(self): + self.assertTrackingTransform([0, 1, 2, 3]) + + def test_flat_tuple(self): + self.assertTrackingTransform((0, 1, 2, 3)) + + def test_nested_object(self): + self.assertTrackingTransform((0, [1, 2], {"a": 3, "b": (4, 5)})) diff --git a/rollbar/test/test_shortener_transform.py b/rollbar/test/test_shortener_transform.py index 11fffb2f..acb91a5b 100644 --- a/rollbar/test/test_shortener_transform.py +++ b/rollbar/test/test_shortener_transform.py @@ -6,7 +6,7 @@ from rollbar import DEFAULT_LOCALS_SIZES from rollbar.lib import transforms from rollbar.lib.transforms.shortener import ShortenerTransform -from rollbar.lib.traverse import Sequence +from rollbar.lib.type_info import Sequence from rollbar.test import BaseTest diff --git a/rollbar/test/test_traverse.py b/rollbar/test/test_traverse.py index 7aa36481..db07ee28 100644 --- a/rollbar/test/test_traverse.py +++ b/rollbar/test/test_traverse.py @@ -7,6 +7,7 @@ class NamedTuple(tuple): """ Modeled after NamedTuple and KeyedTuple from SQLAlchemy 0.7 and 0.8. """ + def __new__(cls, vals, labels=None): t = tuple.__new__(cls, vals) if labels: @@ -24,6 +25,7 @@ class RollbarTraverseTest(BaseTest): will cause an Exception while identifying them if they don't implement the _make method. """ + def setUp(self): self.tuple = NamedTuple((1, 2, 3), labels=["one", "two", "three"]) @@ -31,5 +33,5 @@ def test_base_case(self): self.assertEqual(traverse(self.tuple), (1, 2, 3)) def test_bad_object(self): - setattr(self.tuple, '_fields', 'not quite a named tuple') + setattr(self.tuple, "_fields", "not quite a named tuple") self.assertEqual(traverse(self.tuple), (1, 2, 3))