diff --git a/iopipe/contrib/eventinfo/event_types.py b/iopipe/contrib/eventinfo/event_types.py index 394f7652..22d0f9b1 100644 --- a/iopipe/contrib/eventinfo/event_types.py +++ b/iopipe/contrib/eventinfo/event_types.py @@ -1,4 +1,4 @@ -from .util import collect_all_keys, get_value, has_key +from .util import collect_all_keys, get_value, has_key, slugify class EventType(object): @@ -9,6 +9,10 @@ class EventType(object): def __init__(self, event): self.event = event + @property + def slug(self): + return slugify(self.type) + def has_required_keys(self): return all(has_key(self.event, key) for key in self.required_keys) @@ -176,10 +180,12 @@ def has_required_keys(self): EVENT_TYPES = [AlexaSkill, ApiGateway, CloudFront, Firehose, Kinesis, S3, Scheduled, SNS] -def log_for_event_type(event, log): +def metrics_for_event_type(event, context): for EventType in EVENT_TYPES: event_type = EventType(event) if event_type.has_required_keys(): + context.iopipe.label('@iopipe/event-info') + context.iopipe.label('@iopipe/%s' % event_type.slug) event_info = event_type.collect() - [log(k, v) for k, v in event_info.items()] + [context.iopipe.metric(k, v) for k, v in event_info.items()] break diff --git a/iopipe/contrib/eventinfo/plugin.py b/iopipe/contrib/eventinfo/plugin.py index 1c81a024..caa9882e 100644 --- a/iopipe/contrib/eventinfo/plugin.py +++ b/iopipe/contrib/eventinfo/plugin.py @@ -1,6 +1,6 @@ from iopipe.plugins import Plugin -from .event_types import log_for_event_type +from .event_types import metrics_for_event_type class EventInfoPlugin(Plugin): @@ -19,7 +19,7 @@ def pre_invoke(self, event, context): pass def post_invoke(self, event, context): - log_for_event_type(event, context.iopipe.log) + metrics_for_event_type(event, context) def pre_report(self, report): pass diff --git a/iopipe/contrib/eventinfo/util.py b/iopipe/contrib/eventinfo/util.py index ec7f52ec..c8168a47 100644 --- a/iopipe/contrib/eventinfo/util.py +++ b/iopipe/contrib/eventinfo/util.py @@ -1,4 +1,5 @@ import jmespath +import re # Caches previously seen parsers parsers = {} @@ -49,3 +50,8 @@ def flatten(o, path=None): flatten(obj) return out + + +def slugify(name): + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1-\2', s1).lower() diff --git a/iopipe/contrib/logger/plugin.py b/iopipe/contrib/logger/plugin.py index 17281767..21b69771 100644 --- a/iopipe/contrib/logger/plugin.py +++ b/iopipe/contrib/logger/plugin.py @@ -73,16 +73,17 @@ def pre_report(self, report): pass def post_report(self, report): - signed_request = get_signed_request(report, '.log') - if signed_request and 'signedRequest' in signed_request: - upload_log_data(signed_request['signedRequest'], self.handler.stream) - if 'jwtAccess' in signed_request: - plugin = next((p for p in report.plugins if p['name'] == self.name)) - if 'uploads' not in plugin: - plugin['uploads'] = [] - plugin['uploads'].append(signed_request['jwtAccess']) - if self.use_tmp is True: - self.handler.stream.close() + if self.handler.stream.tell() > 0: + signed_request = get_signed_request(report, '.log') + if signed_request and 'signedRequest' in signed_request: + upload_log_data(signed_request['signedRequest'], self.handler.stream) + if 'jwtAccess' in signed_request: + plugin = next((p for p in report.plugins if p['name'] == self.name)) + if 'uploads' not in plugin: + plugin['uploads'] = [] + plugin['uploads'].append(signed_request['jwtAccess']) + if self.use_tmp is True: + self.handler.stream.close() def __del__(self): if self.use_tmp and self.handler and self.handler.stream: diff --git a/iopipe/contrib/logger/wrapper.py b/iopipe/contrib/logger/wrapper.py index ae4a7688..00fc8041 100644 --- a/iopipe/contrib/logger/wrapper.py +++ b/iopipe/contrib/logger/wrapper.py @@ -7,4 +7,9 @@ def __call__(self, key, value): self.context.iopipe.metric(key, value) def __getattr__(self, name): + self.context.iopipe.label('@iopipe/logs') + if name in ['warn', 'warning']: + self.context.iopipe.label('@iopipe/logs-warning') + if name in ['critical', 'error', 'exception']: + self.context.iopipe.label('@iopipe/logs-error') return getattr(self.logger, name) diff --git a/iopipe/contrib/profiler/plugin.py b/iopipe/contrib/profiler/plugin.py index 0a26d013..25fb8aa6 100644 --- a/iopipe/contrib/profiler/plugin.py +++ b/iopipe/contrib/profiler/plugin.py @@ -56,6 +56,7 @@ def post_invoke(self, event, context): def pre_report(self, report): if self.profile is not None: + self.context.iopipe.label('@iopipe/profile') with tempfile.NamedTemporaryFile() as stats_file: self.profile.dump_stats(stats_file.name) signed_request = get_signed_request(report, '.cprofile') diff --git a/iopipe/contrib/trace/marker.py b/iopipe/contrib/trace/marker.py index 911b12c3..0654d039 100644 --- a/iopipe/contrib/trace/marker.py +++ b/iopipe/contrib/trace/marker.py @@ -1,6 +1,7 @@ class Marker(object): - def __init__(self, timeline): + def __init__(self, timeline, context): self.timeline = timeline + self.context = context self.contexts = [] def __call__(self, name): @@ -19,6 +20,7 @@ def __exit__(self, type, value, traceback): def start(self, name): self.timeline.mark('start:%s' % name) + self.context.iopipe.label('@iopipe/trace') def end(self, name): self.timeline.mark('end:%s' % name) diff --git a/iopipe/contrib/trace/plugin.py b/iopipe/contrib/trace/plugin.py index e0f1b8a7..6531a608 100644 --- a/iopipe/contrib/trace/plugin.py +++ b/iopipe/contrib/trace/plugin.py @@ -22,7 +22,7 @@ def post_setup(self, iopipe): pass def pre_invoke(self, event, context): - context.iopipe.register('mark', Marker(self.timeline)) + context.iopipe.register('mark', Marker(self.timeline, context)) def post_invoke(self, event, context): context.iopipe.unregister('mark') diff --git a/iopipe/report.py b/iopipe/report.py index d5859381..3ec1b13b 100644 --- a/iopipe/report.py +++ b/iopipe/report.py @@ -71,7 +71,9 @@ def __init__(self, config, context): 'timestamp': int(time.time() * 1000), } - constants.COLDSTART = False + if constants.COLDSTART is True: + constants.COLDSTART = False + self.labels.add('@iopipe/coldstart') def extract_context_data(self): """ @@ -110,9 +112,12 @@ def retain_error(self, error, frame=None): :param error: The error exception to add to the report. """ - stack = traceback.format_exc() - if frame is not None: + if frame is None: + stack = traceback.format_exc() + self.labels.add('@iopipe/error') + else: stack = '\n'.join(traceback.format_stack(frame)) + self.labels.add('@iopipe/timeout') details = { 'name': type(error).__name__, 'message': '{}'.format(error), @@ -130,6 +135,9 @@ def prepare(self, error=None, frame=None): if error: self.retain_error(error, frame) + if self.custom_metrics: + self.labels.add('@iopipe/metrics') + self.report['environment']['host']['boot_id'] = system.read_bootid() # convert labels to list for sending diff --git a/tests/conftest.py b/tests/conftest.py index 2caadf82..899c5bb6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import json +import mock import numbers import time from decimal import Decimal @@ -24,6 +25,7 @@ def __init__(self, name='handler', version='$LATEST'): self.function_version = version self.invoked_function_arn = 'arn:aws:lambda:us-east-1:1:function:%s:%s' % (name, version) self.remaining_time_in_millis = float('inf') + self.iopipe = mock.Mock() def get_remaining_time_in_millis(self): return self.remaining_time_in_millis @@ -48,9 +50,10 @@ def _assert_valid_schema(obj, schema=None, path=None, optional_fields=None): if not optional_fields: optional_fields = [] - for key in obj: - key_path = '.'.join(path + [key]) - assert key in schema, "%s not in schema" % key_path + if isinstance(obj, dict): + for key in obj: + key_path = '.'.join(path + [key]) + assert key in schema, "%s not in schema" % key_path for key in schema: key_path = '.'.join(path + [key]) @@ -65,7 +68,8 @@ def _assert_valid_schema(obj, schema=None, path=None, optional_fields=None): _assert_valid_schema(obj[key], schema[key], path + [key], optional_fields) elif isinstance(obj[key], list): for item in obj[key]: - _assert_valid_schema(item, schema[key][0], path + [key], optional_fields) + if isinstance(item, dict): + _assert_valid_schema(item, schema[key][0], path + [key], optional_fields) elif schema[key] == 'b': assert isinstance(obj[key], bool), '%s not a boolean' % key_path elif schema[key] == 'i': diff --git a/tests/contrib/eventinfo/test_plugin.py b/tests/contrib/eventinfo/test_plugin.py index 5db8e2ab..1cf21e16 100644 --- a/tests/contrib/eventinfo/test_plugin.py +++ b/tests/contrib/eventinfo/test_plugin.py @@ -15,6 +15,8 @@ def test__eventinfo_plugin__apigw(mock_send_report, handler_with_eventinfo, even assert any([m['name'] == '@iopipe/event-info.eventType' for m in metrics]) assert len(metrics) == 10 + assert '@iopipe/event-info' in iopipe.report.labels + assert '@iopipe/api-gateway' in iopipe.report.labels event_type = [m for m in metrics if m['name'] == '@iopipe/event-info.eventType'] assert len(event_type) == 1 diff --git a/tests/contrib/logger/test_plugin.py b/tests/contrib/logger/test_plugin.py index 9727801b..6a15840e 100644 --- a/tests/contrib/logger/test_plugin.py +++ b/tests/contrib/logger/test_plugin.py @@ -44,6 +44,7 @@ def test__logger_plugin(mock_send_report, mock_get_signed_request, mock_upload_l json_msg = json.loads(line) assert 'timestamp' in json_msg assert isinstance(datetime.strptime(json_msg['timestamp'], '%Y-%m-%d %H:%M:%S.%f'), datetime) + assert '@iopipe/logs' in iopipe.report.labels @mock.patch('iopipe.contrib.logger.plugin.upload_log_data', autospec=True) diff --git a/tests/contrib/profiler/test_plugin.py b/tests/contrib/profiler/test_plugin.py index 3627247c..52843192 100644 --- a/tests/contrib/profiler/test_plugin.py +++ b/tests/contrib/profiler/test_plugin.py @@ -25,3 +25,4 @@ def test__profiler_plugin(mock_send_report, mock_get_signed_request, mock_upload plugin = next((p for p in iopipe.report.plugins if p['name'] == 'profiler')) assert plugin['uploads'][0] == 'foobar' + assert '@iopipe/profile' in iopipe.report.labels diff --git a/tests/contrib/trace/conftest.py b/tests/contrib/trace/conftest.py index a2713b15..d4295124 100644 --- a/tests/contrib/trace/conftest.py +++ b/tests/contrib/trace/conftest.py @@ -51,8 +51,8 @@ def _handler(event, context): @pytest.fixture -def marker(timeline): - return Marker(timeline) +def marker(timeline, mock_context): + return Marker(timeline, mock_context) @pytest.fixture diff --git a/tests/contrib/trace/test_plugin.py b/tests/contrib/trace/test_plugin.py index 6a5cbbca..20005328 100644 --- a/tests/contrib/trace/test_plugin.py +++ b/tests/contrib/trace/test_plugin.py @@ -11,6 +11,7 @@ def test__trace_plugin(mock_send_report, handler_with_trace, mock_context): assert len(iopipe.report.report['performanceEntries']) == 3 assert any([e['name'] == 'measure:foo' for e in iopipe.report.report['performanceEntries']]) + assert '@iopipe/trace' in iopipe.report.labels @mock.patch('iopipe.report.send_report', autospec=True) diff --git a/tests/test_agent.py b/tests/test_agent.py index a5ea398b..996eb0da 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -11,9 +11,11 @@ def test_coldstarts(mock_send_report, handler, mock_context, monkeypatch): handler(None, mock_context) assert iopipe.report.report['coldstart'] is True + assert '@iopipe/coldstart' in iopipe.report.labels handler(None, mock_context) assert iopipe.report.report['coldstart'] is False + assert '@iopipe/coldstart' not in iopipe.report.labels @mock.patch('iopipe.report.send_report', autospec=True) @@ -43,6 +45,7 @@ def test_custom_metrics(mock_send_report, handler_with_events, mock_context): assert len(iopipe.report.custom_metrics) == 7 # Decimals are converted to strings assert iopipe.report.custom_metrics[6]['s'] == '12.300000000000000710542735760100185871124267578125' + assert '@iopipe/metrics' in iopipe.report.labels @mock.patch('iopipe.report.send_report', autospec=True) @@ -67,6 +70,7 @@ def test_erroring(mock_send_report, handler_that_errors, mock_context): assert iopipe.report.report['errors']['name'] == 'ValueError' assert iopipe.report.report['errors']['message'] == 'Behold, a value error' assert isinstance(iopipe.report.report['errors']['stack'], str) + assert '@iopipe/error' in iopipe.report.labels @mock.patch('iopipe.report.send_report', autospec=True) @@ -83,6 +87,7 @@ def test_timeouts(mock_send_report, handler_that_timeouts, mock_context): assert iopipe.report.report['errors']['message'] == 'Timeout Exceeded.' assert iopipe.report.report['errors']['name'] == 'TimeoutError' assert isinstance(iopipe.report.report['errors']['stack'], str) + assert '@iopipe/timeout' in iopipe.report.labels @mock.patch('iopipe.report.send_report', autospec=True)