From b6b8f75c5dc0672104bb3a1975d4894fcbf93a9d Mon Sep 17 00:00:00 2001 From: Jordan Storms Date: Tue, 2 Jun 2020 11:52:45 -0400 Subject: [PATCH 1/3] add memorysize tag to enhanced metrics --- aws/logs_monitoring/enhanced_lambda_metrics.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index d6569186e..3953dfb53 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -41,6 +41,9 @@ + r"Max\s+Memory\s+Used:\s+(?P<{}>\d+)\s+MB".format(MAX_MEMORY_USED_METRIC_NAME) ) +#Pull memorysize tag and cold start from report +TAGS_TO_PARSE_FROM_REPORT = [MEMORY_ALLOCATED_FIELD_NAME] + METRICS_TO_PARSE_FROM_REPORT = [ DURATION_METRIC_NAME, BILLED_DURATION_METRIC_NAME, @@ -440,6 +443,11 @@ def parse_metrics_from_report_log(report_log_line): return [] metrics = [] + tags = [] + + #loop is to account for adding cold start + for tag in TAGS_TO_PARSE_FROM_REPORT: + tags.append(tag + ':' + regex_match.group(tag)) for metric_name in METRICS_TO_PARSE_FROM_REPORT: metric_point_value = float(regex_match.group(metric_name)) @@ -451,6 +459,10 @@ def parse_metrics_from_report_log(report_log_line): "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, metric_name), metric_point_value, ) + + if tags: + dd_metric.add_tags(tags) + metrics.append(dd_metric) estimated_cost_metric_point = DatadogMetricPoint( @@ -460,6 +472,10 @@ def parse_metrics_from_report_log(report_log_line): float(regex_match.group(MEMORY_ALLOCATED_FIELD_NAME)), ), ) + + if tags: + estimated_cost_metric_point.add_tags(tags) + metrics.append(estimated_cost_metric_point) return metrics From 82b253b6d4e28ad2e46a8a670d94806d4da03fd9 Mon Sep 17 00:00:00 2001 From: Jordan Storms Date: Tue, 2 Jun 2020 12:35:04 -0400 Subject: [PATCH 2/3] fixed formatting --- .../enhanced_lambda_metrics.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index 3953dfb53..bdf7ba7f3 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -41,8 +41,10 @@ + r"Max\s+Memory\s+Used:\s+(?P<{}>\d+)\s+MB".format(MAX_MEMORY_USED_METRIC_NAME) ) -#Pull memorysize tag and cold start from report -TAGS_TO_PARSE_FROM_REPORT = [MEMORY_ALLOCATED_FIELD_NAME] +# Pull memorysize tag and cold start from report +TAGS_TO_PARSE_FROM_REPORT = [ + MEMORY_ALLOCATED_FIELD_NAME, +] METRICS_TO_PARSE_FROM_REPORT = [ DURATION_METRIC_NAME, @@ -184,7 +186,8 @@ def submit_to_dd(self): if not timestamp: timestamp = time() - log.debug("Submitting metric {} {} {}".format(self.name, self.value, self.tags)) + log.debug("Submitting metric {} {} {}".format( + self.name, self.value, self.tags)) lambda_stats.distribution( self.name, self.value, timestamp=timestamp, tags=self.tags ) @@ -244,7 +247,8 @@ def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict): key:value colon-separated string built from the dict ex: "creator:swf" """ - key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"], remove_colons=True) + key = sanitize_aws_tag_string( + aws_key_value_tag_dict["Key"], remove_colons=True) value = sanitize_aws_tag_string(aws_key_value_tag_dict.get("Value")) # Value is optional in DD and AWS if not value: @@ -290,17 +294,21 @@ def build_tags_by_arn_cache(): tags_by_arn_cache (dict): each Lambda's tags in a dict keyed by ARN """ tags_by_arn_cache = {} - get_resources_paginator = resource_tagging_client.get_paginator("get_resources") + get_resources_paginator = resource_tagging_client.get_paginator( + "get_resources") try: for page in get_resources_paginator.paginate( - ResourceTypeFilters=[GET_RESOURCES_LAMBDA_FILTER], ResourcesPerPage=100 + ResourceTypeFilters=[ + GET_RESOURCES_LAMBDA_FILTER], ResourcesPerPage=100 ): lambda_stats.distribution( - "{}.get_resources_api_calls".format(ENHANCED_METRICS_NAMESPACE_PREFIX), + "{}.get_resources_api_calls".format( + ENHANCED_METRICS_NAMESPACE_PREFIX), 1, ) - page_tags_by_arn = parse_get_resources_response_for_tags_by_arn(page) + page_tags_by_arn = parse_get_resources_response_for_tags_by_arn( + page) tags_by_arn_cache.update(page_tags_by_arn) except ClientError: @@ -445,7 +453,7 @@ def parse_metrics_from_report_log(report_log_line): metrics = [] tags = [] - #loop is to account for adding cold start + # loop is to account for adding cold start for tag in TAGS_TO_PARSE_FROM_REPORT: tags.append(tag + ':' + regex_match.group(tag)) @@ -466,13 +474,14 @@ def parse_metrics_from_report_log(report_log_line): metrics.append(dd_metric) estimated_cost_metric_point = DatadogMetricPoint( - "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, ESTIMATED_COST_METRIC_NAME), + "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, + ESTIMATED_COST_METRIC_NAME), calculate_estimated_cost( float(regex_match.group(BILLED_DURATION_METRIC_NAME)), float(regex_match.group(MEMORY_ALLOCATED_FIELD_NAME)), ), ) - + if tags: estimated_cost_metric_point.add_tags(tags) From 4df0c49505e9f8be31f80e0f3d17bed5d5a8d5ad Mon Sep 17 00:00:00 2001 From: Jordan Storms Date: Tue, 2 Jun 2020 12:43:21 -0400 Subject: [PATCH 3/3] fixed formatting with black --- .../enhanced_lambda_metrics.py | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index bdf7ba7f3..253ea2bf0 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -186,8 +186,7 @@ def submit_to_dd(self): if not timestamp: timestamp = time() - log.debug("Submitting metric {} {} {}".format( - self.name, self.value, self.tags)) + log.debug("Submitting metric {} {} {}".format(self.name, self.value, self.tags)) lambda_stats.distribution( self.name, self.value, timestamp=timestamp, tags=self.tags ) @@ -247,8 +246,7 @@ def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict): key:value colon-separated string built from the dict ex: "creator:swf" """ - key = sanitize_aws_tag_string( - aws_key_value_tag_dict["Key"], remove_colons=True) + key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"], remove_colons=True) value = sanitize_aws_tag_string(aws_key_value_tag_dict.get("Value")) # Value is optional in DD and AWS if not value: @@ -294,21 +292,17 @@ def build_tags_by_arn_cache(): tags_by_arn_cache (dict): each Lambda's tags in a dict keyed by ARN """ tags_by_arn_cache = {} - get_resources_paginator = resource_tagging_client.get_paginator( - "get_resources") + get_resources_paginator = resource_tagging_client.get_paginator("get_resources") try: for page in get_resources_paginator.paginate( - ResourceTypeFilters=[ - GET_RESOURCES_LAMBDA_FILTER], ResourcesPerPage=100 + ResourceTypeFilters=[GET_RESOURCES_LAMBDA_FILTER], ResourcesPerPage=100 ): lambda_stats.distribution( - "{}.get_resources_api_calls".format( - ENHANCED_METRICS_NAMESPACE_PREFIX), + "{}.get_resources_api_calls".format(ENHANCED_METRICS_NAMESPACE_PREFIX), 1, ) - page_tags_by_arn = parse_get_resources_response_for_tags_by_arn( - page) + page_tags_by_arn = parse_get_resources_response_for_tags_by_arn(page) tags_by_arn_cache.update(page_tags_by_arn) except ClientError: @@ -455,7 +449,7 @@ def parse_metrics_from_report_log(report_log_line): # loop is to account for adding cold start for tag in TAGS_TO_PARSE_FROM_REPORT: - tags.append(tag + ':' + regex_match.group(tag)) + tags.append(tag + ":" + regex_match.group(tag)) for metric_name in METRICS_TO_PARSE_FROM_REPORT: metric_point_value = float(regex_match.group(metric_name)) @@ -474,8 +468,7 @@ def parse_metrics_from_report_log(report_log_line): metrics.append(dd_metric) estimated_cost_metric_point = DatadogMetricPoint( - "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, - ESTIMATED_COST_METRIC_NAME), + "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, ESTIMATED_COST_METRIC_NAME), calculate_estimated_cost( float(regex_match.group(BILLED_DURATION_METRIC_NAME)), float(regex_match.group(MEMORY_ALLOCATED_FIELD_NAME)),