diff --git a/aws/logs_monitoring/enhanced_lambda_metrics.py b/aws/logs_monitoring/enhanced_lambda_metrics.py index 2afad6307..2a8c26f4b 100644 --- a/aws/logs_monitoring/enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/enhanced_lambda_metrics.py @@ -171,10 +171,49 @@ def should_fetch_custom_tags(): return os.environ.get("DD_FETCH_LAMBDA_TAGS", "false").lower() == "true" -def sanitize_aws_tag_string(raw_string): +_other_chars = r"\w:\-\.\/" +Sanitize = re.compile(r"[^%s]" % _other_chars, re.UNICODE).sub +Dedupe = re.compile(r"_+", re.UNICODE).sub +FixInit = re.compile(r"^[_\d]*", re.UNICODE).sub +FirstCapRe = re.compile("(.)([A-Z][a-z]+)") +AllCapRe = re.compile("([a-z0-9])([A-Z])") + + +def convert_camel_case_to_underscore(string): + """ + Convert from CamelCase to camel_case + """ + global FirstCapRe, AllCapRe + string = FirstCapRe.sub(r"\1_\2", string) + return AllCapRe.sub(r"\1_\2", string).lower() + + +def sanitize_aws_tag_string(tag, remove_colons=False): """Convert characters banned from DD but allowed in AWS tags to underscores """ - return re.sub(r"[\+\-=\.:/@]", "_", raw_string) + global Sanitize, Dedupe, FixInit + + # 1. Convert camel case to underscores + # 2. Replaces colons with _ + # 3. Convert to all lowercase unicode string + # 4. Convert bad characters to underscores + # 5. Dedupe contiguous underscores + # 6. Remove initial underscores/digits such that the string + # starts with an alpha char + # FIXME: tag normalization incorrectly supports tags starting + # with a ':', but this behavior should be phased out in future + # as it results in unqueryable data. See dogweb/#11193 + # 7. Truncate to 200 characters + # 8. Strip trailing underscores + tag = convert_camel_case_to_underscore(tag) + if remove_colons: + tag = tag.replace(":", "_") + tag = Dedupe(u"_", Sanitize(u"_", tag.lower())) + first_char = tag[0] + if first_char == u"_" or u"0" <= first_char <= "9": + tag = FixInit(u"", tag) + tag = tag[0:200].rstrip("_") + return tag def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict): @@ -188,7 +227,7 @@ def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict): key:value colon-separated string built from the dict ex: "creator:swf" """ - key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"]) + key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"], remove_colons=True) value = sanitize_aws_tag_string(aws_key_value_tag_dict.get("Value")) # Value is optional in DD and AWS if not value: diff --git a/aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot b/aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot index 08e134638..9d7b520ad 100644 --- a/aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot +++ b/aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot @@ -19,7 +19,7 @@ "invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x", "ddsource": "cloudwatch", "service": "cloudwatch", "host": "testLogGroup" @@ -38,7 +38,7 @@ "invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x", "ddsource": "cloudwatch", "service": "cloudwatch", "host": "testLogGroup" diff --git a/aws/logs_monitoring/tests/snapshots/cloudwatch_log_lambda_invocation.json~snapshot b/aws/logs_monitoring/tests/snapshots/cloudwatch_log_lambda_invocation.json~snapshot index eebd7de63..c317f9f63 100644 --- a/aws/logs_monitoring/tests/snapshots/cloudwatch_log_lambda_invocation.json~snapshot +++ b/aws/logs_monitoring/tests/snapshots/cloudwatch_log_lambda_invocation.json~snapshot @@ -22,7 +22,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -44,7 +44,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -66,7 +66,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -88,7 +88,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -110,7 +110,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -132,7 +132,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -154,7 +154,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -176,7 +176,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -198,7 +198,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -220,7 +220,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -242,7 +242,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -264,7 +264,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -286,7 +286,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -308,7 +308,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -330,7 +330,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" @@ -352,7 +352,7 @@ "arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" }, "ddsourcecategory": "aws", - "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", + "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1", "ddsource": "lambda", "service": "hello-dog-node-dev-hello12x", "host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x" diff --git a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py index 60899ef06..df7814ad5 100644 --- a/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py +++ b/aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py @@ -33,8 +33,20 @@ class TestEnhancedLambdaMetrics(unittest.TestCase): def test_sanitize_tag_string(self): self.assertEqual(sanitize_aws_tag_string("serverless"), "serverless") - self.assertEqual(sanitize_aws_tag_string("ser:ver_less"), "ser_ver_less") - self.assertEqual(sanitize_aws_tag_string("s-erv:erl_ess"), "s_erv_erl_ess") + # Don't replace : \ / . in middle of string + self.assertEqual(sanitize_aws_tag_string("ser-/.ver_less"), "ser-/.ver_less") + # Remove invalid characters + self.assertEqual(sanitize_aws_tag_string("s+e@rv_erl_ess"), "s_e_rv_erl_ess") + # Dedup underscores + self.assertEqual(sanitize_aws_tag_string("serverl___ess"), "serverl_ess") + # Keep colons when remove_colons=False + self.assertEqual(sanitize_aws_tag_string("serv:erless:"), "serv:erless:") + # Substitute colon when remove_colons=True + self.assertEqual( + sanitize_aws_tag_string("serv:erless:", remove_colons=True), "serv_erless" + ) + # Convert camel case + self.assertEqual(sanitize_aws_tag_string("serVerLess"), "ser_ver_less") def test_parse_lambda_tags_from_arn(self): self.assertListEqual( @@ -44,6 +56,7 @@ def test_parse_lambda_tags_from_arn(self): [ "region:us-east-1", "account_id:1234597598159", + "aws_account:1234597598159", "functionname:swf-hello-test", ], ) @@ -192,6 +205,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", ], "timestamp": 10000, @@ -202,6 +216,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", ], "timestamp": 10000, @@ -212,6 +227,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", ], "timestamp": 10000, @@ -222,6 +238,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", ], "timestamp": 10000, @@ -272,6 +289,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", "team:metrics", "monitor:datadog", @@ -286,6 +304,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", "team:metrics", "monitor:datadog", @@ -300,6 +319,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", "team:metrics", "monitor:datadog", @@ -314,6 +334,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache): "tags": [ "region:us-east-1", "account_id:172597598159", + "aws_account:172597598159", "functionname:post-coupon-prod-us", "team:metrics", "monitor:datadog", diff --git a/aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py b/aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py index 55a1eb54c..c72b939aa 100644 --- a/aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py +++ b/aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py @@ -1,6 +1,7 @@ import unittest import os import urllib.request, json +import re recorder_url = os.environ.get("RECORDER_URL", default="") forwarder_url = os.environ.get("FORWARDER_URL", default="") @@ -14,13 +15,20 @@ class TestForwarderSnapshots(unittest.TestCase): def get_recording(self): with urllib.request.urlopen(recorder_url) as url: - data = json.loads(url.read().decode()) + message = self.filter_message(url.read().decode()) + data = json.loads(message) return data def send_log_event(self, event): request = urllib.request.Request(forwarder_url, data=event.encode("utf-8")) urllib.request.urlopen(request) + def filter_message(self, message): + # Remove forwarder_version from output + return re.sub( + r"forwarder_version:\d+\.\d+\.\d+", "forwarder_version:x.x.x", message + ) + def compare_snapshot(self, input_filename, snapshot_filename): with open(input_filename, "r") as input_file: input_data = input_file.read()