Skip to content

Fix enhanced metric tag conversion #240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 31, 2020
+92 −24
Merged

Fix enhanced metric tag conversion #240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fde4523
Fix tests
DarcyRaynerDD Mar 30, 2020
0f0eeaa
Change enhanced tag parsing rules
DarcyRaynerDD Mar 31, 2020
0c93fa5
Update comment
DarcyRaynerDD Mar 31, 2020
06a05ae
Only remove colons from tag keys
DarcyRaynerDD Mar 31, 2020
96cfc98
Fix typo in comment
DarcyRaynerDD Mar 31, 2020
3c122ef
Blank out forwarder version in snapshot tests
DarcyRaynerDD Mar 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 42 additions & 3 deletions aws/logs_monitoring/enhanced_lambda_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,49 @@ def should_fetch_custom_tags():
return os.environ.get("DD_FETCH_LAMBDA_TAGS", "false").lower() == "true"


def sanitize_aws_tag_string(raw_string):
_other_chars = r"\w:\-\.\/"
Sanitize = re.compile(r"[^%s]" % _other_chars, re.UNICODE).sub
Dedupe = re.compile(r"_+", re.UNICODE).sub
FixInit = re.compile(r"^[_\d]*", re.UNICODE).sub
FirstCapRe = re.compile("(.)([A-Z][a-z]+)")
AllCapRe = re.compile("([a-z0-9])([A-Z])")


def convert_camel_case_to_underscore(string):
"""
Convert from CamelCase to camel_case
"""
global FirstCapRe, AllCapRe
string = FirstCapRe.sub(r"\1_\2", string)
return AllCapRe.sub(r"\1_\2", string).lower()


def sanitize_aws_tag_string(tag, remove_colons=False):
"""Convert characters banned from DD but allowed in AWS tags to underscores
"""
return re.sub(r"[\+\-=\.:/@]", "_", raw_string)
global Sanitize, Dedupe, FixInit

# 1. Convert camel case to underscores
# 2. Replaces colons with _
# 3. Convert to all lowercase unicode string
# 4. Convert bad characters to underscores
# 5. Dedupe contiguous underscores
# 6. Remove initial underscores/digits such that the string
# starts with an alpha char
# FIXME: tag normalization incorrectly supports tags starting
# with a ':', but this behavior should be phased out in future
# as it results in unqueryable data. See dogweb/#11193
# 7. Truncate to 200 characters
# 8. Strip trailing underscores
tag = convert_camel_case_to_underscore(tag)
if remove_colons:
tag = tag.replace(":", "_")
tag = Dedupe(u"_", Sanitize(u"_", tag.lower()))
first_char = tag[0]
if first_char == u"_" or u"0" <= first_char <= "9":
tag = FixInit(u"", tag)
tag = tag[0:200].rstrip("_")
return tag


def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict):
Expand All @@ -188,7 +227,7 @@ def get_dd_tag_string_from_aws_dict(aws_key_value_tag_dict):
key:value colon-separated string built from the dict
ex: "creator:swf"
"""
key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"])
key = sanitize_aws_tag_string(aws_key_value_tag_dict["Key"], remove_colons=True)
value = sanitize_aws_tag_string(aws_key_value_tag_dict.get("Value"))
# Value is optional in DD and AWS
if not value:
Expand Down
4 changes: 2 additions & 2 deletions aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x",
"ddsource": "cloudwatch",
"service": "cloudwatch",
"host": "testLogGroup"
Expand All @@ -38,7 +38,7 @@
"invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x",
"ddsource": "cloudwatch",
"service": "cloudwatch",
"host": "testLogGroup"
Expand Down
32 changes: 16 additions & 16 deletions aws/logs_monitoring/tests/snapshots/cloudwatch_log_lambda_invocation.json~snapshot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -44,7 +44,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -66,7 +66,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -88,7 +88,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -110,7 +110,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -132,7 +132,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -154,7 +154,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -176,7 +176,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -198,7 +198,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -220,7 +220,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -242,7 +242,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -264,7 +264,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -286,7 +286,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -308,7 +308,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -330,7 +330,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand All @@ -352,7 +352,7 @@
"arn": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
},
"ddsourcecategory": "aws",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:3.4.0,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:hello-dog-node-dev-hello12x,region:us-east-1",
"ddsource": "lambda",
"service": "hello-dog-node-dev-hello12x",
"host": "arn:aws:lambda:us-east-1:0:function:hello-dog-node-dev-hello12x"
Expand Down
25 changes: 23 additions & 2 deletions aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,20 @@ class TestEnhancedLambdaMetrics(unittest.TestCase):

def test_sanitize_tag_string(self):
self.assertEqual(sanitize_aws_tag_string("serverless"), "serverless")
self.assertEqual(sanitize_aws_tag_string("ser:ver_less"), "ser_ver_less")
self.assertEqual(sanitize_aws_tag_string("s-erv:erl_ess"), "s_erv_erl_ess")
# Don't replace : \ / . in middle of string
self.assertEqual(sanitize_aws_tag_string("ser-/.ver_less"), "ser-/.ver_less")
# Remove invalid characters
self.assertEqual(sanitize_aws_tag_string("s+e@rv_erl_ess"), "s_e_rv_erl_ess")
# Dedup underscores
self.assertEqual(sanitize_aws_tag_string("serverl___ess"), "serverl_ess")
# Keep colons when remove_colons=False
self.assertEqual(sanitize_aws_tag_string("serv:erless:"), "serv:erless:")
# Substitute colon when remove_colons=True
self.assertEqual(
sanitize_aws_tag_string("serv:erless:", remove_colons=True), "serv_erless"
)
# Convert camel case
self.assertEqual(sanitize_aws_tag_string("serVerLess"), "ser_ver_less")

def test_parse_lambda_tags_from_arn(self):
self.assertListEqual(
Expand All @@ -44,6 +56,7 @@ def test_parse_lambda_tags_from_arn(self):
[
"region:us-east-1",
"account_id:1234597598159",
"aws_account:1234597598159",
"functionname:swf-hello-test",
],
)
Expand Down Expand Up @@ -192,6 +205,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
],
"timestamp": 10000,
Expand All @@ -202,6 +216,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
],
"timestamp": 10000,
Expand All @@ -212,6 +227,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
],
"timestamp": 10000,
Expand All @@ -222,6 +238,7 @@ def test_generate_enhanced_lambda_metrics(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
],
"timestamp": 10000,
Expand Down Expand Up @@ -272,6 +289,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
"team:metrics",
"monitor:datadog",
Expand All @@ -286,6 +304,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
"team:metrics",
"monitor:datadog",
Expand All @@ -300,6 +319,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
"team:metrics",
"monitor:datadog",
Expand All @@ -314,6 +334,7 @@ def test_generate_enhanced_lambda_metrics_with_tags(self, mock_build_cache):
"tags": [
"region:us-east-1",
"account_id:172597598159",
"aws_account:172597598159",
"functionname:post-coupon-prod-us",
"team:metrics",
"monitor:datadog",
Expand Down
10 changes: 9 additions & 1 deletion aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import unittest
import os
import urllib.request, json
import re

recorder_url = os.environ.get("RECORDER_URL", default="")
forwarder_url = os.environ.get("FORWARDER_URL", default="")
Expand All @@ -14,13 +15,20 @@
class TestForwarderSnapshots(unittest.TestCase):
def get_recording(self):
with urllib.request.urlopen(recorder_url) as url:
data = json.loads(url.read().decode())
message = self.filter_message(url.read().decode())
data = json.loads(message)
return data

def send_log_event(self, event):
request = urllib.request.Request(forwarder_url, data=event.encode("utf-8"))
urllib.request.urlopen(request)

def filter_message(self, message):
# Remove forwarder_version from output
return re.sub(
r"forwarder_version:\d+\.\d+\.\d+", "forwarder_version:x.x.x", message
)

def compare_snapshot(self, input_filename, snapshot_filename):
with open(input_filename, "r") as input_file:
input_data = input_file.read()
Expand Down