Run integration tests on wrapped Forwarder (captures metrics)

aws/logs_monitoring/tests/snapshots/cloudwatch_log.json~snapshot

@@ -19,7 +19,7 @@
             "invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>",
           "ddsource": "cloudwatch",
           "service": "cloudwatch",
           "host": "testLogGroup"
@@ -38,7 +38,7 @@
             "invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>",
           "ddsource": "cloudwatch",
           "service": "cloudwatch",
           "host": "testLogGroup"

aws/logs_monitoring/tests/snapshots/cloudwatch_log_coldstart.json~snapshot

@@ -22,7 +22,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
@@ -44,7 +44,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
@@ -66,7 +66,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"

aws/logs_monitoring/tests/snapshots/cloudwatch_log_custom_tags.json~snapshot

@@ -19,7 +19,7 @@
             "invoked_function_arn": "arn:aws:lambda:us-east-1:0:function:test"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,custom_tag1:value1,custom_tag2:value2",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,custom_tag1:value1,custom_tag2:value2",
           "ddsource": "cloudwatch",
           "service": "cloudwatch",
           "host": "testLogGroup"

aws/logs_monitoring/tests/snapshots/cloudwatch_log_timeout.json~snapshot

@@ -22,7 +22,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
@@ -44,7 +44,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
@@ -66,7 +66,7 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
@@ -88,12 +88,35 @@
             "arn": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
           },
           "ddsourcecategory": "aws",
-          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:x.x.x,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
+          "ddtags": "forwardername:test,forwarder_memorysize:1536,forwarder_version:<redacted from snapshot>,env:none,account_id:0,aws_account:0,functionname:storms-cloudwatch-event,region:us-east-1,service:storms-cloudwatch-event",
           "ddsource": "lambda",
           "service": "storms-cloudwatch-event",
           "host": "arn:aws:lambda:us-east-1:0:function:storms-cloudwatch-event"
         }
       ]
+    },
+    {
+      "path": "/api/v1/distribution_points?api_key=abcdefghijklmnopqrstuvwxyz012345",
+      "verb": "POST",
+      "content-type": "application/json",
+      "data": {
+        "series": [
+          {
+            "metric": "aws.lambda.enhanced.timeouts",
+            "points": "<redacted from snapshot>",
+            "type": "distribution",
+            "host": null,
+            "device": null,
+            "tags": [
+              "region:us-east-1",
+              "account_id:0",
+              "aws_account:0",
+              "functionname:storms-cloudwatch-event"
+            ],
+            "interval": 10
+          }
+        ]
+      }
     }
   ]
 }

aws/logs_monitoring/tools/Dockerfile_integration

@@ -5,6 +5,3 @@ ARG forwarder
 
 # Add the code into /var/task (will unzip files)
 ADD $forwarder /var/task/
-
-ENV DD_API_KEY "fake-api-key"
-ENV DD_SITE "datadog.com"

aws/logs_monitoring/tools/docker-compose.yml

@@ -3,14 +3,15 @@ version: "3.7"
 services:
   forwarder:
     image: datadog-log-forwarder:${PYTHON_RUNTIME:-python3.7}
-    command: lambda_function.datadog_forwarder
+    command: lambda_function.lambda_handler
     environment:
       AWS_ACCOUNT_ID: 0000000000
       DOCKER_LAMBDA_STAY_OPEN: 1
       DD_LOG_LEVEL: info
       DD_API_KEY: abcdefghijklmnopqrstuvwxyz012345 # Must be 32 characters exactly
       DD_URL: forwarder-recorder # Used for logs intake
       DD_PORT: 8080 # API port to use
+      DD_SITE: datadog.com
       DD_API_URL: http://forwarder-recorder:8080
       DD_LOGS_INTAKE_URL: forwarder-recorder:8080
       DD_TRACE_INTAKE_URL: http://forwarder-recorder:8080

aws/logs_monitoring/tools/test_harness/recorder/recorder.py

@@ -12,7 +12,7 @@
 
 class RecorderHandler(BaseHTTPRequestHandler):
     def __init__(self, request, client_address, server):
-        super().__init__(request, client_address, server)()
+        super().__init__(request, client_address, server)
 
     def handle_request(self):
         global events
@@ -52,7 +52,6 @@ def handle_request(self):
         self.wfile.write(response.encode("utf-8"))
         return
 
-    # Handler for the GET requests
     def do_GET(self):
         self.handle_request()
 

aws/logs_monitoring/tools/test_harness/tests/test_snapshots.py

@@ -1,9 +1,11 @@
 import unittest
 import base64
 import os
-import urllib.request, json
+import urllib.request
+import json
 import re
 import gzip
+from time import sleep
 
 recorder_url = os.environ.get("RECORDER_URL", default="")
 forwarder_url = os.environ.get("FORWARDER_URL", default="")
@@ -20,7 +22,7 @@ class TestForwarderSnapshots(unittest.TestCase):
 
     def get_recording(self):
         with urllib.request.urlopen(recorder_url) as url:
-            message = self.filter_message(url.read().decode())
+            message = self.filter_snapshot(url.read().decode())
             data = json.loads(message)
         return data
 
@@ -35,11 +37,22 @@ def send_log_event(self, event):
         request = urllib.request.Request(forwarder_url, data=event.encode("utf-8"))
         urllib.request.urlopen(request)
 
-    def filter_message(self, message):
-        # Remove forwarder_version from output
-        return re.sub(
-            r"forwarder_version:\d+\.\d+\.\d+", "forwarder_version:x.x.x", message
+    def filter_snapshot(self, snapshot):
+        # Remove things that can vary during each test run
+        # forwarder_version
+        snapshot = re.sub(
+            r"forwarder_version:\d+\.\d+\.\d+",
+            "forwarder_version:<redacted from snapshot>",
+            snapshot,
         )
+        # Metric points
+        snapshot = re.sub(
+            r"\"points\":.*?,(?=\s*\")",
+            '"points": "<redacted from snapshot>",',
+            snapshot,
+            flags=re.MULTILINE,
+        )
+        return snapshot
 
     def compare_snapshot(self, input_filename, snapshot_filename):
         with open(input_filename, "r") as input_file: