From dfc5ea1b31883e0f64ea596392d70fdd04a1b0c4 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 18 Nov 2020 15:22:28 +0100 Subject: [PATCH 1/4] Detect API Gateway v2 trigger events on AWS Lambda --- instana/instrumentation/aws/triggers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/instana/instrumentation/aws/triggers.py b/instana/instrumentation/aws/triggers.py index bce5401f..2198352f 100644 --- a/instana/instrumentation/aws/triggers.py +++ b/instana/instrumentation/aws/triggers.py @@ -26,6 +26,20 @@ def is_api_gateway_proxy_trigger(event): return True +def is_api_gateway_v2_proxy_trigger(event): + for key in ["version", "requestContext"]: + if key not in event: + return False + + if event["version"] != "2.0": + return False + + for key in ["apiId", "stage", "http"]: + if key not in event["requestContext"]: + return False + + return True + def is_application_load_balancer_trigger(event): if 'requestContext' in event and 'elb' in event['requestContext']: return True From 9bfc9d6ac082777f4b084b30a011ba3459bd574f Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 18 Nov 2020 15:28:40 +0100 Subject: [PATCH 2/4] Extract trace context from API Gateway v2 events --- instana/instrumentation/aws/triggers.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/instana/instrumentation/aws/triggers.py b/instana/instrumentation/aws/triggers.py index 2198352f..d5c733a4 100644 --- a/instana/instrumentation/aws/triggers.py +++ b/instana/instrumentation/aws/triggers.py @@ -13,7 +13,11 @@ def get_context(tracer, event): # TODO: Search for more types of trigger context - if is_api_gateway_proxy_trigger(event) or is_application_load_balancer_trigger(event): + is_proxy_event = is_api_gateway_proxy_trigger(event) or \ + is_api_gateway_v2_proxy_trigger(event) or \ + is_application_load_balancer_trigger(event) + + if is_proxy_event: return tracer.extract('http_headers', event['headers']) return tracer.extract('http_headers', event) From 2ab6a59f1e97157b6800b549f02e763b5ce5d0f9 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 18 Nov 2020 15:29:54 +0100 Subject: [PATCH 3/4] Extract tags from API Gateway v2 trigger events for the entry span --- instana/instrumentation/aws/triggers.py | 17 +++++ tests/data/lambda/api_gateway_v2_event.json | 75 +++++++++++++++++++++ tests/platforms/test_lambda.py | 67 ++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 tests/data/lambda/api_gateway_v2_event.json diff --git a/instana/instrumentation/aws/triggers.py b/instana/instrumentation/aws/triggers.py index d5c733a4..16b833a8 100644 --- a/instana/instrumentation/aws/triggers.py +++ b/instana/instrumentation/aws/triggers.py @@ -161,6 +161,23 @@ def enrich_lambda_span(agent, span, event, context): if agent.options.extra_http_headers is not None: capture_extra_headers(event, span, agent.options.extra_http_headers) + elif is_api_gateway_v2_proxy_trigger(event): + logger.debug("Detected as API Gateway v2.0 Proxy Trigger") + + reqCtx = event["requestContext"] + + # trim optional HTTP method prefix + route_path = event["routeKey"].split(" ", 2)[-1] + + span.set_tag(STR_LAMBDA_TRIGGER, 'aws:api.gateway') + span.set_tag('http.method', reqCtx["http"]["method"]) + span.set_tag('http.url', reqCtx["http"]["path"]) + span.set_tag('http.path_tpl', route_path) + span.set_tag('http.params', read_http_query_params(event)) + + if agent.options.extra_http_headers is not None: + capture_extra_headers(event, span, agent.options.extra_http_headers) + elif is_application_load_balancer_trigger(event): logger.debug("Detected as Application Load Balancer Trigger") span.set_tag(STR_LAMBDA_TRIGGER, 'aws:application.load.balancer') diff --git a/tests/data/lambda/api_gateway_v2_event.json b/tests/data/lambda/api_gateway_v2_event.json new file mode 100644 index 00000000..2f9fad31 --- /dev/null +++ b/tests/data/lambda/api_gateway_v2_event.json @@ -0,0 +1,75 @@ +{ + "version": "2.0", + "routeKey": "ANY /my/{resource}", + "rawPath": "/my/path", + "rawQueryString": "parameter1=value1¶meter1=value2¶meter2=value", + "cookies": [ + "cookie1", + "cookie2" + ], + "headers": { + "Header1": "value1", + "Header2": "value1,value2", + "X-Instana-T": "0000000000001234", + "X-Instana-S": "0000000000004567", + "X-Instana-L": "1", + "X-Instana-Synthetic": "1", + "X-Custom-Header-1": "value1", + "x-custom-header-2": "value2" + }, + "queryStringParameters": { + "secret": "key", + "q": "term" + }, + "requestContext": { + "accountId": "123456789012", + "apiId": "api-id", + "authentication": { + "clientCert": { + "clientCertPem": "CERT_CONTENT", + "subjectDN": "www.example.com", + "issuerDN": "Example issuer", + "serialNumber": "a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1", + "validity": { + "notBefore": "May 28 12:30:02 2019 GMT", + "notAfter": "Aug 5 09:36:04 2021 GMT" + } + } + }, + "authorizer": { + "jwt": { + "claims": { + "claim1": "value1", + "claim2": "value2" + }, + "scopes": [ + "scope1", + "scope2" + ] + } + }, + "domainName": "id.execute-api.us-east-1.amazonaws.com", + "domainPrefix": "id", + "http": { + "method": "POST", + "path": "/my/path", + "protocol": "HTTP/1.1", + "sourceIp": "IP", + "userAgent": "agent" + }, + "requestId": "id", + "routeKey": "$default", + "stage": "$default", + "time": "12/Mar/2020:19:03:58 +0000", + "timeEpoch": 1583348638390 + }, + "body": "Hello from Lambda", + "pathParameters": { + "parameter1": "value1" + }, + "isBase64Encoded": false, + "stageVariables": { + "stageVariable1": "value1", + "stageVariable2": "value2" + } +} diff --git a/tests/platforms/test_lambda.py b/tests/platforms/test_lambda.py index cd45e98b..a0d43b68 100644 --- a/tests/platforms/test_lambda.py +++ b/tests/platforms/test_lambda.py @@ -300,6 +300,73 @@ def test_api_gateway_trigger_tracing(self): else: self.assertEqual("foo=['bar']", span.data['http']['params']) + def test_api_gateway_v2_trigger_tracing(self): + with open(self.pwd + '/../data/lambda/api_gateway_v2_event.json', 'r') as json_file: + event = json.load(json_file) + + self.create_agent_and_setup_tracer() + + # Call the Instana Lambda Handler as we do in the real world. It will initiate tracing and then + # figure out the original (the users') Lambda Handler and execute it. + # The original Lambda handler is set in os.environ["LAMBDA_HANDLER"] + result = lambda_handler(event, self.context) + + assert isinstance(result, dict) + assert 'headers' in result + assert 'Server-Timing' in result['headers'] + + time.sleep(1) + payload = self.agent.collector.prepare_payload() + + self.assertTrue("metrics" in payload) + self.assertTrue("spans" in payload) + self.assertEqual(2, len(payload.keys())) + + self.assertTrue(isinstance(payload['metrics']['plugins'], list)) + self.assertTrue(len(payload['metrics']['plugins']) == 1) + plugin_data = payload['metrics']['plugins'][0] + + self.assertEqual('com.instana.plugin.aws.lambda', plugin_data['name']) + self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', plugin_data['entityId']) + + self.assertEqual(1, len(payload['spans'])) + + span = payload['spans'][0] + self.assertEqual('aws.lambda.entry', span.n) + self.assertEqual('0000000000001234', span.t) + self.assertIsNotNone(span.s) + self.assertEqual('0000000000004567', span.p) + self.assertIsNotNone(span.ts) + self.assertIsNotNone(span.d) + + server_timing_value = "intid;desc=%s" % span.t + assert result['headers']['Server-Timing'] == server_timing_value + + self.assertEqual({'hl': True, 'cp': 'aws', 'e': 'arn:aws:lambda:us-east-2:12345:function:TestPython:1'}, + span.f) + + self.assertTrue(span.sy) + + self.assertIsNone(span.ec) + self.assertIsNone(span.data['lambda']['error']) + + self.assertEqual('arn:aws:lambda:us-east-2:12345:function:TestPython:1', span.data['lambda']['arn']) + self.assertEqual(None, span.data['lambda']['alias']) + self.assertEqual('python', span.data['lambda']['runtime']) + self.assertEqual('TestPython', span.data['lambda']['functionName']) + self.assertEqual('1', span.data['lambda']['functionVersion']) + self.assertIsNone(span.data['service']) + + self.assertEqual('aws:api.gateway', span.data['lambda']['trigger']) + self.assertEqual('POST', span.data['http']['method']) + self.assertEqual('/my/path', span.data['http']['url']) + self.assertEqual('/my/{resource}', span.data['http']['path_tpl']) + if sys.version[:3] == '2.7': + self.assertEqual(u"q=term&secret=key", span.data['http']['params']) + else: + self.assertEqual("secret=key&q=term", span.data['http']['params']) + + def test_application_lb_trigger_tracing(self): with open(self.pwd + '/../data/lambda/api_gateway_event.json', 'r') as json_file: event = json.load(json_file) From b6f16d5e809fc6bf5a4e11fd86a3d7e25ff58ead Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Fri, 20 Nov 2020 15:24:12 +0100 Subject: [PATCH 4/4] Safely read HTTP headers from AWL Lambda proxy event payloads --- instana/instrumentation/aws/triggers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instana/instrumentation/aws/triggers.py b/instana/instrumentation/aws/triggers.py index 16b833a8..f8c6774a 100644 --- a/instana/instrumentation/aws/triggers.py +++ b/instana/instrumentation/aws/triggers.py @@ -18,7 +18,7 @@ def get_context(tracer, event): is_application_load_balancer_trigger(event) if is_proxy_event: - return tracer.extract('http_headers', event['headers']) + return tracer.extract('http_headers', event.get('headers', {})) return tracer.extract('http_headers', event)