From 76013a75fdcde9ff6596bd56cc4cc61c2d6dc150 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Mon, 22 Jan 2024 15:45:17 +0530 Subject: [PATCH 1/2] boto3: initial code for capturing request headers Signed-off-by: Varsha GS --- instana/instrumentation/boto3_inst.py | 17 ++++++- tests/clients/boto3/test_boto3_s3.py | 69 ++++++++++++++++++++++++++- 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/instana/instrumentation/boto3_inst.py b/instana/instrumentation/boto3_inst.py index cf6511b5..f64c27f3 100644 --- a/instana/instrumentation/boto3_inst.py +++ b/instana/instrumentation/boto3_inst.py @@ -8,7 +8,7 @@ import inspect from ..log import logger -from ..singletons import tracer +from ..singletons import tracer, agent from ..util.traceutils import get_active_tracer try: @@ -16,6 +16,17 @@ import boto3 from boto3.s3 import inject + def extract_custom_headers(span, headers): + if agent.options.extra_http_headers is None: + return + try: + for custom_header in agent.options.extra_http_headers: + if custom_header in headers: + span.set_tag("http.header.%s" % custom_header, headers[custom_header]) + + except Exception: + logger.debug("extract_custom_headers: ", exc_info=True) + def lambda_inject_context(payload, scope): """ @@ -40,6 +51,9 @@ def make_api_call_with_instana(wrapped, instance, arg_list, kwargs): # pylint: disable=protected-access active_tracer = get_active_tracer() + print("\nWrapping BaseClient._make_api_call with instrumentation") + logger.info("\nkwargs inside make_api_call_with_instana: %s", kwargs) + # If we're not tracing, just return if active_tracer is None: return wrapped(*arg_list, **kwargs) @@ -71,6 +85,7 @@ def make_api_call_with_instana(wrapped, instance, arg_list, kwargs): result = wrapped(*arg_list, **kwargs) if isinstance(result, dict): + ## can use response['ResponseMetadata']['HTTPHeaders'] to attach response headers http_dict = result.get('ResponseMetadata') if isinstance(http_dict, dict): status = http_dict.get('HTTPStatusCode') diff --git a/tests/clients/boto3/test_boto3_s3.py b/tests/clients/boto3/test_boto3_s3.py index 4b6d6005..b8e3e5fe 100644 --- a/tests/clients/boto3/test_boto3_s3.py +++ b/tests/clients/boto3/test_boto3_s3.py @@ -9,7 +9,7 @@ from moto import mock_s3 -from instana.singletons import tracer +from instana.singletons import tracer, agent from ...helpers import get_first_span_by_filter pwd = os.path.dirname(os.path.abspath(__file__)) @@ -271,3 +271,70 @@ def test_s3_download_file_obj(s3): assert boto_span.data['boto3']['reg'] == 'us-east-1' assert boto_span.data['http']['method'] == 'POST' assert boto_span.data['http']['url'] == 'https://s3.amazonaws.com:443/download_fileobj' + + +def test_request_header_capture(s3): + + original_extra_http_headers = agent.options.extra_http_headers + agent.options.extra_http_headers = ['X-Capture-This', 'X-Capture-That'] + + # Access the event system on the S3 client + event_system = s3.meta.events + + # Create a function that adds a custom header and prints all headers. + def add_custom_header_before_call(params, **kwargs): + params['headers'] = { + 'X-Capture-This': 'this', + 'X-Capture-That': 'that' + } + headers = params['headers'] + print(f'params headers in add_custom_header_before_call: {headers}') + + print("\nRegistering add_custom_header_before_call to event before-call") + # Register the function to an event. + event_system.register('before-call', add_custom_header_before_call) + + # Create a function that prints the request information. + def inspect_request_created(request, operation_name, **kwargs): + print("headers after request is created:") + for header in request.headers: + if header.startswith("X-Capture"): + print(header, sep='\t') + + + event_system.register('request-created', inspect_request_created) + + with tracer.start_active_span('test'): + result = s3.create_bucket(Bucket="aws_bucket_name") + + # result = s3.list_buckets() + # # print(result) + # assert len(result['Buckets']) == 1 + # assert result['Buckets'][0]['Name'] == 'aws_bucket_name' + + # spans = tracer.recorder.queued_spans() + # assert len(spans) == 2 + + # filter = lambda span: span.n == "sdk" + # test_span = get_first_span_by_filter(spans, filter) + # assert (test_span) + + # filter = lambda span: span.n == "boto3" + # boto_span = get_first_span_by_filter(spans, filter) + # assert (boto_span) + + # assert (boto_span.t == test_span.t) + # assert (boto_span.p == test_span.s) + + # assert (test_span.ec is None) + # assert (boto_span.ec is None) + + # assert boto_span.data['boto3']['op'] == 'CreateBucket' + # assert boto_span.data['boto3']['ep'] == 'https://s3.amazonaws.com' + # assert boto_span.data['boto3']['reg'] == 'us-east-1' + # assert boto_span.data['boto3']['payload'] == {'Bucket': 'aws_bucket_name'} + # assert boto_span.data['http']['status'] == 200 + # assert boto_span.data['http']['method'] == 'POST' + # assert boto_span.data['http']['url'] == 'https://s3.amazonaws.com:443/CreateBucket' + + agent.options.extra_http_headers = original_extra_http_headers From 0f268dcfa846aad3f3658ecc3b8a0039c10000e3 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Tue, 23 Jan 2024 16:27:11 +0530 Subject: [PATCH 2/2] boto3: capture response headers Signed-off-by: Varsha GS --- instana/instrumentation/boto3_inst.py | 4 +- tests/clients/boto3/test_boto3_s3.py | 85 ++++++++++++--------------- 2 files changed, 41 insertions(+), 48 deletions(-) diff --git a/instana/instrumentation/boto3_inst.py b/instana/instrumentation/boto3_inst.py index f64c27f3..e0b3df5e 100644 --- a/instana/instrumentation/boto3_inst.py +++ b/instana/instrumentation/boto3_inst.py @@ -52,7 +52,6 @@ def make_api_call_with_instana(wrapped, instance, arg_list, kwargs): active_tracer = get_active_tracer() print("\nWrapping BaseClient._make_api_call with instrumentation") - logger.info("\nkwargs inside make_api_call_with_instana: %s", kwargs) # If we're not tracing, just return if active_tracer is None: @@ -85,12 +84,13 @@ def make_api_call_with_instana(wrapped, instance, arg_list, kwargs): result = wrapped(*arg_list, **kwargs) if isinstance(result, dict): - ## can use response['ResponseMetadata']['HTTPHeaders'] to attach response headers http_dict = result.get('ResponseMetadata') if isinstance(http_dict, dict): status = http_dict.get('HTTPStatusCode') if status is not None: scope.span.set_tag('http.status_code', status) + headers = http_dict.get('HTTPHeaders') + extract_custom_headers(scope.span, headers) return result except Exception as exc: diff --git a/tests/clients/boto3/test_boto3_s3.py b/tests/clients/boto3/test_boto3_s3.py index b8e3e5fe..5970048e 100644 --- a/tests/clients/boto3/test_boto3_s3.py +++ b/tests/clients/boto3/test_boto3_s3.py @@ -90,7 +90,7 @@ def test_s3_list_buckets(s3): result = s3.list_buckets() assert len(result['Buckets']) == 0 - assert result['ResponseMetadata']['HTTPStatusCode'] is 200 + assert result['ResponseMetadata']['HTTPStatusCode'] == 200 spans = tracer.recorder.queued_spans() assert len(spans) == 2 @@ -273,68 +273,61 @@ def test_s3_download_file_obj(s3): assert boto_span.data['http']['url'] == 'https://s3.amazonaws.com:443/download_fileobj' -def test_request_header_capture(s3): +def test_response_header_capture(s3): original_extra_http_headers = agent.options.extra_http_headers - agent.options.extra_http_headers = ['X-Capture-This', 'X-Capture-That'] + agent.options.extra_http_headers = ['X-Capture-This-Too', 'X-Capture-That-Too'] # Access the event system on the S3 client event_system = s3.meta.events - # Create a function that adds a custom header and prints all headers. - def add_custom_header_before_call(params, **kwargs): - params['headers'] = { - 'X-Capture-This': 'this', - 'X-Capture-That': 'that' - } - headers = params['headers'] - print(f'params headers in add_custom_header_before_call: {headers}') + response_headers = { + "X-Capture-This-Too": "this too", + "X-Capture-That-Too": "that too", + } - print("\nRegistering add_custom_header_before_call to event before-call") - # Register the function to an event. - event_system.register('before-call', add_custom_header_before_call) + # Create a function that modifies the after-call event args. + def modify_after_call_args(http_response, parsed, model, **kwargs): + parsed['ResponseMetadata']['HTTPHeaders'].update(response_headers) - # Create a function that prints the request information. - def inspect_request_created(request, operation_name, **kwargs): - print("headers after request is created:") - for header in request.headers: - if header.startswith("X-Capture"): - print(header, sep='\t') - - - event_system.register('request-created', inspect_request_created) + # Register the function to an event + event_system.register('after-call', modify_after_call_args) with tracer.start_active_span('test'): result = s3.create_bucket(Bucket="aws_bucket_name") - # result = s3.list_buckets() - # # print(result) - # assert len(result['Buckets']) == 1 - # assert result['Buckets'][0]['Name'] == 'aws_bucket_name' + result = s3.list_buckets() + assert len(result['Buckets']) == 1 + assert result['Buckets'][0]['Name'] == 'aws_bucket_name' - # spans = tracer.recorder.queued_spans() - # assert len(spans) == 2 + spans = tracer.recorder.queued_spans() + assert len(spans) == 2 - # filter = lambda span: span.n == "sdk" - # test_span = get_first_span_by_filter(spans, filter) - # assert (test_span) + filter = lambda span: span.n == "sdk" + test_span = get_first_span_by_filter(spans, filter) + assert (test_span) - # filter = lambda span: span.n == "boto3" - # boto_span = get_first_span_by_filter(spans, filter) - # assert (boto_span) + filter = lambda span: span.n == "boto3" + boto_span = get_first_span_by_filter(spans, filter) + assert (boto_span) - # assert (boto_span.t == test_span.t) - # assert (boto_span.p == test_span.s) + assert (boto_span.t == test_span.t) + assert (boto_span.p == test_span.s) + + assert (test_span.ec is None) + assert (boto_span.ec is None) - # assert (test_span.ec is None) - # assert (boto_span.ec is None) + assert boto_span.data['boto3']['op'] == 'CreateBucket' + assert boto_span.data['boto3']['ep'] == 'https://s3.amazonaws.com' + assert boto_span.data['boto3']['reg'] == 'us-east-1' + assert boto_span.data['boto3']['payload'] == {'Bucket': 'aws_bucket_name'} + assert boto_span.data['http']['status'] == 200 + assert boto_span.data['http']['method'] == 'POST' + assert boto_span.data['http']['url'] == 'https://s3.amazonaws.com:443/CreateBucket' - # assert boto_span.data['boto3']['op'] == 'CreateBucket' - # assert boto_span.data['boto3']['ep'] == 'https://s3.amazonaws.com' - # assert boto_span.data['boto3']['reg'] == 'us-east-1' - # assert boto_span.data['boto3']['payload'] == {'Bucket': 'aws_bucket_name'} - # assert boto_span.data['http']['status'] == 200 - # assert boto_span.data['http']['method'] == 'POST' - # assert boto_span.data['http']['url'] == 'https://s3.amazonaws.com:443/CreateBucket' + assert ("X-Capture-This-Too" in boto_span.data["http"]["header"]) + assert ("this too" == boto_span.data["http"]["header"]["X-Capture-This-Too"]) + assert ("X-Capture-That-Too" in boto_span.data["http"]["header"]) + assert ("that too" == boto_span.data["http"]["header"]["X-Capture-That-Too"]) agent.options.extra_http_headers = original_extra_http_headers