From 1d029e73d8390a91473f7dde71c07b7ed9677278 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 7 May 2024 10:55:13 -0700 Subject: [PATCH 1/8] Instrument aiobotocore --- newrelic/config.py | 6 + newrelic/hooks/external_aiobotocore.py | 48 +++++ tests/external_aiobotocore/conftest.py | 151 ++++++++++++++++ .../test_aiobotocore_dynamodb.py | 167 ++++++++++++++++++ .../test_aiobotocore_s3.py | 123 +++++++++++++ .../test_aiobotocore_sns.py | 73 ++++++++ .../test_aiobotocore_sqs.py | 113 ++++++++++++ tox.ini | 7 + 8 files changed, 688 insertions(+) create mode 100644 newrelic/hooks/external_aiobotocore.py create mode 100644 tests/external_aiobotocore/conftest.py create mode 100644 tests/external_aiobotocore/test_aiobotocore_dynamodb.py create mode 100644 tests/external_aiobotocore/test_aiobotocore_s3.py create mode 100644 tests/external_aiobotocore/test_aiobotocore_sns.py create mode 100644 tests/external_aiobotocore/test_aiobotocore_sqs.py diff --git a/newrelic/config.py b/newrelic/config.py index 55b012e58..08a5c538b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -4489,6 +4489,12 @@ def _process_module_builtin_defaults(): "instrument_gearman_worker", ) + _process_module_definition( + "aiobotocore.endpoint", + "newrelic.hooks.external_aiobotocore", + "instrument_aiobotocore_endpoint", + ) + _process_module_definition( "botocore.endpoint", "newrelic.hooks.external_botocore", diff --git a/newrelic/hooks/external_aiobotocore.py b/newrelic/hooks/external_aiobotocore.py new file mode 100644 index 000000000..7c8be883b --- /dev/null +++ b/newrelic/hooks/external_aiobotocore.py @@ -0,0 +1,48 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.external_trace import ExternalTrace +from newrelic.common.object_wrapper import wrap_function_wrapper + + +def _bind_make_request_params(operation_model, request_dict, *args, **kwargs): + return operation_model, request_dict + + +def bind__send_request(request_dict, operation_model, *args, **kwargs): + return operation_model, request_dict + + +async def wrap_endpoint_make_request(wrapped, instance, args, kwargs): + operation_model, request_dict = _bind_make_request_params(*args, **kwargs) + url = request_dict.get("url") + method = request_dict.get("method") + + with ExternalTrace(library="aiobotocore", url=url, method=method, source=wrapped) as trace: + try: + trace._add_agent_attribute("aws.operation", operation_model.name) + except: + pass + + result = await wrapped(*args, **kwargs) + try: + request_id = result[1]["ResponseMetadata"]["RequestId"] + trace._add_agent_attribute("aws.requestId", request_id) + except: + pass + return result + + +def instrument_aiobotocore_endpoint(module): + wrap_function_wrapper(module, "AioEndpoint.make_request", wrap_endpoint_make_request) diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py new file mode 100644 index 000000000..53548dfda --- /dev/null +++ b/tests/external_aiobotocore/conftest.py @@ -0,0 +1,151 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import logging +import socket +import threading + +import moto.server +import werkzeug.serving +from testing_support.fixture.event_loop import ( # noqa: F401, pylint: disable=W061 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +PORT = 4443 +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec +HOST = "127.0.0.1" + + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (external_aiobotocore)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (external_aiobotocore)"], +) + + +def get_free_tcp_port(release_socket: bool = False): + sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sckt.bind((HOST, 0)) + _, port = sckt.getsockname() # address, port + if release_socket: + sckt.close() + return port + + return sckt, port + + +class MotoService: + """Will Create MotoService. + Service is ref-counted so there will only be one per process. Real Service will + be returned by `__aenter__`.""" + + _services = dict() # {name: instance} + + def __init__(self, service_name: str, port: int = None, ssl: bool = False): + self._service_name = service_name + + if port: + self._socket = None + self._port = port + else: + self._socket, self._port = get_free_tcp_port() + + self._thread = None + self._logger = logging.getLogger("MotoService") + self._refcount = None + self._ip_address = HOST + self._server = None + self._ssl_ctx = werkzeug.serving.generate_adhoc_ssl_context() if ssl else None + self._schema = "http" if not self._ssl_ctx else "https" + + @property + def endpoint_url(self): + return f"{self._schema}://{self._ip_address}:{self._port}" + + def __call__(self, func): + async def wrapper(*args, **kwargs): + await self._start() + try: + result = await func(*args, **kwargs) + finally: + await self._stop() + return result + + functools.update_wrapper(wrapper, func) + wrapper.__wrapped__ = func + return wrapper + + async def __aenter__(self): + svc = self._services.get(self._service_name) + if svc is None: + self._services[self._service_name] = self + self._refcount = 1 + await self._start() + return self + else: + svc._refcount += 1 + return svc + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self._refcount -= 1 + + if self._socket: + self._socket.close() + self._socket = None + + if self._refcount == 0: + del self._services[self._service_name] + await self._stop() + + def _server_entry(self): + self._main_app = moto.server.DomainDispatcherApplication( + moto.server.create_backend_app # , service=self._service_name + ) + self._main_app.debug = True + + if self._socket: + self._socket.close() # release right before we use it + self._socket = None + + self._server = werkzeug.serving.make_server( + self._ip_address, + self._port, + self._main_app, + True, + ssl_context=self._ssl_ctx, + ) + self._server.serve_forever() + + async def _start(self): + self._thread = threading.Thread(target=self._server_entry, daemon=True) + self._thread.start() + + async def _stop(self): + if self._server: + self._server.shutdown() + + self._thread.join() diff --git a/tests/external_aiobotocore/test_aiobotocore_dynamodb.py b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py new file mode 100644 index 000000000..a15bf2faf --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py @@ -0,0 +1,167 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W061 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TEST_TABLE = "python-agent-test" + +_dynamodb_scoped_metrics = [ + ("Datastore/statement/DynamoDB/%s/create_table" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/put_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/get_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/update_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/query" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/scan" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/delete_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/delete_table" % TEST_TABLE, 1), +] + +_dynamodb_rollup_metrics = [ + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/DynamoDB/all", 8), + ("Datastore/DynamoDB/allOther", 8), +] + + +# aws.requestId count disabled due to variability in count. +# Flaky due to waiter function, which "aws.operation" == "DescribeTable" +# This is a polling function, so in real time, this value could fluctuate +# @validate_span_events(expected_agents=("aws.requestId",), count=9) +# @validate_span_events(exact_agents={"aws.operation": "DescribeTable"}, count=2) +@validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) +@validate_transaction_metrics( + "test_aiobotocore_dynamodb:test_aiobotocore_dynamodb", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_dynamodb(loop): + async def _test(): + async with MotoService("dynamodb", port=PORT): + session = get_session() + + async with session.create_client( + "dynamodb", + region_name="us-east-1", + endpoint_url="http://localhost:%d" % PORT, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + resp = await client.create_table( + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "Id", "KeyType": "HASH"}, + {"AttributeName": "Foo", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={ + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, + ) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + # Wait for table to be created + waiter = client.get_waiter("table_exists") + await waiter.wait(TableName=TEST_TABLE) + + # Put item + resp = await client.put_item( + TableName=TEST_TABLE, + Item={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Get item + resp = await client.get_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["Item"]["Foo"]["S"] == "hello_world" + + # Update item + resp = await client.update_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + AttributeUpdates={ + "Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}, + }, + ReturnValues="ALL_NEW", + ) + assert resp["Attributes"]["Foo2"] + + # Query for item + resp = await client.query( + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, + ) + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["Foo"]["S"] == "hello_world" + + # Scan + resp = await client.scan(TableName=TEST_TABLE) + assert len(resp["Items"]) == 1 + + # Delete item + resp = await client.delete_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Delete table + resp = await client.delete_table(TableName=TEST_TABLE) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_s3.py b/tests/external_aiobotocore/test_aiobotocore_s3.py new file mode 100644 index 000000000..473546432 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_s3.py @@ -0,0 +1,123 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import aiobotocore +from conftest import ( # noqa: F401, pylint: disable=W061 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TEST_BUCKET = "python-agent-test" +FILENAME = "dummy.bin" +FOLDER = "aiobotocore" +ENDPOINT = "localhost:%s" % PORT +KEY = "{}/{}".format(FOLDER, FILENAME) +EXPECTED_BUCKET_URL = "http://%s/%s" % (ENDPOINT, TEST_BUCKET) +EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/" + KEY + + +_s3_scoped_metrics = [ + ("External/%s/aiobotocore/GET" % ENDPOINT, 5), + ("External/%s/aiobotocore/PUT" % ENDPOINT, 2), + ("External/%s/aiobotocore/DELETE" % ENDPOINT, 2), +] + +_s3_rollup_metrics = [ + ("External/all", 9), + ("External/allOther", 9), + ("External/%s/all" % ENDPOINT, 9), + ("External/%s/aiobotocore/GET" % ENDPOINT, 5), + ("External/%s/aiobotocore/PUT" % ENDPOINT, 2), + ("External/%s/aiobotocore/DELETE" % ENDPOINT, 2), +] + + +@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=2) +@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1) +@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=4) +@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=4) +@validate_transaction_metrics( + "test_aiobotocore_s3:test_aiobotocore_s3", + scoped_metrics=_s3_scoped_metrics, + rollup_metrics=_s3_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_s3(loop): + async def _test(): + + data = b"hello_world" + + async with MotoService("s3", port=PORT): + + session = aiobotocore.session.get_session() + + async with session.create_client( # nosec + "s3", + region_name="us-east-1", + endpoint_url="http://localhost:%d" % PORT, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + # Create bucket + await client.create_bucket( + Bucket=TEST_BUCKET, + ) + + # List buckets + await client.list_buckets() + + # Upload object to s3 + resp = await client.put_object(Bucket=TEST_BUCKET, Key=KEY, Body=data) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # List objects from bucket + await client.list_objects(Bucket=TEST_BUCKET) + + # Getting s3 object properties of uploaded file + resp = await client.get_object_acl(Bucket=TEST_BUCKET, Key=KEY) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Get object from s3 + response = await client.get_object(Bucket=TEST_BUCKET, Key=KEY) + # this will ensure the connection is correctly re-used/closed + async with response["Body"] as stream: + assert await stream.read() == data + + # List s3 objects using paginator + paginator = client.get_paginator("list_objects") + async for result in paginator.paginate(Bucket=TEST_BUCKET, Prefix=FOLDER): + for content in result.get("Contents", []): + assert content + + # Delete object from s3 + await client.delete_object(Bucket=TEST_BUCKET, Key=KEY) + + # Delete bucket from s3 + await client.delete_bucket(Bucket=TEST_BUCKET) + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_sns.py b/tests/external_aiobotocore/test_aiobotocore_sns.py new file mode 100644 index 000000000..9a7acf403 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_sns.py @@ -0,0 +1,73 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W061 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TOPIC = "arn:aws:sns:us-east-1:123456789012:some-topic" +sns_metrics = [ + ("MessageBroker/SNS/Topic/Produce/Named/%s" % TOPIC, 1), + ("MessageBroker/SNS/Topic/Produce/Named/PhoneNumber", 1), +] + + +@validate_span_events(expected_agents=("aws.requestId",), count=4) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=2) +@validate_transaction_metrics( + "test_aiobotocore_sns:test_publish_to_sns", + scoped_metrics=sns_metrics, + rollup_metrics=sns_metrics, + background_task=True, +) +@background_task() +def test_publish_to_sns(loop): + async def _test(): + + async with MotoService("sns", port=PORT): + session = get_session() + + async with session.create_client( + "sns", + region_name="us-east-1", + endpoint_url="http://localhost:%d" % PORT, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + topic_arn = await client.create_topic(Name="some-topic") + topic_arn_name = topic_arn["TopicArn"] + + kwargs = {"TopicArn": topic_arn_name} + published_message = await client.publish(Message="my message", **kwargs) + assert "MessageId" in published_message + + await client.subscribe(TopicArn=topic_arn_name, Protocol="sms", Endpoint="5555555555") + + published_message = await client.publish(PhoneNumber="5555555555", Message="my msg") + assert "MessageId" in published_message + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_sqs.py b/tests/external_aiobotocore/test_aiobotocore_sqs.py new file mode 100644 index 000000000..e7e1e72f8 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_sqs.py @@ -0,0 +1,113 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W061 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +URL = "localhost:%s" % PORT +TEST_QUEUE = "python-agent-test" + +_sqs_scoped_metrics = [ + ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), + ("External/%s/aiobotocore/POST" % URL, 6), +] + +_sqs_rollup_metrics = [ + ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), + # ("MessageBroker/SQS/Queue/Consume/Named/%s" % TEST_QUEUE, 1), + ("External/all", 6), + ("External/allOther", 6), + ("External/%s/all" % URL, 6), + ("External/%s/aiobotocore/POST" % URL, 6), +] + + +@validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ListQueues"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) +# @validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessageBatch"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PurgeQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteQueue"}, count=1) +@validate_transaction_metrics( + "test_aiobotocore_sqs:test_aiobotocore_sqs", + scoped_metrics=_sqs_scoped_metrics, + rollup_metrics=_sqs_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_sqs(loop): + async def _test(): + async with MotoService("sqs", port=PORT): + session = get_session() + + async with session.create_client( + "sqs", + region_name="us-east-1", + endpoint_url="http://localhost:%d" % PORT, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + response = await client.create_queue(QueueName=TEST_QUEUE) + + queue_url = response["QueueUrl"] + + # List queues + response = await client.list_queues() + for queue_name in response.get("QueueUrls", []): + assert queue_name + + # Send message + resp = await client.send_message(QueueUrl=queue_url, MessageBody="hello_world") + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Receive message + # This works in an actual application, but not with the proxy set up + # for the test. The aiobotocore's proxy requests redirect to aiohttp's + # request call which results in double instrumentation and invalid XML + # generation in response_dict["headers"]["body"] + # resp = await client.receive_message(QueueUrl=queue_url) + # assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Send message batch + messages = [ + {"Id": "1", "MessageBody": "message 1"}, + {"Id": "2", "MessageBody": "message 2"}, + {"Id": "3", "MessageBody": "message 3"}, + ] + resp = await client.send_message_batch(QueueUrl=queue_url, Entries=messages) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Purge queue + resp = await client.purge_queue(QueueUrl=queue_url) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Delete queue + resp = await client.delete_queue(QueueUrl=queue_url) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + loop.run_until_complete(_test()) diff --git a/tox.ini b/tox.ini index 1c4bdf8f6..4484babce 100644 --- a/tox.ini +++ b/tox.ini @@ -102,6 +102,7 @@ envlist = python-cross_agent-{py27,py37,py38,py39,py310,py311,py312}-{with,without}_extensions, python-cross_agent-pypy27-without_extensions, python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,py312,pypy27,pypy310}, + python-external_aiobotocore-{py38,py39,py310,py311,py312}-aiobotocorelatest, python-external_botocore-{py38,py39,py310,py311,py312}-botocorelatest, python-external_botocore-{py311}-botocorelatest-langchain, python-external_botocore-py310-botocore0125, @@ -258,6 +259,11 @@ deps = datastore_redis-redislatest: redis datastore_rediscluster-redislatest: redis datastore_redis-redis0400: redis<4.1 + external_aiobotocore-aiobotocorelatest: aiobotocore[awscli] + external_aiobotocore-aiobotocorelatest: flask + external_aiobotocore-aiobotocorelatest: flask-cors + external_aiobotocore-aiobotocorelatest: moto[all] + external_aiobotocore-aiobotocorelatest: aiohttp external_botocore-botocorelatest: botocore external_botocore-botocorelatest: boto3 external_botocore-botocorelatest-langchain: langchain @@ -454,6 +460,7 @@ changedir = datastore_redis: tests/datastore_redis datastore_rediscluster: tests/datastore_rediscluster datastore_sqlite: tests/datastore_sqlite + external_aiobotocore: tests/external_aiobotocore external_botocore: tests/external_botocore external_feedparser: tests/external_feedparser external_http: tests/external_http From 8d24725bace58fdfa6c8c0334db18051093e9f75 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 7 May 2024 11:15:43 -0700 Subject: [PATCH 2/8] Replace __version__ in flask instrumentation to avoid deprecation --- newrelic/hooks/framework_flask.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/newrelic/hooks/framework_flask.py b/newrelic/hooks/framework_flask.py index 6ef45e6af..0da056a53 100644 --- a/newrelic/hooks/framework_flask.py +++ b/newrelic/hooks/framework_flask.py @@ -28,12 +28,9 @@ from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version - -def framework_details(): - import flask - - return ("Flask", getattr(flask, "__version__", None)) +FLASK_VERSION = ("Flask", get_package_version("flask")) def status_code(exc, value, tb): @@ -276,7 +273,7 @@ def instrument_flask_views(module): def instrument_flask_app(module): - wrap_wsgi_application(module, "Flask.wsgi_app", framework=framework_details) + wrap_wsgi_application(module, "Flask.wsgi_app", framework=FLASK_VERSION) wrap_function_wrapper(module, "Flask.add_url_rule", _nr_wrapper_Flask_add_url_rule_input_) From 9a901040488fa8a76c7e790ddc40cb7f8574bce5 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 14 May 2024 14:41:02 -0700 Subject: [PATCH 3/8] Disable browser monitoring --- tests/external_aiobotocore/conftest.py | 1 + .../test_aiobotocore_sqs.py | 29 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py index 53548dfda..69a477c81 100644 --- a/tests/external_aiobotocore/conftest.py +++ b/tests/external_aiobotocore/conftest.py @@ -39,6 +39,7 @@ "transaction_tracer.stack_trace_threshold": 0.0, "debug.log_data_collector_payloads": True, "debug.record_transaction_failure": True, + "browser_monitoring.enabled": False, } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (external_aiobotocore)", diff --git a/tests/external_aiobotocore/test_aiobotocore_sqs.py b/tests/external_aiobotocore/test_aiobotocore_sqs.py index e7e1e72f8..a9e4e39dc 100644 --- a/tests/external_aiobotocore/test_aiobotocore_sqs.py +++ b/tests/external_aiobotocore/test_aiobotocore_sqs.py @@ -32,23 +32,23 @@ _sqs_scoped_metrics = [ ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), - ("External/%s/aiobotocore/POST" % URL, 6), + ("External/%s/aiobotocore/POST" % URL, 7), ] _sqs_rollup_metrics = [ ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), - # ("MessageBroker/SQS/Queue/Consume/Named/%s" % TEST_QUEUE, 1), - ("External/all", 6), - ("External/allOther", 6), - ("External/%s/all" % URL, 6), - ("External/%s/aiobotocore/POST" % URL, 6), + ("MessageBroker/SQS/Queue/Consume/Named/%s" % TEST_QUEUE, 1), + ("External/all", 7), + ("External/allOther", 7), + ("External/%s/all" % URL, 7), + ("External/%s/aiobotocore/POST" % URL, 7), ] @validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ListQueues"}, count=1) @validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) -# @validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) @validate_span_events(exact_agents={"aws.operation": "SendMessageBatch"}, count=1) @validate_span_events(exact_agents={"aws.operation": "PurgeQueue"}, count=1) @validate_span_events(exact_agents={"aws.operation": "DeleteQueue"}, count=1) @@ -82,16 +82,17 @@ async def _test(): assert queue_name # Send message - resp = await client.send_message(QueueUrl=queue_url, MessageBody="hello_world") + resp = await client.send_message( + QueueUrl=queue_url, + MessageBody="hello_world", + ) assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Receive message - # This works in an actual application, but not with the proxy set up - # for the test. The aiobotocore's proxy requests redirect to aiohttp's - # request call which results in double instrumentation and invalid XML - # generation in response_dict["headers"]["body"] - # resp = await client.receive_message(QueueUrl=queue_url) - # assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + resp = await client.receive_message( + QueueUrl=queue_url, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Send message batch messages = [ From da6b566c329b48ece6900148a6be8d47bca62069 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 14 May 2024 15:16:14 -0700 Subject: [PATCH 4/8] Fix typo --- newrelic/api/web_transaction.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/newrelic/api/web_transaction.py b/newrelic/api/web_transaction.py index 5416f2e80..dac89f630 100644 --- a/newrelic/api/web_transaction.py +++ b/newrelic/api/web_transaction.py @@ -33,8 +33,7 @@ ) from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object -from newrelic.core.attribute import create_attributes, process_user_attribute -from newrelic.core.attribute_filter import DST_BROWSER_MONITORING, DST_NONE +from newrelic.core.attribute_filter import DST_BROWSER_MONITORING from newrelic.packages import six _logger = logging.getLogger(__name__) @@ -457,15 +456,15 @@ def browser_timing_header(self, nonce=None): # create the data structure that pull all our data in - broswer_agent_configuration = self.browser_monitoring_intrinsics(obfuscation_key) + browser_agent_configuration = self.browser_monitoring_intrinsics(obfuscation_key) if attributes: attributes = obfuscate(json_encode(attributes), obfuscation_key) - broswer_agent_configuration["atts"] = attributes + browser_agent_configuration["atts"] = attributes header = _js_agent_header_fragment % ( _encode_nonce(nonce), - json_encode(broswer_agent_configuration), + json_encode(browser_agent_configuration), self._settings.js_agent_loader, ) From bb75e160d79596edf172b81ad6168e0c0d947dc4 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 14 May 2024 15:17:09 -0700 Subject: [PATCH 5/8] Disable browser monitoring with aiobotocore --- newrelic/hooks/external_aiobotocore.py | 6 ++++++ tests/external_aiobotocore/conftest.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/newrelic/hooks/external_aiobotocore.py b/newrelic/hooks/external_aiobotocore.py index 7c8be883b..6103dab94 100644 --- a/newrelic/hooks/external_aiobotocore.py +++ b/newrelic/hooks/external_aiobotocore.py @@ -31,6 +31,12 @@ async def wrap_endpoint_make_request(wrapped, instance, args, kwargs): with ExternalTrace(library="aiobotocore", url=url, method=method, source=wrapped) as trace: try: + # Because AIOBotocore's proxy functionality uses aiohttp + # and urllib3 under the hood, New Relic has portions that + # are classified as Web Transactions. This means that + # browser monitoring will now be true. However, this will + # inject unwanted JS Agent Header Fragments into SQS responses. + trace.settings.browser_monitoring.enabled = False trace._add_agent_attribute("aws.operation", operation_model.name) except: pass diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py index 69a477c81..53548dfda 100644 --- a/tests/external_aiobotocore/conftest.py +++ b/tests/external_aiobotocore/conftest.py @@ -39,7 +39,6 @@ "transaction_tracer.stack_trace_threshold": 0.0, "debug.log_data_collector_payloads": True, "debug.record_transaction_failure": True, - "browser_monitoring.enabled": False, } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (external_aiobotocore)", From 95cf91886ed9f9bb1ee0fc926c29d28bc1e8409a Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Tue, 14 May 2024 15:37:59 -0700 Subject: [PATCH 6/8] Fix linter errors --- newrelic/api/web_transaction.py | 2 +- tests/external_aiobotocore/conftest.py | 4 ++-- tests/external_aiobotocore/test_aiobotocore_dynamodb.py | 2 +- tests/external_aiobotocore/test_aiobotocore_s3.py | 2 +- tests/external_aiobotocore/test_aiobotocore_sns.py | 2 +- tests/external_aiobotocore/test_aiobotocore_sqs.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/newrelic/api/web_transaction.py b/newrelic/api/web_transaction.py index dac89f630..3b7a06e19 100644 --- a/newrelic/api/web_transaction.py +++ b/newrelic/api/web_transaction.py @@ -567,7 +567,7 @@ def __iter__(self): yield "content-length", self.environ["CONTENT_LENGTH"] elif key == "CONTENT_TYPE": yield "content-type", self.environ["CONTENT_TYPE"] - elif key == "HTTP_CONTENT_LENGTH" or key == "HTTP_CONTENT_TYPE": + elif key in ("HTTP_CONTENT_LENGTH", "HTTP_CONTENT_TYPE"): # These keys are illegal and should be ignored continue elif key.startswith("HTTP_"): diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py index 53548dfda..558167ec2 100644 --- a/tests/external_aiobotocore/conftest.py +++ b/tests/external_aiobotocore/conftest.py @@ -19,7 +19,7 @@ import moto.server import werkzeug.serving -from testing_support.fixture.event_loop import ( # noqa: F401, pylint: disable=W061 +from testing_support.fixture.event_loop import ( # noqa: F401, pylint: disable=W0611 event_loop as loop, ) from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 @@ -63,7 +63,7 @@ class MotoService: Service is ref-counted so there will only be one per process. Real Service will be returned by `__aenter__`.""" - _services = dict() # {name: instance} + _services = {} # {name: instance} def __init__(self, service_name: str, port: int = None, ssl: bool = False): self._service_name = service_name diff --git a/tests/external_aiobotocore/test_aiobotocore_dynamodb.py b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py index a15bf2faf..a38cb384a 100644 --- a/tests/external_aiobotocore/test_aiobotocore_dynamodb.py +++ b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py @@ -13,7 +13,7 @@ # limitations under the License. from aiobotocore.session import get_session -from conftest import ( # noqa: F401, pylint: disable=W061 +from conftest import ( # noqa: F401, pylint: disable=W0611 AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, PORT, diff --git a/tests/external_aiobotocore/test_aiobotocore_s3.py b/tests/external_aiobotocore/test_aiobotocore_s3.py index 473546432..7db5379c4 100644 --- a/tests/external_aiobotocore/test_aiobotocore_s3.py +++ b/tests/external_aiobotocore/test_aiobotocore_s3.py @@ -13,7 +13,7 @@ # limitations under the License. import aiobotocore -from conftest import ( # noqa: F401, pylint: disable=W061 +from conftest import ( # noqa: F401, pylint: disable=W0611 AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, PORT, diff --git a/tests/external_aiobotocore/test_aiobotocore_sns.py b/tests/external_aiobotocore/test_aiobotocore_sns.py index 9a7acf403..29ae3a87b 100644 --- a/tests/external_aiobotocore/test_aiobotocore_sns.py +++ b/tests/external_aiobotocore/test_aiobotocore_sns.py @@ -13,7 +13,7 @@ # limitations under the License. from aiobotocore.session import get_session -from conftest import ( # noqa: F401, pylint: disable=W061 +from conftest import ( # noqa: F401, pylint: disable=W0611 AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, PORT, diff --git a/tests/external_aiobotocore/test_aiobotocore_sqs.py b/tests/external_aiobotocore/test_aiobotocore_sqs.py index a9e4e39dc..6d3acba65 100644 --- a/tests/external_aiobotocore/test_aiobotocore_sqs.py +++ b/tests/external_aiobotocore/test_aiobotocore_sqs.py @@ -13,7 +13,7 @@ # limitations under the License. from aiobotocore.session import get_session -from conftest import ( # noqa: F401, pylint: disable=W061 +from conftest import ( # noqa: F401, pylint: disable=W0611 AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, PORT, From 4c19af1648c302c3998a2401c1e3e34812c40f6b Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Thu, 16 May 2024 15:33:30 -0700 Subject: [PATCH 7/8] Revert to disabling settings in conftest --- newrelic/hooks/external_aiobotocore.py | 6 ------ tests/external_aiobotocore/conftest.py | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/newrelic/hooks/external_aiobotocore.py b/newrelic/hooks/external_aiobotocore.py index 6103dab94..7c8be883b 100644 --- a/newrelic/hooks/external_aiobotocore.py +++ b/newrelic/hooks/external_aiobotocore.py @@ -31,12 +31,6 @@ async def wrap_endpoint_make_request(wrapped, instance, args, kwargs): with ExternalTrace(library="aiobotocore", url=url, method=method, source=wrapped) as trace: try: - # Because AIOBotocore's proxy functionality uses aiohttp - # and urllib3 under the hood, New Relic has portions that - # are classified as Web Transactions. This means that - # browser monitoring will now be true. However, this will - # inject unwanted JS Agent Header Fragments into SQS responses. - trace.settings.browser_monitoring.enabled = False trace._add_agent_attribute("aws.operation", operation_model.name) except: pass diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py index 558167ec2..b664d004a 100644 --- a/tests/external_aiobotocore/conftest.py +++ b/tests/external_aiobotocore/conftest.py @@ -39,6 +39,7 @@ "transaction_tracer.stack_trace_threshold": 0.0, "debug.log_data_collector_payloads": True, "debug.record_transaction_failure": True, + "browser_monitoring.enabled": False, } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (external_aiobotocore)", From f956c29d55393dc119e3d1c5d618b71272ff16d2 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei Date: Wed, 22 May 2024 16:00:52 -0700 Subject: [PATCH 8/8] Remove browser monitoring disabling flag --- tests/external_aiobotocore/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py index b664d004a..558167ec2 100644 --- a/tests/external_aiobotocore/conftest.py +++ b/tests/external_aiobotocore/conftest.py @@ -39,7 +39,6 @@ "transaction_tracer.stack_trace_threshold": 0.0, "debug.log_data_collector_payloads": True, "debug.record_transaction_failure": True, - "browser_monitoring.enabled": False, } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (external_aiobotocore)",