From 4d3aa6dff17108ccb603d6c544111d803a4bedba Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 12:08:03 +0200 Subject: [PATCH 01/16] Log requests to the Google Cloud Storage API made with google.cloud.storage --- .../instrumentation/google/cloud/storage.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 instana/instrumentation/google/cloud/storage.py diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py new file mode 100644 index 00000000..c91791b5 --- /dev/null +++ b/instana/instrumentation/google/cloud/storage.py @@ -0,0 +1,19 @@ +from __future__ import absolute_import + +import wrapt +from ....log import logger + +try: + from google.cloud import storage + + def collect_params(api_request): + logger.debug('uninstrumented Google Cloud Storage API request: %s' % api_request) + + def execute_with_instana(wrapped, instance, args, kwargs): + print(collect_params(kwargs)) + + return wrapped(*args, **kwargs) + + wrapt.wrap_function_wrapper('google.cloud.storage._http', 'Connection.api_request', execute_with_instana) +except ImportError: + pass From fba58416e1e7749914cb5324513fbaa0b6e9c33f Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 14:59:39 +0200 Subject: [PATCH 02/16] Collect GCS API bucket operation tags --- .../instrumentation/google/cloud/storage.py | 128 +++++++++++++++++- setup.py | 1 + tests/clients/test_google-cloud-storage.py | 117 ++++++++++++++++ 3 files changed, 242 insertions(+), 4 deletions(-) create mode 100644 tests/clients/test_google-cloud-storage.py diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index c91791b5..d3813ae8 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -1,18 +1,138 @@ from __future__ import absolute_import import wrapt +import re +from urllib.parse import unquote + from ....log import logger +from ....singletons import tracer try: from google.cloud import storage - def collect_params(api_request): - logger.debug('uninstrumented Google Cloud Storage API request: %s' % api_request) + # A map of GCS API param collectors organized as: + # request_method -> path_matcher -> collector + # + # * request method - the HTTP method used to make an API request (GET, POST, etc.) + # * path_matcher - either a string or a regex applied to the API request path (string values match first). + # * collector - a lambda returning a dict of span from API request query string. + # parameters and request body data. If a regex is used as a path matcher, the match result + # will be provided as a third argument. + _storage_api = { + 'GET': { + # Bucket operations + '/b': lambda params, data: { + 'gcs.op': 'buckets.list', + 'gcs.projectId': params.get('project', None) + }, + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.get', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { + 'gcs.op': 'buckets.getIamPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam/testPermissions$'): lambda params, data, match: { + 'gcs.op': 'buckets.testIamPermissions', + 'gcs.bucket': unquote(match.group('bucket')), + }, + }, + 'POST': { + # Bucket operations + '/b': lambda params, data: { + 'gcs.op': 'buckets.insert', + 'gcs.projectId': params.get('project', None), + 'gcs.bucket': data.get('name', None), + }, + re.compile('^/b/(?P[^/]+)/lockRetentionPolicy$'): lambda params, data, match: { + 'gcs.op': 'buckets.lockRetentionPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + }, + 'PATCH': { + # Bucket operations + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.patch', + 'gcs.bucket': unquote(match.group('bucket')), + }, + }, + 'PUT': { + # Bucket operations + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.update', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { + 'gcs.op': 'buckets.setIamPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + }, + 'DELETE': { + # Bucket operations + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.delete', + 'gcs.bucket': unquote(match.group('bucket')), + }, + } + } + + def collect_tags(api_request): + """ + Extract span tags from Google Cloud Storage API request. Returns None if the request is not + supported. + + :param: dict + :return: dict or None + """ + method, path = api_request.get('method', None), api_request.get('path', None) + + if method not in _storage_api: + return + + params = api_request.get('query_params', {}) + data = api_request.get('data', {}) + + if path in _storage_api[method]: + # check is any of string keys matches the path exactly + return _storage_api[method][path](params, data) + else: + # look for a regex that matches the string + for (matcher, collect) in _storage_api[method].items(): + if not isinstance(matcher, re.Pattern): + continue + + m = matcher.match(path) + if m is None: + continue + + return collect(params, data, m) def execute_with_instana(wrapped, instance, args, kwargs): - print(collect_params(kwargs)) + parent_span = tracer.active_span + + # return early if we're not tracing + if parent_span is None: + return wrapped(*args, **kwargs) + + tags = collect_tags(kwargs) + + # don't trace if the call is not instrumented + if tags is None: + logger.debug('uninstrumented Google Cloud Storage API request: %s' % kwargs) + return wrapped(*args, **kwargs) + + with tracer.start_active_span('gcs', child_of=parent_span) as scope: + for (k, v) in tags.items(): + scope.span.set_tag(k, v) - return wrapped(*args, **kwargs) + try: + kv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_exception(e) + raise + else: + return kv wrapt.wrap_function_wrapper('google.cloud.storage._http', 'Connection.api_request', execute_with_instana) except ImportError: diff --git a/setup.py b/setup.py index 3b80905d..d211f460 100644 --- a/setup.py +++ b/setup.py @@ -98,6 +98,7 @@ def check_setuptools(): 'nose>=1.0', 'flask>=0.12.2', 'grpcio>=1.18.0', + 'google-cloud-storage>=1.31.1;python_version>="3.5"', 'lxml>=3.4', 'mock>=2.0.0', 'mysqlclient>=1.3.14;python_version>="3.5"', diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py new file mode 100644 index 00000000..7b4e26c7 --- /dev/null +++ b/tests/clients/test_google-cloud-storage.py @@ -0,0 +1,117 @@ +from __future__ import absolute_import + +import unittest + +from instana.instrumentation.google.cloud.storage import collect_tags + +class TestGoogleCloudStorage(unittest.TestCase): + def test_collect_tags(self): + test_cases = { + 'buckets.list': ( + { + 'method': 'GET', + 'path': '/b', + 'query_params': {'project': 'test-project'} + }, + { + 'gcs.op': 'buckets.list', + 'gcs.projectId': 'test-project' + } + ), + 'buckets.insert':( + { + 'method': 'POST', + 'path': '/b', + 'query_params': {'project': 'test-project'}, + 'data': {'name': 'test bucket'} + }, + { + 'gcs.op': 'buckets.insert', + 'gcs.projectId': 'test-project', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.get': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket' + }, + { + 'gcs.op': 'buckets.get', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.patch': ( + { + 'method': 'PATCH', + 'path': '/b/test%20bucket' + }, + { + 'gcs.op': 'buckets.patch', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.update': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket' + }, + { + 'gcs.op': 'buckets.update', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.getIamPolicy': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/iam' + }, + { + 'gcs.op': 'buckets.getIamPolicy', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.setIamPolicy': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket/iam' + }, + { + 'gcs.op': 'buckets.setIamPolicy', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.testIamPermissions': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/iam/testPermissions' + }, + { + 'gcs.op': 'buckets.testIamPermissions', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.lockRetentionPolicy': ( + { + 'method': 'POST', + 'path': '/b/test%20bucket/lockRetentionPolicy' + }, + { + 'gcs.op': 'buckets.lockRetentionPolicy', + 'gcs.bucket': 'test bucket' + } + ), + 'buckets.delete': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket' + }, + { + 'gcs.op': 'buckets.update', + 'gcs.bucket': 'test bucket' + } + ) + } + + for (op, (request, expected)) in test_cases.items(): + self.assertEqual(expected, collect_tags(request), msg=op) From f4644506be4514aaa11b3e711e1f0ef94de3af91 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 16:20:49 +0200 Subject: [PATCH 03/16] Collect GCS API blob operation tags --- .../instrumentation/google/cloud/storage.py | 109 +++++++++++++++ tests/clients/test_google-cloud-storage.py | 126 ++++++++++++++++++ 2 files changed, 235 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index d3813ae8..019a1a4a 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -37,6 +37,16 @@ 'gcs.op': 'buckets.testIamPermissions', 'gcs.bucket': unquote(match.group('bucket')), }, + # Object/blob operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': params.get('alt', 'json') == 'media' and 'objects.get' or 'objects.attrs', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + }, + re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { + 'gcs.op': 'objects.list', + 'gcs.bucket': unquote(match.group('bucket')) + } }, 'POST': { # Bucket operations @@ -49,6 +59,34 @@ 'gcs.op': 'buckets.lockRetentionPolicy', 'gcs.bucket': unquote(match.group('bucket')), }, + # Object/blob operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/compose$'): lambda params, data, match: { + 'gcs.op': 'objects.compose', + 'gcs.destinationBucket': unquote(match.group('bucket')), + 'gcs.destinationObject': unquote(match.group('object')), + 'gcs.sourceObjects': ','.join( + ['%s/%s' % (unquote(match.group('bucket')), o.name) for o in data.get('sourceObjects', [])] + ) + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/copyTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.copy', + 'gcs.destinationBucket': unquote(match.group('destBucket')), + 'gcs.destinationObject': unquote(match.group('destObject')), + 'gcs.sourceBucket': unquote(match.group('srcBucket')), + 'gcs.sourceObject': unquote(match.group('srcObject')), + }, + re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { + 'gcs.op': 'objects.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': params.get('name', data.get('name', None)), + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/rewriteTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.rewrite', + 'gcs.destinationBucket': unquote(match.group('destBucket')), + 'gcs.destinationObject': unquote(match.group('destObject')), + 'gcs.sourceBucket': unquote(match.group('srcBucket')), + 'gcs.sourceObject': unquote(match.group('srcObject')), + } }, 'PATCH': { # Bucket operations @@ -56,6 +94,12 @@ 'gcs.op': 'buckets.patch', 'gcs.bucket': unquote(match.group('bucket')), }, + # Object/blob operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + } }, 'PUT': { # Bucket operations @@ -67,6 +111,12 @@ 'gcs.op': 'buckets.setIamPolicy', 'gcs.bucket': unquote(match.group('bucket')), }, + # Object/blob operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + } }, 'DELETE': { # Bucket operations @@ -74,6 +124,12 @@ 'gcs.op': 'buckets.delete', 'gcs.bucket': unquote(match.group('bucket')), }, + # Object/blob operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + } } } @@ -134,6 +190,59 @@ def execute_with_instana(wrapped, instance, args, kwargs): else: return kv + def download_with_instana(wrapped, instance, args, kwargs): + parent_span = tracer.active_span + + # return early if we're not tracing + if parent_span is None: + return wrapped(*args, **kwargs) + + with tracer.start_active_span('gcs', child_of=parent_span) as scope: + scope.span.set_tag('gcs.op', 'objects.get') + scope.span.set_tag('gcs.bucket', instance.bucket.name) + scope.span.set_tag('gcs.object', instance.name) + + start = len(args) > 4 and args[4] or kwargs.get('start', None) + if start is None: + start = '' + + end = len(args) > 5 and args[5] or kwargs.get('end', None) + if end is None: + end = '' + + if start != '' or end != '': + scope.span.set_tag('gcs.range', '-'.join((start, end))) + + try: + kv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_exception(e) + raise + else: + return kv + + def upload_with_instana(wrapped, instance, args, kwargs): + parent_span = tracer.active_span + + # return early if we're not tracing + if parent_span is None: + return wrapped(*args, **kwargs) + + with tracer.start_active_span('gcs', child_of=parent_span) as scope: + scope.span.set_tag('gcs.op', 'objects.insert') + scope.span.set_tag('gcs.bucket', instance.bucket.name) + scope.span.set_tag('gcs.object', instance.name) + + try: + kv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_exception(e) + raise + else: + return kv + wrapt.wrap_function_wrapper('google.cloud.storage._http', 'Connection.api_request', execute_with_instana) + wrapt.wrap_function_wrapper('google.cloud.storage.blob', 'Blob._do_download', download_with_instana) + wrapt.wrap_function_wrapper('google.cloud.storage.blob', 'Blob._do_upload', upload_with_instana) except ImportError: pass diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 7b4e26c7..02cdd17f 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -5,6 +5,10 @@ from instana.instrumentation.google.cloud.storage import collect_tags class TestGoogleCloudStorage(unittest.TestCase): + class Blob: + def __init__(self, name): + self.name = name + def test_collect_tags(self): test_cases = { 'buckets.list': ( @@ -110,6 +114,128 @@ def test_collect_tags(self): 'gcs.op': 'buckets.update', 'gcs.bucket': 'test bucket' } + ), + 'objects.compose': ( + { + 'method': 'POST', + 'path': '/b/test%20bucket/o/dest%20object/compose', + 'data': { + 'sourceObjects': [ + TestGoogleCloudStorage.Blob('object1'), + TestGoogleCloudStorage.Blob('object2'), + ] + } + }, + { + 'gcs.op': 'objects.compose', + 'gcs.destinationBucket': 'test bucket', + 'gcs.destinationObject': 'dest object', + 'gcs.sourceObjects': 'test bucket/object1,test bucket/object2' + } + ), + 'objects.copy': ( + { + 'method': 'POST', + 'path': '/b/src%20bucket/o/src%20object/copyTo/b/dest%20bucket/o/dest%20object' + }, + { + 'gcs.op': 'objects.copy', + 'gcs.destinationBucket': 'dest bucket', + 'gcs.destinationObject': 'dest object', + 'gcs.sourceBucket': 'src bucket', + 'gcs.sourceObject': 'src object' + } + ), + 'objects.delete': ( + { + 'method': 'DELETE', + 'path': '/b/test%20bucket/o/test%20object' + }, + { + 'gcs.op': 'objects.delete', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objects.attrs': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/o/test%20object' + }, + { + 'gcs.op': 'objects.attrs', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objects.get': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/o/test%20object', + 'query_params': {'alt': 'media'} + }, + { + 'gcs.op': 'objects.get', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objects.insert': ( + { + 'method': 'POST', + 'path': '/b/test%20bucket/o', + 'query_params': {'name': 'test object'} + }, + { + 'gcs.op': 'objects.insert', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objects.list': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/o' + }, + { + 'gcs.op': 'objects.list', + 'gcs.bucket': 'test bucket' + } + ), + 'objects.patch': ( + { + 'method': 'PATCH', + 'path': '/b/test%20bucket/o/test%20object' + }, + { + 'gcs.op': 'objects.patch', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objects.rewrite': ( + { + 'method': 'POST', + 'path': '/b/src%20bucket/o/src%20object/rewriteTo/b/dest%20bucket/o/dest%20object' + }, + { + 'gcs.op': 'objects.rewrite', + 'gcs.destinationBucket': 'dest bucket', + 'gcs.destinationObject': 'dest object', + 'gcs.sourceBucket': 'src bucket', + 'gcs.sourceObject': 'src object' + } + ), + 'objects.update': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket/o/test%20object' + }, + { + 'gcs.op': 'objects.update', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } ) } From 00803ed9c202207e21c297b3d2f923cbf06b91f9 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 16:37:56 +0200 Subject: [PATCH 04/16] Collect GCS API channel operation tags --- instana/instrumentation/google/cloud/storage.py | 5 +++++ tests/clients/test_google-cloud-storage.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index 019a1a4a..e5c0237b 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -86,6 +86,11 @@ 'gcs.destinationObject': unquote(match.group('destObject')), 'gcs.sourceBucket': unquote(match.group('srcBucket')), 'gcs.sourceObject': unquote(match.group('srcObject')), + }, + # Channel operations + '/channels/stop': lambda params, data: { + 'gcs.op': 'channels.stop', + 'gcs.entity': data.get('id', None) } }, 'PATCH': { diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 02cdd17f..75dc86b1 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -236,6 +236,17 @@ def test_collect_tags(self): 'gcs.bucket': 'test bucket', 'gcs.object': 'test object' } + ), + 'channels.stop': ( + { + 'method': 'POST', + 'path': '/channels/stop', + 'data': {'id': 'test channel'} + }, + { + 'gcs.op': 'channels.stop', + 'gcs.entity': 'test channel' + } ) } From 1668dbba209f7fd69ed7745024a4d4d2e4603715 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:00:40 +0200 Subject: [PATCH 05/16] Collect GCS API default object ACL operation tags --- .../instrumentation/google/cloud/storage.py | 34 ++++++++++ tests/clients/test_google-cloud-storage.py | 66 +++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index e5c0237b..61eac8c5 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -46,6 +46,16 @@ re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { 'gcs.op': 'objects.list', 'gcs.bucket': unquote(match.group('bucket')) + }, + # Default object ACLs operations + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.get', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) + }, + re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.list', + 'gcs.bucket': unquote(match.group('bucket')), } }, 'POST': { @@ -91,6 +101,12 @@ '/channels/stop': lambda params, data: { 'gcs.op': 'channels.stop', 'gcs.entity': data.get('id', None) + }, + # Default object ACLs operations + re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': data.get('entity', None) } }, 'PATCH': { @@ -104,6 +120,12 @@ 'gcs.op': 'objects.patch', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), + }, + # Default object ACLs operations + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) } }, 'PUT': { @@ -121,6 +143,12 @@ 'gcs.op': 'objects.update', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), + }, + # Default object ACLs operations + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) } }, 'DELETE': { @@ -134,6 +162,12 @@ 'gcs.op': 'objects.delete', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), + }, + # Default object ACLs operations + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) } } } diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 75dc86b1..c2af09d3 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -247,6 +247,72 @@ def test_collect_tags(self): 'gcs.op': 'channels.stop', 'gcs.entity': 'test channel' } + ), + 'defaultAcls.delete': ( + { + 'method': 'DELETE', + 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' + }, + { + 'gcs.op': 'defaultAcls.delete', + 'gcs.bucket': 'test bucket', + 'gcs.entity': 'user-test@example.com' + } + ), + 'defaultAcls.get': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' + }, + { + 'gcs.op': 'defaultAcls.get', + 'gcs.bucket': 'test bucket', + 'gcs.entity': 'user-test@example.com' + } + ), + 'defaultAcls.insert': ( + { + 'method': 'POST', + 'path': '/b/test%20bucket/defaultObjectAcl', + 'data': {'entity': 'user-test@example.com'} + }, + { + 'gcs.op': 'defaultAcls.insert', + 'gcs.bucket': 'test bucket', + 'gcs.entity': 'user-test@example.com' + } + ), + 'defaultAcls.list': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/defaultObjectAcl' + }, + { + 'gcs.op': 'defaultAcls.list', + 'gcs.bucket': 'test bucket' + } + ), + 'defaultAcls.patch': ( + { + 'method': 'PATCH', + 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' + }, + { + 'gcs.op': 'defaultAcls.patch', + 'gcs.bucket': 'test bucket', + 'gcs.entity': 'user-test@example.com' + } + ), + 'defaultAcls.update': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' + }, + { + 'gcs.op': 'defaultAcls.update', + 'gcs.bucket': 'test bucket', + 'gcs.entity': 'user-test@example.com' + } ) } From d8f091cd33d6a406d1f5515c89f4d3efc01a0321 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:13:51 +0200 Subject: [PATCH 06/16] Collect GCS API object ACL operation tags --- .../instrumentation/google/cloud/storage.py | 40 +++++++++++ tests/clients/test_google-cloud-storage.py | 72 +++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index 61eac8c5..d13f4663 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -56,6 +56,18 @@ re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { 'gcs.op': 'defaultAcls.list', 'gcs.bucket': unquote(match.group('bucket')), + }, + # Object ACL operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.get', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.list', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')) } }, 'POST': { @@ -107,6 +119,13 @@ 'gcs.op': 'defaultAcls.insert', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.entity': data.get('entity', None) + }, + # Object ACL operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': data.get('entity', None) } }, 'PATCH': { @@ -126,6 +145,13 @@ 'gcs.op': 'defaultAcls.patch', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.entity': unquote(match.group('entity')) + }, + # Object ACL operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) } }, 'PUT': { @@ -149,6 +175,13 @@ 'gcs.op': 'defaultAcls.update', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.entity': unquote(match.group('entity')) + }, + # Object ACL operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) } }, 'DELETE': { @@ -168,6 +201,13 @@ 'gcs.op': 'defaultAcls.delete', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.entity': unquote(match.group('entity')) + }, + # Object ACL operations + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) } } } diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index c2af09d3..577555e0 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -313,6 +313,78 @@ def test_collect_tags(self): 'gcs.bucket': 'test bucket', 'gcs.entity': 'user-test@example.com' } + ), + 'objectAcls.delete': ( + { + 'method': 'DELETE', + 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' + }, + { + 'gcs.op': 'objectAcls.delete', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object', + 'gcs.entity': 'user-test@example.com' + } + ), + 'objectAcls.get': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' + }, + { + 'gcs.op': 'objectAcls.get', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object', + 'gcs.entity': 'user-test@example.com' + } + ), + 'objectAcls.insert': ( + { + 'method': 'POST', + 'path': '/b/test%20bucket/o/test%20object/acl', + 'data': {'entity': 'user-test@example.com'} + }, + { + 'gcs.op': 'objectAcls.insert', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object', + 'gcs.entity': 'user-test@example.com' + } + ), + 'objectAcls.list': ( + { + 'method': 'GET', + 'path': '/b/test%20bucket/o/test%20object/acl' + }, + { + 'gcs.op': 'objectAcls.list', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object' + } + ), + 'objectAcls.patch': ( + { + 'method': 'PATCH', + 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' + }, + { + 'gcs.op': 'objectAcls.patch', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object', + 'gcs.entity': 'user-test@example.com' + } + ), + 'objectAcls.update': ( + { + 'method': 'PUT', + 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' + }, + { + 'gcs.op': 'objectAcls.update', + 'gcs.bucket': 'test bucket', + 'gcs.object': 'test object', + 'gcs.entity': 'user-test@example.com' + } ) } From efad4da2e28ced80186ca2d37983e8ed53baa7a0 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:31:43 +0200 Subject: [PATCH 07/16] Collect GCS API HMAC keys operation tags --- .../instrumentation/google/cloud/storage.py | 27 ++++++++++ tests/clients/test_google-cloud-storage.py | 53 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index d13f4663..94de56e8 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -68,6 +68,16 @@ 'gcs.op': 'objectAcls.list', 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')) + }, + # HMAC keys operations + re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.list', + 'gcs.projectId': unquote(match.group('project')) + }, + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.get', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) } }, 'POST': { @@ -126,6 +136,11 @@ 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), 'gcs.entity': data.get('entity', None) + }, + # HMAC keys operations + re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.create', + 'gcs.projectId': unquote(match.group('project')) } }, 'PATCH': { @@ -182,6 +197,12 @@ 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), 'gcs.entity': unquote(match.group('entity')) + }, + # HMAC keys operations + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.update', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) } }, 'DELETE': { @@ -208,6 +229,12 @@ 'gcs.bucket': unquote(match.group('bucket')), 'gcs.object': unquote(match.group('object')), 'gcs.entity': unquote(match.group('entity')) + }, + # HMAC keys operations + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.delete', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) } } } diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 577555e0..48c50fcd 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -385,6 +385,59 @@ def test_collect_tags(self): 'gcs.object': 'test object', 'gcs.entity': 'user-test@example.com' } + ), + 'hmacKeys.create': ( + { + 'method': 'POST', + 'path': '/projects/test%20project/hmacKeys' + }, + { + 'gcs.op': 'hmacKeys.create', + 'gcs.projectId': 'test project' + } + ), + 'hmacKeys.delete': ( + { + 'method': 'DELETE', + 'path': '/projects/test%20project/hmacKeys/test%20key' + }, + { + 'gcs.op': 'hmacKeys.delete', + 'gcs.projectId': 'test project', + 'gcs.accessId': 'test key' + } + ), + 'hmacKeys.get': ( + { + 'method': 'GET', + 'path': '/projects/test%20project/hmacKeys/test%20key' + }, + { + 'gcs.op': 'hmacKeys.get', + 'gcs.projectId': 'test project', + 'gcs.accessId': 'test key' + } + ), + 'hmacKeys.list': ( + { + 'method': 'GET', + 'path': '/projects/test%20project/hmacKeys' + }, + { + 'gcs.op': 'hmacKeys.list', + 'gcs.projectId': 'test project' + } + ), + 'hmacKeys.update': ( + { + 'method': 'PUT', + 'path': '/projects/test%20project/hmacKeys/test%20key' + }, + { + 'gcs.op': 'hmacKeys.update', + 'gcs.projectId': 'test project', + 'gcs.accessId': 'test key' + } ) } From 074de2d99cea8afa810d09abac0ff9ca7585c9f5 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:33:38 +0200 Subject: [PATCH 08/16] Collect GCS API service account operation tags --- instana/instrumentation/google/cloud/storage.py | 5 +++++ tests/clients/test_google-cloud-storage.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index 94de56e8..a3aed01d 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -78,6 +78,11 @@ 'gcs.op': 'hmacKeys.get', 'gcs.projectId': unquote(match.group('project')), 'gcs.accessId': unquote(match.group('accessId')) + }, + # Service account operations + re.compile('^/projects/(?P[^/]+)/serviceAccount$'): lambda params, data, match: { + 'gcs.op': 'serviceAccount.get', + 'gcs.projectId': unquote(match.group('project')) } }, 'POST': { diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 48c50fcd..34f394bd 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -438,6 +438,16 @@ def test_collect_tags(self): 'gcs.projectId': 'test project', 'gcs.accessId': 'test key' } + ), + 'serviceAccount.get': ( + { + 'method': 'GET', + 'path': '/projects/test%20project/serviceAccount' + }, + { + 'gcs.op': 'serviceAccount.get', + 'gcs.projectId': 'test project' + } ) } From 38dc44d4236695fa74a62769247c31d30b7b4c94 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Thu, 24 Sep 2020 12:58:10 +0200 Subject: [PATCH 09/16] Collect GCS batch operation tags --- .../instrumentation/google/cloud/storage.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index a3aed01d..4afddfa8 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -276,6 +276,10 @@ def collect_tags(api_request): return collect(params, data, m) def execute_with_instana(wrapped, instance, args, kwargs): + # batch requests are traced with finish_batch_with_instana() + if isinstance(instance, storage.Batch): + return wrapped(*args, **kwargs) + parent_span = tracer.active_span # return early if we're not tracing @@ -352,8 +356,29 @@ def upload_with_instana(wrapped, instance, args, kwargs): else: return kv + def finish_batch_with_instana(wrapped, instance, args, kwargs): + parent_span = tracer.active_span + + # return early if we're not tracing + if parent_span is None: + return wrapped(*args, **kwargs) + + with tracer.start_active_span('gcs', child_of=parent_span) as scope: + scope.span.set_tag('gcs.op', 'batch') + scope.span.set_tag('gcs.projectId', instance._client.project) + scope.span.set_tag('gcs.numberOfOperations', len(instance._requests)) + + try: + kv = wrapped(*args, **kwargs) + except Exception as e: + scope.span.log_exception(e) + raise + else: + return kv + wrapt.wrap_function_wrapper('google.cloud.storage._http', 'Connection.api_request', execute_with_instana) wrapt.wrap_function_wrapper('google.cloud.storage.blob', 'Blob._do_download', download_with_instana) wrapt.wrap_function_wrapper('google.cloud.storage.blob', 'Blob._do_upload', upload_with_instana) + wrapt.wrap_function_wrapper('google.cloud.storage.batch', 'Batch.finish', finish_batch_with_instana) except ImportError: pass From 83de35d4b1310b4fbfb0598d03a78bbec71ea643 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:42:55 +0200 Subject: [PATCH 10/16] Move GCS API tag collectors into a separate file --- .../google/cloud/collectors.py | 317 ++++++++++++++++++ .../instrumentation/google/cloud/storage.py | 271 ++------------- 2 files changed, 337 insertions(+), 251 deletions(-) create mode 100644 instana/instrumentation/google/cloud/collectors.py diff --git a/instana/instrumentation/google/cloud/collectors.py b/instana/instrumentation/google/cloud/collectors.py new file mode 100644 index 00000000..a6d4a9e3 --- /dev/null +++ b/instana/instrumentation/google/cloud/collectors.py @@ -0,0 +1,317 @@ +import re + +try: + # Python 3 + from urllib.parse import unquote +except ImportError: + # Python 2 + from urllib import unquote + +# _storage_api defines a conversion of Google Storage JSON API requests into span tags as follows: +# request_method -> path_matcher -> collector +# +# * request method - the HTTP method used to make an API request (GET, POST, etc.) +# * path_matcher - either a string or a regex applied to the API request path (string values match first). +# * collector - a lambda returning a dict of span from API request query string. +# parameters and request body data. If a regex is used as a path matcher, the match result +# will be provided as a third argument. +# +# The API documentation can be found at https://cloud.google.com/storage/docs/json_api +_storage_api = { + 'GET': { + ##################### + # Bucket operations # + ##################### + '/b': lambda params, data: { + 'gcs.op': 'buckets.list', + 'gcs.projectId': params.get('project', None) + }, + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.get', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { + 'gcs.op': 'buckets.getIamPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam/testPermissions$'): lambda params, data, match: { + 'gcs.op': 'buckets.testIamPermissions', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ########################## + # Object/blob operations # + ########################## + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': params.get('alt', 'json') == 'media' and 'objects.get' or 'objects.attrs', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + }, + re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { + 'gcs.op': 'objects.list', + 'gcs.bucket': unquote(match.group('bucket')) + }, + + ################################## + # Default object ACLs operations # + ################################## + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.get', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) + }, + re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.list', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ######################### + # Object ACL operations # + ######################### + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.get', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.list', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')) + }, + + ######################## + # HMAC keys operations # + ######################## + re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.list', + 'gcs.projectId': unquote(match.group('project')) + }, + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.get', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) + }, + + ############################## + # Service account operations # + ############################## + re.compile('^/projects/(?P[^/]+)/serviceAccount$'): lambda params, data, match: { + 'gcs.op': 'serviceAccount.get', + 'gcs.projectId': unquote(match.group('project')) + } + }, + 'POST': { + ##################### + # Bucket operations # + ##################### + '/b': lambda params, data: { + 'gcs.op': 'buckets.insert', + 'gcs.projectId': params.get('project', None), + 'gcs.bucket': data.get('name', None), + }, + re.compile('^/b/(?P[^/]+)/lockRetentionPolicy$'): lambda params, data, match: { + 'gcs.op': 'buckets.lockRetentionPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ########################## + # Object/blob operations # + ########################## + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/compose$'): lambda params, data, match: { + 'gcs.op': 'objects.compose', + 'gcs.destinationBucket': unquote(match.group('bucket')), + 'gcs.destinationObject': unquote(match.group('object')), + 'gcs.sourceObjects': ','.join( + ['%s/%s' % (unquote(match.group('bucket')), o['name']) for o in data.get('sourceObjects', []) if 'name' in o] + ) + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/copyTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.copy', + 'gcs.destinationBucket': unquote(match.group('destBucket')), + 'gcs.destinationObject': unquote(match.group('destObject')), + 'gcs.sourceBucket': unquote(match.group('srcBucket')), + 'gcs.sourceObject': unquote(match.group('srcObject')), + }, + re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { + 'gcs.op': 'objects.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': params.get('name', data.get('name', None)), + }, + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/rewriteTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.rewrite', + 'gcs.destinationBucket': unquote(match.group('destBucket')), + 'gcs.destinationObject': unquote(match.group('destObject')), + 'gcs.sourceBucket': unquote(match.group('srcBucket')), + 'gcs.sourceObject': unquote(match.group('srcObject')), + }, + + ###################### + # Channel operations # + ###################### + '/channels/stop': lambda params, data: { + 'gcs.op': 'channels.stop', + 'gcs.entity': data.get('id', None) + }, + + ################################## + # Default object ACLs operations # + ################################## + re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': data.get('entity', None) + }, + + ######################### + # Object ACL operations # + ######################### + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.insert', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': data.get('entity', None) + }, + + ######################## + # HMAC keys operations # + ######################## + re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.create', + 'gcs.projectId': unquote(match.group('project')) + } + }, + 'PATCH': { + ##################### + # Bucket operations # + ##################### + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.patch', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ########################## + # Object/blob operations # + ########################## + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + }, + + ################################## + # Default object ACLs operations # + ################################## + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) + }, + + ######################### + # Object ACL operations # + ######################### + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.patch', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) + } + }, + 'PUT': { + ##################### + # Bucket operations # + ##################### + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.update', + 'gcs.bucket': unquote(match.group('bucket')), + }, + re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { + 'gcs.op': 'buckets.setIamPolicy', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ########################## + # Object/blob operations # + ########################## + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + }, + + ################################## + # Default object ACLs operations # + ################################## + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) + }, + + ######################### + # Object ACL operations # + ######################### + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.update', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) + }, + + ######################## + # HMAC keys operations # + ######################## + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.update', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) + } + }, + 'DELETE': { + ##################### + # Bucket operations # + ##################### + re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'buckets.delete', + 'gcs.bucket': unquote(match.group('bucket')), + }, + + ########################## + # Object/blob operations # + ########################## + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objects.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + }, + + ################################## + # Default object ACLs operations # + ################################## + re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'defaultAcls.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.entity': unquote(match.group('entity')) + }, + + ######################### + # Object ACL operations # + ######################### + re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'objectAcls.delete', + 'gcs.bucket': unquote(match.group('bucket')), + 'gcs.object': unquote(match.group('object')), + 'gcs.entity': unquote(match.group('entity')) + }, + + ######################## + # HMAC keys operations # + ######################## + re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { + 'gcs.op': 'hmacKeys.delete', + 'gcs.projectId': unquote(match.group('project')), + 'gcs.accessId': unquote(match.group('accessId')) + } + } +} diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index 4afddfa8..e45388db 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -2,249 +2,15 @@ import wrapt import re -from urllib.parse import unquote from ....log import logger from ....singletons import tracer +from .collectors import _storage_api try: from google.cloud import storage - # A map of GCS API param collectors organized as: - # request_method -> path_matcher -> collector - # - # * request method - the HTTP method used to make an API request (GET, POST, etc.) - # * path_matcher - either a string or a regex applied to the API request path (string values match first). - # * collector - a lambda returning a dict of span from API request query string. - # parameters and request body data. If a regex is used as a path matcher, the match result - # will be provided as a third argument. - _storage_api = { - 'GET': { - # Bucket operations - '/b': lambda params, data: { - 'gcs.op': 'buckets.list', - 'gcs.projectId': params.get('project', None) - }, - re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'buckets.get', - 'gcs.bucket': unquote(match.group('bucket')), - }, - re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { - 'gcs.op': 'buckets.getIamPolicy', - 'gcs.bucket': unquote(match.group('bucket')), - }, - re.compile('^/b/(?P[^/]+)/iam/testPermissions$'): lambda params, data, match: { - 'gcs.op': 'buckets.testIamPermissions', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object/blob operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': params.get('alt', 'json') == 'media' and 'objects.get' or 'objects.attrs', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - }, - re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { - 'gcs.op': 'objects.list', - 'gcs.bucket': unquote(match.group('bucket')) - }, - # Default object ACLs operations - re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.get', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.entity': unquote(match.group('entity')) - }, - re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.list', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object ACL operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.get', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - 'gcs.entity': unquote(match.group('entity')) - }, - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.list', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')) - }, - # HMAC keys operations - re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { - 'gcs.op': 'hmacKeys.list', - 'gcs.projectId': unquote(match.group('project')) - }, - re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'hmacKeys.get', - 'gcs.projectId': unquote(match.group('project')), - 'gcs.accessId': unquote(match.group('accessId')) - }, - # Service account operations - re.compile('^/projects/(?P[^/]+)/serviceAccount$'): lambda params, data, match: { - 'gcs.op': 'serviceAccount.get', - 'gcs.projectId': unquote(match.group('project')) - } - }, - 'POST': { - # Bucket operations - '/b': lambda params, data: { - 'gcs.op': 'buckets.insert', - 'gcs.projectId': params.get('project', None), - 'gcs.bucket': data.get('name', None), - }, - re.compile('^/b/(?P[^/]+)/lockRetentionPolicy$'): lambda params, data, match: { - 'gcs.op': 'buckets.lockRetentionPolicy', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object/blob operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/compose$'): lambda params, data, match: { - 'gcs.op': 'objects.compose', - 'gcs.destinationBucket': unquote(match.group('bucket')), - 'gcs.destinationObject': unquote(match.group('object')), - 'gcs.sourceObjects': ','.join( - ['%s/%s' % (unquote(match.group('bucket')), o.name) for o in data.get('sourceObjects', [])] - ) - }, - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/copyTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objects.copy', - 'gcs.destinationBucket': unquote(match.group('destBucket')), - 'gcs.destinationObject': unquote(match.group('destObject')), - 'gcs.sourceBucket': unquote(match.group('srcBucket')), - 'gcs.sourceObject': unquote(match.group('srcObject')), - }, - re.compile('^/b/(?P[^/]+)/o$'): lambda params, data, match: { - 'gcs.op': 'objects.insert', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': params.get('name', data.get('name', None)), - }, - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/rewriteTo/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objects.rewrite', - 'gcs.destinationBucket': unquote(match.group('destBucket')), - 'gcs.destinationObject': unquote(match.group('destObject')), - 'gcs.sourceBucket': unquote(match.group('srcBucket')), - 'gcs.sourceObject': unquote(match.group('srcObject')), - }, - # Channel operations - '/channels/stop': lambda params, data: { - 'gcs.op': 'channels.stop', - 'gcs.entity': data.get('id', None) - }, - # Default object ACLs operations - re.compile('^/b/(?P[^/]+)/defaultObjectAcl$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.insert', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.entity': data.get('entity', None) - }, - # Object ACL operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.insert', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - 'gcs.entity': data.get('entity', None) - }, - # HMAC keys operations - re.compile('^/projects/(?P[^/]+)/hmacKeys$'): lambda params, data, match: { - 'gcs.op': 'hmacKeys.create', - 'gcs.projectId': unquote(match.group('project')) - } - }, - 'PATCH': { - # Bucket operations - re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'buckets.patch', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object/blob operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objects.patch', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - }, - # Default object ACLs operations - re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.patch', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.entity': unquote(match.group('entity')) - }, - # Object ACL operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.patch', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - 'gcs.entity': unquote(match.group('entity')) - } - }, - 'PUT': { - # Bucket operations - re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'buckets.update', - 'gcs.bucket': unquote(match.group('bucket')), - }, - re.compile('^/b/(?P[^/]+)/iam$'): lambda params, data, match: { - 'gcs.op': 'buckets.setIamPolicy', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object/blob operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objects.update', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - }, - # Default object ACLs operations - re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.update', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.entity': unquote(match.group('entity')) - }, - # Object ACL operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.update', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - 'gcs.entity': unquote(match.group('entity')) - }, - # HMAC keys operations - re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'hmacKeys.update', - 'gcs.projectId': unquote(match.group('project')), - 'gcs.accessId': unquote(match.group('accessId')) - } - }, - 'DELETE': { - # Bucket operations - re.compile('^/b/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'buckets.delete', - 'gcs.bucket': unquote(match.group('bucket')), - }, - # Object/blob operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objects.delete', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - }, - # Default object ACLs operations - re.compile('^/b/(?P[^/]+)/defaultObjectAcl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'defaultAcls.delete', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.entity': unquote(match.group('entity')) - }, - # Object ACL operations - re.compile('^/b/(?P[^/]+)/o/(?P[^/]+)/acl/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'objectAcls.delete', - 'gcs.bucket': unquote(match.group('bucket')), - 'gcs.object': unquote(match.group('object')), - 'gcs.entity': unquote(match.group('entity')) - }, - # HMAC keys operations - re.compile('^/projects/(?P[^/]+)/hmacKeys/(?P[^/]+)$'): lambda params, data, match: { - 'gcs.op': 'hmacKeys.delete', - 'gcs.projectId': unquote(match.group('project')), - 'gcs.accessId': unquote(match.group('accessId')) - } - } - } - - def collect_tags(api_request): + def _collect_tags(api_request): """ Extract span tags from Google Cloud Storage API request. Returns None if the request is not supported. @@ -257,23 +23,26 @@ def collect_tags(api_request): if method not in _storage_api: return - params = api_request.get('query_params', {}) - data = api_request.get('data', {}) + try: + params = api_request.get('query_params', {}) + data = api_request.get('data', {}) - if path in _storage_api[method]: - # check is any of string keys matches the path exactly - return _storage_api[method][path](params, data) - else: - # look for a regex that matches the string - for (matcher, collect) in _storage_api[method].items(): - if not isinstance(matcher, re.Pattern): - continue + if path in _storage_api[method]: + # check is any of string keys matches the path exactly + return _storage_api[method][path](params, data) + else: + # look for a regex that matches the string + for (matcher, collect) in _storage_api[method].items(): + if not isinstance(matcher, re.Pattern): + continue - m = matcher.match(path) - if m is None: - continue + m = matcher.match(path) + if m is None: + continue - return collect(params, data, m) + return collect(params, data, m) + except Exception: + logger.debug("instana.instrumentation.google.cloud.storage._collect_tags: ", exc_info=True) def execute_with_instana(wrapped, instance, args, kwargs): # batch requests are traced with finish_batch_with_instana() @@ -286,7 +55,7 @@ def execute_with_instana(wrapped, instance, args, kwargs): if parent_span is None: return wrapped(*args, **kwargs) - tags = collect_tags(kwargs) + tags = _collect_tags(kwargs) # don't trace if the call is not instrumented if tags is None: From c5190c72eaf85c3d72629e29f5796354fe20a452 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 17:46:48 +0200 Subject: [PATCH 11/16] Auto-instrument google-cloud-storage client --- instana/__init__.py | 1 + instana/instrumentation/google/cloud/storage.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/instana/__init__.py b/instana/__init__.py index f484a23f..eafb26b6 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -155,6 +155,7 @@ def boot_agent(): from .instrumentation import urllib3 from .instrumentation.django import middleware from .instrumentation import pymongo + from .instrumentation.google.cloud import storage # Hooks from .hooks import hook_uwsgi diff --git a/instana/instrumentation/google/cloud/storage.py b/instana/instrumentation/google/cloud/storage.py index e45388db..f3748893 100644 --- a/instana/instrumentation/google/cloud/storage.py +++ b/instana/instrumentation/google/cloud/storage.py @@ -10,6 +10,8 @@ try: from google.cloud import storage + logger.debug('Instrumenting google-cloud-storage') + def _collect_tags(api_request): """ Extract span tags from Google Cloud Storage API request. Returns None if the request is not From 4138347b77c6515fefc77b1d8d16b35413abbd09 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Wed, 23 Sep 2020 18:13:36 +0200 Subject: [PATCH 12/16] Instrument google-cloud-storage for Python 3 only The library has dropped support for Python 2 --- instana/__init__.py | 4 +++- tests/clients/test_google-cloud-storage.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/instana/__init__.py b/instana/__init__.py index eafb26b6..394f1f8e 100644 --- a/instana/__init__.py +++ b/instana/__init__.py @@ -137,6 +137,9 @@ def boot_agent(): else: from .instrumentation import mysqlclient + if sys.version_info[0] >= 3: + from .instrumentation.google.cloud import storage + from .instrumentation.celery import hooks from .instrumentation import cassandra_inst @@ -155,7 +158,6 @@ def boot_agent(): from .instrumentation import urllib3 from .instrumentation.django import middleware from .instrumentation import pymongo - from .instrumentation.google.cloud import storage # Hooks from .hooks import hook_uwsgi diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 34f394bd..54c9a480 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -1,9 +1,13 @@ from __future__ import absolute_import +import sys import unittest +import pytest -from instana.instrumentation.google.cloud.storage import collect_tags +if sys.version_info[0] >= 3: + from instana.instrumentation.google.cloud.storage import collect_tags +@pytest.mark.skipif(sys.version_info[0] < 3, reason="google-cloud-storage has dropped support for Python 2") class TestGoogleCloudStorage(unittest.TestCase): class Blob: def __init__(self, name): From f34b0e16883f83c4a128268119b699a0bf0d5d22 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Thu, 24 Sep 2020 16:12:15 +0200 Subject: [PATCH 13/16] Register GCS span format --- instana/recorder.py | 2 +- instana/span.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/instana/recorder.py b/instana/recorder.py index 437c1862..3034ae12 100644 --- a/instana/recorder.py +++ b/instana/recorder.py @@ -18,7 +18,7 @@ class StanRecorder(object): THREAD_NAME = "Instana Span Reporting" REGISTERED_SPANS = ("aiohttp-client", "aiohttp-server", "aws.lambda.entry", "cassandra", - "celery-client", "celery-worker", "couchbase", "django", "log", + "celery-client", "celery-worker", "couchbase", "django", "gcs", "log", "memcache", "mongo", "mysql", "postgres", "pymongo", "rabbitmq", "redis", "render", "rpc-client", "rpc-server", "sqlalchemy", "soap", "tornado-client", "tornado-server", "urllib3", "wsgi") diff --git a/instana/span.py b/instana/span.py index 2b5ac4cf..96d38881 100644 --- a/instana/span.py +++ b/instana/span.py @@ -232,7 +232,7 @@ class RegisteredSpan(BaseSpan): EXIT_SPANS = ("aiohttp-client", "cassandra", "celery-client", "couchbase", "log", "memcache", "mongo", "mysql", "postgres", "rabbitmq", "redis", "rpc-client", "sqlalchemy", - "soap", "tornado-client", "urllib3", "pymongo") + "soap", "tornado-client", "urllib3", "pymongo", "gcs") ENTRY_SPANS = ("aiohttp-server", "aws.lambda.entry", "celery-worker", "django", "wsgi", "rabbitmq", "rpc-server", "tornado-server") @@ -420,6 +420,21 @@ def _populate_exit_span_data(self, span): self.data["mongo"]["json"] = span.tags.pop('json', None) self.data["mongo"]["error"] = span.tags.pop('error', None) + elif span.operation_name == "gcs": + self.data["gcs"]["op"] = span.tags.pop('gcs.op') + self.data["gcs"]["bucket"] = span.tags.pop('gcs.bucket', None) + self.data["gcs"]["object"] = span.tags.pop('gcs.object', None) + self.data["gcs"]["entity"] = span.tags.pop('gcs.entity', None) + self.data["gcs"]["range"] = span.tags.pop('gcs.range', None) + self.data["gcs"]["sourceBucket"] = span.tags.pop('gcs.sourceBucket', None) + self.data["gcs"]["sourceObject"] = span.tags.pop('gcs.sourceObject', None) + self.data["gcs"]["sourceObjects"] = span.tags.pop('gcs.sourceObjects', None) + self.data["gcs"]["destinationBucket"] = span.tags.pop('gcs.destinationBucket', None) + self.data["gcs"]["destinationObject"] = span.tags.pop('gcs.destinationObject', None) + self.data["gcs"]["numberOfOperations"] = span.tags.pop('gcs.numberOfOperations', None) + self.data["gcs"]["projectId"] = span.tags.pop('gcs.projectId', None) + self.data["gcs"]["accessId"] = span.tags.pop('gcs.accessId', None) + elif span.operation_name == "log": # use last special key values for l in span.logs: From d6840713f34c7b8a21596e76f17deb92f3784ab8 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Thu, 24 Sep 2020 16:53:02 +0200 Subject: [PATCH 14/16] Add trace context propagation test util --- tests/test_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6c5539cd..1f3b61ec 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -22,3 +22,8 @@ def test_validate_url(): assert(validate_url("http:boligrafo") is False) assert(validate_url(None) is False) +class _TraceContextMixin: + def assertTraceContextPropagated(self, parent_span, child_span): + self.assertEqual(parent_span.t, child_span.t) + self.assertEqual(parent_span.s, child_span.p) + self.assertNotEqual(parent_span.s, child_span.s) From 36f5bba7638b2e694204562473a95d0b2b8ae699 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Thu, 24 Sep 2020 19:21:28 +0200 Subject: [PATCH 15/16] Add Google Cloud Storage instrumentation tests --- tests/clients/test_google-cloud-storage.py | 1375 +++++++++++++------- 1 file changed, 927 insertions(+), 448 deletions(-) diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 54c9a480..5f9d544d 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -3,457 +3,936 @@ import sys import unittest import pytest +import json +import requests + +from instana.singletons import tracer +from ..test_utils import _TraceContextMixin + +from mock import patch, Mock +from six.moves import http_client if sys.version_info[0] >= 3: - from instana.instrumentation.google.cloud.storage import collect_tags + from google.cloud import storage + from google.api_core import iam @pytest.mark.skipif(sys.version_info[0] < 3, reason="google-cloud-storage has dropped support for Python 2") -class TestGoogleCloudStorage(unittest.TestCase): - class Blob: - def __init__(self, name): - self.name = name - - def test_collect_tags(self): - test_cases = { - 'buckets.list': ( - { - 'method': 'GET', - 'path': '/b', - 'query_params': {'project': 'test-project'} - }, - { - 'gcs.op': 'buckets.list', - 'gcs.projectId': 'test-project' - } - ), - 'buckets.insert':( - { - 'method': 'POST', - 'path': '/b', - 'query_params': {'project': 'test-project'}, - 'data': {'name': 'test bucket'} - }, - { - 'gcs.op': 'buckets.insert', - 'gcs.projectId': 'test-project', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.get': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket' - }, - { - 'gcs.op': 'buckets.get', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.patch': ( - { - 'method': 'PATCH', - 'path': '/b/test%20bucket' - }, - { - 'gcs.op': 'buckets.patch', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.update': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket' - }, - { - 'gcs.op': 'buckets.update', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.getIamPolicy': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/iam' - }, - { - 'gcs.op': 'buckets.getIamPolicy', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.setIamPolicy': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket/iam' - }, - { - 'gcs.op': 'buckets.setIamPolicy', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.testIamPermissions': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/iam/testPermissions' - }, - { - 'gcs.op': 'buckets.testIamPermissions', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.lockRetentionPolicy': ( - { - 'method': 'POST', - 'path': '/b/test%20bucket/lockRetentionPolicy' - }, - { - 'gcs.op': 'buckets.lockRetentionPolicy', - 'gcs.bucket': 'test bucket' - } - ), - 'buckets.delete': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket' - }, - { - 'gcs.op': 'buckets.update', - 'gcs.bucket': 'test bucket' - } - ), - 'objects.compose': ( - { - 'method': 'POST', - 'path': '/b/test%20bucket/o/dest%20object/compose', - 'data': { - 'sourceObjects': [ - TestGoogleCloudStorage.Blob('object1'), - TestGoogleCloudStorage.Blob('object2'), - ] - } - }, - { - 'gcs.op': 'objects.compose', - 'gcs.destinationBucket': 'test bucket', - 'gcs.destinationObject': 'dest object', - 'gcs.sourceObjects': 'test bucket/object1,test bucket/object2' - } - ), - 'objects.copy': ( - { - 'method': 'POST', - 'path': '/b/src%20bucket/o/src%20object/copyTo/b/dest%20bucket/o/dest%20object' - }, - { - 'gcs.op': 'objects.copy', - 'gcs.destinationBucket': 'dest bucket', - 'gcs.destinationObject': 'dest object', - 'gcs.sourceBucket': 'src bucket', - 'gcs.sourceObject': 'src object' - } - ), - 'objects.delete': ( - { - 'method': 'DELETE', - 'path': '/b/test%20bucket/o/test%20object' - }, - { - 'gcs.op': 'objects.delete', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objects.attrs': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/o/test%20object' - }, - { - 'gcs.op': 'objects.attrs', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objects.get': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/o/test%20object', - 'query_params': {'alt': 'media'} - }, - { - 'gcs.op': 'objects.get', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objects.insert': ( - { - 'method': 'POST', - 'path': '/b/test%20bucket/o', - 'query_params': {'name': 'test object'} - }, - { - 'gcs.op': 'objects.insert', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objects.list': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/o' - }, - { - 'gcs.op': 'objects.list', - 'gcs.bucket': 'test bucket' - } - ), - 'objects.patch': ( - { - 'method': 'PATCH', - 'path': '/b/test%20bucket/o/test%20object' - }, - { - 'gcs.op': 'objects.patch', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objects.rewrite': ( - { - 'method': 'POST', - 'path': '/b/src%20bucket/o/src%20object/rewriteTo/b/dest%20bucket/o/dest%20object' - }, - { - 'gcs.op': 'objects.rewrite', - 'gcs.destinationBucket': 'dest bucket', - 'gcs.destinationObject': 'dest object', - 'gcs.sourceBucket': 'src bucket', - 'gcs.sourceObject': 'src object' - } - ), - 'objects.update': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket/o/test%20object' - }, - { - 'gcs.op': 'objects.update', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'channels.stop': ( - { - 'method': 'POST', - 'path': '/channels/stop', - 'data': {'id': 'test channel'} - }, - { - 'gcs.op': 'channels.stop', - 'gcs.entity': 'test channel' - } - ), - 'defaultAcls.delete': ( - { - 'method': 'DELETE', - 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' - }, - { - 'gcs.op': 'defaultAcls.delete', - 'gcs.bucket': 'test bucket', - 'gcs.entity': 'user-test@example.com' - } - ), - 'defaultAcls.get': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' - }, - { - 'gcs.op': 'defaultAcls.get', - 'gcs.bucket': 'test bucket', - 'gcs.entity': 'user-test@example.com' - } - ), - 'defaultAcls.insert': ( - { - 'method': 'POST', - 'path': '/b/test%20bucket/defaultObjectAcl', - 'data': {'entity': 'user-test@example.com'} - }, - { - 'gcs.op': 'defaultAcls.insert', - 'gcs.bucket': 'test bucket', - 'gcs.entity': 'user-test@example.com' - } - ), - 'defaultAcls.list': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/defaultObjectAcl' - }, - { - 'gcs.op': 'defaultAcls.list', - 'gcs.bucket': 'test bucket' - } - ), - 'defaultAcls.patch': ( - { - 'method': 'PATCH', - 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' - }, - { - 'gcs.op': 'defaultAcls.patch', - 'gcs.bucket': 'test bucket', - 'gcs.entity': 'user-test@example.com' - } - ), - 'defaultAcls.update': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket/defaultObjectAcl/user-test%40example.com' - }, - { - 'gcs.op': 'defaultAcls.update', - 'gcs.bucket': 'test bucket', - 'gcs.entity': 'user-test@example.com' - } - ), - 'objectAcls.delete': ( - { - 'method': 'DELETE', - 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' - }, - { - 'gcs.op': 'objectAcls.delete', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object', - 'gcs.entity': 'user-test@example.com' - } - ), - 'objectAcls.get': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' - }, - { - 'gcs.op': 'objectAcls.get', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object', - 'gcs.entity': 'user-test@example.com' - } - ), - 'objectAcls.insert': ( - { - 'method': 'POST', - 'path': '/b/test%20bucket/o/test%20object/acl', - 'data': {'entity': 'user-test@example.com'} - }, - { - 'gcs.op': 'objectAcls.insert', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object', - 'gcs.entity': 'user-test@example.com' - } - ), - 'objectAcls.list': ( - { - 'method': 'GET', - 'path': '/b/test%20bucket/o/test%20object/acl' - }, - { - 'gcs.op': 'objectAcls.list', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object' - } - ), - 'objectAcls.patch': ( - { - 'method': 'PATCH', - 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' - }, - { - 'gcs.op': 'objectAcls.patch', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object', - 'gcs.entity': 'user-test@example.com' - } - ), - 'objectAcls.update': ( - { - 'method': 'PUT', - 'path': '/b/test%20bucket/o/test%20object/acl/user-test%40example.com' - }, - { - 'gcs.op': 'objectAcls.update', - 'gcs.bucket': 'test bucket', - 'gcs.object': 'test object', - 'gcs.entity': 'user-test@example.com' - } - ), - 'hmacKeys.create': ( - { - 'method': 'POST', - 'path': '/projects/test%20project/hmacKeys' - }, - { - 'gcs.op': 'hmacKeys.create', - 'gcs.projectId': 'test project' - } - ), - 'hmacKeys.delete': ( - { - 'method': 'DELETE', - 'path': '/projects/test%20project/hmacKeys/test%20key' - }, - { - 'gcs.op': 'hmacKeys.delete', - 'gcs.projectId': 'test project', - 'gcs.accessId': 'test key' - } - ), - 'hmacKeys.get': ( - { - 'method': 'GET', - 'path': '/projects/test%20project/hmacKeys/test%20key' - }, - { - 'gcs.op': 'hmacKeys.get', - 'gcs.projectId': 'test project', - 'gcs.accessId': 'test key' - } - ), - 'hmacKeys.list': ( - { - 'method': 'GET', - 'path': '/projects/test%20project/hmacKeys' - }, - { - 'gcs.op': 'hmacKeys.list', - 'gcs.projectId': 'test project' - } - ), - 'hmacKeys.update': ( - { - 'method': 'PUT', - 'path': '/projects/test%20project/hmacKeys/test%20key' - }, - { - 'gcs.op': 'hmacKeys.update', - 'gcs.projectId': 'test project', - 'gcs.accessId': 'test key' - } - ), - 'serviceAccount.get': ( - { - 'method': 'GET', - 'path': '/projects/test%20project/serviceAccount' - }, - { - 'gcs.op': 'serviceAccount.get', - 'gcs.projectId': 'test project' - } +class TestGoogleCloudStorage(unittest.TestCase, _TraceContextMixin): + def setUp(self): + self.recorder = tracer.recorder + self.recorder.clear_spans() + + @patch('requests.Session.request') + def test_buckets_list(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#buckets", "items": []}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + buckets = client.list_buckets() + self.assertEqual(0, self.recorder.queue_size(), msg='span has been created before the actual request') + + # trigger the iterator + for b in buckets: + pass + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.list', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + + @patch('requests.Session.request') + def test_buckets_insert(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#bucket"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.create_bucket('test bucket') + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.insert', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_get(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#bucket"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.get_bucket('test bucket') + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertEqual(test_span.t, gcs_span.t) + self.assertEqual(test_span.s, gcs_span.p) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.get', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_patch(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#bucket"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').patch() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.patch', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_update(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#bucket"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').update() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.update', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_get_iam_policy(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#policy"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').get_iam_policy() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.getIamPolicy', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_set_iam_policy(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#policy"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').set_iam_policy(iam.Policy()) + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.setIamPolicy', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_test_iam_permissions(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#testIamPermissionsResponse"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').test_iam_permissions('test-permission') + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.testIamPermissions', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_lock_retention_policy(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#bucket", "metageneration": 1, "retentionPolicy": {"isLocked": False}}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + bucket = client.bucket('test bucket') + bucket.reload() + + with tracer.start_active_span('test'): + bucket.lock_retention_policy() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.lockRetentionPolicy', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_buckets_delete(self, mock_requests): + mock_requests.return_value = self._mock_response() + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').delete() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('buckets.delete', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_objects_compose(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('dest object').compose([ + storage.blob.Blob('object 1', 'test bucket'), + storage.blob.Blob('object 2', 'test bucket') + ]) + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.compose', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["destinationBucket"]) + self.assertEqual('dest object', gcs_span.data["gcs"]["destinationObject"]) + self.assertEqual('test bucket/object 1,test bucket/object 2', gcs_span.data["gcs"]["sourceObjects"]) + + @patch('requests.Session.request') + def test_objects_copy(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + bucket = client.bucket('src bucket') + + with tracer.start_active_span('test'): + bucket.copy_blob( + bucket.blob('src object'), + client.bucket('dest bucket'), + new_name='dest object' + ) + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.copy', gcs_span.data["gcs"]["op"]) + self.assertEqual('dest bucket', gcs_span.data["gcs"]["destinationBucket"]) + self.assertEqual('dest object', gcs_span.data["gcs"]["destinationObject"]) + self.assertEqual('src bucket', gcs_span.data["gcs"]["sourceBucket"]) + self.assertEqual('src object', gcs_span.data["gcs"]["sourceObject"]) + + @patch('requests.Session.request') + def test_objects_delete(self, mock_requests): + mock_requests.return_value = self._mock_response() + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').delete() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.delete', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_objects_attrs(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').exists() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.attrs', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_objects_get(self, mock_requests): + mock_requests.return_value = self._mock_response( + content=b'CONTENT', + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').download_as_bytes( + raw_download=True ) - } - for (op, (request, expected)) in test_cases.items(): - self.assertEqual(expected, collect_tags(request), msg=op) + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.get', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_objects_insert(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').upload_from_string('CONTENT') + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.insert', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_objects_list(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + blobs = client.bucket('test bucket').list_blobs() + self.assertEqual(0, self.recorder.queue_size(), msg='span has been created before the actual request') + + for b in blobs: pass + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.list', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_objects_patch(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').patch() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.patch', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_objects_rewrite(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#rewriteResponse", "totalBytesRewritten": 0, "objectSize": 0, "done": True, "resource": {}}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('dest bucket').blob('dest object').rewrite( + client.bucket('src bucket').blob('src object') + ) + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.rewrite', gcs_span.data["gcs"]["op"]) + self.assertEqual('dest bucket', gcs_span.data["gcs"]["destinationBucket"]) + self.assertEqual('dest object', gcs_span.data["gcs"]["destinationObject"]) + self.assertEqual('src bucket', gcs_span.data["gcs"]["sourceBucket"]) + self.assertEqual('src object', gcs_span.data["gcs"]["sourceObject"]) + + @patch('requests.Session.request') + def test_objects_update(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#object"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').update() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objects.update', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_default_acls_list(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#objectAccessControls", "items": []}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').default_object_acl.get_entities() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('defaultAcls.list', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + + @patch('requests.Session.request') + def test_object_acls_list(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#objectAccessControls", "items": []}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.bucket('test bucket').blob('test object').acl.get_entities() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('objectAcls.list', gcs_span.data["gcs"]["op"]) + self.assertEqual('test bucket', gcs_span.data["gcs"]["bucket"]) + self.assertEqual('test object', gcs_span.data["gcs"]["object"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_create(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#hmacKey", "metadata": {}, "secret": ""}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.create_hmac_key('test@example.com') + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('hmacKeys.create', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_delete(self, mock_requests): + mock_requests.return_value = self._mock_response() + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + key = storage.hmac_key.HMACKeyMetadata(client, access_id='test key') + key.state = storage.hmac_key.HMACKeyMetadata.INACTIVE_STATE + key.delete() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('hmacKeys.delete', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + self.assertEqual('test key', gcs_span.data["gcs"]["accessId"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_get(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#hmacKey", "metadata": {}, "secret": ""}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + storage.hmac_key.HMACKeyMetadata(client, access_id='test key').exists() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('hmacKeys.get', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + self.assertEqual('test key', gcs_span.data["gcs"]["accessId"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_list(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#hmacKeysMetadata", "items": []}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + keys = client.list_hmac_keys() + self.assertEqual(0, self.recorder.queue_size(), msg='span has been created before the actual request') + + for k in keys: pass + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('hmacKeys.list', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_update(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"kind": "storage#hmacKey", "metadata": {}, "secret": ""}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + storage.hmac_key.HMACKeyMetadata(client, access_id='test key').update() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('hmacKeys.update', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + self.assertEqual('test key', gcs_span.data["gcs"]["accessId"]) + + @patch('requests.Session.request') + def test_object_hmac_keys_update(self, mock_requests): + mock_requests.return_value = self._mock_response( + json_content={"email_address": "test@example.com", "kind": "storage#serviceAccount"}, + status_code=http_client.OK + ) + + client = self._client(project='test-project') + + with tracer.start_active_span('test'): + client.get_service_account_email() + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + self.assertIsNone(tracer.active_span) + + gcs_span = spans[0] + test_span = spans[1] + + self.assertTraceContextPropagated(test_span, gcs_span) + + self.assertEqual('gcs',gcs_span.n) + self.assertEqual(2, gcs_span.k) + self.assertIsNone(gcs_span.ec) + + self.assertEqual('serviceAccount.get', gcs_span.data["gcs"]["op"]) + self.assertEqual('test-project', gcs_span.data["gcs"]["projectId"]) + + @patch('requests.Session.request') + def test_batch_operation(self, mock_requests): + mock_requests.return_value = self._mock_response( + _TWO_PART_BATCH_RESPONSE, + status_code=http_client.OK, + headers={"content-type": 'multipart/mixed; boundary="DEADBEEF="'} + ) + + client = self._client(project='test-project') + bucket = client.bucket('test-bucket') + + with tracer.start_active_span('test'): + with client.batch(): + for obj in ['obj1', 'obj2']: + bucket.delete_blob(obj) + + spans = self.recorder.queued_spans() + + self.assertEqual(2, len(spans)) + + def _client(self, *args, **kwargs): + # override the HTTP client to bypass the authorization + kwargs['_http'] = kwargs.get('_http', requests.Session()) + + return storage.Client(*args, **kwargs) + + def _mock_response(self, content=b'', status_code=http_client.NO_CONTENT, json_content=None, headers={}): + resp = Mock() + resp.status_code = status_code + resp.headers = headers + resp.content = content + resp.__enter__ = Mock(return_value=resp) + resp.__exit__ = Mock() + + if json_content is not None: + if resp.content == b'': + resp.content = json.dumps(json_content) + + resp.json = Mock(return_value=json_content) + + return resp + +_TWO_PART_BATCH_RESPONSE = b"""\ +--DEADBEEF= +Content-Type: application/json +Content-ID: + +HTTP/1.1 204 No Content + +Content-Type: application/json; charset=UTF-8 +Content-Length: 0 + +--DEADBEEF= +Content-Type: application/json +Content-ID: + +HTTP/1.1 204 No Content + +Content-Type: application/json; charset=UTF-8 +Content-Length: 0 + +--DEADBEEF=-- +""" From 88079b936112589c12f4bb28697c28a4829c58c5 Mon Sep 17 00:00:00 2001 From: Andrey Slotin Date: Mon, 28 Sep 2020 13:02:23 +0200 Subject: [PATCH 16/16] Lower the min supported version of google-cloud-storage to 1.24.0 --- setup.py | 2 +- tests/clients/test_google-cloud-storage.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d211f460..78fece24 100644 --- a/setup.py +++ b/setup.py @@ -98,7 +98,7 @@ def check_setuptools(): 'nose>=1.0', 'flask>=0.12.2', 'grpcio>=1.18.0', - 'google-cloud-storage>=1.31.1;python_version>="3.5"', + 'google-cloud-storage>=1.24.0;python_version>="3.5"', 'lxml>=3.4', 'mock>=2.0.0', 'mysqlclient>=1.3.14;python_version>="3.5"', diff --git a/tests/clients/test_google-cloud-storage.py b/tests/clients/test_google-cloud-storage.py index 5f9d544d..c3ec3243 100644 --- a/tests/clients/test_google-cloud-storage.py +++ b/tests/clients/test_google-cloud-storage.py @@ -5,6 +5,7 @@ import pytest import json import requests +import io from instana.singletons import tracer from ..test_utils import _TraceContextMixin @@ -457,7 +458,8 @@ def test_objects_get(self, mock_requests): client = self._client(project='test-project') with tracer.start_active_span('test'): - client.bucket('test bucket').blob('test object').download_as_bytes( + client.bucket('test bucket').blob('test object').download_to_file( + io.BytesIO(), raw_download=True )