diff --git a/README.rst b/README.rst index 502cea439..61acb808f 100644 --- a/README.rst +++ b/README.rst @@ -418,8 +418,29 @@ method, and status will be collected. You can enable Requests integration by specifying ``'requests'`` to ``trace_integrations``. +It's possible to configure a list of URL you don't want traced. By default the request to exporter +won't be traced. It's configurable by giving an array of hostname/port to the attribute +``blacklist_hostnames`` in OpenCensus context's attributes: + +.. code:: python + + execution_context.set_opencensus_attr('blacklist_hostnames',['hostname:port']) + +Only the hostname must be specified if only the hostname is specified in the URL request. + .. _Requests package: https://pypi.python.org/pypi/requests +Httplib +~~~~~~~~ + +Census can trace HTTP requests made with the httplib library. + +You can enable Requests integration by specifying ``'httplib'`` to ``trace_integrations``. + +It's possible to configure a list of URL you don't want traced. See requests integration +for more information. The only difference is that you need to specify hostname and port +every time. + Google Cloud Client Libraries ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/opencensus/trace/ext/django/config.py b/opencensus/trace/ext/django/config.py index 68e701834..c17729b69 100644 --- a/opencensus/trace/ext/django/config.py +++ b/opencensus/trace/ext/django/config.py @@ -37,6 +37,7 @@ 'ZIPKIN_EXPORTER_PORT': 9411, 'ZIPKIN_EXPORTER_PROTOCOL': 'http', 'OCAGENT_TRACE_EXPORTER_ENDPOINT': None, + 'BLACKLIST_HOSTNAMES': None, 'TRANSPORT': 'opencensus.trace.exporters.transports.sync.SyncTransport', } diff --git a/opencensus/trace/ext/django/middleware.py b/opencensus/trace/ext/django/middleware.py index ac90890fa..3258f0c9e 100644 --- a/opencensus/trace/ext/django/middleware.py +++ b/opencensus/trace/ext/django/middleware.py @@ -45,6 +45,7 @@ ZIPKIN_EXPORTER_PORT = 'ZIPKIN_EXPORTER_PORT' ZIPKIN_EXPORTER_PROTOCOL = 'ZIPKIN_EXPORTER_PROTOCOL' OCAGENT_TRACE_EXPORTER_ENDPOINT = 'OCAGENT_TRACE_EXPORTER_ENDPOINT' +BLACKLIST_HOSTNAMES = 'BLACKLIST_HOSTNAMES' log = logging.getLogger(__name__) @@ -161,6 +162,9 @@ def __init__(self, get_response=None): else: self.exporter = self._exporter(transport=transport) + self.blacklist_hostnames = settings.params.get( + BLACKLIST_HOSTNAMES, None) + # Initialize the propagator self.propagator = self._propagator() @@ -179,6 +183,10 @@ def process_request(self, request): REQUEST_THREAD_LOCAL_KEY, request) + execution_context.set_opencensus_attr( + 'blacklist_hostnames', + self.blacklist_hostnames) + try: # Start tracing this request span_context = self.propagator.from_headers( diff --git a/opencensus/trace/ext/flask/flask_middleware.py b/opencensus/trace/ext/flask/flask_middleware.py index d7d509048..f88acd495 100644 --- a/opencensus/trace/ext/flask/flask_middleware.py +++ b/opencensus/trace/ext/flask/flask_middleware.py @@ -45,6 +45,7 @@ ZIPKIN_EXPORTER_PORT = 'ZIPKIN_EXPORTER_PORT' ZIPKIN_EXPORTER_PROTOCOL = 'ZIPKIN_EXPORTER_PROTOCOL' OCAGENT_TRACE_EXPORTER_ENDPOINT = 'OCAGENT_TRACE_EXPORTER_ENDPOINT' +BLACKLIST_HOSTNAMES = 'BLACKLIST_HOSTNAMES' log = logging.getLogger(__name__) @@ -165,6 +166,7 @@ def init_app(self, app): else: self.exporter = self.exporter(transport=transport) + self.blacklist_hostnames = params.get(BLACKLIST_HOSTNAMES, None) # Initialize the propagator if inspect.isclass(self.propagator): self.propagator = self.propagator() @@ -203,6 +205,9 @@ def _before_request(self): tracer.add_attribute_to_current_span( HTTP_METHOD, flask.request.method) tracer.add_attribute_to_current_span(HTTP_URL, flask.request.url) + execution_context.set_opencensus_attr( + 'blacklist_hostnames', + self.blacklist_hostnames) except Exception: # pragma: NO COVER log.error('Failed to trace request', exc_info=True) diff --git a/opencensus/trace/ext/httplib/trace.py b/opencensus/trace/ext/httplib/trace.py index 87973c2bf..8d4de8873 100644 --- a/opencensus/trace/ext/httplib/trace.py +++ b/opencensus/trace/ext/httplib/trace.py @@ -18,6 +18,7 @@ from opencensus.trace import attributes_helper from opencensus.trace import execution_context from opencensus.trace import span as span_module +from opencensus.trace.ext import utils PYTHON2 = sys.version_info.major == 2 @@ -61,6 +62,12 @@ def wrap_httplib_request(request_func): def call(self, method, url, body, headers, *args, **kwargs): _tracer = execution_context.get_opencensus_tracer() + blacklist_hostnames = execution_context.get_opencensus_attr( + 'blacklist_hostnames') + dest_url = '{}:{}'.format(self._dns_host, self.port) + if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): + return request_func(self, method, url, body, + headers, *args, **kwargs) _span = _tracer.start_span() _span.span_kind = span_module.SpanKind.CLIENT _span.name = '[httplib]{}'.format(request_func.__name__) @@ -100,7 +107,7 @@ def call(self, *args, **kwargs): span = _tracer.current_span() # No corresponding request span is found, request not traced. - if span.span_id != current_span_id: + if not span or span.span_id != current_span_id: return response_func(self, *args, **kwargs) result = response_func(self, *args, **kwargs) diff --git a/opencensus/trace/ext/requests/trace.py b/opencensus/trace/ext/requests/trace.py index f9f266edd..f9fc9d0a4 100644 --- a/opencensus/trace/ext/requests/trace.py +++ b/opencensus/trace/ext/requests/trace.py @@ -15,10 +15,15 @@ import logging import requests import wrapt +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse from opencensus.trace import attributes_helper from opencensus.trace import execution_context from opencensus.trace import span as span_module +from opencensus.trace.ext import utils log = logging.getLogger(__name__) @@ -56,6 +61,16 @@ def trace_integration(tracer=None): def wrap_requests(requests_func): """Wrap the requests function to trace it.""" def call(url, *args, **kwargs): + blacklist_hostnames = execution_context.get_opencensus_attr( + 'blacklist_hostnames') + parsed_url = urlparse(url) + if parsed_url.port is None: + dest_url = parsed_url.hostname + else: + dest_url = '{}:{}'.format(parsed_url.hostname, parsed_url.port) + if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): + return requests_func(url, *args, **kwargs) + _tracer = execution_context.get_opencensus_tracer() _span = _tracer.start_span() _span.name = '[requests]{}'.format(requests_func.__name__) @@ -80,6 +95,17 @@ def wrap_session_request(wrapped, instance, args, kwargs): """Wrap the session function to trace it.""" method = kwargs.get('method') or args[0] url = kwargs.get('url') or args[1] + + blacklist_hostnames = execution_context.get_opencensus_attr( + 'blacklist_hostnames') + parsed_url = urlparse(url) + if parsed_url.port is None: + dest_url = parsed_url.hostname + else: + dest_url = '{}:{}'.format(parsed_url.hostname, parsed_url.port) + if utils.disable_tracing_hostname(dest_url, blacklist_hostnames): + return wrapped(*args, **kwargs) + _tracer = execution_context.get_opencensus_tracer() _span = _tracer.start_span() diff --git a/opencensus/trace/ext/utils.py b/opencensus/trace/ext/utils.py index 005936f24..1a39c8c0e 100644 --- a/opencensus/trace/ext/utils.py +++ b/opencensus/trace/ext/utils.py @@ -14,6 +14,8 @@ import re +from opencensus.trace import execution_context + # By default the blacklist urls are not tracing, currently just include the # health check url. The paths are literal string matched instead of regular # expressions. Do not include the '/' at the beginning of the path. @@ -63,3 +65,31 @@ def disable_tracing_url(url, blacklist_paths=None): return True return False + + +def disable_tracing_hostname(url, blacklist_hostnames=None): + """Disable tracing for the provided blacklist URLs, by default not tracing + the exporter url. + + If the url path starts with the blacklisted path, return True. + + :type blacklist_hostnames: list + :param blacklist_hostnames: URL that not tracing. + + :rtype: bool + :returns: True if not tracing, False if tracing. + """ + if blacklist_hostnames is None: + # Exporter host_name are not traced by default + _tracer = execution_context.get_opencensus_tracer() + try: + blacklist_hostnames = [ + '{}:{}'.format( + _tracer.exporter.host_name, + _tracer.exporter.port + ) + ] + except(AttributeError): + blacklist_hostnames = [] + + return url in blacklist_hostnames diff --git a/tests/unit/trace/ext/django/test_config.py b/tests/unit/trace/ext/django/test_config.py index 6f8a230e6..c797033ee 100644 --- a/tests/unit/trace/ext/django/test_config.py +++ b/tests/unit/trace/ext/django/test_config.py @@ -86,6 +86,7 @@ def test__set_default_configs(self): 'ZIPKIN_EXPORTER_PORT': 9411, 'ZIPKIN_EXPORTER_PROTOCOL': 'http', 'OCAGENT_TRACE_EXPORTER_ENDPOINT': None, + 'BLACKLIST_HOSTNAMES': None, 'TRANSPORT': 'opencensus.trace.exporters.transports.sync.SyncTransport', } diff --git a/tests/unit/trace/ext/httplib/test_httplib_trace.py b/tests/unit/trace/ext/httplib/test_httplib_trace.py index 07cc7e2fd..d5727a8b3 100644 --- a/tests/unit/trace/ext/httplib/test_httplib_trace.py +++ b/tests/unit/trace/ext/httplib/test_httplib_trace.py @@ -99,7 +99,85 @@ def test_wrap_httplib_request(self): self.assertEqual(expected_attributes, mock_tracer.span.attributes) self.assertEqual(expected_name, mock_tracer.span.name) - self.assertEqual(span_module.SpanKind.CLIENT, mock_tracer.span.span_kind) + self.assertEqual(span_module.SpanKind.CLIENT, mock_tracer.span.span_kind) + + def test_wrap_httplib_request_blacklist_ok(self): + mock_span = mock.Mock() + span_id = '1234' + mock_span.span_id = span_id + mock_tracer = MockTracer(mock_span) + mock_request_func = mock.Mock() + mock_request_func.__name__ = 'request' + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=None) + + wrapped = trace.wrap_httplib_request(mock_request_func) + + mock_self = mock.Mock() + method = 'GET' + url = 'http://localhost:8080' + body = None + headers = {} + + with patch_tracer, patch_attr: + wrapped(mock_self, method, url, body, headers) + + expected_attributes = { + 'http.url': url, + 'http.method': method} + expected_name = '[httplib]request' + + mock_request_func.assert_called_with( + mock_self, method, url, body, { + 'traceparent': '00-123-456-01', + } + ) + + def test_wrap_httplib_request_blacklist_nok(self): + mock_span = mock.Mock() + span_id = '1234' + mock_span.span_id = span_id + mock_tracer = MockTracer(mock_span) + mock_request_func = mock.Mock() + mock_request_func.__name__ = 'request' + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=['localhost:8080']) + + wrapped = trace.wrap_httplib_request(mock_request_func) + + mock_self = mock.Mock() + mock_self._dns_host = 'localhost' + mock_self.port = '8080' + method = 'GET' + url = 'http://{}:{}'.format(mock_self._dns_host, mock_self.port) + body = None + headers = {} + + with patch_tracer, patch_attr: + wrapped(mock_self, method, url, body, headers) + + expected_attributes = { + 'http.url': url, + 'http.method': method} + expected_name = '[httplib]request' + + mock_request_func.assert_called_with( + mock_self, method, url, body, {} + ) def test_wrap_httplib_response(self): mock_span = mock.Mock() diff --git a/tests/unit/trace/ext/requests/test_requests_trace.py b/tests/unit/trace/ext/requests/test_requests_trace.py index b8f187108..fe85c8728 100644 --- a/tests/unit/trace/ext/requests/test_requests_trace.py +++ b/tests/unit/trace/ext/requests/test_requests_trace.py @@ -106,11 +106,67 @@ def test_wrap_requests(self): mock_tracer.current_span.attributes) self.assertEqual(expected_name, mock_tracer.current_span.name) + def test_wrap_requests_blacklist_ok(self): + mock_return = mock.Mock() + mock_return.status_code = 200 + return_value = mock_return + mock_func = mock.Mock() + mock_func.__name__ = 'get' + mock_func.return_value = return_value + mock_tracer = MockTracer() + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=['localhost:8080']) + + wrapped = trace.wrap_requests(mock_func) + + url = 'http://localhost' + + with patch_tracer, patch_attr: + wrapped(url) + + expected_name = '[requests]get' + + self.assertEqual(expected_name, mock_tracer.current_span.name) + + def test_wrap_requests_blacklist_nok(self): + mock_return = mock.Mock() + mock_return.status_code = 200 + return_value = mock_return + mock_func = mock.Mock() + mock_func.__name__ = 'get' + mock_func.return_value = return_value + mock_tracer = MockTracer() + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=['localhost:8080']) + + wrapped = trace.wrap_requests(mock_func) + + url = 'http://localhost:8080' + + with patch_tracer, patch_attr: + wrapped(url) + + self.assertEqual(None, mock_tracer.current_span) + def test_wrap_session_request(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) mock_tracer = MockTracer(propagator=mock.Mock( - to_headers= lambda x: {'x-trace': 'some-value'}) + to_headers=lambda x: {'x-trace': 'some-value'}) ) patch = mock.patch( @@ -123,7 +179,7 @@ def test_wrap_session_request(self): kwargs = {} with patch: - result = trace.wrap_session_request( + trace.wrap_session_request( wrapped, 'Session.request', (request_method, url), kwargs) expected_attributes = { @@ -138,12 +194,66 @@ def test_wrap_session_request(self): self.assertEqual(kwargs['headers']['x-trace'], 'some-value') self.assertEqual(expected_name, mock_tracer.current_span.name) + def test_wrap_session_request_blacklist_ok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer(propagator=mock.Mock( + to_headers=lambda x: {'x-trace': 'some-value'}) + ) + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=None) + + url = 'http://localhost' + request_method = 'POST' + + with patch_tracer, patch_attr: + trace.wrap_session_request( + wrapped, 'Session.request', (request_method, url), {}) + + expected_name = '[requests]POST' + self.assertEqual(expected_name, mock_tracer.current_span.name) + + def test_wrap_session_request_blacklist_nok(self): + def wrapped(*args, **kwargs): + result = mock.Mock() + result.status_code = 200 + return result + + mock_tracer = MockTracer() + + patch_tracer = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_tracer', + return_value=mock_tracer) + patch_attr = mock.patch( + 'opencensus.trace.ext.requests.trace.execution_context.' + 'get_opencensus_attr', + return_value=['localhost:8080']) + + url = 'http://localhost:8080' + request_method = 'POST' + + with patch_tracer, patch_attr: + trace.wrap_session_request( + wrapped, 'Session.request', (request_method, url), {}) + self.assertEqual(None, mock_tracer.current_span) + def test_header_is_passed_in(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) mock_tracer = MockTracer(propagator=mock.Mock( - to_headers= lambda x: {'x-trace': 'some-value'}) + to_headers=lambda x: {'x-trace': 'some-value'}) ) - + patch = mock.patch( 'opencensus.trace.ext.requests.trace.execution_context.' 'get_opencensus_tracer', @@ -154,7 +264,7 @@ def test_header_is_passed_in(self): kwargs = {} with patch: - result = trace.wrap_session_request( + trace.wrap_session_request( wrapped, 'Session.request', (request_method, url), kwargs) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') @@ -162,7 +272,7 @@ def test_header_is_passed_in(self): def test_headers_are_preserved(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) mock_tracer = MockTracer(propagator=mock.Mock( - to_headers= lambda x: {'x-trace': 'some-value'}) + to_headers=lambda x: {'x-trace': 'some-value'}) ) patch = mock.patch( @@ -175,17 +285,16 @@ def test_headers_are_preserved(self): kwargs = {'headers': {'key': 'value'}} with patch: - result = trace.wrap_session_request( + trace.wrap_session_request( wrapped, 'Session.request', (request_method, url), kwargs) self.assertEqual(kwargs['headers']['key'], 'value') self.assertEqual(kwargs['headers']['x-trace'], 'some-value') - def test_tracer_headers_are_overwritten(self): wrapped = mock.Mock(return_value=mock.Mock(status_code=200)) mock_tracer = MockTracer(propagator=mock.Mock( - to_headers= lambda x: {'x-trace': 'some-value'}) + to_headers=lambda x: {'x-trace': 'some-value'}) ) patch = mock.patch( @@ -198,11 +307,12 @@ def test_tracer_headers_are_overwritten(self): kwargs = {'headers': {'x-trace': 'original-value'}} with patch: - result = trace.wrap_session_request( + trace.wrap_session_request( wrapped, 'Session.request', (request_method, url), kwargs) self.assertEqual(kwargs['headers']['x-trace'], 'some-value') + class MockTracer(object): def __init__(self, propagator=None): self.current_span = None diff --git a/tests/unit/trace/ext/test_ext_utils.py b/tests/unit/trace/ext/test_ext_utils.py index 86dfb831c..091a528b1 100644 --- a/tests/unit/trace/ext/test_ext_utils.py +++ b/tests/unit/trace/ext/test_ext_utils.py @@ -56,3 +56,20 @@ def test_disable_tracing_url_explicit(self): disable_tracing = utils.disable_tracing_url(url, blacklist_paths) self.assertTrue(disable_tracing) + + def test_disable_tracing_hostname_default(self): + url = '127.0.0.1:8080' + + disable_tracing = utils.disable_tracing_hostname(url) + self.assertFalse(disable_tracing) + + def test_disable_tracing_hostname_explicit(self): + blacklist_paths = ['127.0.0.1', '192.168.0.1:80'] + + url = '127.0.0.1:8080' + disable_tracing = utils.disable_tracing_hostname(url, blacklist_paths) + self.assertFalse(disable_tracing) + + url = '127.0.0.1:80' + disable_tracing = utils.disable_tracing_hostname(url, blacklist_paths) + self.assertFalse(disable_tracing)