Implement http client interceptors (#2)

* initial implementation of http clien interceptors
* add interceptors for urllib and urllib2
* unify code for urllib and urllib2
* more code deduplication
* docu + bug fixes
* fix import path
* update CHANGELOG + bump version

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.12.0
+- Add HTTP client interceptors for urllib, urllib2, urllib3 and requests that automatically track dependencies
+- Change default type to HTTP for track_dependency
+
 ## 0.11.13
 - Fix envelope type name of AvailabilityData
 

README.rst

@@ -263,6 +263,56 @@ Since operation_id is being set as a property of telemetry client, the client sh
 
     # exceptions will cause a flush of all un-sent telemetry items
 
+**Track dependency telemetry for HTTP requests with requests**
+
+.. code:: python
+
+    from applicationinsights.client import enable_for_requests
+    import requests
+
+    enable_for_requests('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+    requests.get("https://www.python.org/")
+    # a dependency telemetry will be sent to the Application Insights service
+
+**Track dependency telemetry for HTTP requests with urllib**
+
+.. code:: python
+
+    from applicationinsights.client import enable_for_urllib
+    import urllib.requests
+
+    enable_for_urllib('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+    urllib.request.urlopen("https://www.python.org/")
+    # a dependency telemetry will be sent to the Application Insights service
+
+**Track dependency telemetry for HTTP requests with urllib2**
+
+.. code:: python
+
+    from applicationinsights.client import enable_for_urllib2
+    import urllib2
+
+    enable_for_urllib2('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+    urllib2.urlopen("https://www.python.org/")
+    # a dependency telemetry will be sent to the Application Insights service
+
+
+**Track dependency telemetry for HTTP requests with urllib3**
+
+.. code:: python
+
+    from applicationinsights.client import enable_for_urllib3
+    import urllib3.requests
+
+    enable_for_urllib3('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+    urllib3.PoolManager().request("GET", "https://www.python.org/")
+    # a dependency telemetry will be sent to the Application Insights service
+
+
 **Integrating with Flask**
 
 .. code:: python

applicationinsights/channel/SenderBase.py

@@ -11,6 +11,8 @@
 
 DEFAULT_ENDPOINT_URL = 'https://dc.services.visualstudio.com/v2/track'
 
+default_openers = HTTPClient._opener
+
 
 class SenderBase(object):
     """The base class for all types of senders for use in conjunction with an implementation of :class:`QueueBase`.
@@ -135,7 +137,13 @@ def send(self, data_to_send):
 
         request = HTTPClient.Request(self._service_endpoint_uri, bytearray(
             request_payload, 'utf-8'), {'Accept': 'application/json', 'Content-Type': 'application/json; charset=utf-8'})
+
+        current_opener = HTTPClient._opener
+
         try:
+            # we need to set the default urllib openers here
+            # otherwise we would intercept the HTTP calls to Application Insights and create dependency telemetry for them as well
+            HTTPClient._opener = default_openers
             response = HTTPClient.urlopen(request, timeout=self._timeout)
             status_code = response.getcode()
             if 200 <= status_code < 300:
@@ -145,6 +153,8 @@ def send(self, data_to_send):
                 return
         except Exception as e:
             pass
+        finally:
+            HTTPClient._opener = current_opener
 
         # Add our unsent data back on to the queue
         for data in data_to_send:

applicationinsights/channel/contracts/RemoteDependencyData.py

@@ -19,7 +19,7 @@ class RemoteDependencyData(object):
         ('success', True),
         ('data', None),
         ('target', None),
-        ('type', None),
+        ('type', 'HTTP'),
         ('properties', {}),
         ('measurements', {})
     ])

applicationinsights/channel/contracts/Utils.py

@@ -7,7 +7,7 @@ def _write_complex_object(defaults, values):
         default = defaults[key]
         if key in values:
             value = values[key]
-            if value == None:
+            if value is None:
                 value = default
         elif default:
             value = default

applicationinsights/client/__init__.py

@@ -0,0 +1 @@
+from .enable import *

applicationinsights/client/enable.py

@@ -0,0 +1,224 @@
+import sys
+
+from applicationinsights import TelemetryClient
+from applicationinsights.channel import SynchronousSender, SynchronousQueue, TelemetryChannel
+
+import time
+
+current_milli_time = lambda: int(round(time.time() * 1000))
+
+
+def _track_dependency(tc, always_flush, host, method, url, duration, response_code):
+    success = response_code < 400
+
+    tc.track_dependency(
+        name=host,
+        data="{} {}".format(method, url),
+        target=url,
+        duration=duration,
+        success=success,
+        result_code=response_code
+    )
+
+    if always_flush:
+        tc.flush()
+
+
+def __enable_for_urllib3(http_connection_pool_class, https_connection_pool_class, instrumentation_key,
+                         telemetry_channel, always_flush):
+    if not instrumentation_key:
+        raise Exception('Instrumentation key was required but not provided')
+
+    if telemetry_channel is None:
+        sender = SynchronousSender()
+        queue = SynchronousQueue(sender)
+        telemetry_channel = TelemetryChannel(None, queue)
+
+    client = TelemetryClient(instrumentation_key, telemetry_channel)
+
+    orig_http_urlopen_method = http_connection_pool_class.urlopen
+    orig_https_urlopen_method = https_connection_pool_class.urlopen
+
+    def custom_urlopen_wrapper(urlopen_func):
+        def custom_urlopen(*args, **kwargs):
+            start_time = current_milli_time()
+            response = urlopen_func(*args, **kwargs)
+            try:  # make sure to always return the response
+                duration = current_milli_time() - start_time
+                try:
+                    method = args[1]
+                except IndexError:
+                    method = kwargs['method']
+
+                try:
+                    url = args[2]
+                except IndexError:
+                    url = kwargs['url']
+
+                _track_dependency(client, always_flush, args[0].host, method, url, duration, response.status)
+            finally:
+                return response
+
+        return custom_urlopen
+
+    http_connection_pool_class.urlopen = custom_urlopen_wrapper(orig_http_urlopen_method)
+    https_connection_pool_class.urlopen = custom_urlopen_wrapper(orig_https_urlopen_method)
+
+
+def enable_for_urllib3(instrumentation_key, telemetry_channel=None, always_flush=False):
+    """Enables the automatic collection of dependency telemetries for HTTP calls with urllib3.
+
+    .. code:: python
+
+        from applicationinsights.client import enable_for_urllib3
+        import urllib3.requests
+
+        enable_for_urllib3('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+        urllib3.PoolManager().request("GET", "https://www.python.org/")
+        # a dependency telemetry will be sent to the Application Insights service
+
+    Args:
+        instrumentation_key (str). the instrumentation key to use while sending telemetry to the service.
+        telemetry_channel (TelemetryChannel). a custom telemetry channel to use
+        always_flush (bool). if true every HTTP call will flush the dependency telemetry
+    """
+    from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+    __enable_for_urllib3(HTTPConnectionPool, HTTPSConnectionPool, instrumentation_key, telemetry_channel, always_flush)
+
+
+def enable_for_requests(instrumentation_key, telemetry_channel=None, always_flush=False):
+    """Enables the automatic collection of dependency telemetries for HTTP calls with requests.
+
+    .. code:: python
+
+        from applicationinsights.client import enable_for_requests
+        import requests
+
+        enable_for_requests('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+        requests.get("https://www.python.org/")
+        # a dependency telemetry will be sent to the Application Insights service
+
+    Args:
+        instrumentation_key (str). the instrumentation key to use while sending telemetry to the service.
+        telemetry_channel (TelemetryChannel). a custom telemetry channel to use
+        always_flush (bool). if true every HTTP call will flush the dependency telemetry
+    """
+    from requests.packages.urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
+    __enable_for_urllib3(HTTPConnectionPool, HTTPSConnectionPool, instrumentation_key, telemetry_channel, always_flush)
+
+
+def _track_for_urllib(tc, always_flush, start_time, req, resp):
+    duration = current_milli_time() - start_time
+    method = req.get_method()
+
+    if sys.version_info.major > 2:
+        url = req.selector
+        status = resp.status
+    else:
+        url = req.get_selector()
+        status = resp.code
+
+    _track_dependency(tc, always_flush, req.host, method, url, duration, status)
+
+
+def __enable_for_urllib(base_http_handler_class, base_https_handler_class, instrumentation_key, telemetry_channel=None,
+                        always_flush=False):
+    pass
+    if not instrumentation_key:
+        raise Exception('Instrumentation key was required but not provided')
+
+    if telemetry_channel is None:
+        sender = SynchronousSender()
+        queue = SynchronousQueue(sender)
+        telemetry_channel = TelemetryChannel(None, queue)
+
+    client = TelemetryClient(instrumentation_key, telemetry_channel)
+
+    class AppInsightsHTTPHandler(base_http_handler_class, object):
+        def http_open(self, req):
+            start_time = current_milli_time()
+            response = super(AppInsightsHTTPHandler, self).http_open(req)
+
+            try:
+                _track_for_urllib(client, always_flush, start_time, req, response)
+            finally:
+                return response
+
+    class AppInsightsHTTPSHandler(base_https_handler_class, object):
+        def https_open(self, req):
+            start_time = current_milli_time()
+            response = super(AppInsightsHTTPSHandler, self).https_open(req)
+
+            try:
+                _track_for_urllib(client, always_flush, start_time, req, response)
+            finally:
+                return response
+
+    return AppInsightsHTTPHandler, AppInsightsHTTPSHandler
+
+
+def enable_for_urllib(instrumentation_key, telemetry_channel=None, always_flush=False):
+    """Enables the automatic collection of dependency telemetries for HTTP calls with urllib.
+
+    .. code:: python
+
+        from applicationinsights.client import enable_for_urllib
+        import urllib.requests
+
+        enable_for_urllib('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+        urllib.request.urlopen("https://www.python.org/")
+        # a dependency telemetry will be sent to the Application Insights service
+
+    Args:
+        instrumentation_key (str). the instrumentation key to use while sending telemetry to the service.
+        telemetry_channel (TelemetryChannel). a custom telemetry channel to use
+        always_flush (bool). if true every HTTP call will flush the dependency telemetry
+    """
+    import urllib.request
+
+    http_handler, https_handler = __enable_for_urllib(
+        urllib.request.HTTPHandler,
+        urllib.request.HTTPSHandler,
+        instrumentation_key,
+        telemetry_channel,
+        always_flush
+    )
+
+    urllib.request.install_opener(
+        urllib.request.build_opener(http_handler, https_handler)
+    )
+
+
+def enable_for_urllib2(instrumentation_key, telemetry_channel=None, always_flush=False):
+    """Enables the automatic collection of dependency telemetries for HTTP calls with urllib2.
+
+    .. code:: python
+
+        from applicationinsights.client import enable_for_urllib2
+        import urllib2
+
+        enable_for_urllib2('<YOUR INSTRUMENTATION KEY GOES HERE>')
+
+        urllib2.urlopen("https://www.python.org/")
+        # a dependency telemetry will be sent to the Application Insights service
+
+    Args:
+        instrumentation_key (str). the instrumentation key to use while sending telemetry to the service.
+        telemetry_channel (TelemetryChannel). a custom telemetry channel to use
+        always_flush (bool). if true every HTTP call will flush the dependency telemetry
+    """
+    import urllib2
+
+    http_handler, https_handler = __enable_for_urllib(
+        urllib2.HTTPHandler,
+        urllib2.HTTPSHandler,
+        instrumentation_key,
+        telemetry_channel,
+        always_flush
+    )
+    urllib2.install_opener(
+        urllib2.build_opener(http_handler, https_handler)
+    )

doc/applicationinsights.client.rst

@@ -0,0 +1,13 @@
+.. toctree::
+    :maxdepth: 2
+    :hidden:
+
+applicationinsights.exceptions module
+=====================================
+
+enable function
+---------------
+.. autofunction:: applicationinsights.client.enable_for_requests
+.. autofunction:: applicationinsights.client.enable_for_urllib3
+.. autofunction:: applicationinsights.client.enable_for_urllib2
+.. autofunction:: applicationinsights.client.enable_for_urllib

doc/applicationinsights.rst

@@ -9,6 +9,7 @@ applicationinsights module
     applicationinsights.requests
     applicationinsights.django
     applicationinsights.exceptions
+    applicationinsights.client
 
 TelemetryClient class
 ----------------------

setup.py

@@ -14,7 +14,7 @@
     # Versions should comply with PEP440.  For a discussion on single-sourcing
     # the version across setup.py and the project code, see
     # http://packaging.python.org/en/latest/tutorial.html#version
-    version='0.11.14',
+    version='0.12.0',
 
     description='This project extends the Application Insights API surface to support Python.',
     long_description=long_description,
@@ -62,6 +62,12 @@
     # You can just specify the packages manually here if your project is
     # simple. Or you can use find_packages().
     packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
+    extras_require={
+       "requests-client": ["requests"],
+       "urllib3-client": ["urllib3"]
+    },
+
+    tests_require=["httpretty", "requests", "urllib3"],
 
     test_suite='tests.applicationinsights_tests'
 )

tests/applicationinsights_tests/TestTelemetryClient.py

@@ -1,6 +1,5 @@
 from applicationinsights import TelemetryClient, channel
 import unittest
-import inspect
 import json
 import sys