Add ssl context support (#291)

geopy/compat.py

@@ -2,7 +2,9 @@
 Compatibility...
 """
 
+import inspect
 import sys
+import warnings
 
 py3k = sys.version_info >= (3, 0)
 
@@ -50,7 +52,7 @@ def isfinite(x):
     from urllib.parse import (urlencode, quote, # pylint: disable=W0611,F0401,W0611,E0611
                               urlparse, parse_qs)
     from urllib.request import (Request, urlopen, # pylint: disable=W0611,F0401,W0611,E0611
-                                build_opener, ProxyHandler,
+                                build_opener, ProxyHandler, HTTPSHandler,
                                 URLError,
                                 HTTPPasswordMgrWithDefaultRealm,
                                 HTTPBasicAuthHandler)
@@ -74,7 +76,7 @@ def iteritems(d):
 else: # pragma: no cover
     from urllib import urlencode as original_urlencode, quote # pylint: disable=W0611,F0401,W0611,E0611
     from urllib2 import (Request, HTTPError,   # pylint: disable=W0611,F0401,W0611,E0611
-                         ProxyHandler, URLError, urlopen,
+                         ProxyHandler, HTTPSHandler, URLError, urlopen,
                          build_opener,
                          HTTPPasswordMgrWithDefaultRealm,
                          HTTPBasicAuthHandler)
@@ -120,3 +122,31 @@ def iteritems(d):
         For Python3
         """
         return d.iteritems()
+
+
+def _is_urllib_context_supported(HTTPSHandler_=HTTPSHandler):
+    context_arg = 'context'
+    if py3k:
+        argspec = inspect.getfullargspec(HTTPSHandler_.__init__)
+        return context_arg in argspec.args or context_arg in argspec.kwonlyargs
+    else:
+        return context_arg in inspect.getargspec(HTTPSHandler_.__init__).args
+
+
+_URLLIB_SUPPORTS_SSL_CONTEXT = _is_urllib_context_supported()
+
+
+def build_opener_with_context(context=None, *handlers):
+    # `context` has been added in Python 2.7.9 and 3.4.3.
+    if _URLLIB_SUPPORTS_SSL_CONTEXT:
+        https_handler = HTTPSHandler(context=context)
+    else:
+        warnings.warn(
+            ("SSL context is not supported in your environment for urllib "
+             "calls. Perhaps your Python version is obsolete? "
+             "This probably means that TLS verification doesn't happen, "
+             "which is insecure. Please consider upgrading your Python "
+             "interpreter version."),
+            UserWarning)
+        https_handler = HTTPSHandler()
+    return build_opener(https_handler, *handlers)

geopy/geocoders/arcgis.py

@@ -41,6 +41,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Create a ArcGIS-based geocoder.
@@ -78,6 +79,12 @@ def __init__(
         :param str format_string:
             See :attr:`geopy.geocoders.options.default_format_string`.
 
+            .. versionadded:: 1.14.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
             .. versionadded:: 1.14.0
         """
         super(ArcGIS, self).__init__(
@@ -86,6 +93,7 @@ def __init__(
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         if username or password or referer:
             if not (username and password and referer):

geopy/geocoders/baidu.py

@@ -29,6 +29,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Initialize a customized Baidu geocoder using the v2 API.
@@ -59,6 +60,12 @@ def __init__(
         :param str format_string:
             See :attr:`geopy.geocoders.options.default_format_string`.
 
+            .. versionadded:: 1.14.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
             .. versionadded:: 1.14.0
         """
         super(Baidu, self).__init__(
@@ -67,6 +74,7 @@ def __init__(
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         self.api_key = api_key
         self.api = '%s://api.map.baidu.com/geocoder/v2/' % self.scheme

geopy/geocoders/base.py

@@ -4,15 +4,15 @@
 
 from ssl import SSLError
 from socket import timeout as SocketTimeout
+import functools
 import json
 import warnings
 
 from geopy.compat import (
     string_compare,
     HTTPError,
     py3k,
-    urlopen as urllib_urlopen,
-    build_opener,
+    build_opener_with_context,
     ProxyHandler,
     URLError,
     Request,
@@ -85,6 +85,28 @@ class options(object):
     default_user_agent = "geopy/%s" % __version__
     """User-Agent header to send with the requests to geocoder API."""
 
+    default_ssl_context = None
+    """An :class:`ssl.SSLContext` instance with custom TLS verification
+    settings. Pass ``None`` to use the interpreter's defaults (starting from
+    Python 2.7.9 and 3.4.3 that is to use the system's trusted CA certificates;
+    the older versions don't support TLS verification completely).
+
+    For older versions of Python (before 2.7.9 and 3.4.3) this argument is
+    ignored, as ``urlopen`` doesn't accept an ssl context there, and a warning
+    is issued.
+
+    To disable TLS certificate verification completely::
+
+        >>> import ssl
+        >>> import geopy.geocoders
+        >>> ctx = ssl.create_default_context()
+        >>> ctx.check_hostname = False
+        >>> ctx.verify_mode = ssl.CERT_NONE
+        >>> geopy.geocoders.options.default_ssl_context = ctx
+
+    See docs for the :class:`ssl.SSLContext` class for more examples.
+    """
+
 
 # Create an object which `repr` returns 'DEFAULT_SENTINEL'. Sphinx (docs) uses
 # this value when generating method's signature.
@@ -118,6 +140,7 @@ def __init__(
             timeout=DEFAULT_SENTINEL,
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Mostly-common geocoder validation, proxies, &c. Not all geocoders
@@ -134,14 +157,19 @@ def __init__(
         self.proxies = (proxies if proxies is not DEFAULT_SENTINEL
                         else options.default_proxies)
         self.headers = {'User-Agent': user_agent or options.default_user_agent}
+        self.ssl_context = (ssl_context if ssl_context is not DEFAULT_SENTINEL
+                            else options.default_ssl_context)
 
         if self.proxies:
-            opener = build_opener(
-                ProxyHandler(self.proxies)
+            opener = build_opener_with_context(
+                self.ssl_context,
+                ProxyHandler(self.proxies),
             )
-            self.urlopen = opener.open
         else:
-            self.urlopen = urllib_urlopen
+            opener = build_opener_with_context(
+                self.ssl_context,
+            )
+        self.urlopen = opener.open
 
     @staticmethod
     def _coerce_point_to_string(point):
@@ -181,8 +209,9 @@ def _call_geocoder(
         """
 
         if requester:
-            # Don't construct an urllib's Request for a custom requester
-            req = url
+            req = url  # Don't construct an urllib's Request for a custom requester
+            requester = functools.partial(requester, context=self.ssl_context,
+                                          headers=self.headers)
         else:
             if isinstance(url, Request):
                 # copy Request

geopy/geocoders/bing.py

@@ -39,6 +39,7 @@ def __init__(
             timeout=DEFAULT_SENTINEL,
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """Initialize a customized Bing geocoder with location-specific
         address information and your Bing Maps API key.
@@ -61,13 +62,20 @@ def __init__(
             See :attr:`geopy.geocoders.options.default_user_agent`.
 
             .. versionadded:: 1.12.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
+            .. versionadded:: 1.14.0
         """
         super(Bing, self).__init__(
             format_string=format_string,
             scheme=scheme,
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         self.api_key = api_key
         self.api = "%s://dev.virtualearth.net/REST/v1/Locations" % self.scheme

geopy/geocoders/databc.py

@@ -24,6 +24,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Create a DataBC-based geocoder.
@@ -45,6 +46,12 @@ def __init__(
         :param str format_string:
             See :attr:`geopy.geocoders.options.default_format_string`.
 
+            .. versionadded:: 1.14.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
             .. versionadded:: 1.14.0
         """
         super(DataBC, self).__init__(
@@ -53,6 +60,7 @@ def __init__(
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         self.api = '%s://apps.gov.bc.ca/pub/geocoder/addresses.geojson' % self.scheme
 

geopy/geocoders/geocodefarm.py

@@ -28,6 +28,7 @@ def __init__(
             timeout=DEFAULT_SENTINEL,
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Create a geocoder for GeocodeFarm.
@@ -48,13 +49,20 @@ def __init__(
             See :attr:`geopy.geocoders.options.default_user_agent`.
 
             .. versionadded:: 1.12.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
+            .. versionadded:: 1.14.0
         """
         super(GeocodeFarm, self).__init__(
             format_string=format_string,
             scheme='https',
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         self.api_key = api_key
         self.api = (

geopy/geocoders/geonames.py

@@ -33,6 +33,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         :param str country_bias:
@@ -54,6 +55,12 @@ def __init__(
         :param str format_string:
             See :attr:`geopy.geocoders.options.default_format_string`.
 
+            .. versionadded:: 1.14.0
+
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
             .. versionadded:: 1.14.0
         """
         super(GeoNames, self).__init__(
@@ -62,6 +69,7 @@ def __init__(
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         if username is None:
             raise ConfigurationError(

geopy/geocoders/googlev3.py

@@ -48,6 +48,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
             channel='',
     ):
         """
@@ -87,6 +88,12 @@ def __init__(
 
             .. versionadded:: 1.14.0
 
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
+            .. versionadded:: 1.14.0
+
         :param str channel: If using premier, the channel identifier.
 
             .. versionadded:: 1.12.0
@@ -97,6 +104,7 @@ def __init__(
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
         if client_id and not secret_key:
             raise ConfigurationError('Must provide secret_key with client_id.')

geopy/geocoders/ignfrance.py

@@ -48,6 +48,7 @@ def __init__(
             proxies=DEFAULT_SENTINEL,
             user_agent=None,
             format_string=None,
+            ssl_context=DEFAULT_SENTINEL,
     ):
         """
         Initialize a customized IGN France geocoder.
@@ -89,13 +90,20 @@ def __init__(
 
             .. versionadded:: 1.14.0
 
+        :type ssl_context: :class:`ssl.SSLContext`
+        :param ssl_context:
+            See :attr:`geopy.geocoders.options.default_ssl_context`.
+
+            .. versionadded:: 1.14.0
+
         """
         super(IGNFrance, self).__init__(
             format_string=format_string,
             scheme=scheme,
             timeout=timeout,
             proxies=proxies,
             user_agent=user_agent,
+            ssl_context=ssl_context,
         )
 
         # Catch if no api key with username and password