Upgrade urllib3 to 1.9.x

requests/packages/urllib3/__init__.py

@@ -1,9 +1,3 @@
-# urllib3/__init__.py
-# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
-#
-# This module is part of urllib3 and is released under
-# the MIT License: http://www.opensource.org/licenses/mit-license.php
-
 """
 urllib3 - Thread-safe connection pooling and re-using.
 """
@@ -23,7 +17,10 @@
 from .filepost import encode_multipart_formdata
 from .poolmanager import PoolManager, ProxyManager, proxy_from_url
 from .response import HTTPResponse
-from .util import make_headers, get_host, Timeout
+from .util.request import make_headers
+from .util.url import get_host
+from .util.timeout import Timeout
+from .util.retry import Retry
 
 
 # Set default logging handler to avoid "No handler found" warnings.
@@ -51,8 +48,19 @@ def add_stderr_logger(level=logging.DEBUG):
     handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
     logger.addHandler(handler)
     logger.setLevel(level)
-    logger.debug('Added an stderr logging handler to logger: %s' % __name__)
+    logger.debug('Added a stderr logging handler to logger: %s' % __name__)
     return handler
 
 # ... Clean up.
 del NullHandler
+
+
+# Set security warning to only go off once by default.
+import warnings
+warnings.simplefilter('module', exceptions.InsecureRequestWarning)
+
+def disable_warnings(category=exceptions.HTTPWarning):
+    """
+    Helper for quickly disabling all urllib3 warnings.
+    """
+    warnings.simplefilter('ignore', category)

requests/packages/urllib3/_collections.py

@@ -1,9 +1,3 @@
-# urllib3/_collections.py
-# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
-#
-# This module is part of urllib3 and is released under
-# the MIT License: http://www.opensource.org/licenses/mit-license.php
-
 from collections import Mapping, MutableMapping
 try:
     from threading import RLock
@@ -116,7 +110,7 @@ class HTTPHeaderDict(MutableMapping):
     A ``dict`` like container for storing HTTP Headers.
 
     Field names are stored and compared case-insensitively in compliance with
-    RFC 2616. Iteration provides the first case-sensitive key seen for each
+    RFC 7230. Iteration provides the first case-sensitive key seen for each
     case-insensitive pair.
 
     Using ``__setitem__`` syntax overwrites fields that compare equal

requests/packages/urllib3/connection.py

@@ -1,52 +1,44 @@
-# urllib3/connection.py
-# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
-#
-# This module is part of urllib3 and is released under
-# the MIT License: http://www.opensource.org/licenses/mit-license.php
-
 import sys
 import socket
 from socket import timeout as SocketTimeout
 
-try: # Python 3
+try:  # Python 3
     from http.client import HTTPConnection as _HTTPConnection, HTTPException
 except ImportError:
     from httplib import HTTPConnection as _HTTPConnection, HTTPException
 
+
 class DummyConnection(object):
     "Used to detect a failed ConnectionCls import."
     pass
 
-try: # Compiled with SSL?
-    ssl = None
+
+try:  # Compiled with SSL?
     HTTPSConnection = DummyConnection
+    import ssl
+    BaseSSLError = ssl.SSLError
+except (ImportError, AttributeError):  # Platform-specific: No SSL.
+    ssl = None
 
     class BaseSSLError(BaseException):
         pass
 
-    try: # Python 3
-        from http.client import HTTPSConnection as _HTTPSConnection
-    except ImportError:
-        from httplib import HTTPSConnection as _HTTPSConnection
-
-    import ssl
-    BaseSSLError = ssl.SSLError
-
-except (ImportError, AttributeError): # Platform-specific: No SSL.
-    pass
 
 from .exceptions import (
     ConnectTimeoutError,
 )
 from .packages.ssl_match_hostname import match_hostname
 from .packages import six
-from .util import (
-    assert_fingerprint,
+
+from .util.ssl_ import (
     resolve_cert_reqs,
     resolve_ssl_version,
     ssl_wrap_socket,
+    assert_fingerprint,
 )
 
+from .util import connection
+
 
 port_by_scheme = {
     'http': 80,
@@ -58,38 +50,79 @@ class HTTPConnection(_HTTPConnection, object):
     """
     Based on httplib.HTTPConnection but provides an extra constructor
     backwards-compatibility layer between older and newer Pythons.
+
+    Additional keyword parameters are used to configure attributes of the connection.
+    Accepted parameters include:
+
+      - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
+      - ``source_address``: Set the source address for the current connection.
+
+        .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x
+
+      - ``socket_options``: Set specific options on the underlying socket. If not specified, then
+        defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
+        Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
+
+        For example, if you wish to enable TCP Keep Alive in addition to the defaults,
+        you might pass::
+
+            HTTPConnection.default_socket_options + [
+                (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
+            ]
+
+        Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
     """
 
     default_port = port_by_scheme['http']
 
-    # By default, disable Nagle's Algorithm.
-    tcp_nodelay = 1
+    #: Disable Nagle's algorithm by default.
+    #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
+    default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
+
+    #: Whether this connection verifies the host's certificate.
+    is_verified = False
 
     def __init__(self, *args, **kw):
         if six.PY3:  # Python 3
             kw.pop('strict', None)
-        if sys.version_info < (2, 7):  # Python 2.6 and older
-            kw.pop('source_address', None)
 
         # Pre-set source_address in case we have an older Python like 2.6.
         self.source_address = kw.get('source_address')
 
+        if sys.version_info < (2, 7):  # Python 2.6
+            # _HTTPConnection on Python 2.6 will balk at this keyword arg, but
+            # not newer versions. We can still use it when creating a
+            # connection though, so we pop it *after* we have saved it as
+            # self.source_address.
+            kw.pop('source_address', None)
+
+        #: The socket options provided by the user. If no options are
+        #: provided, we use the default options.
+        self.socket_options = kw.pop('socket_options', self.default_socket_options)
+
         # Superclass also sets self.source_address in Python 2.7+.
-        _HTTPConnection.__init__(self, *args, **kw)  
+        _HTTPConnection.__init__(self, *args, **kw)
 
     def _new_conn(self):
         """ Establish a socket connection and set nodelay settings on it.
 
-        :return: a new socket connection
+        :return: New socket connection.
         """
-        extra_args = []
-        if self.source_address:  # Python 2.7+
-            extra_args.append(self.source_address)
+        extra_kw = {}
+        if self.source_address:
+            extra_kw['source_address'] = self.source_address
 
-        conn = socket.create_connection(
-            (self.host, self.port), self.timeout, *extra_args)
-        conn.setsockopt(
-            socket.IPPROTO_TCP, socket.TCP_NODELAY, self.tcp_nodelay)
+        if self.socket_options:
+            extra_kw['socket_options'] = self.socket_options
+
+        try:
+            conn = connection.create_connection(
+                (self.host, self.port), self.timeout, **extra_kw)
+
+        except SocketTimeout:
+            raise ConnectTimeoutError(
+                self, "Connection to %s timed out. (connect timeout=%s)" %
+                (self.host, self.timeout))
 
         return conn
 
@@ -101,6 +134,8 @@ def _prepare_conn(self, conn):
         if getattr(self, '_tunnel_host', None):
             # TODO: Fix tunnel so it doesn't depend on self.sock state.
             self._tunnel()
+            # Mark this connection as not reusable
+            self.auto_open = 0
 
     def connect(self):
         conn = self._new_conn()
@@ -137,7 +172,6 @@ class VerifiedHTTPSConnection(HTTPSConnection):
     cert_reqs = None
     ca_certs = None
     ssl_version = None
-    conn_kw = {}
 
     def set_cert(self, key_file=None, cert_file=None,
                  cert_reqs=None, ca_certs=None,
@@ -152,18 +186,7 @@ def set_cert(self, key_file=None, cert_file=None,
 
     def connect(self):
         # Add certificate verification
-
-        try:
-            sock = socket.create_connection(
-                address=(self.host, self.port), timeout=self.timeout,
-                **self.conn_kw)
-        except SocketTimeout:
-            raise ConnectTimeoutError(
-                self, "Connection to %s timed out. (connect timeout=%s)" %
-                (self.host, self.timeout))
-
-        sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY,
-                        self.tcp_nodelay)
+        conn = self._new_conn()
 
         resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
         resolved_ssl_version = resolve_ssl_version(self.ssl_version)
@@ -173,17 +196,19 @@ def connect(self):
             # _tunnel_host was added in Python 2.6.3
             # (See: http://hg.python.org/cpython/rev/0f57b30a152f)
 
-            self.sock = sock
+            self.sock = conn
             # Calls self._set_hostport(), so self.host is
             # self._tunnel_host below.
             self._tunnel()
+            # Mark this connection as not reusable
+            self.auto_open = 0
 
             # Override the host with the one we're requesting data from.
             hostname = self._tunnel_host
 
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
-        self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
+        self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
                                     cert_reqs=resolved_cert_reqs,
                                     ca_certs=self.ca_certs,
                                     server_hostname=hostname,
@@ -197,6 +222,8 @@ def connect(self):
                 match_hostname(self.sock.getpeercert(),
                                self.assert_hostname or hostname)
 
+        self.is_verified = resolved_cert_reqs == ssl.CERT_REQUIRED
+
 
 if ssl:
     # Make a copy for testing.