Update urllib3 to 1.19

Author: Lukasa

requests/packages/urllib3/__init__.py

@@ -32,7 +32,7 @@ def emit(self, record):
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.16'
+__version__ = '1.19'
 
 __all__ = (
     'HTTPConnectionPool',

requests/packages/urllib3/connection.py

@@ -7,13 +7,8 @@
 from socket import error as SocketError, timeout as SocketTimeout
 import warnings
 from .packages import six
-
-try:  # Python 3
-    from http.client import HTTPConnection as _HTTPConnection
-    from http.client import HTTPException  # noqa: unused in this module
-except ImportError:
-    from httplib import HTTPConnection as _HTTPConnection
-    from httplib import HTTPException  # noqa: unused in this module
+from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
+from .packages.six.moves.http_client import HTTPException  # noqa: F401
 
 try:  # Compiled with SSL?
     import ssl
@@ -44,8 +39,9 @@ class ConnectionError(Exception):
 from .util.ssl_ import (
     resolve_cert_reqs,
     resolve_ssl_version,
-    ssl_wrap_socket,
     assert_fingerprint,
+    create_urllib3_context,
+    ssl_wrap_socket
 )
 
 
@@ -174,7 +170,13 @@ def request_chunked(self, method, url, body=None, headers=None):
         """
         headers = HTTPHeaderDict(headers if headers is not None else {})
         skip_accept_encoding = 'accept-encoding' in headers
-        self.putrequest(method, url, skip_accept_encoding=skip_accept_encoding)
+        skip_host = 'host' in headers
+        self.putrequest(
+            method,
+            url,
+            skip_accept_encoding=skip_accept_encoding,
+            skip_host=skip_host
+        )
         for header, value in headers.items():
             self.putheader(header, value)
         if 'transfer-encoding' not in headers:
@@ -203,14 +205,18 @@ def request_chunked(self, method, url, body=None, headers=None):
 class HTTPSConnection(HTTPConnection):
     default_port = port_by_scheme['https']
 
+    ssl_version = None
+
     def __init__(self, host, port=None, key_file=None, cert_file=None,
-                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kw):
+                 strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+                 ssl_context=None, **kw):
 
         HTTPConnection.__init__(self, host, port, strict=strict,
                                 timeout=timeout, **kw)
 
         self.key_file = key_file
         self.cert_file = cert_file
+        self.ssl_context = ssl_context
 
         # Required property for Google AppEngine 1.9.0 which otherwise causes
         # HTTPS requests to go out as HTTP. (See Issue #356)
@@ -219,7 +225,19 @@ def __init__(self, host, port=None, key_file=None, cert_file=None,
     def connect(self):
         conn = self._new_conn()
         self._prepare_conn(conn)
-        self.sock = ssl.wrap_socket(conn, self.key_file, self.cert_file)
+
+        if self.ssl_context is None:
+            self.ssl_context = create_urllib3_context(
+                ssl_version=resolve_ssl_version(None),
+                cert_reqs=resolve_cert_reqs(None),
+            )
+
+        self.sock = ssl_wrap_socket(
+            sock=conn,
+            keyfile=self.key_file,
+            certfile=self.cert_file,
+            ssl_context=self.ssl_context,
+        )
 
 
 class VerifiedHTTPSConnection(HTTPSConnection):
@@ -237,9 +255,18 @@ def set_cert(self, key_file=None, cert_file=None,
                  cert_reqs=None, ca_certs=None,
                  assert_hostname=None, assert_fingerprint=None,
                  ca_cert_dir=None):
-
-        if (ca_certs or ca_cert_dir) and cert_reqs is None:
-            cert_reqs = 'CERT_REQUIRED'
+        """
+        This method should only be called once, before the connection is used.
+        """
+        # If cert_reqs is not provided, we can try to guess. If the user gave
+        # us a cert database, we assume they want to use it: otherwise, if
+        # they gave us an SSL Context object we should use whatever is set for
+        # it.
+        if cert_reqs is None:
+            if ca_certs or ca_cert_dir:
+                cert_reqs = 'CERT_REQUIRED'
+            elif self.ssl_context is not None:
+                cert_reqs = self.ssl_context.verify_mode
 
         self.key_file = key_file
         self.cert_file = cert_file
@@ -253,9 +280,6 @@ def connect(self):
         # Add certificate verification
         conn = self._new_conn()
 
-        resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
-        resolved_ssl_version = resolve_ssl_version(self.ssl_version)
-
         hostname = self.host
         if getattr(self, '_tunnel_host', None):
             # _tunnel_host was added in Python 2.6.3
@@ -281,17 +305,27 @@ def connect(self):
 
         # Wrap socket using verification with the root certs in
         # trusted_root_certs
-        self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
-                                    cert_reqs=resolved_cert_reqs,
-                                    ca_certs=self.ca_certs,
-                                    ca_cert_dir=self.ca_cert_dir,
-                                    server_hostname=hostname,
-                                    ssl_version=resolved_ssl_version)
+        if self.ssl_context is None:
+            self.ssl_context = create_urllib3_context(
+                ssl_version=resolve_ssl_version(self.ssl_version),
+                cert_reqs=resolve_cert_reqs(self.cert_reqs),
+            )
+
+        context = self.ssl_context
+        context.verify_mode = resolve_cert_reqs(self.cert_reqs)
+        self.sock = ssl_wrap_socket(
+            sock=conn,
+            keyfile=self.key_file,
+            certfile=self.cert_file,
+            ca_certs=self.ca_certs,
+            ca_cert_dir=self.ca_cert_dir,
+            server_hostname=hostname,
+            ssl_context=context)
 
         if self.assert_fingerprint:
             assert_fingerprint(self.sock.getpeercert(binary_form=True),
                                self.assert_fingerprint)
-        elif resolved_cert_reqs != ssl.CERT_NONE \
+        elif context.verify_mode != ssl.CERT_NONE \
                 and self.assert_hostname is not False:
             cert = self.sock.getpeercert()
             if not cert.get('subjectAltName', ()):
@@ -304,8 +338,10 @@ def connect(self):
                 )
             _match_hostname(cert, self.assert_hostname or hostname)
 
-        self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED or
-                            self.assert_fingerprint is not None)
+        self.is_verified = (
+            context.verify_mode == ssl.CERT_REQUIRED or
+            self.assert_fingerprint is not None
+        )
 
 
 def _match_hostname(cert, asserted_hostname):

requests/packages/urllib3/connectionpool.py

@@ -7,13 +7,6 @@
 from socket import error as SocketError, timeout as SocketTimeout
 import socket
 
-try:  # Python 3
-    from queue import LifoQueue, Empty, Full
-except ImportError:
-    from Queue import LifoQueue, Empty, Full
-    # Queue is imported for side effects on MS Windows
-    import Queue as _unused_module_Queue  # noqa: unused
-
 
 from .exceptions import (
     ClosedPoolError,
@@ -32,6 +25,7 @@
 )
 from .packages.ssl_match_hostname import CertificateError
 from .packages import six
+from .packages.six.moves.queue import LifoQueue, Empty, Full
 from .connection import (
     port_by_scheme,
     DummyConnection,
@@ -48,6 +42,10 @@
 from .util.url import get_host, Url
 
 
+if six.PY2:
+    # Queue is imported for side effects on MS Windows
+    import Queue as _unused_module_Queue  # noqa: F401
+
 xrange = six.moves.xrange
 
 log = logging.getLogger(__name__)
@@ -210,8 +208,8 @@ def _new_conn(self):
         Return a fresh :class:`HTTPConnection`.
         """
         self.num_connections += 1
-        log.info("Starting new HTTP connection (%d): %s",
-                 self.num_connections, self.host)
+        log.debug("Starting new HTTP connection (%d): %s",
+                  self.num_connections, self.host)
 
         conn = self.ConnectionCls(host=self.host, port=self.port,
                                   timeout=self.timeout.connect_timeout,
@@ -246,7 +244,7 @@ def _get_conn(self, timeout=None):
 
         # If this is a persistent connection, check if it got disconnected
         if conn and is_connection_dropped(conn):
-            log.info("Resetting dropped connection: %s", self.host)
+            log.debug("Resetting dropped connection: %s", self.host)
             conn.close()
             if getattr(conn, 'auto_open', 1) == 0:
                 # This is a proxied connection that has been mutated by
@@ -397,8 +395,9 @@ def _make_request(self, conn, method, url, timeout=_Default, chunked=False,
 
         # AppEngine doesn't have a version attr.
         http_version = getattr(conn, '_http_vsn_str', 'HTTP/?')
-        log.debug("\"%s %s %s\" %s %s", method, url, http_version,
-                  httplib_response.status, httplib_response.length)
+        log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port,
+                  method, url, http_version, httplib_response.status,
+                  httplib_response.length)
 
         try:
             assert_header_parsing(httplib_response.msg)
@@ -600,10 +599,14 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
             # mess.
             response_conn = conn if not release_conn else None
 
+            # Pass method to Response for length checking
+            response_kw['request_method'] = method
+
             # Import httplib's response into our own wrapper object
             response = self.ResponseCls.from_httplib(httplib_response,
                                                      pool=self,
                                                      connection=response_conn,
+                                                     retries=retries,
                                                      **response_kw)
 
             # Everything went great!
@@ -683,7 +686,8 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
                     raise
                 return response
 
-            log.info("Redirecting %s -> %s", url, redirect_location)
+            retries.sleep_for_retry(response)
+            log.debug("Redirecting %s -> %s", url, redirect_location)
             return self.urlopen(
                 method, redirect_location, body, headers,
                 retries=retries, redirect=redirect,
@@ -692,7 +696,8 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
                 release_conn=release_conn, **response_kw)
 
         # Check if we should retry the HTTP response.
-        if retries.is_forced_retry(method, status_code=response.status):
+        has_retry_after = bool(response.getheader('Retry-After'))
+        if retries.is_retry(method, response.status, has_retry_after):
             try:
                 retries = retries.increment(method, url, response=response, _pool=self)
             except MaxRetryError:
@@ -702,8 +707,8 @@ def urlopen(self, method, url, body=None, headers=None, retries=None,
                     response.release_conn()
                     raise
                 return response
-            retries.sleep()
-            log.info("Forced retry: %s", url)
+            retries.sleep(response)
+            log.debug("Retry: %s", url)
             return self.urlopen(
                 method, url, body, headers,
                 retries=retries, redirect=redirect,
@@ -775,7 +780,6 @@ def _prepare_conn(self, conn):
                           assert_hostname=self.assert_hostname,
                           assert_fingerprint=self.assert_fingerprint)
             conn.ssl_version = self.ssl_version
-
         return conn
 
     def _prepare_proxy(self, conn):
@@ -801,8 +805,8 @@ def _new_conn(self):
         Return a fresh :class:`httplib.HTTPSConnection`.
         """
         self.num_connections += 1
-        log.info("Starting new HTTPS connection (%d): %s",
-                 self.num_connections, self.host)
+        log.debug("Starting new HTTPS connection (%d): %s",
+                  self.num_connections, self.host)
 
         if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
             raise SSLError("Can't connect to HTTPS URL because the SSL "
@@ -834,7 +838,8 @@ def _validate_conn(self, conn):
             warnings.warn((
                 'Unverified HTTPS request is being made. '
                 'Adding certificate verification is strongly advised. See: '
-                'https://urllib3.readthedocs.io/en/latest/security.html'),
+                'https://urllib3.readthedocs.io/en/latest/advanced-usage.html'
+                '#ssl-warnings'),
                 InsecureRequestWarning)