Merge branch 'master' into release

# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.
#
# It looks like you may be committing a merge.
# If this is not correct, please remove the file
#	.git/MERGE_HEAD
# and try again.

# Please enter the commit message for your changes. Lines starting
# with '#' will be ignored, and an empty message aborts the commit.
# On branch release
# Your branch is up-to-date with 'origin/release'.
#
# All conflicts fixed but you are still merging.
#
# Changes to be committed:
#	modified:   CHANGES.rst
#	modified:   CONTRIBUTORS.txt
#	modified:   MANIFEST.in
#	new file:   docs/doc-requirements.txt
#	modified:   docs/index.rst
#	modified:   dummyserver/handlers.py
#	modified:   setup.py
#	modified:   test/__init__.py
#	modified:   test/test_poolmanager.py
#	modified:   test/test_util.py
#	modified:   test/with_dummyserver/test_connectionpool.py
#	modified:   test/with_dummyserver/test_https.py
#	modified:   urllib3/connection.py
#	modified:   urllib3/connectionpool.py
#	modified:   urllib3/contrib/pyopenssl.py
#	deleted:    urllib3/util.py
#	new file:   urllib3/util/__init__.py
#	new file:   urllib3/util/connection.py
#	new file:   urllib3/util/request.py
#	new file:   urllib3/util/response.py
#	new file:   urllib3/util/ssl_.py
#	new file:   urllib3/util/timeout.py
#	new file:   urllib3/util/url.py
#

CHANGES.rst

@@ -1,6 +1,19 @@
 Changes
 =======
 
+1.8 (2014-03-04)
+++++++++++++++++
+
+* Fix AppEngine bug of HTTPS requests going out as HTTP. (Issue #356)
+
+* Don't install ``dummyserver`` into ``site-packages`` as it's only needed
+  for the test suite. (Issue #362)
+
+* Added support for specifying ``source_address``. (Issue #352)
+
+* ... [Short description of non-trivial change.] (Issue #)
+
+
 1.8 (2014-03-04)
 ++++++++++++++++
 

CONTRIBUTORS.txt

@@ -111,5 +111,11 @@ In chronological order:
   * HTTPHeaderDict and associated tests and docs
   * Bugfixes, docs, test coverage
 
+* Tahia Khan <http://tahia.tk/>
+  * Added Timeout examples in docs
+
+* Arthur Grunseid <http://grunseid.com>
+  * source_address support and tests (with https://github.com/bui)
+
 * [Your name or handle] <[email or website]>
   * [Brief summary of your changes]

MANIFEST.in

@@ -1 +1,2 @@
 include README.rst CHANGES.rst LICENSE.txt CONTRIBUTORS.txt test-requirements.txt
+recursive-include dummyserver *.*

docs/doc-requirements.txt

@@ -0,0 +1,11 @@
+ndg-httpsclient==0.3.2
+pyasn1==0.1.7
+Sphinx==1.2.2
+Jinja2==2.7.2
+MarkupSafe==0.19
+Pygments==1.6
+cryptography==0.2.2
+six==1.6.1
+cffi==0.8.2
+docutils==0.11
+pycparser==2.10

docs/index.rst

@@ -152,6 +152,35 @@ should use a :class:`~urllib3.poolmanager.PoolManager`.
 A :class:`~urllib3.connectionpool.ConnectionPool` is composed of a collection
 of :class:`httplib.HTTPConnection` objects.
 
+Timeout
+-------
+
+A timeout can be set to abort socket operations on individual connections after
+the specified duration. The timeout can be defined as a float or an instance of
+:class:`~urllib3.util.timeout.Timeout` which gives more granular configuration
+over how much time is allowed for different stages of the request. This can be
+set for the entire pool or per-request.
+
+::
+    >>> from urllib3 import PoolManager, Timeout
+
+    >>> # Manager with 3 seconds combined timeout.
+    >>> http = PoolManager(timeout=3.0)
+    >>> r = http.request('GET', 'http://httpbin.org/delay/1')
+
+    >>> # Manager with 2 second timeout for the read phase, no limit for the rest.
+    >>> http = PoolManager(timeout=Timeout(read=2.0)) 
+    >>> r = http.request('GET', 'http://httpbin.org/delay/1')
+
+    >>> # Manager with no timeout but a request with a timeout of 1 seconds for
+    >>> # the connect phase and 2 seconds for the read phase.
+    >>> http = PoolManager()
+    >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(connect=1.0, read=2.0))
+
+    >>> # Same Manager but request with a 5 second total timeout.
+    >>> r = http.request('GET', 'http://httpbin.org/delay/1', timeout=Timeout(total=5.0))
+
+
 Foundation
 ----------
 

dummyserver/handlers.py

@@ -70,6 +70,10 @@ def index(self, _request):
         "Render simple message"
         return Response("Dummy server!")
 
+    def source_address(self, request):
+        """Return the requester's IP address."""
+        return Response(request.remote_ip)
+
     def set_up(self, request):
         test_type = request.params.get('test_type')
         test_id = request.params.get('test_id')

setup.py

@@ -44,7 +44,7 @@
       author_email='andrey.petrov@shazow.net',
       url='http://urllib3.readthedocs.org/',
       license='MIT',
-      packages=['urllib3', 'dummyserver',
+      packages=['urllib3',
                 'urllib3.packages', 'urllib3.packages.ssl_match_hostname',
                 'urllib3.contrib',
                 ],

test/__init__.py

@@ -1,3 +1,4 @@
+import sys
 import errno
 import functools
 import socket
@@ -8,7 +9,39 @@
 from urllib3.packages import six
 
 
-def onlyPY3(test):
+# We need a host that will not immediately close the connection with a TCP
+# Reset. SO suggests this hostname
+TARPIT_HOST = '10.255.255.1'
+
+VALID_SOURCE_ADDRESSES = [('::1', 0), ('127.0.0.1', 0)]
+# RFC 5737: 192.0.2.0/24 is for testing only.
+# RFC 3849: 2001:db8::/32 is for documentation only.
+INVALID_SOURCE_ADDRESSES = [('192.0.2.255', 0), ('2001:db8::1', 0)]
+
+
+def onlyPy26OrOlder(test):
+    """Skips this test unless you are on Python2.6.x or earlier."""
+
+    @functools.wraps(test)
+    def wrapper(*args, **kwargs):
+        msg = "{name} requires Python2.7.x+ to run".format(name=test.__name__)
+        if sys.version_info > (2, 6):
+            raise SkipTest(msg)
+        return test(*args, **kwargs)
+    return wrapper
+
+def onlyPy27OrNewer(test):
+    """Skips this test unless you are on Python2.7.x or later."""
+
+    @functools.wraps(test)
+    def wrapper(*args, **kwargs):
+        msg = "{name} requires Python2.7.x+ to run".format(name=test.__name__)
+        if sys.version_info < (2, 7):
+            raise SkipTest(msg)
+        return test(*args, **kwargs)
+    return wrapper
+
+def onlyPy3(test):
     """Skips this test unless you are on Python3.x"""
 
     @functools.wraps(test)

test/test_poolmanager.py

@@ -2,7 +2,10 @@
 
 from urllib3.poolmanager import PoolManager
 from urllib3 import connection_from_url
-from urllib3.exceptions import ClosedPoolError
+from urllib3.exceptions import (
+    ClosedPoolError,
+    LocationParseError,
+)
 
 
 class TestPoolManager(unittest.TestCase):
@@ -63,6 +66,9 @@ def test_manager_clear(self):
         self.assertEqual(len(p.pools), 0)
 
 
+    def test_nohost(self):
+        p = PoolManager(5)
+        self.assertRaises(LocationParseError, p.connection_from_url, 'http://@')
 
 
 if __name__ == '__main__':

test/test_util.py

@@ -77,6 +77,7 @@ def test_invalid_host(self):
         for location in invalid_host:
             self.assertRaises(LocationParseError, get_host, location)
 
+
     def test_parse_url(self):
         url_host_map = {
             'http://google.com/mail': Url('http', host='google.com', path='/mail'),
@@ -107,6 +108,7 @@ def test_parse_url(self):
             'http://foo:bar@localhost/': Url('http', auth='foo:bar', host='localhost', path='/'),
             'http://foo@localhost/': Url('http', auth='foo', host='localhost', path='/'),
             'http://foo:bar@baz@localhost/': Url('http', auth='foo:bar@baz', host='localhost', path='/'),
+            'http://@': Url('http', host=None, auth='')
         }
         for url, expected_url in url_host_map.items():
             returned_url = parse_url(url)
@@ -231,7 +233,7 @@ def test_invalid_timeouts(self):
             self.assertTrue('int or float' in str(e))
 
 
-    @patch('urllib3.util.current_time')
+    @patch('urllib3.util.timeout.current_time')
     def test_timeout(self, current_time):
         timeout = Timeout(total=3)
 
@@ -278,7 +280,7 @@ def test_timeout_str(self):
         self.assertEqual(str(timeout), "Timeout(connect=1, read=None, total=3)")
 
 
-    @patch('urllib3.util.current_time')
+    @patch('urllib3.util.timeout.current_time')
     def test_timeout_elapsed(self, current_time):
         current_time.return_value = TIMEOUT_EPOCH
         timeout = Timeout(total=3)

test/with_dummyserver/test_connectionpool.py

@@ -10,7 +10,9 @@
 except:
     from urllib import urlencode
 
-from test import requires_network, onlyPY3
+from test import (
+    onlyPy3, onlyPy27OrNewer, onlyPy26OrOlder, requires_network, TARPIT_HOST,
+    VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES)
 from urllib3 import (
     encode_multipart_formdata,
     HTTPConnectionPool,
@@ -23,7 +25,7 @@
     MaxRetryError,
     ReadTimeoutError,
 )
-from urllib3.packages.six import u
+from urllib3.packages.six import b, u, string_types
 from urllib3 import util
 
 import tornado
@@ -35,10 +37,6 @@
 log.setLevel(logging.NOTSET)
 log.addHandler(logging.StreamHandler(sys.stdout))
 
-# We need a host that will not immediately close the connection with a TCP
-# Reset. SO suggests this hostname
-TARPIT_HOST = '10.255.255.1'
-
 
 class TestConnectionPool(HTTPDummyServerTestCase):
 
@@ -536,7 +534,32 @@ def test_dns_error(self):
         pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001)
         self.assertRaises(MaxRetryError, pool.request, 'GET', '/test', retries=2)
 
-    @onlyPY3
+    @onlyPy26OrOlder
+    def test_source_address_ignored(self):
+        # source_address is ignored in Python 2.6 and older.
+        for addr in INVALID_SOURCE_ADDRESSES:
+            pool = HTTPConnectionPool(
+                self.host, self.port, source_address=addr)
+            r = pool.request('GET', '/source_address')
+            assert r.status == 200
+
+    @onlyPy27OrNewer
+    def test_source_address(self):
+        for addr in VALID_SOURCE_ADDRESSES:
+            pool = HTTPConnectionPool(
+                self.host, self.port, source_address=addr)
+            r = pool.request('GET', '/source_address')
+            assert r.data == b(addr[0])
+
+    @onlyPy27OrNewer
+    def test_source_address_error(self):
+        for addr in INVALID_SOURCE_ADDRESSES:
+            pool = HTTPConnectionPool(
+                self.host, self.port, source_address=addr)
+            self.assertRaises(
+                MaxRetryError, pool.request, 'GET', '/source_address')
+
+    @onlyPy3
     def test_httplib_headers_case_insensitive(self):
         HEADERS = {'Content-Length': '0', 'Content-type': 'text/plain',
                     'Server': 'TornadoServer/%s' % tornado.version}

test/with_dummyserver/test_https.py

@@ -9,25 +9,25 @@
 from dummyserver.testcase import HTTPSDummyServerTestCase
 from dummyserver.server import DEFAULT_CA, DEFAULT_CA_BAD, DEFAULT_CERTS
 
-from test import requires_network
-
+from test import (
+    onlyPy3, onlyPy27OrNewer, onlyPy26OrOlder, requires_network, TARPIT_HOST,
+    VALID_SOURCE_ADDRESSES, INVALID_SOURCE_ADDRESSES)
 from urllib3 import HTTPSConnectionPool
+from urllib3.packages.six import b, string_types
 import urllib3.connection
 from urllib3.connection import (
     VerifiedHTTPSConnection,
     UnverifiedHTTPSConnection,
 )
-from urllib3.exceptions import SSLError, ConnectTimeoutError, ReadTimeoutError
+from urllib3.exceptions import (
+    SSLError, MaxRetryError, ReadTimeoutError, ConnectTimeoutError)
 from urllib3.util import Timeout
 
 
 log = logging.getLogger('urllib3.connectionpool')
 log.setLevel(logging.NOTSET)
 log.addHandler(logging.StreamHandler(sys.stdout))
 
-# We need a host that will not immediately close the connection with a TCP
-# Reset. SO suggests this hostname
-TARPIT_HOST = '10.255.255.1'
 
 class TestHTTPS(HTTPSDummyServerTestCase):
     def setUp(self):
@@ -121,14 +121,14 @@ def test_ssl_unverified_with_ca_certs(self):
     @requires_network
     def test_ssl_verified_with_platform_ca_certs(self):
         """
-        This test check that whe rely on platform CA file to validate
-        authenticity of SSL certificate. Since this file is used by many
-        components of the OS, such as curl, apt-get, etc., we decided to not
-        touch it, in order to not compromise the security of the OS
-        running the test suite (typically urllib3 developer's OS).
-
-        This test assume that httpbin.org use a certificate signed
-        by a well known Certificate Authority.
+        We should rely on the platform CA file to validate authenticity of SSL
+        certificates. Since this file is used by many components of the OS,
+        such as curl, apt-get, etc., we decided to not touch it, in order to
+        not compromise the security of the OS running the test suite (typically
+        urllib3 developer's OS).
+
+        This test assumes that httpbin.org uses a certificate signed by a well
+        known Certificate Authority.
         """
         try:
             import urllib3.contrib.pyopenssl
@@ -137,7 +137,8 @@ def test_ssl_verified_with_platform_ca_certs(self):
         if (urllib3.connection.ssl_wrap_socket is
                 urllib3.contrib.pyopenssl.orig_connection_ssl_wrap_socket):
             # Not patched
-            raise SkipTest('This test needs pyopenssl support')
+            raise SkipTest('This test should only be run after pyopenssl '
+                           'monkey patching')
 
         https_pool = HTTPSConnectionPool('httpbin.org', 443,
                                          cert_reqs=ssl.CERT_REQUIRED)
@@ -268,7 +269,6 @@ def test_tunnel_old_python(self):
         del conn._tunnel_host
         self._pool._make_request(conn, 'GET', '/')
 
-
     @requires_network
     def test_enhanced_timeout(self):
         def new_pool(timeout, cert_reqs='CERT_REQUIRED'):
@@ -303,6 +303,37 @@ def test_enhanced_ssl_connection(self):
                                         '7A:F2:8A:D7:1E:07:33:67:DE'
         https_pool._make_request(conn, 'GET', '/')
 
+    @onlyPy26OrOlder
+    def test_source_address_ignored(self):
+        # source_address is ignored in Python 2.6 and earlier.
+        for addr in INVALID_SOURCE_ADDRESSES:
+            https_pool = HTTPSConnectionPool(
+                self.host, self.port, cert_reqs='CERT_REQUIRED',
+                source_address=addr)
+            https_pool.ca_certs = DEFAULT_CA
+            r = https_pool.request('GET', '/source_address')
+            assert r.status == 200
+
+    @onlyPy27OrNewer
+    def test_source_address(self):
+        for addr in VALID_SOURCE_ADDRESSES:
+            https_pool = HTTPSConnectionPool(
+                self.host, self.port, cert_reqs='CERT_REQUIRED',
+                source_address=addr)
+            https_pool.ca_certs = DEFAULT_CA
+            r = https_pool.request('GET', '/source_address')
+            assert r.data == b(addr[0])
+
+    @onlyPy27OrNewer
+    def test_source_address_error(self):
+        for addr in INVALID_SOURCE_ADDRESSES:
+            https_pool = HTTPSConnectionPool(
+                self.host, self.port, cert_reqs='CERT_REQUIRED',
+                source_address=addr)
+            https_pool.ca_certs = DEFAULT_CA
+            self.assertRaises(
+                MaxRetryError, https_pool.request, 'GET', '/source_address')
+
 
 class TestHTTPS_TLSv1(HTTPSDummyServerTestCase):
     certs = DEFAULT_CERTS.copy()