Merge branch master into release

urllib3 · Jun 16, 2012 · b922f87 · b922f87
2 parents 5366c01 + 358c789
commit b922f87
Show file tree

Hide file tree

Showing 9 changed files with 192 additions and 129 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -5,6 +5,6 @@ omit = urllib3/packages/*
 exclude_lines =
     .* # Platform-specific.*
     except ImportError:
-    .*:.* # Python 3
+    .*:.* # Python \d.*
     pass
     .* # Abstract
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,6 +1,20 @@
 Changes
 =======
 
+
+1.4 (2012-06-16)
+++++++++++++++++
+
+* Minor AppEngine-related fixes.
+
+* Switched from ``mimetools.choose_boundary`` to ``uuid.uuid4()``.
+
+* Improved url parsing. (Issue #73)
+
+* IPv6 url support. (Issue #72)
+
+
+>>>>>>> master
 1.3 (2012-03-25)
 ++++++++++++++++
 

diff --git a/setup.py b/setup.py
@@ -45,7 +45,7 @@
       url='http://urllib3.readthedocs.org/',
       license='MIT',
       packages=['urllib3', 'dummyserver', 'urllib3.packages',
-                'urllib3.packages.ssl_match_hostname', 'urllib3.packages.mimetools_choose_boundary',
+                'urllib3.packages.ssl_match_hostname',
                 ],
       requires=requirements,
       tests_require=tests_requirements,

diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py
@@ -1,30 +1,10 @@
 import unittest
 
 from urllib3.connectionpool import connection_from_url, HTTPConnectionPool
-from urllib3.util import get_host, make_headers
-from urllib3.exceptions import EmptyPoolError, LocationParseError
+from urllib3.exceptions import EmptyPoolError
 
 
 class TestConnectionPool(unittest.TestCase):
-    def test_get_host(self):
-        url_host_map = {
-            'http://google.com/mail': ('http', 'google.com', None),
-            'http://google.com/mail/': ('http', 'google.com', None),
-            'google.com/mail': ('http', 'google.com', None),
-            'http://google.com/': ('http', 'google.com', None),
-            'http://google.com': ('http', 'google.com', None),
-            'http://www.google.com': ('http', 'www.google.com', None),
-            'http://mail.google.com': ('http', 'mail.google.com', None),
-            'http://google.com:8000/mail/': ('http', 'google.com', 8000),
-            'http://google.com:8000': ('http', 'google.com', 8000),
-            'https://google.com': ('https', 'google.com', None),
-            'https://google.com:8000': ('https', 'google.com', 8000),
-            'http://user:password@127.0.0.1:1234': ('http', '127.0.0.1', 1234),
-        }
-        for url, expected_host in url_host_map.items():
-            returned_host = get_host(url)
-            self.assertEquals(returned_host, expected_host)
-
     def test_same_host(self):
         same_host = [
             ('http://google.com/', '/'),
@@ -50,45 +30,6 @@ def test_same_host(self):
             c = connection_from_url(a)
             self.assertFalse(c.is_same_host(b), "%s =? %s" % (a, b))
 
-    def test_invalid_host(self):
-        # TODO: Add more tests
-        invalid_host = [
-            'http://google.com:foo',
-        ]
-
-        for location in invalid_host:
-            self.assertRaises(LocationParseError, get_host, location)
-
-
-    def test_make_headers(self):
-        self.assertEqual(
-            make_headers(accept_encoding=True),
-            {'accept-encoding': 'gzip,deflate'})
-
-        self.assertEqual(
-            make_headers(accept_encoding='foo,bar'),
-            {'accept-encoding': 'foo,bar'})
-
-        self.assertEqual(
-            make_headers(accept_encoding=['foo', 'bar']),
-            {'accept-encoding': 'foo,bar'})
-
-        self.assertEqual(
-            make_headers(accept_encoding=True, user_agent='banana'),
-            {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'})
-
-        self.assertEqual(
-            make_headers(user_agent='banana'),
-            {'user-agent': 'banana'})
-
-        self.assertEqual(
-            make_headers(keep_alive=True),
-            {'connection': 'keep-alive'})
-
-        self.assertEqual(
-            make_headers(basic_auth='foo:bar'),
-            {'authorization': 'Basic Zm9vOmJhcg=='})
-
     def test_max_connections(self):
         pool = HTTPConnectionPool(host='localhost', maxsize=1, block=True)
 

diff --git a/test/test_util.py b/test/test_util.py
@@ -0,0 +1,102 @@
+import unittest
+
+from urllib3.util import get_host, make_headers, split_first
+from urllib3.exceptions import LocationParseError
+
+
+class TestUtil(unittest.TestCase):
+    def test_get_host(self):
+        url_host_map = {
+            # Hosts
+            'http://google.com/mail': ('http', 'google.com', None),
+            'http://google.com/mail/': ('http', 'google.com', None),
+            'google.com/mail': ('http', 'google.com', None),
+            'http://google.com/': ('http', 'google.com', None),
+            'http://google.com': ('http', 'google.com', None),
+            'http://www.google.com': ('http', 'www.google.com', None),
+            'http://mail.google.com': ('http', 'mail.google.com', None),
+            'http://google.com:8000/mail/': ('http', 'google.com', 8000),
+            'http://google.com:8000': ('http', 'google.com', 8000),
+            'https://google.com': ('https', 'google.com', None),
+            'https://google.com:8000': ('https', 'google.com', 8000),
+            'http://user:password@127.0.0.1:1234': ('http', '127.0.0.1', 1234),
+            'http://google.com/foo=http://bar:42/baz': ('http', 'google.com', None),
+            'http://google.com?foo=http://bar:42/baz': ('http', 'google.com', None),
+            'http://google.com#foo=http://bar:42/baz': ('http', 'google.com', None),
+
+            # IPv4
+            '173.194.35.7': ('http', '173.194.35.7', None),
+            'http://173.194.35.7': ('http', '173.194.35.7', None),
+            'http://173.194.35.7/test': ('http', '173.194.35.7', None),
+            'http://173.194.35.7:80': ('http', '173.194.35.7', 80),
+            'http://173.194.35.7:80/test': ('http', '173.194.35.7', 80),
+
+            # IPv6
+            '[2a00:1450:4001:c01::67]': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]/test': ('http', '2a00:1450:4001:c01::67', None),
+            'http://[2a00:1450:4001:c01::67]:80': ('http', '2a00:1450:4001:c01::67', 80),
+            'http://[2a00:1450:4001:c01::67]:80/test': ('http', '2a00:1450:4001:c01::67', 80),
+
+            # More IPv6 from http://www.ietf.org/rfc/rfc2732.txt
+            'http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8000/index.html': ('http', 'FEDC:BA98:7654:3210:FEDC:BA98:7654:3210', 8000),
+            'http://[1080:0:0:0:8:800:200C:417A]/index.html': ('http', '1080:0:0:0:8:800:200C:417A', None),
+            'http://[3ffe:2a00:100:7031::1]': ('http', '3ffe:2a00:100:7031::1', None),
+            'http://[1080::8:800:200C:417A]/foo': ('http', '1080::8:800:200C:417A', None),
+            'http://[::192.9.5.5]/ipng': ('http', '::192.9.5.5', None),
+            'http://[::FFFF:129.144.52.38]:42/index.html': ('http', '::FFFF:129.144.52.38', 42),
+            'http://[2010:836B:4179::836B:4179]': ('http', '2010:836B:4179::836B:4179', None),
+        }
+        for url, expected_host in url_host_map.items():
+            returned_host = get_host(url)
+            self.assertEquals(returned_host, expected_host)
+
+    def test_invalid_host(self):
+        # TODO: Add more tests
+        invalid_host = [
+            'http://google.com:foo',
+        ]
+
+        for location in invalid_host:
+            self.assertRaises(LocationParseError, get_host, location)
+
+
+    def test_make_headers(self):
+        self.assertEqual(
+            make_headers(accept_encoding=True),
+            {'accept-encoding': 'gzip,deflate'})
+
+        self.assertEqual(
+            make_headers(accept_encoding='foo,bar'),
+            {'accept-encoding': 'foo,bar'})
+
+        self.assertEqual(
+            make_headers(accept_encoding=['foo', 'bar']),
+            {'accept-encoding': 'foo,bar'})
+
+        self.assertEqual(
+            make_headers(accept_encoding=True, user_agent='banana'),
+            {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'})
+
+        self.assertEqual(
+            make_headers(user_agent='banana'),
+            {'user-agent': 'banana'})
+
+        self.assertEqual(
+            make_headers(keep_alive=True),
+            {'connection': 'keep-alive'})
+
+        self.assertEqual(
+            make_headers(basic_auth='foo:bar'),
+            {'authorization': 'Basic Zm9vOmJhcg=='})
+
+
+    def test_split_first(self):
+        test_cases = {
+            ('abcd', 'b'): ('a', 'cd'),
+            ('abcd', 'cb'): ('a', 'cd'),
+            ('abcd', ''): ('abcd', ''),
+        }
+        for input, expected in test_cases.iteritems():
+            output = split_first(*input)
+            self.assertEqual(output, expected)
diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py
@@ -258,12 +258,16 @@ def _make_request(self, conn, method, url, timeout=_Default,
         if sock:
             sock.settimeout(timeout)
 
-        httplib_response = conn.getresponse()
-
-        log.debug("\"%s %s %s\" %s %s" %
-                  (method, url,
-                   conn._http_vsn_str, # pylint: disable-msg=W0212
-                   httplib_response.status, httplib_response.length))
+        try: # Python 2.7+, use buffering of HTTP responses
+            httplib_response = conn.getresponse(buffering=True)
+        except TypeError: # Python 2.6 and older
+            httplib_response = conn.getresponse()
+
+        # AppEngine doesn't have a version attr.
+        http_version = getattr(conn, '_http_vsn_str', 'HTTP/?'),
+        log.debug("\"%s %s %s\" %s %s" % (method, url, http_version,
+                                          httplib_response.status,
+                                          httplib_response.length))
 
         return httplib_response
 

diff --git a/urllib3/filepost.py b/urllib3/filepost.py
@@ -7,11 +7,7 @@
 import codecs
 import mimetypes
 
-try:
-    from mimetools import choose_boundary
-except ImportError:
-    from .packages.mimetools_choose_boundary import choose_boundary
-
+from uuid import uuid4
 from io import BytesIO
 
 from .packages import six
@@ -20,6 +16,13 @@
 writer = codecs.lookup('utf-8')[3]
 
 
+def choose_boundary():
+    """
+    Our embarassingly-simple replacement for mimetools.choose_boundary.
+    """
+    return uuid4().hex
+
+
 def get_content_type(filename):
     return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
 

diff --git a/urllib3/packages/mimetools_choose_boundary/__init__.py b/urllib3/packages/mimetools_choose_boundary/__init__.py