Permalink
Browse files

Merge branch master into release, with version bump.

  • Loading branch information...
2 parents ac74f71 + d8149ba commit f7eaa46ff1a96c5f0d209d8f2cedb48e759f522b @shazow committed Aug 2, 2012
View
@@ -1,6 +1,30 @@
Changes
=======
+1.5 (2012-08-02)
+++++++++++++++++
+
+* Added ``urllib3.add_stderr_logger()`` for quickly enabling STDERR debug
+ logging in urllib3.
+
+* Native full URL parsing (including auth, path, query, fragment) available in
+ ``urllib3.util.parse_url(url)``.
+
+* Built-in redirect will switch method to 'GET' if status code is 303.
+ (Issue #11)
+
+* ``urllib3.PoolManager`` strips the scheme and host before sending the request
+ uri. (Issue #8)
+
+* New ``urllib3.exceptions.DecodeError`` exception for when automatic decoding,
+ based on the Content-Type header, fails.
+
+* Fixed bug with pool depletion and leaking connections (Issue #76). Added
+ explicit connection closing on pool eviction. Added
+ ``urllib3.PoolManager.clear()``.
+
+* 99% -> 100% unit test coverage.
+
1.4 (2012-06-16)
++++++++++++++++
View
@@ -39,5 +39,11 @@ In chronological order:
* brandon-rhodes <http://rhodesmill.org/brandon>
* Design review, bugfixes, test coverage.
+* studer <theo.studer@gmail.com>
+ * IPv6 url support and test coverage
+
+* Shivaram Lingamneni <slingamn@cs.stanford.edu>
+ * Support for explicitly closing pooled connections
+
* [Your name or handle] <[email or website]>
* [Brief summary of your changes]
View
@@ -9,7 +9,7 @@ Highlights
- Supports gzip and deflate decoding.
- Thread-safe and sanity-safe.
- Works with AppEngine, gevent, and eventlib.
-- Tested on Python 2.6+ and Python 3.2+, 99% unit test coverage.
+- Tested on Python 2.6+ and Python 3.2+, 100% unit test coverage.
- Small and easy to understand codebase perfect for extending and building upon.
For a more comprehensive solution, have a look at
`Requests <http://python-requests.org/>`_ which is also powered by urllib3.
View
@@ -4,6 +4,13 @@ Helpers
Useful methods for working with :mod:`httplib`, completely decoupled from
code specific to **urllib3**.
+
+Utilities
+---------
+
+.. automodule:: urllib3.util
+ :members:
+
Filepost
--------
View
@@ -30,7 +30,7 @@ Highlights
- Thread-safe and sanity-safe.
-- Tested on Python 2.6+ and Python 3.2+, 99% unit test coverage.
+- Tested on Python 2.6+ and Python 3.2+, 100% unit test coverage.
- Works with AppEngine, gevent, and eventlib.
@@ -145,14 +145,20 @@ def encodingrequest(self, request):
data = b"hello, world!"
encoding = request.headers.get('Accept-Encoding', '')
headers = None
- if 'gzip' in encoding:
+ if encoding == 'gzip':
headers = [('Content-Encoding', 'gzip')]
file_ = BytesIO()
gzip.GzipFile('', mode='w', fileobj=file_).write(data)
data = file_.getvalue()
- elif 'deflate' in encoding:
+ elif encoding == 'deflate':
headers = [('Content-Encoding', 'deflate')]
data = zlib.compress(data)
+ elif encoding == 'garbage-gzip':
+ headers = [('Content-Encoding', 'gzip')]
+ data = 'garbage'
+ elif encoding == 'garbage-deflate':
+ headers = [('Content-Encoding', 'deflate')]
+ data = 'garbage'
return Response(data, headers=headers)
def shutdown(self, request):
@@ -36,19 +36,7 @@ def test_expire(self):
d[5] = '5'
# Check state
- self.assertEqual(list(d.keys()), [0, 2, 3, 4, 5])
-
- def test_pruning(self):
- d = Container(5)
-
- for i in xrange(5):
- d[i] = str(i)
-
- # Contend 2 entries for the most-used slot to balloon the heap
- for i in xrange(100):
- d.get(i % 2)
-
- self.assertTrue(len(d.access_log) <= d.CLEANUP_FACTOR * d._maxsize)
+ self.assertEqual(list(d.keys()), [2, 3, 4, 0, 5])
def test_same_key(self):
d = Container(5)
@@ -57,24 +45,22 @@ def test_same_key(self):
d['foo'] = i
self.assertEqual(list(d.keys()), ['foo'])
-
- d._prune_invalidated_entries()
-
- self.assertEqual(len(d.access_log), 1)
+ self.assertEqual(len(d), 1)
def test_access_ordering(self):
d = Container(5)
for i in xrange(10):
d[i] = True
- self.assertEqual(d._get_ordered_access_keys(), [9,8,7,6,5])
+ # Keys should be ordered by access time
+ self.assertEqual(list(d.keys()), [5, 6, 7, 8, 9])
new_order = [7,8,6,9,5]
- for k in reversed(new_order):
+ for k in new_order:
d[k]
- self.assertEqual(d._get_ordered_access_keys(), new_order)
+ self.assertEqual(list(d.keys()), new_order)
def test_delete(self):
d = Container(5)
@@ -107,6 +93,35 @@ def test_get(self):
self.assertRaises(KeyError, lambda: d[5])
+ def test_disposal(self):
+ evicted_items = []
+
+ def dispose_func(arg):
+ # Save the evicted datum for inspection
+ evicted_items.append(arg)
+
+ d = Container(5, dispose_func=dispose_func)
+ for i in xrange(5):
+ d[i] = i
+ self.assertEqual(list(d.keys()), list(xrange(5)))
+ self.assertEqual(evicted_items, []) # Nothing disposed
+
+ d[5] = 5
+ self.assertEqual(list(d.keys()), list(xrange(1, 6)))
+ self.assertEqual(evicted_items, [0])
+
+ del d[1]
+ self.assertEqual(evicted_items, [0, 1])
+
+ d.clear()
+ self.assertEqual(evicted_items, [0, 1, 2, 3, 4, 5])
+
+ def test_iter(self):
+ d = Container()
+
+ with self.assertRaises(NotImplementedError):
+ for i in d:
+ self.fail("Iteration shouldn't be implemented.")
if __name__ == '__main__':
unittest.main()
@@ -1,7 +1,25 @@
import unittest
from urllib3.connectionpool import connection_from_url, HTTPConnectionPool
-from urllib3.exceptions import EmptyPoolError
+from urllib3.packages.ssl_match_hostname import CertificateError
+from urllib3.exceptions import (
+ ClosedPoolError,
+ EmptyPoolError,
+ HostChangedError,
+ MaxRetryError,
+ SSLError,
+ TimeoutError,
+)
+
+from socket import timeout as SocketTimeout
+from ssl import SSLError as BaseSSLError
+
+try: # Python 3
+ from queue import Empty
+ from http.client import HTTPException
+except ImportError:
+ from Queue import Empty
+ from httplib import HTTPException
class TestConnectionPool(unittest.TestCase):
@@ -68,6 +86,67 @@ def test_exception_str(self):
str(EmptyPoolError(HTTPConnectionPool(host='localhost'), "Test.")),
"HTTPConnectionPool(host='localhost', port=None): Test.")
+ def test_pool_size(self):
+ POOL_SIZE = 1
+ pool = HTTPConnectionPool(host='localhost', maxsize=POOL_SIZE, block=True)
+
+ def _raise(ex):
+ raise ex()
+
+ def _test(exception, expect):
+ pool._make_request = lambda *args, **kwargs: _raise(exception)
+ with self.assertRaises(expect):
+ pool.request('GET', '/')
+
+ self.assertEqual(pool.pool.qsize(), POOL_SIZE)
+
+ #make sure that all of the exceptions return the connection to the pool
+ _test(Empty, TimeoutError)
+ _test(SocketTimeout, TimeoutError)
+ _test(BaseSSLError, SSLError)
+ _test(CertificateError, SSLError)
+
+ # The pool should never be empty, and with these two exceptions being raised,
+ # a retry will be triggered, but that retry will fail, eventually raising
+ # MaxRetryError, not EmptyPoolError
+ # See: https://github.com/shazow/urllib3/issues/76
+ pool._make_request = lambda *args, **kwargs: _raise(HTTPException)
+ with self.assertRaises(MaxRetryError):
+ pool.request('GET', '/', retries=1, pool_timeout=0.01)
+ self.assertEqual(pool.pool.qsize(), POOL_SIZE)
+
+ def test_assert_same_host(self):
+ c = connection_from_url('http://google.com:80')
+
+ with self.assertRaises(HostChangedError):
+ c.request('GET', 'http://yahoo.com:80', assert_same_host=True)
+
+ def test_pool_close(self):
+ pool = connection_from_url('http://google.com:80')
+
+ # Populate with some connections
+ conn1 = pool._get_conn()
+ conn2 = pool._get_conn()
+ conn3 = pool._get_conn()
+ pool._put_conn(conn1)
+ pool._put_conn(conn2)
+
+ old_pool_queue = pool.pool
+
+ pool.close()
+ self.assertEqual(pool.pool, None)
+
+ with self.assertRaises(ClosedPoolError):
+ pool._get_conn()
+
+ pool._put_conn(conn3)
+
+ with self.assertRaises(ClosedPoolError):
+ pool._get_conn()
+
+ with self.assertRaises(Empty):
+ old_pool_queue.get(block=False)
+
if __name__ == '__main__':
unittest.main()
@@ -2,6 +2,7 @@
from urllib3.poolmanager import PoolManager
from urllib3 import connection_from_url
+from urllib3.exceptions import ClosedPoolError
class TestPoolManager(unittest.TestCase):
@@ -42,6 +43,29 @@ def test_many_urls(self):
self.assertEqual(len(connections), 5)
+ def test_manager_clear(self):
+ p = PoolManager(5)
+
+ conn_pool = p.connection_from_url('http://google.com')
+ self.assertEqual(len(p.pools), 1)
+
+ conn = conn_pool._get_conn()
+
+ p.clear()
+ self.assertEqual(len(p.pools), 0)
+
+ with self.assertRaises(ClosedPoolError):
+ conn_pool._get_conn()
+
+ conn_pool._put_conn(conn)
+
+ with self.assertRaises(ClosedPoolError):
+ conn_pool._get_conn()
+
+ self.assertEqual(len(p.pools), 0)
+
+
+
if __name__ == '__main__':
unittest.main()
@@ -1,9 +1,9 @@
import unittest
-import zlib
from io import BytesIO
from urllib3.response import HTTPResponse
+from urllib3.exceptions import DecodeError
class TestLegacyResponse(unittest.TestCase):
def test_getheaders(self):
@@ -50,7 +50,7 @@ def test_no_preload(self):
def test_decode_bad_data(self):
fp = BytesIO(b'\x00' * 10)
- self.assertRaises(zlib.error, HTTPResponse, fp, headers={
+ self.assertRaises(DecodeError, HTTPResponse, fp, headers={
'content-encoding': 'deflate'
})
Oops, something went wrong.

0 comments on commit f7eaa46

Please sign in to comment.