From 29d2814c31145b8ee4198d0dd3410a24eb90b0ca Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 5 Feb 2012 12:42:12 -0800 Subject: [PATCH 01/24] Gonna try keeping a separate release branch with real version numbers. --- urllib3/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/__init__.py b/urllib3/__init__.py index 87ea65b7b4..b7c0a88a17 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -10,7 +10,7 @@ __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.2.1' +__version__ = 'dev' from .connectionpool import ( From d8ff66d0758f8752b78d415a6200a0242acafb64 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 5 Feb 2012 13:13:15 -0800 Subject: [PATCH 02/24] Refactor helpers into util.py, removed deprecated api calls. --- .coveragerc | 1 + CHANGES.rst | 8 +++ test/test_connectionpool.py | 8 +-- urllib3/__init__.py | 16 ++--- urllib3/connectionpool.py | 120 ++-------------------------------- urllib3/request.py | 21 +----- urllib3/util.py | 125 ++++++++++++++++++++++++++++++++++++ 7 files changed, 146 insertions(+), 153 deletions(-) create mode 100644 urllib3/util.py diff --git a/.coveragerc b/.coveragerc index e8e30c193e..e323dcb555 100644 --- a/.coveragerc +++ b/.coveragerc @@ -7,3 +7,4 @@ exclude_lines = except ImportError: try:.* # Python 3 pass + .* # Abstract diff --git a/CHANGES.rst b/CHANGES.rst index 7ee7f1df1d..310fd86db2 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changes ======= +dev (master branch) ++++++++++++++++++++ + +* Removed pre-1.0 deprecated API. + +* Refactored helpers into a ``urllib3.util`` submodule. + + 1.2.1 (2012-02-05) ++++++++++++++++++ diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index 4281d4262d..d8d1c2bfbf 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -1,11 +1,7 @@ import unittest -from urllib3.connectionpool import ( - connection_from_url, - get_host, - HTTPConnectionPool, - make_headers) - +from urllib3.connectionpool import connection_from_url, HTTPConnectionPool +from urllib3.util import get_host, make_headers from urllib3.exceptions import EmptyPoolError, LocationParseError diff --git a/urllib3/__init__.py b/urllib3/__init__.py index b7c0a88a17..81e76f50ba 100644 --- a/urllib3/__init__.py +++ b/urllib3/__init__.py @@ -16,20 +16,14 @@ from .connectionpool import ( HTTPConnectionPool, HTTPSConnectionPool, - connection_from_url, - get_host, - make_headers) - - -from .exceptions import ( - HTTPError, - MaxRetryError, - SSLError, - TimeoutError) + connection_from_url +) +from . import exceptions +from .filepost import encode_multipart_formdata from .poolmanager import PoolManager, ProxyManager, proxy_from_url from .response import HTTPResponse -from .filepost import encode_multipart_formdata +from .util import make_headers, get_host # Set default logging handler to avoid "No handler found" warnings. diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index 39e652ed44..a586b496ac 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -7,15 +7,8 @@ import logging import socket -from base64 import b64encode from socket import error as SocketError, timeout as SocketTimeout -try: - from select import poll, POLLIN -except ImportError: # Doesn't exist on OSX and other platforms - from select import select - poll = False - try: # Python 3 from http.client import HTTPConnection, HTTPException from http.client import HTTP_PORT, HTTPS_PORT @@ -46,13 +39,12 @@ pass -from .packages.ssl_match_hostname import match_hostname, CertificateError from .request import RequestMethods from .response import HTTPResponse +from .util import get_host, is_connection_dropped from .exceptions import ( EmptyPoolError, HostChangedError, - LocationParseError, MaxRetryError, SSLError, TimeoutError, @@ -61,6 +53,7 @@ from .packages.ssl_match_hostname import match_hostname, CertificateError from .packages import six + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -72,6 +65,7 @@ 'https': HTTPS_PORT, } + ## Connection objects (extension of httplib) class VerifiedHTTPSConnection(HTTPSConnection): @@ -107,6 +101,7 @@ def connect(self): if self.ca_certs: match_hostname(self.sock.getpeercert(), self.host) + ## Pool objects class ConnectionPool(object): @@ -495,94 +490,6 @@ def _new_conn(self): return connection -## Helpers - -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None): - """ - Shortcuts for generating request headers. - - :param keep_alive: - If ``True``, adds 'connection: keep-alive' header. - - :param accept_encoding: - Can be a boolean, list, or string. - ``True`` translates to 'gzip,deflate'. - List will get joined by comma. - String will be used as provided. - - :param user_agent: - String representing the user-agent you want, such as - "python-urllib3/0.6" - - :param basic_auth: - Colon-separated username:password string for 'authorization: basic ...' - auth header. - - Example: :: - - >>> make_headers(keep_alive=True, user_agent="Batman/1.0") - {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} - >>> make_headers(accept_encoding=True) - {'accept-encoding': 'gzip,deflate'} - """ - headers = {} - if accept_encoding: - if isinstance(accept_encoding, str): - pass - elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) - else: - accept_encoding = 'gzip,deflate' - headers['accept-encoding'] = accept_encoding - - if user_agent: - headers['user-agent'] = user_agent - - if keep_alive: - headers['connection'] = 'keep-alive' - - if basic_auth: - headers['authorization'] = 'Basic ' + \ - b64encode(six.b(basic_auth)).decode('utf-8') - - return headers - - -def get_host(url): - """ - Given a url, return its scheme, host and port (None if it's not there). - - For example: :: - - >>> get_host('http://google.com/mail/') - ('http', 'google.com', None) - >>> get_host('google.com:80') - ('http', 'google.com', 80) - """ - - # This code is actually similar to urlparse.urlsplit, but much - # simplified for our needs. - port = None - scheme = 'http' - - if '://' in url: - scheme, url = url.split('://', 1) - if '/' in url: - url, _path = url.split('/', 1) - if '@' in url: - _auth, url = url.split('@', 1) - if ':' in url: - url, port = url.split(':', 1) - - if not port.isdigit(): - raise LocationParseError("Failed to parse: %s") - - port = int(port) - - return scheme, url, port - - def connection_from_url(url, **kw): """ Given a url, return an :class:`.ConnectionPool` instance of its host. @@ -608,22 +515,3 @@ def connection_from_url(url, **kw): return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) - - -def is_connection_dropped(conn): - """ - Returns True if the connection is dropped and should be closed. - - :param conn: - ``HTTPConnection`` object. - """ - if not poll: # Platform-specific - return select([conn.sock], [], [], 0.0)[0] - - # This version is better on platforms that support it. - p = poll() - p.register(conn.sock, POLLIN) - for (fno, ev) in p.poll(0.0): - if fno == conn.sock.fileno(): - # Either data is buffered (bad), or the connection is dropped. - return True diff --git a/urllib3/request.py b/urllib3/request.py index 5ea26a0f04..569ac96625 100644 --- a/urllib3/request.py +++ b/urllib3/request.py @@ -44,7 +44,7 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, - **kw): + **kw): # Abstract raise NotImplemented("Classes extending RequestMethods must implement " "their own ``urlopen`` method.") @@ -126,22 +126,3 @@ def request_encode_body(self, method, url, fields=None, headers=None, return self.urlopen(method, url, body=body, headers=headers, **urlopen_kw) - - # Deprecated: - - def get_url(self, url, fields=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_url('GET', url, fields=fields, - **urlopen_kw) - - def post_url(self, url, fields=None, headers=None, **urlopen_kw): - """ - .. deprecated:: 1.0 - Use :meth:`request` instead. - """ - return self.request_encode_body('POST', url, fields=fields, - headers=headers, - **urlopen_kw) diff --git a/urllib3/util.py b/urllib3/util.py new file mode 100644 index 0000000000..dadbc1d492 --- /dev/null +++ b/urllib3/util.py @@ -0,0 +1,125 @@ +# urllib3/util.py +# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +# +# This module is part of urllib3 and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php + + +from base64 import b64encode + +try: + from select import poll, POLLIN +except ImportError: # Doesn't exist on OSX and other platforms + from select import select + poll = False + +from .packages import six + +from .exceptions import LocationParseError + + +def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, + basic_auth=None): + """ + Shortcuts for generating request headers. + + :param keep_alive: + If ``True``, adds 'connection: keep-alive' header. + + :param accept_encoding: + Can be a boolean, list, or string. + ``True`` translates to 'gzip,deflate'. + List will get joined by comma. + String will be used as provided. + + :param user_agent: + String representing the user-agent you want, such as + "python-urllib3/0.6" + + :param basic_auth: + Colon-separated username:password string for 'authorization: basic ...' + auth header. + + Example: :: + + >>> make_headers(keep_alive=True, user_agent="Batman/1.0") + {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} + >>> make_headers(accept_encoding=True) + {'accept-encoding': 'gzip,deflate'} + """ + headers = {} + if accept_encoding: + if isinstance(accept_encoding, str): + pass + elif isinstance(accept_encoding, list): + accept_encoding = ','.join(accept_encoding) + else: + accept_encoding = 'gzip,deflate' + headers['accept-encoding'] = accept_encoding + + if user_agent: + headers['user-agent'] = user_agent + + if keep_alive: + headers['connection'] = 'keep-alive' + + if basic_auth: + headers['authorization'] = 'Basic ' + \ + b64encode(six.b(basic_auth)).decode('utf-8') + + return headers + + +def get_host(url): + """ + Given a url, return its scheme, host and port (None if it's not there). + + For example: :: + + >>> get_host('http://google.com/mail/') + ('http', 'google.com', None) + >>> get_host('google.com:80') + ('http', 'google.com', 80) + """ + + # This code is actually similar to urlparse.urlsplit, but much + # simplified for our needs. + port = None + scheme = 'http' + + if '://' in url: + scheme, url = url.split('://', 1) + if '/' in url: + url, _path = url.split('/', 1) + if '@' in url: + _auth, url = url.split('@', 1) + if ':' in url: + url, port = url.split(':', 1) + + if not port.isdigit(): + raise LocationParseError("Failed to parse: %s") + + port = int(port) + + return scheme, url, port + + + + +def is_connection_dropped(conn): + """ + Returns True if the connection is dropped and should be closed. + + :param conn: + ``HTTPConnection`` object. + """ + if not poll: # Platform-specific + return select([conn.sock], [], [], 0.0)[0] + + # This version is better on platforms that support it. + p = poll() + p.register(conn.sock, POLLIN) + for (fno, ev) in p.poll(0.0): + if fno == conn.sock.fileno(): + # Either data is buffered (bad), or the connection is dropped. + return True From ccf71ac38112f1a195d90fdc17eded72694b058a Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 5 Feb 2012 13:13:58 -0800 Subject: [PATCH 03/24] Oops, forgot to remove test that used a deprecated api. --- test/test_connectionpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_connectionpool.py b/test/test_connectionpool.py index d8d1c2bfbf..c32c6dcec0 100644 --- a/test/test_connectionpool.py +++ b/test/test_connectionpool.py @@ -101,7 +101,7 @@ def test_max_connections(self): pass try: - pool.get_url('/', pool_timeout=0.01) + pool.request('GET', '/', pool_timeout=0.01) self.fail("Managed to get a connection without EmptyPoolError") except EmptyPoolError: pass From 90c3ffbbcc422d77d30b7ed9ba3ee9cea3475742 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Mon, 6 Feb 2012 08:02:49 -0800 Subject: [PATCH 04/24] Merge. --- CHANGES.rst | 6 ++++++ MANIFEST.in | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 310fd86db2..c6b2e22f37 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,12 @@ dev (master branch) * Refactored helpers into a ``urllib3.util`` submodule. +1.2.2 (2012-02-06) +++++++++++++++++++ + +* Fixed packaging bug of not shipping ``test-requirements.txt``. (Issue #47) + + 1.2.1 (2012-02-05) ++++++++++++++++++ diff --git a/MANIFEST.in b/MANIFEST.in index 34a924a7d9..d1abae25fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include README.rst CHANGES.rst LICENSE.txt CONTRIBUTORS.txt +include README.rst CHANGES.rst LICENSE.txt CONTRIBUTORS.txt test-requirements.txt From d1d5fd1fd1d23cf68035e465ee395f580f6aebbd Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 25 Feb 2012 15:19:28 -0800 Subject: [PATCH 05/24] Support multiple fields with the same name for multipart encoding. (Fixed #48) --- test/with_dummyserver/test_connectionpool.py | 15 +++++++++++++++ urllib3/filepost.py | 14 +++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index cbe69e8c28..899568f2bf 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -60,6 +60,21 @@ def test_upload(self): r = self.pool.request('POST', '/upload', fields=fields) self.assertEqual(r.status, 200, r.data) + def test_one_name_multiple_values(self): + fields = [ + ('foo', 'a'), + ('foo', 'b'), + ] + + # urlencode + r = self.pool.request('GET', '/echo', fields=fields) + self.assertEqual(r.data, 'foo=a&foo=b') + + # multipart + r = self.pool.request('POST', '/echo', fields=fields) + self.assertEqual(r.data.count('name="foo"'), 2) + + def test_unicode_upload(self): fieldname = u('myfile') filename = u('\xe2\x99\xa5.txt') diff --git a/urllib3/filepost.py b/urllib3/filepost.py index e1ec8af792..1160ce9d47 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -24,6 +24,18 @@ def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' +def iter_fields(fields): + """ + Iterate over fields. + + Supports list of (k, v) tuples and dicts. + """ + if isinstance(fields, dict): + return ((k, v) for k, v in six.iteritems(fields)) + + return ((k,v) for k, v in fields) + + def encode_multipart_formdata(fields, boundary=None): """ Encode a dictionary of ``fields`` using the multipart/form-data mime format. @@ -42,7 +54,7 @@ def encode_multipart_formdata(fields, boundary=None): if boundary is None: boundary = choose_boundary() - for fieldname, value in six.iteritems(fields): + for fieldname, value in iter_fields(fields): body.write(b('--%s\r\n' % (boundary))) if isinstance(value, tuple): From 35212b7e7632fa8e16aae701d6cbbddb666ded38 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 25 Feb 2012 15:21:20 -0800 Subject: [PATCH 06/24] py32 support. --- test/with_dummyserver/test_connectionpool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 899568f2bf..012ef1fd93 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -68,11 +68,11 @@ def test_one_name_multiple_values(self): # urlencode r = self.pool.request('GET', '/echo', fields=fields) - self.assertEqual(r.data, 'foo=a&foo=b') + self.assertEqual(r.data, b'foo=a&foo=b') # multipart r = self.pool.request('POST', '/echo', fields=fields) - self.assertEqual(r.data.count('name="foo"'), 2) + self.assertEqual(r.data.count(b'name="foo"'), 2) def test_unicode_upload(self): From a602b64856bade9392124169eb2fbccf5b1eb27a Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 25 Feb 2012 19:46:04 -0800 Subject: [PATCH 07/24] Docs and changes update. --- CHANGES.rst | 3 +++ urllib3/filepost.py | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index c6b2e22f37..9ee1ca28e9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,9 @@ dev (master branch) * Refactored helpers into a ``urllib3.util`` submodule. +* Fixed multipart encoding to support list-of-tuples for keys with multiple + values. (Issue #48) + 1.2.2 (2012-02-06) ++++++++++++++++++ diff --git a/urllib3/filepost.py b/urllib3/filepost.py index 1160ce9d47..3f7d770764 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -41,10 +41,10 @@ def encode_multipart_formdata(fields, boundary=None): Encode a dictionary of ``fields`` using the multipart/form-data mime format. :param fields: - Dictionary of fields. The key is treated as the field name, and the - value as the body of the form-data. If the value is a tuple of two - elements, then the first element is treated as the filename of the - form-data section. + Dictionary of fields or list of (key, value) field tuples. + The key is treated as the field name, and the value as the body of the + form-data. If the value is a tuple of two elements, then the first + element is treated as the filename of the form-data section. :param boundary: If not specified, then a random boundary will be generated using From f26ebc19a14809f56b6a615d0e7b5c9ca02870a9 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 2 Mar 2012 12:27:58 -0500 Subject: [PATCH 08/24] Added a clause in HTTPResponse.from_httplib to merge HTTP headers with the same name. This is necessary because of an API change in Python 3's HTTPResponse. --- urllib3/response.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/urllib3/response.py b/urllib3/response.py index 4dd431ec55..e6843e1821 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -171,11 +171,20 @@ def from_httplib(ResponseCls, r, **response_kw): with ``original_response=r``. """ + # comma-seperate header values with identical header names + headers = {} + for k, v in r.getheaders(): + # In Python 3, the header keys are returned capitalised + k = k.lower() + if k in headers: + headers[k] = headers[k] + ", " + v + else: + headers[k] = v + # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, 'strict', 0) return ResponseCls(body=r, - # In Python 3, the header keys are returned capitalised - headers=dict((k.lower(), v) for k,v in r.getheaders()), + headers=headers, status=r.status, version=r.version, reason=r.reason, From d8013cb111644a06eb5cb9bccce174a1a996078d Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 3 Mar 2012 13:44:23 -0800 Subject: [PATCH 09/24] Failing test in py32 for cookies. --- test/with_dummyserver/test_socketlevel.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/with_dummyserver/test_socketlevel.py b/test/with_dummyserver/test_socketlevel.py index de5d24b138..46510281f0 100644 --- a/test/with_dummyserver/test_socketlevel.py +++ b/test/with_dummyserver/test_socketlevel.py @@ -1,11 +1,32 @@ from urllib3 import HTTPConnectionPool -from urllib3.poolmanager import ProxyManager, proxy_from_url +from urllib3.poolmanager import proxy_from_url from dummyserver.testcase import SocketDummyServerTestCase from threading import Event +class TestCookies(SocketDummyServerTestCase): + + def test_multi_setcookie(self): + def multicookie_response_handler(listener): + sock = listener.accept()[0] + + buf = b'' + while not buf.endswith(b'\r\n\r\n'): + buf += sock.recv(65536) + + sock.send(b'HTTP/1.1 200 OK\r\n' + b'Set-Cookie: foo=1\r\n' + b'Set-Cookie: bar=1\r\n' + b'\r\n') + + self._start_server(multicookie_response_handler) + pool = HTTPConnectionPool(self.host, self.port) + r = pool.request('GET', '/', retries=0) + self.assertEquals(r.headers, {'set-cookie': 'foo=1, bar=1'}) + + class TestSocketClosing(SocketDummyServerTestCase): def test_recovery_when_server_closes_connection(self): From 08e86e5d5043d898ac1de8c4f117f95a22b6d874 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 3 Mar 2012 13:52:04 -0800 Subject: [PATCH 10/24] Cleanup. --- .coveragerc | 2 +- urllib3/response.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.coveragerc b/.coveragerc index e323dcb555..1fafb7aa0f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,6 @@ omit = urllib3/packages/* exclude_lines = .* # Platform-specific.* except ImportError: - try:.* # Python 3 + .*:.* # Python 3 pass .* # Abstract diff --git a/urllib3/response.py b/urllib3/response.py index e6843e1821..5fab824383 100644 --- a/urllib3/response.py +++ b/urllib3/response.py @@ -171,15 +171,17 @@ def from_httplib(ResponseCls, r, **response_kw): with ``original_response=r``. """ - # comma-seperate header values with identical header names + # Normalize headers between different versions of Python headers = {} for k, v in r.getheaders(): - # In Python 3, the header keys are returned capitalised + # Python 3: Header keys are returned capitalised k = k.lower() - if k in headers: - headers[k] = headers[k] + ", " + v - else: - headers[k] = v + + has_value = headers.get(k) + if has_value: # Python 3: Repeating header keys are unmerged. + v = ', '.join([has_value, v]) + + headers[k] = v # HTTPResponse objects in Python 3 don't have a .strict attribute strict = getattr(r, 'strict', 0) From b077073e9117fadf467a232cec62309ed017050a Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 3 Mar 2012 13:53:21 -0800 Subject: [PATCH 11/24] Updated changelog. --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 9ee1ca28e9..59f395c41f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,6 +11,9 @@ dev (master branch) * Fixed multipart encoding to support list-of-tuples for keys with multiple values. (Issue #48) +* Fixed multiple Set-Cookie headers in response not getting merged properly in + Python 3. (Issue #53) + 1.2.2 (2012-02-06) ++++++++++++++++++ From 5b1a8b763187f3d560eb452bfe6b297149945f01 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 3 Mar 2012 14:39:14 -0800 Subject: [PATCH 12/24] More filepost tests. --- test/test_collections.py | 1 + test/test_filepost.py | 80 ++++++++++++++++++++ test/with_dummyserver/test_connectionpool.py | 11 +-- urllib3/filepost.py | 2 +- 4 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 test/test_filepost.py diff --git a/test/test_collections.py b/test/test_collections.py index f8275e04ab..6cb5aca283 100644 --- a/test/test_collections.py +++ b/test/test_collections.py @@ -4,6 +4,7 @@ from urllib3.packages import six xrange = six.moves.xrange + class TestLRUContainer(unittest.TestCase): def test_maxsize(self): d = Container(5) diff --git a/test/test_filepost.py b/test/test_filepost.py new file mode 100644 index 0000000000..dc89a84b02 --- /dev/null +++ b/test/test_filepost.py @@ -0,0 +1,80 @@ +import unittest + +from urllib3.filepost import encode_multipart_formdata, iter_fields + + +BOUNDARY = '---boundary---' + + +class TestIterfields(unittest.TestCase): + + def test_dict(self): + for fieldname, value in iter_fields(dict(a='b')): + self.assertEqual((fieldname, value), ('a', 'b')) + + self.assertEqual( + list(sorted(iter_fields(dict(a='b', c='d')))), + [('a', 'b'), ('c', 'd')]) + + def test_tuple_list(self): + for fieldname, value in iter_fields([('a', 'b')]): + self.assertEqual((fieldname, value), ('a', 'b')) + + self.assertEqual( + list(iter_fields([('a', 'b'), ('c', 'd')])), + [('a', 'b'), ('c', 'd')]) + + +class TestMultipartEncoding(unittest.TestCase): + + def test_input_datastructures(self): + fieldsets = [ + dict(k='v', k2='v2'), + [('k', 'v'), ('k2', 'v2')], + ] + + for fields in fieldsets: + encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY) + self.assertEqual(encoded.count(BOUNDARY), 3) + + + def test_field_encoding(self): + fieldsets = [ + [('k', 'v'), ('k2', 'v2')], + [(b'k', b'v'), (b'k2', b'v2')], + [(b'k', b'v'), ('k2', 'v2')], + ] + + for fields in fieldsets: + encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) + + self.assertEqual(encoded, + b'-----boundary---\r\n' + b'Content-Disposition: form-data; name="k"\r\n' + b'Content-Type: text/plain\r\n' + b'\r\n' + b'v\r\n' + b'-----boundary---\r\n' + b'Content-Disposition: form-data; name="k2"\r\n' + b'Content-Type: text/plain\r\n' + b'\r\n' + b'v2\r\n-----boundary-----\r\n') + + self.assertEqual(content_type, + b'multipart/form-data; boundary=---boundary---') + + + def test_filename(self): + fields = [('k', ('somename', 'v'))] + + encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) + + self.assertEqual(encoded, + b'-----boundary---\r\n' + b'Content-Disposition: form-data; name="k"; filename="somename"\r\n' + b'Content-Type: application/octet-stream\r\n' + b'\r\n' + b'v\r\n-----boundary-----\r\n') + + self.assertEqual(content_type, + b'multipart/form-data; boundary=---boundary---') diff --git a/test/with_dummyserver/test_connectionpool.py b/test/with_dummyserver/test_connectionpool.py index 012ef1fd93..22bf93de45 100644 --- a/test/with_dummyserver/test_connectionpool.py +++ b/test/with_dummyserver/test_connectionpool.py @@ -101,13 +101,10 @@ def test_timeout(self): pass def test_redirect(self): - r = self.pool.request('GET', '/redirect', - fields={'target': '/'}, - redirect=False) + r = self.pool.request('GET', '/redirect', fields={'target': '/'}, redirect=False) self.assertEqual(r.status, 303) - r = self.pool.request('GET', '/redirect', - fields={'target': '/'}) + r = self.pool.request('GET', '/redirect', fields={'target': '/'}) self.assertEqual(r.status, 200) self.assertEqual(r.data, b'Dummy server!') @@ -184,9 +181,7 @@ def test_keepalive_close(self): def test_post_with_urlencode(self): data = {'banana': 'hammock', 'lol': 'cat'} - r = self.pool.request('POST', '/echo', - fields=data, - encode_multipart=False) + r = self.pool.request('POST', '/echo', fields=data, encode_multipart=False) self.assertEqual(r.data.decode('utf-8'), urlencode(data)) def test_post_with_multipart(self): diff --git a/urllib3/filepost.py b/urllib3/filepost.py index 3f7d770764..fabf5c2250 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -33,7 +33,7 @@ def iter_fields(fields): if isinstance(fields, dict): return ((k, v) for k, v in six.iteritems(fields)) - return ((k,v) for k, v in fields) + return ((k, v) for k, v in fields) def encode_multipart_formdata(fields, boundary=None): From cd330f10740c6455e21c8699c1a7763a458141f6 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Mon, 12 Mar 2012 05:42:39 -0700 Subject: [PATCH 13/24] making dummyserver work with IPv4 by explicitly passing the server host address. Should work fine on IPv6, too. fixes #59 --- dummyserver/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dummyserver/server.py b/dummyserver/server.py index 529850f480..6c0943c062 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -83,7 +83,7 @@ def _start_server(self): else: http_server = tornado.httpserver.HTTPServer(container) - http_server.listen(self.port) + http_server.listen(self.port, address=self.host) return http_server def run(self): @@ -106,7 +106,7 @@ def stop(self): if len(sys.argv) > 1: url = sys.argv[1] - print("Starting WGI server at: %s" % url) + print("Starting WSGI server at: %s" % url) scheme, host, port = get_host(url) t = TornadoServerThread(scheme=scheme, host=host, port=port) From 70b1cdff41788805d6481941da4af6533633b589 Mon Sep 17 00:00:00 2001 From: David Wolever Date: Mon, 12 Mar 2012 13:48:18 -0700 Subject: [PATCH 14/24] Fixing bugs in `PoolManager`'s docstring. --- urllib3/poolmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/urllib3/poolmanager.py b/urllib3/poolmanager.py index d42f35bc25..310ea21d96 100644 --- a/urllib3/poolmanager.py +++ b/urllib3/poolmanager.py @@ -39,11 +39,11 @@ class PoolManager(RequestMethods): Example: :: - >>> manager = PoolManager() + >>> manager = PoolManager(num_pools=2) >>> r = manager.urlopen("http://google.com/") >>> r = manager.urlopen("http://google.com/mail") >>> r = manager.urlopen("http://yahoo.com/") - >>> len(r.pools) + >>> len(manager.pools) 2 """ From 6870e0440355971e4bf59fca6ec8e87091f70e64 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Mon, 12 Mar 2012 05:42:39 -0700 Subject: [PATCH 15/24] making dummyserver work with IPv4 by explicitly passing the server host address. Should work fine on IPv6, too. fixes issue #59 --- dummyserver/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dummyserver/server.py b/dummyserver/server.py index 529850f480..6c0943c062 100755 --- a/dummyserver/server.py +++ b/dummyserver/server.py @@ -83,7 +83,7 @@ def _start_server(self): else: http_server = tornado.httpserver.HTTPServer(container) - http_server.listen(self.port) + http_server.listen(self.port, address=self.host) return http_server def run(self): @@ -106,7 +106,7 @@ def stop(self): if len(sys.argv) > 1: url = sys.argv[1] - print("Starting WGI server at: %s" % url) + print("Starting WSGI server at: %s" % url) scheme, host, port = get_host(url) t = TornadoServerThread(scheme=scheme, host=host, port=port) From 3177886e08b43e4880848aaa4cb09af5c017617d Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 17 Mar 2012 11:41:17 -0700 Subject: [PATCH 16/24] (#61) First step towards AppEngine support: Make select optional. --- urllib3/util.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/urllib3/util.py b/urllib3/util.py index dadbc1d492..60a45c1739 100644 --- a/urllib3/util.py +++ b/urllib3/util.py @@ -9,12 +9,14 @@ try: from select import poll, POLLIN -except ImportError: # Doesn't exist on OSX and other platforms - from select import select +except ImportError: # `poll` doesn't exist on OSX and other platforms poll = False + try: + from select import select + except ImportError: # `select` doesn't exist on AppEngine. + select = False from .packages import six - from .exceptions import LocationParseError @@ -114,6 +116,11 @@ def is_connection_dropped(conn): ``HTTPConnection`` object. """ if not poll: # Platform-specific + if not select: #Platform-specific + # For environments like AppEngine which handle its own socket reuse, we + # can assume they will be recycled transparently to us. + return False + return select([conn.sock], [], [], 0.0)[0] # This version is better on platforms that support it. From f50101619e43e97ee300474a28386b8bcaa04b37 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 17 Mar 2012 12:06:50 -0700 Subject: [PATCH 17/24] (#61) Make conn.sock optional. Py27 AppEngine seems to work now. --- urllib3/connectionpool.py | 12 ++++++++---- urllib3/util.py | 17 +++++++++++------ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index a586b496ac..579c685606 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -35,7 +35,7 @@ import ssl BaseSSLError = ssl.SSLError -except ImportError: +except (ImportError, AttributeError): pass @@ -207,7 +207,7 @@ def _get_conn(self, timeout=None): conn = self.pool.get(block=self.block, timeout=timeout) # If this is a persistent connection, check if it got disconnected - if conn and conn.sock and is_connection_dropped(conn): + if conn and is_connection_dropped(conn): log.info("Resetting dropped connection: %s" % self.host) conn.close() @@ -251,9 +251,13 @@ def _make_request(self, conn, method, url, timeout=_Default, timeout = self.timeout conn.timeout = timeout # This only does anything in Py26+ - conn.request(method, url, **httplib_request_kw) - conn.sock.settimeout(timeout) + + # Set timeout + sock = getattr(conn, 'sock', False) # AppEngine doesn't have sock attr. + if sock: + sock.settimeout(timeout) + httplib_response = conn.getresponse() log.debug("\"%s %s %s\" %s %s" % diff --git a/urllib3/util.py b/urllib3/util.py index 60a45c1739..6bed12aeec 100644 --- a/urllib3/util.py +++ b/urllib3/util.py @@ -114,19 +114,24 @@ def is_connection_dropped(conn): :param conn: ``HTTPConnection`` object. + + Note: For platforms like AppEngine, this will always return ``False`` to + let the platform handle connection recycling transparently for us. """ + sock = getattr(conn, 'sock', False) + if not sock: #Platform-specific: AppEngine + return False + if not poll: # Platform-specific - if not select: #Platform-specific - # For environments like AppEngine which handle its own socket reuse, we - # can assume they will be recycled transparently to us. + if not select: #Platform-specific: AppEngine return False - return select([conn.sock], [], [], 0.0)[0] + return select([sock], [], [], 0.0)[0] # This version is better on platforms that support it. p = poll() - p.register(conn.sock, POLLIN) + p.register(sock, POLLIN) for (fno, ev) in p.poll(0.0): - if fno == conn.sock.fileno(): + if fno == sock.fileno(): # Either data is buffered (bad), or the connection is dropped. return True From c5b7fd787cc3f24c8d10ed10ca0fadc00598491b Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sat, 17 Mar 2012 12:11:54 -0700 Subject: [PATCH 18/24] CHANGES. --- CHANGES.rst | 2 ++ urllib3/util.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 59f395c41f..315e4e4d2b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,8 @@ dev (master branch) * Fixed multiple Set-Cookie headers in response not getting merged properly in Python 3. (Issue #53) +* AppEngine support with Py27. (Issue #61) + 1.2.2 (2012-02-06) ++++++++++++++++++ diff --git a/urllib3/util.py b/urllib3/util.py index 6bed12aeec..6ff89083e4 100644 --- a/urllib3/util.py +++ b/urllib3/util.py @@ -107,7 +107,6 @@ def get_host(url): - def is_connection_dropped(conn): """ Returns True if the connection is dropped and should be closed. From d84078adc8cdb1a5cfaf6db9e5aa1ec71366f148 Mon Sep 17 00:00:00 2001 From: Ian Danforth Date: Sun, 18 Mar 2012 18:19:54 -0700 Subject: [PATCH 19/24] Fix Sphinx complaint. Methods named 'request' are very common and cause cross-reference warnings. --- urllib3/connectionpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/connectionpool.py b/urllib3/connectionpool.py index 579c685606..c3cb3b125f 100644 --- a/urllib3/connectionpool.py +++ b/urllib3/connectionpool.py @@ -294,7 +294,7 @@ def urlopen(self, method, url, body=None, headers=None, retries=3, .. note:: More commonly, it's appropriate to use a convenience method provided - by :class:`.RequestMethods`, such as :meth:`.request`. + by :class:`.RequestMethods`, such as :meth:`request`. .. note:: From de20783d2836e95488d89e5d6a5b3db30d0867df Mon Sep 17 00:00:00 2001 From: Kevin von Horn Date: Sat, 24 Mar 2012 09:34:35 -0700 Subject: [PATCH 20/24] Added url to LocationParseError message urllib3/util.py + Added the url that couldn't be parsed to LocationParseError message. --- urllib3/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/urllib3/util.py b/urllib3/util.py index 6ff89083e4..2684a2fd40 100644 --- a/urllib3/util.py +++ b/urllib3/util.py @@ -99,7 +99,7 @@ def get_host(url): url, port = url.split(':', 1) if not port.isdigit(): - raise LocationParseError("Failed to parse: %s") + raise LocationParseError("Failed to parse: %s" % url) port = int(port) From 32248ff578ae03e44bf2d5002ce0afeacf382a4c Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 25 Mar 2012 11:10:21 -0700 Subject: [PATCH 21/24] Py3 byte fixen in formencode. --- test/test_filepost.py | 29 +++++++++++++++++------------ urllib3/filepost.py | 10 +++++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/test/test_filepost.py b/test/test_filepost.py index dc89a84b02..b805ef989b 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -1,9 +1,10 @@ import unittest from urllib3.filepost import encode_multipart_formdata, iter_fields +from urllib3.packages.six import b -BOUNDARY = '---boundary---' +BOUNDARY = '!! test boundary !!' class TestIterfields(unittest.TestCase): @@ -35,46 +36,50 @@ def test_input_datastructures(self): for fields in fieldsets: encoded, _ = encode_multipart_formdata(fields, boundary=BOUNDARY) - self.assertEqual(encoded.count(BOUNDARY), 3) + self.assertEqual(encoded.count(b(BOUNDARY)), 3) def test_field_encoding(self): fieldsets = [ [('k', 'v'), ('k2', 'v2')], - [(b'k', b'v'), (b'k2', b'v2')], - [(b'k', b'v'), ('k2', 'v2')], + [('k', b'v'), ('k2', b'v2')], + [('k', b'v'), ('k2', 'v2')], ] for fields in fieldsets: encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) self.assertEqual(encoded, - b'-----boundary---\r\n' + b'--' + b(BOUNDARY) + b'\r\n' b'Content-Disposition: form-data; name="k"\r\n' b'Content-Type: text/plain\r\n' b'\r\n' b'v\r\n' - b'-----boundary---\r\n' + b'--' + b(BOUNDARY) + b'\r\n' b'Content-Disposition: form-data; name="k2"\r\n' b'Content-Type: text/plain\r\n' b'\r\n' - b'v2\r\n-----boundary-----\r\n') + b'v2\r\n' + b'--' + b(BOUNDARY) + b'--\r\n' + , fields) self.assertEqual(content_type, - b'multipart/form-data; boundary=---boundary---') + b'multipart/form-data; boundary=' + b(BOUNDARY)) def test_filename(self): - fields = [('k', ('somename', 'v'))] + fields = [('k', ('somename', b'v'))] encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY) self.assertEqual(encoded, - b'-----boundary---\r\n' + b'--' + b(BOUNDARY) + b'\r\n' b'Content-Disposition: form-data; name="k"; filename="somename"\r\n' b'Content-Type: application/octet-stream\r\n' b'\r\n' - b'v\r\n-----boundary-----\r\n') + b'v\r\n' + b'--' + b(BOUNDARY) + b'--\r\n' + ) self.assertEqual(content_type, - b'multipart/form-data; boundary=---boundary---') + b'multipart/form-data; boundary=' + b(BOUNDARY)) diff --git a/urllib3/filepost.py b/urllib3/filepost.py index fabf5c2250..4e533aefad 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -15,7 +15,7 @@ from io import BytesIO from .packages import six -from .packages.six import b +from .packages.six import b, u writer = codecs.lookup('utf-8')[3] @@ -41,10 +41,10 @@ def encode_multipart_formdata(fields, boundary=None): Encode a dictionary of ``fields`` using the multipart/form-data mime format. :param fields: - Dictionary of fields or list of (key, value) field tuples. - The key is treated as the field name, and the value as the body of the - form-data. If the value is a tuple of two elements, then the first - element is treated as the filename of the form-data section. + Dictionary of fields or list of (key, value) field tuples. The key is + treated as the field name, and the value as the body of the form-data + bytes. If the value is a tuple of two elements, then the first element + is treated as the filename of the form-data section. :param boundary: If not specified, then a random boundary will be generated using From c6f4c783f1b51607198abc8e2639b274f4596a6e Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 25 Mar 2012 11:12:58 -0700 Subject: [PATCH 22/24] Unicode note on fieldname/filename encode. --- test/test_filepost.py | 6 +++--- urllib3/filepost.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_filepost.py b/test/test_filepost.py index b805ef989b..c2517782bd 100644 --- a/test/test_filepost.py +++ b/test/test_filepost.py @@ -1,7 +1,7 @@ import unittest from urllib3.filepost import encode_multipart_formdata, iter_fields -from urllib3.packages.six import b +from urllib3.packages.six import b, u BOUNDARY = '!! test boundary !!' @@ -42,8 +42,8 @@ def test_input_datastructures(self): def test_field_encoding(self): fieldsets = [ [('k', 'v'), ('k2', 'v2')], - [('k', b'v'), ('k2', b'v2')], - [('k', b'v'), ('k2', 'v2')], + [('k', b'v'), (u('k2'), b'v2')], + [('k', b'v'), (u('k2'), 'v2')], ] for fields in fieldsets: diff --git a/urllib3/filepost.py b/urllib3/filepost.py index 4e533aefad..344a1030cc 100644 --- a/urllib3/filepost.py +++ b/urllib3/filepost.py @@ -15,7 +15,7 @@ from io import BytesIO from .packages import six -from .packages.six import b, u +from .packages.six import b writer = codecs.lookup('utf-8')[3] @@ -46,6 +46,8 @@ def encode_multipart_formdata(fields, boundary=None): bytes. If the value is a tuple of two elements, then the first element is treated as the filename of the form-data section. + Field names and filenames must be unicode. + :param boundary: If not specified, then a random boundary will be generated using :func:`mimetools.choose_boundary`. From 50e61cc4bd9f7d40ac9cb36074e2e295d68772c7 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 25 Mar 2012 11:14:41 -0700 Subject: [PATCH 23/24] Updated readme. --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 315e4e4d2b..ffba6be0eb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -16,6 +16,9 @@ dev (master branch) * AppEngine support with Py27. (Issue #61) +* Minor ``encode_multipart_formdata`` fixes related to Python 3 strings vs + bytes. + 1.2.2 (2012-02-06) ++++++++++++++++++ From 968263b87f5b594bd00141383ee4b0048427c496 Mon Sep 17 00:00:00 2001 From: Andrey Petrov Date: Sun, 25 Mar 2012 11:16:41 -0700 Subject: [PATCH 24/24] Version bump. --- CHANGES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index ffba6be0eb..a3ef6546b6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,10 @@ Changes dev (master branch) +++++++++++++++++++ + +1.3 (2012-03-25) +++++++++++++++++ + * Removed pre-1.0 deprecated API. * Refactored helpers into a ``urllib3.util`` submodule.