Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Allow default HTTPRequest attributes to be set globally via configure.

Closes #379.
  • Loading branch information...
commit 1a5b337552cf0f6062f2ac9ac401bbc8b2fc7b03 1 parent aeff86a
@bdarnell bdarnell authored
View
8 tornado/curl_httpclient.py
@@ -31,12 +31,15 @@
from tornado import stack_context
from tornado.escape import utf8
-from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
+from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
class CurlAsyncHTTPClient(AsyncHTTPClient):
- def initialize(self, io_loop=None, max_clients=10):
+ def initialize(self, io_loop=None, max_clients=10, defaults=None):
self.io_loop = io_loop
+ self.defaults = dict(HTTPRequest._DEFAULTS)
+ if defaults is not None:
+ self.defaults.update(defaults)
self._multi = pycurl.CurlMulti()
self._multi.setopt(pycurl.M_TIMERFUNCTION, self._set_timeout)
self._multi.setopt(pycurl.M_SOCKETFUNCTION, self._handle_socket)
@@ -77,6 +80,7 @@ def close(self):
def fetch(self, request, callback, **kwargs):
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
+ request = _RequestProxy(request, self.defaults)
self._requests.append((request, stack_context.wrap(callback)))
self._process_queue()
self._set_timeout(0)
View
44 tornado/httpclient.py
@@ -40,7 +40,7 @@
from tornado.escape import utf8
from tornado import httputil, stack_context
from tornado.ioloop import IOLoop
-from tornado.util import import_object, Configurable
+from tornado.util import Configurable
class HTTPClient(object):
@@ -195,16 +195,30 @@ def configure(cls, impl, **kwargs):
class HTTPRequest(object):
"""HTTP client request object."""
+
+ # Default values for HTTPRequest parameters.
+ # Merged with the values on the request object by AsyncHTTPClient
+ # implementations.
+ _DEFAULTS = dict(
+ connect_timeout=20.0,
+ request_timeout=20.0,
+ follow_redirects=True,
+ max_redirects=5,
+ use_gzip=True,
+ proxy_password='',
+ allow_nonstandard_methods=False,
+ validate_cert=True)
+
def __init__(self, url, method="GET", headers=None, body=None,
auth_username=None, auth_password=None,
- connect_timeout=20.0, request_timeout=20.0,
- if_modified_since=None, follow_redirects=True,
- max_redirects=5, user_agent=None, use_gzip=True,
+ connect_timeout=None, request_timeout=None,
+ if_modified_since=None, follow_redirects=None,
+ max_redirects=None, user_agent=None, use_gzip=None,
network_interface=None, streaming_callback=None,
header_callback=None, prepare_curl_callback=None,
proxy_host=None, proxy_port=None, proxy_username=None,
- proxy_password='', allow_nonstandard_methods=False,
- validate_cert=True, ca_certs=None,
+ proxy_password=None, allow_nonstandard_methods=None,
+ validate_cert=None, ca_certs=None,
allow_ipv6=None,
client_key=None, client_cert=None):
"""Creates an `HTTPRequest`.
@@ -393,6 +407,24 @@ def __init__(self, code, message=None, response=None):
self.response = response
Exception.__init__(self, "HTTP %d: %s" % (self.code, message))
+class _RequestProxy(object):
+ """Combines an object with a dictionary of defaults.
+
+ Used internally by AsyncHTTPClient implementations.
+ """
+ def __init__(self, request, defaults):
+ self.request = request
+ self.defaults = defaults
+
+ def __getattr__(self, name):
+ request_attr = getattr(self.request, name)
+ if request_attr is not None:
+ return request_attr
+ elif self.defaults is not None:
+ return self.defaults.get(name, None)
+ else:
+ return None
+
def main():
from tornado.options import define, options, parse_command_line
View
13 tornado/simple_httpclient.py
@@ -2,7 +2,7 @@
from __future__ import absolute_import, division, with_statement
from tornado.escape import utf8, _unicode, native_str
-from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main
+from tornado.httpclient import HTTPRequest, HTTPResponse, HTTPError, AsyncHTTPClient, main, _RequestProxy
from tornado.httputil import HTTPHeaders
from tornado.iostream import IOStream, SSLIOStream
from tornado.netutil import Resolver
@@ -53,7 +53,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient):
"""
def initialize(self, io_loop=None, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
- resolver=None):
+ resolver=None, defaults=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
@@ -80,6 +80,9 @@ def initialize(self, io_loop=None, max_clients=10,
self.hostname_mapping = hostname_mapping
self.max_buffer_size = max_buffer_size
self.resolver = resolver or Resolver(io_loop=io_loop)
+ self.defaults = dict(HTTPRequest._DEFAULTS)
+ if defaults is not None:
+ self.defaults.update(defaults)
def fetch(self, request, callback, **kwargs):
if not isinstance(request, HTTPRequest):
@@ -88,6 +91,7 @@ def fetch(self, request, callback, **kwargs):
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = HTTPHeaders(request.headers)
+ request = _RequestProxy(request, self.defaults)
callback = stack_context.wrap(callback)
self.queue.append((request, callback))
self._process_queue()
@@ -396,10 +400,11 @@ def _on_body(self, data):
if (self.request.follow_redirects and
self.request.max_redirects > 0 and
self.code in (301, 302, 303, 307)):
- new_request = copy.copy(self.request)
+ assert isinstance(self.request, _RequestProxy)
+ new_request = copy.copy(self.request.request)
new_request.url = urlparse.urljoin(self.request.url,
self.headers["Location"])
- new_request.max_redirects -= 1
+ new_request.max_redirects = self.request.max_redirects - 1
del new_request.headers["Host"]
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
# Client SHOULD make a GET request after a 303.
View
52 tornado/test/httpclient_test.py
@@ -9,10 +9,12 @@
import re
from tornado.escape import utf8
+from tornado.httpclient import HTTPRequest, _RequestProxy
from tornado.iostream import IOStream
from tornado import netutil
from tornado.stack_context import ExceptionStackContext
from tornado.testing import AsyncHTTPTestCase, bind_unused_port
+from tornado.test.util import unittest
from tornado.util import b, bytes_type
from tornado.web import Application, RequestHandler, url
@@ -55,6 +57,12 @@ class EchoPostHandler(RequestHandler):
def post(self):
self.write(self.request.body)
+
+class UserAgentHandler(RequestHandler):
+ def get(self):
+ self.write(self.request.headers.get('User-Agent', 'User agent not set'))
+
+
# These tests end up getting run redundantly: once here with the default
# HTTPClient implementation, and then again in each implementation's own
# test suite.
@@ -69,6 +77,7 @@ def get_app(self):
url("/auth", AuthHandler),
url("/countdown/([0-9]+)", CountdownHandler, name="countdown"),
url("/echopost", EchoPostHandler),
+ url("/user_agent", UserAgentHandler),
], gzip=True)
def test_hello_world(self):
@@ -249,3 +258,46 @@ def header_callback(header_line):
self.fetch('/chunk', header_callback=header_callback)
self.assertEqual(len(exc_info), 1)
self.assertIs(exc_info[0][0], ZeroDivisionError)
+
+ def test_configure_defaults(self):
+ defaults = dict(user_agent='TestDefaultUserAgent')
+ # Construct a new instance of the configured client class
+ client = self.http_client.__class__(self.io_loop, force_instance=True,
+ defaults=defaults)
+ client.fetch(self.get_url('/user_agent'), callback=self.stop)
+ response = self.wait()
+ self.assertEqual(response.body, b('TestDefaultUserAgent'))
+
+
+class RequestProxyTest(unittest.TestCase):
+ def test_request_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/',
+ user_agent='foo'),
+ dict())
+ self.assertEqual(proxy.user_agent, 'foo')
+
+ def test_default_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict(network_interface='foo'))
+ self.assertEqual(proxy.network_interface, 'foo')
+
+ def test_both_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/',
+ proxy_host='foo'),
+ dict(proxy_host='bar'))
+ self.assertEqual(proxy.proxy_host, 'foo')
+
+ def test_neither_set(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict())
+ self.assertIs(proxy.auth_username, None)
+
+ def test_bad_attribute(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'),
+ dict())
+ with self.assertRaises(AttributeError):
+ proxy.foo
+
+ def test_defaults_none(self):
+ proxy = _RequestProxy(HTTPRequest('http://example.com/'), None)
+ self.assertIs(proxy.auth_username, None)
View
11 tornado/test/httpserver_test.py
@@ -178,10 +178,13 @@ def get_app(self):
def raw_fetch(self, headers, body):
client = SimpleAsyncHTTPClient(self.io_loop)
- conn = RawRequestHTTPConnection(self.io_loop, client,
- httpclient.HTTPRequest(self.get_url("/")),
- None, self.stop,
- 1024 * 1024)
+ conn = RawRequestHTTPConnection(
+ self.io_loop, client,
+ httpclient._RequestProxy(
+ httpclient.HTTPRequest(self.get_url("/")),
+ dict(httpclient.HTTPRequest._DEFAULTS)),
+ None, self.stop,
+ 1024 * 1024)
conn.set_request(
b("\r\n").join(headers +
[utf8("Content-Length: %d\r\n" % len(body))]) +
View
10 website/sphinx/releases/next.rst
@@ -172,3 +172,13 @@ In progress
* Secondary `AsyncHTTPClient` callbacks (``streaming_callback``,
``header_callback``, and ``prepare_curl_callback``) now respect
`StackContext`.
+* `AsyncHTTPClient.configure` and all `AsyncHTTPClient` constructors
+ now take a ``defaults`` keyword argument. This argument should be a
+ dictionary, and its values will be used in place of corresponding
+ attributes of `HTTPRequest` that are not set.
+* All unset attributes of `tornado.httpclient.HTTPRequest` are now ``None``.
+ The default values of some attributes (``connect_timeout``,
+ ``request_timeout``, ``follow_redirects``, ``max_redirects``,
+ ``use_gzip``, ``proxy_password``, ``allow_nonstandard_methods``,
+ and ``validate_cert`` have been moved from `HTTPRequest` to the
+ client implementations.
Please sign in to comment.
Something went wrong with that request. Please try again.