From ac7c16d7b7c3031892288469f2f7618a8e63c54d Mon Sep 17 00:00:00 2001 From: Carl Waldbieser Date: Wed, 29 Jul 2015 20:47:25 -0400 Subject: [PATCH 1/4] Revert "Added short docs and a test for providing a custom agent." Glyph pointed out that the various treq request functions (`treq.get()`, `treq.post()`, etc.) don't need to be modified. Instead, the documentation should suggest that you use a `treq.client.HTTPClient` instance directly if you need to customize its behavior. This reverts commit 972b84f6c3ca31f6ba6672290e41e1cd2f6808d2. --- docs/howto.rst | 14 -------------- treq/test/test_api.py | 7 +------ 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/docs/howto.rst b/docs/howto.rst index 9600d828..18cb6060 100644 --- a/docs/howto.rst +++ b/docs/howto.rst @@ -101,17 +101,3 @@ access as `requests cookies ` - -Agent Customization -------------------- - -`treq` creates its own `twisted.web.client.Agent` with reasonable defaults, but -you may want to provide your own custom agent. A custom agent can be passed -to the various `treq` request methods using the ``agent`` keyword argument. - -.. code-block:: python - - custom_agent = create_custom_agent(trusted_certs, timeout) - treq.get(url, agent=custom_agent) - - diff --git a/treq/test/test_api.py b/treq/test/test_api.py index eef74b55..1590d1da 100644 --- a/treq/test/test_api.py +++ b/treq/test/test_api.py @@ -1,6 +1,7 @@ import mock from treq.test.util import TestCase + import treq from treq._utils import set_global_pool @@ -43,9 +44,3 @@ def test_cached_pool(self): treq.get('http://test.com') self.Agent.assert_called_with(mock.ANY, pool=pool) - - def test_custom_agent(self): - custom_agent = mock.Mock() - treq.get('https://www.example.org/', agent=custom_agent) - self.HTTPClient.assertCalledWith(custom_agent) - From 09379ff4c2a1ced053b4896feb9325a99055b208 Mon Sep 17 00:00:00 2001 From: Carl Waldbieser Date: Wed, 29 Jul 2015 21:59:30 -0400 Subject: [PATCH 2/4] Updated documentation in an attempt to make it more clear that `treq.client.HTTPClient` is meant to be used. --- docs/api.rst | 15 +++++++++++++++ docs/examples/basic_auth.py | 3 ++- docs/examples/basic_get.py | 3 ++- docs/examples/basic_post.py | 3 ++- docs/examples/custom_agent.py | 19 +++++++++++++++++++ docs/examples/disable_redirects.py | 3 ++- docs/examples/download_file.py | 3 ++- docs/examples/query_params.py | 11 ++++++----- docs/examples/redirects.py | 3 ++- docs/examples/response_history.py | 3 ++- docs/examples/using_cookies.py | 5 +++-- docs/howto.rst | 27 ++++++++++++++++++++++++--- 12 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 docs/examples/custom_agent.py diff --git a/docs/api.rst b/docs/api.rst index 31f8f946..6fa69386 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,6 +1,11 @@ Making Requests =============== +The :py:mod:`treq` module provides several shortcut functions for making +requests. These functions all create a default +:py:class:`treq.client.HTTPClient` instance and pass their arguments to +the appropriate :py:class:`~treq.client.HTTPClient` method. + .. module:: treq .. autofunction:: request @@ -19,6 +24,16 @@ Accessing Content .. autofunction:: text_content .. autofunction:: json_content +The HTTP Client +=============== + +.. module:: treq.client + +.. class:: HTTPClient(agent, cookiejar=None, data_to_body_producer=IBodyProducer) + +:py:class:`~treq.client.HTTPClient` has methods that match the signatures of the +shortcut request functions in the :py:mod:`treq` module. + Responses ========= diff --git a/docs/examples/basic_auth.py b/docs/examples/basic_auth.py index 73a99188..d07ce38f 100644 --- a/docs/examples/basic_auth.py +++ b/docs/examples/basic_auth.py @@ -5,7 +5,8 @@ def main(reactor, *args): - d = treq.get( + http_client = treq + d = http_client.get( 'http://httpbin.org/basic-auth/treq/treq', auth=('treq', 'treq') ) diff --git a/docs/examples/basic_get.py b/docs/examples/basic_get.py index c92c8217..9523dd00 100644 --- a/docs/examples/basic_get.py +++ b/docs/examples/basic_get.py @@ -5,7 +5,8 @@ def main(reactor, *args): - d = treq.get('http://httpbin.org/get') + http_client = treq + d = http_client.get('http://httpbin.org/get') d.addCallback(print_response) return d diff --git a/docs/examples/basic_post.py b/docs/examples/basic_post.py index dfc9477d..80299be1 100644 --- a/docs/examples/basic_post.py +++ b/docs/examples/basic_post.py @@ -7,7 +7,8 @@ def main(reactor, *args): - d = treq.post('http://httpbin.org/post', + http_client = treq + d = http_client.post('http://httpbin.org/post', json.dumps({"msg": "Hello!"}), headers={'Content-Type': ['application/json']}) d.addCallback(print_response) diff --git a/docs/examples/custom_agent.py b/docs/examples/custom_agent.py new file mode 100644 index 00000000..40e91165 --- /dev/null +++ b/docs/examples/custom_agent.py @@ -0,0 +1,19 @@ +from treq.client import HTTPClient +from _utils import print_response +from twisted.internet.task import react +from twisted.web.client import Agent + +def make_custom_agent(reactor): + return Agent(reactor) + +def main(reactor, *args): + agent = make_custom_agent(reactor) + http_client = HTTPClient(agent) + d = http_client.get( + 'https://secure.example.net/area51', + auth=('admin', "you'll never guess!")) + d.addCallback(print_response) + return d + +react(main, []) + diff --git a/docs/examples/disable_redirects.py b/docs/examples/disable_redirects.py index dac64e00..0a962937 100644 --- a/docs/examples/disable_redirects.py +++ b/docs/examples/disable_redirects.py @@ -5,7 +5,8 @@ def main(reactor, *args): - d = treq.get('http://httpbin.org/redirect/1', allow_redirects=False) + http_client = treq + d = http_client.get('http://httpbin.org/redirect/1', allow_redirects=False) d.addCallback(print_response) return d diff --git a/docs/examples/download_file.py b/docs/examples/download_file.py index 66d7c4cb..4051b853 100644 --- a/docs/examples/download_file.py +++ b/docs/examples/download_file.py @@ -5,7 +5,8 @@ def download_file(reactor, url, destination_filename): destination = file(destination_filename, 'w') - d = treq.get(url) + http_client = treq + d = http_client.get(url) d.addCallback(treq.collect, destination.write) d.addBoth(lambda _: destination.close()) return d diff --git a/docs/examples/query_params.py b/docs/examples/query_params.py index 2d56a39d..15c360c5 100644 --- a/docs/examples/query_params.py +++ b/docs/examples/query_params.py @@ -7,31 +7,32 @@ @inlineCallbacks def main(reactor): print 'List of tuples' - resp = yield treq.get('http://httpbin.org/get', + http_client = treq + resp = yield http_client.get('http://httpbin.org/get', params=[('foo', 'bar'), ('baz', 'bax')]) content = yield treq.content(resp) print content print 'Single value dictionary' - resp = yield treq.get('http://httpbin.org/get', + resp = yield http_client.get('http://httpbin.org/get', params={'foo': 'bar', 'baz': 'bax'}) content = yield treq.content(resp) print content print 'Multi value dictionary' - resp = yield treq.get('http://httpbin.org/get', + resp = yield http_client.get('http://httpbin.org/get', params={'foo': ['bar', 'baz', 'bax']}) content = yield treq.content(resp) print content print 'Mixed value dictionary' - resp = yield treq.get('http://httpbin.org/get', + resp = yield http_client.get('http://httpbin.org/get', params={'foo': ['bar', 'baz'], 'bax': 'quux'}) content = yield treq.content(resp) print content print 'Preserved query parameters' - resp = yield treq.get('http://httpbin.org/get?foo=bar', + resp = yield http_client.get('http://httpbin.org/get?foo=bar', params={'baz': 'bax'}) content = yield treq.content(resp) print content diff --git a/docs/examples/redirects.py b/docs/examples/redirects.py index a9a66603..fb57a244 100644 --- a/docs/examples/redirects.py +++ b/docs/examples/redirects.py @@ -5,7 +5,8 @@ def main(reactor, *args): - d = treq.get('http://httpbin.org/redirect/1') + http_client = treq + d = http_client.get('http://httpbin.org/redirect/1') d.addCallback(print_response) return d diff --git a/docs/examples/response_history.py b/docs/examples/response_history.py index 61a2c164..857d21c9 100644 --- a/docs/examples/response_history.py +++ b/docs/examples/response_history.py @@ -5,7 +5,8 @@ def main(reactor, *args): - d = treq.get('http://httpbin.org/redirect/1') + http_client = treq + d = http_client.get('http://httpbin.org/redirect/1') def cb(response): print 'Response history:', response.history() diff --git a/docs/examples/using_cookies.py b/docs/examples/using_cookies.py index a963716d..2be1854c 100644 --- a/docs/examples/using_cookies.py +++ b/docs/examples/using_cookies.py @@ -5,14 +5,15 @@ def main(reactor, *args): - d = treq.get('http://httpbin.org/cookies/set?hello=world') + http_client = treq + d = http_client.get('http://httpbin.org/cookies/set?hello=world') def _get_jar(resp): jar = resp.cookies() print 'The server set our hello cookie to: {0}'.format(jar['hello']) - return treq.get('http://httpbin.org/cookies', cookies=jar) + return http_client.get('http://httpbin.org/cookies', cookies=jar) d.addCallback(_get_jar) d.addCallback(print_response) diff --git a/docs/howto.rst b/docs/howto.rst index 18cb6060..18dbfd42 100644 --- a/docs/howto.rst +++ b/docs/howto.rst @@ -22,7 +22,7 @@ Full example: :download:`download_file.py ` Query Parameters ---------------- -:py:func:`treq.request` supports a ``params`` keyword argument which will +:py:func:`treq.HTTPClient.request` supports a ``params`` keyword argument which will be urlencoded and added to the ``url`` argument in addition to any query parameters that may already exist. @@ -91,9 +91,10 @@ Cookies Cookies can be set by passing a ``dict`` or ``cookielib.CookieJar`` instance via the ``cookies`` keyword argument. Later cookies set by the server can be -retrieved using the :py:func:`treq.cookies` function. +retrieved using the :py:meth:`~treq.response.Response.cookies` method of the +response. -The the object returned by :py:func:`treq.cookies` supports the same key/value +The the object returned by :py:meth:`~treq.Response.cookies` supports the same key/value access as `requests cookies `_ .. literalinclude:: examples/using_cookies.py @@ -101,3 +102,23 @@ access as `requests cookies ` + +Customizing the Twisted Agent +----------------------------- + +The main :py:mod:`treq` module has helper functions that automatically instantiate +an instance of :py:class:`treq.client.HTTPClient`. You can create an instance +of :py:class:`~treq.client.HTTPClient` directly in order to customize the +paramaters used to initialize it. +Internally, the :py:class:`~treq.client.HTTPClient` wraps an instance of +:py:class:`twisted.web.client.Agent`. When you create an instance of +:py:class:`~treq.client.HTTPClient`, you must initialize it with an instance of +:py:class:`~twisted.web.client.Agent`. This allows you to customize its +behavior. + +.. literalinclude:: examples/custom_agent.py + :linenos: + :lines: 6-19 + +Full example: :download:`custom_agent.py ` + From 652dadec3e565a4d53afa1726edbe9e6e223656b Mon Sep 17 00:00:00 2001 From: Carl Waldbieser Date: Fri, 31 Jul 2015 22:06:14 -0400 Subject: [PATCH 3/4] Changed the other examples back to using `treq.$REQUUEST_METHOD` directly. --- docs/examples/basic_auth.py | 3 +-- docs/examples/basic_get.py | 3 +-- docs/examples/basic_post.py | 3 +-- docs/examples/disable_redirects.py | 3 +-- docs/examples/download_file.py | 3 +-- docs/examples/query_params.py | 11 +++++------ docs/examples/redirects.py | 3 +-- docs/examples/response_history.py | 3 +-- docs/examples/using_cookies.py | 5 ++--- 9 files changed, 14 insertions(+), 23 deletions(-) diff --git a/docs/examples/basic_auth.py b/docs/examples/basic_auth.py index d07ce38f..73a99188 100644 --- a/docs/examples/basic_auth.py +++ b/docs/examples/basic_auth.py @@ -5,8 +5,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.get( + d = treq.get( 'http://httpbin.org/basic-auth/treq/treq', auth=('treq', 'treq') ) diff --git a/docs/examples/basic_get.py b/docs/examples/basic_get.py index 9523dd00..c92c8217 100644 --- a/docs/examples/basic_get.py +++ b/docs/examples/basic_get.py @@ -5,8 +5,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.get('http://httpbin.org/get') + d = treq.get('http://httpbin.org/get') d.addCallback(print_response) return d diff --git a/docs/examples/basic_post.py b/docs/examples/basic_post.py index 80299be1..dfc9477d 100644 --- a/docs/examples/basic_post.py +++ b/docs/examples/basic_post.py @@ -7,8 +7,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.post('http://httpbin.org/post', + d = treq.post('http://httpbin.org/post', json.dumps({"msg": "Hello!"}), headers={'Content-Type': ['application/json']}) d.addCallback(print_response) diff --git a/docs/examples/disable_redirects.py b/docs/examples/disable_redirects.py index 0a962937..dac64e00 100644 --- a/docs/examples/disable_redirects.py +++ b/docs/examples/disable_redirects.py @@ -5,8 +5,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.get('http://httpbin.org/redirect/1', allow_redirects=False) + d = treq.get('http://httpbin.org/redirect/1', allow_redirects=False) d.addCallback(print_response) return d diff --git a/docs/examples/download_file.py b/docs/examples/download_file.py index 4051b853..66d7c4cb 100644 --- a/docs/examples/download_file.py +++ b/docs/examples/download_file.py @@ -5,8 +5,7 @@ def download_file(reactor, url, destination_filename): destination = file(destination_filename, 'w') - http_client = treq - d = http_client.get(url) + d = treq.get(url) d.addCallback(treq.collect, destination.write) d.addBoth(lambda _: destination.close()) return d diff --git a/docs/examples/query_params.py b/docs/examples/query_params.py index 15c360c5..2d56a39d 100644 --- a/docs/examples/query_params.py +++ b/docs/examples/query_params.py @@ -7,32 +7,31 @@ @inlineCallbacks def main(reactor): print 'List of tuples' - http_client = treq - resp = yield http_client.get('http://httpbin.org/get', + resp = yield treq.get('http://httpbin.org/get', params=[('foo', 'bar'), ('baz', 'bax')]) content = yield treq.content(resp) print content print 'Single value dictionary' - resp = yield http_client.get('http://httpbin.org/get', + resp = yield treq.get('http://httpbin.org/get', params={'foo': 'bar', 'baz': 'bax'}) content = yield treq.content(resp) print content print 'Multi value dictionary' - resp = yield http_client.get('http://httpbin.org/get', + resp = yield treq.get('http://httpbin.org/get', params={'foo': ['bar', 'baz', 'bax']}) content = yield treq.content(resp) print content print 'Mixed value dictionary' - resp = yield http_client.get('http://httpbin.org/get', + resp = yield treq.get('http://httpbin.org/get', params={'foo': ['bar', 'baz'], 'bax': 'quux'}) content = yield treq.content(resp) print content print 'Preserved query parameters' - resp = yield http_client.get('http://httpbin.org/get?foo=bar', + resp = yield treq.get('http://httpbin.org/get?foo=bar', params={'baz': 'bax'}) content = yield treq.content(resp) print content diff --git a/docs/examples/redirects.py b/docs/examples/redirects.py index fb57a244..a9a66603 100644 --- a/docs/examples/redirects.py +++ b/docs/examples/redirects.py @@ -5,8 +5,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.get('http://httpbin.org/redirect/1') + d = treq.get('http://httpbin.org/redirect/1') d.addCallback(print_response) return d diff --git a/docs/examples/response_history.py b/docs/examples/response_history.py index 857d21c9..61a2c164 100644 --- a/docs/examples/response_history.py +++ b/docs/examples/response_history.py @@ -5,8 +5,7 @@ def main(reactor, *args): - http_client = treq - d = http_client.get('http://httpbin.org/redirect/1') + d = treq.get('http://httpbin.org/redirect/1') def cb(response): print 'Response history:', response.history() diff --git a/docs/examples/using_cookies.py b/docs/examples/using_cookies.py index 2be1854c..a963716d 100644 --- a/docs/examples/using_cookies.py +++ b/docs/examples/using_cookies.py @@ -5,15 +5,14 @@ def main(reactor, *args): - http_client = treq - d = http_client.get('http://httpbin.org/cookies/set?hello=world') + d = treq.get('http://httpbin.org/cookies/set?hello=world') def _get_jar(resp): jar = resp.cookies() print 'The server set our hello cookie to: {0}'.format(jar['hello']) - return http_client.get('http://httpbin.org/cookies', cookies=jar) + return treq.get('http://httpbin.org/cookies', cookies=jar) d.addCallback(_get_jar) d.addCallback(print_response) From 9d939cbd2b000d3aab9c4019d5f6ec3210b09141 Mon Sep 17 00:00:00 2001 From: Carl Waldbieser Date: Sat, 10 Oct 2015 15:50:04 -0400 Subject: [PATCH 4/4] If you want to customize the behavior, instantiate `HTTPClient` directly. --- treq/api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/treq/api.py b/treq/api.py index 069ca3cb..68bc77e6 100644 --- a/treq/api.py +++ b/treq/api.py @@ -106,11 +106,9 @@ def request(method, url, **kwargs): # def _client(*args, **kwargs): - agent = kwargs.get('agent') - if agent is None: - reactor = default_reactor(kwargs.get('reactor')) - pool = default_pool(reactor, - kwargs.get('pool'), - kwargs.get('persistent')) - agent = Agent(reactor, pool=pool) + reactor = default_reactor(kwargs.get('reactor')) + pool = default_pool(reactor, + kwargs.get('pool'), + kwargs.get('persistent')) + agent = Agent(reactor, pool=pool) return HTTPClient(agent)