Skip to content
Browse files

Merge remote-tracking branch 'origin/develop' into develop

  • Loading branch information...
2 parents a69cf7e + 6497d11 commit c8f166f696327dc8ec07a248863fcc35fafa2038 @kennethreitz committed Sep 2, 2012
Showing with 35 additions and 18 deletions.
  1. +8 −14 docs/user/advanced.rst
  2. +3 −4 requests/models.py
  3. +6 −0 requests/utils.py
  4. +18 −0 tests/test_requests.py
View
22 docs/user/advanced.rst
@@ -116,26 +116,20 @@ If you specify a wrong path or an invalid cert::
Body Content Workflow
---------------------
-By default, when you make a request, the body of the response isn't downloaded immediately. The response headers are downloaded when you make a request, but the content isn't downloaded until you access the :class:`Response.content` attribute.
-
-Let's walk through it::
+By default, when you make a request, the body of the response is downloaded immediately. You can override this behavior and defer downloading the response body until you access the :class:`Response.content` attribute with the ``prefetch`` parameter::
tarball_url = 'https://github.com/kennethreitz/requests/tarball/master'
- r = requests.get(tarball_url)
-
-The request has been made, but the connection is still open. The response body has not been downloaded yet.
-
-::
-
- r.content
+ r = requests.get(tarball_url, prefetch=False)
-The content has been downloaded and cached.
+At this point only the response headers have been downloaded and the connection remains open, hence allowing us to make content retrieval conditional::
-You can override this default behavior with the ``prefetch`` parameter::
+ if int(r.headers['content-length']) < TOO_LONG:
+ content = r.content
+ ...
- r = requests.get(tarball_url, prefetch=True)
- # Blocks until all of request body has been downloaded.
+You can further control the workflow by use of the :class:`Response.iter_content` and :class:`Response.iter_lines` methods, or reading from the underlying urllib3 :class:`urllib3.HTTPResponse` at :class:`Response.raw`.
+Note that in versions prior to 0.13.6 the ``prefetch`` default was set to ``False``.
Configuring Requests
--------------------
View
7 requests/models.py
@@ -31,7 +31,7 @@
from .utils import (
get_encoding_from_headers, stream_untransfer, guess_filename, requote_uri,
stream_decode_response_unicode, get_netrc_auth, get_environ_proxies,
- to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links)
+ to_key_val_list, DEFAULT_CA_BUNDLE_PATH, parse_header_links, iter_slices)
from .compat import (
cookielib, urlparse, urlunparse, urljoin, urlsplit, urlencode, str, bytes,
StringIO, is_py2, chardet, json, builtin_str, numeric_types)
@@ -730,9 +730,8 @@ def iter_content(self, chunk_size=1, decode_unicode=False):
length of each item returned as decoding can take place.
"""
if self._content_consumed:
- raise RuntimeError(
- 'The content for this response was already consumed'
- )
+ # simulate reading small chunks of the content
+ return iter_slices(self._content, chunk_size)
def generate():
while 1:
View
6 requests/utils.py
@@ -360,6 +360,12 @@ def stream_decode_response_unicode(iterator, r):
if rv:
yield rv
+def iter_slices(string, slice_length):
+ """Iterate over slices of a string."""
+ pos = 0
+ while pos < len(string):
+ yield string[pos:pos+slice_length]
+ pos += slice_length
def get_unicode_from_response(r):
"""Returns the requested content back in unicode.
View
18 tests/test_requests.py
@@ -924,6 +924,24 @@ def test_iter_lines(self):
joined = lines[0] + '\n' + lines[1] + '\r\n' + lines[2]
self.assertEqual(joined, quote)
+ def test_permissive_iter_content(self):
+ """Test that iter_content and iter_lines work even after the body has been fetched."""
+ r = get(httpbin('stream', '10'), prefetch=True)
+ assert r._content_consumed
+ # iter_lines should still work without crashing
+ self.assertEqual(len(list(r.iter_lines())), 10)
+
+ # iter_content should return a one-item iterator over the whole content
+ iter_content_list = list(r.iter_content(chunk_size=1))
+ self.assertTrue(all(len(item) == 1 for item in iter_content_list))
+ # when joined, it should be exactly the original content
+ self.assertEqual(bytes().join(iter_content_list), r.content)
+
+ # test different chunk sizes:
+ for chunk_size in range(2, 20):
+ self.assertEqual(bytes().join(r.iter_content(chunk_size=chunk_size)), r.content)
+
+
# def test_safe_mode(self):
# safe = requests.session(config=dict(safe_mode=True))

0 comments on commit c8f166f

Please sign in to comment.
Something went wrong with that request. Please try again.