Permalink
Browse files

Fixed #7581 -- Added streaming responses.

Thanks mrmachine and everyone else involved on this long-standing ticket.
  • Loading branch information...
1 parent 300d052 commit 4b27813198ae31892f1159d437e492f7745761a0 @aaugustin aaugustin committed Oct 20, 2012
View
@@ -528,45 +528,48 @@ def parse_cookie(cookie):
class BadHeaderError(ValueError):
pass
-class HttpResponse(object):
- """A basic HTTP response, with content and dictionary-accessed headers."""
+class HttpResponseBase(object):
+ """
+ An HTTP response base class with dictionary-accessed headers.
+
+ This class doesn't handle content. It should not be used directly.
+ Use the HttpResponse and StreamingHttpResponse subclasses instead.
+ """
status_code = 200
- def __init__(self, content='', content_type=None, status=None,
- mimetype=None):
+ def __init__(self, content_type=None, status=None, mimetype=None):
# _headers is a mapping of the lower-case name to the original case of
# the header (required for working with legacy systems) and the header
# value. Both the name of the header and its value are ASCII strings.
self._headers = {}
self._charset = settings.DEFAULT_CHARSET
+ self._closable_objects = []
if mimetype:
warnings.warn("Using mimetype keyword argument is deprecated, use"
" content_type instead", PendingDeprecationWarning)
content_type = mimetype
if not content_type:
content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE,
self._charset)
- # content is a bytestring. See the content property methods.
- self.content = content
self.cookies = SimpleCookie()
if status:
self.status_code = status
self['Content-Type'] = content_type
- def serialize(self):
- """Full HTTP message, including headers, as a bytestring."""
+ def serialize_headers(self):
+ """HTTP headers as a bytestring."""
headers = [
('%s: %s' % (key, value)).encode('us-ascii')
for key, value in self._headers.values()
]
- return b'\r\n'.join(headers) + b'\r\n\r\n' + self.content
+ return b'\r\n'.join(headers)
if six.PY3:
- __bytes__ = serialize
+ __bytes__ = serialize_headers
else:
- __str__ = serialize
+ __str__ = serialize_headers
def _convert_to_charset(self, value, charset, mime_encode=False):
"""Converts headers key/value to ascii/latin1 native strings.
@@ -690,24 +693,75 @@ def delete_cookie(self, key, path='/', domain=None):
self.set_cookie(key, max_age=0, path=path, domain=domain,
expires='Thu, 01-Jan-1970 00:00:00 GMT')
+ # Common methods used by subclasses
+
+ def make_bytes(self, value):
+ """Turn a value into a bytestring encoded in the output charset."""
+ # For backwards compatibility, this method supports values that are
+ # unlikely to occur in real applications. It has grown complex and
+ # should be refactored. It also overlaps __next__. See #18796.
+ if self.has_header('Content-Encoding'):
+ if isinstance(value, int):
+ value = six.text_type(value)
+ if isinstance(value, six.text_type):
+ value = value.encode('ascii')
+ # force conversion to bytes in case chunk is a subclass
+ return bytes(value)
+ else:
+ return force_bytes(value, self._charset)
+
+ # These methods partially implement the file-like object interface.
+ # See http://docs.python.org/lib/bltin-file-objects.html
+
+ # The WSGI server must call this method upon completion of the request.
+ # See http://blog.dscpl.com.au/2012/10/obligations-for-calling-close-on.html
+ def close(self):
+ for closable in self._closable_objects:
+ closable.close()
+
+ def write(self, content):
+ raise Exception("This %s instance is not writable" % self.__class__.__name__)
+
+ def flush(self):
+ pass
+
+ def tell(self):
+ raise Exception("This %s instance cannot tell its position" % self.__class__.__name__)
+
+class HttpResponse(HttpResponseBase):
+ """
+ An HTTP response class with a string as content.
+
+ This content that can be read, appended to or replaced.
+ """
+
+ streaming = False
+
+ def __init__(self, content='', *args, **kwargs):
+ super(HttpResponse, self).__init__(*args, **kwargs)
+ # Content is a bytestring. See the `content` property methods.
+ self.content = content
+
+ def serialize(self):
+ """Full HTTP message, including headers, as a bytestring."""
+ return self.serialize_headers() + b'\r\n\r\n' + self.content
+
+ if six.PY3:
+ __bytes__ = serialize
+ else:
+ __str__ = serialize
+
@property
def content(self):
- if self.has_header('Content-Encoding'):
- def make_bytes(value):
- if isinstance(value, int):
- value = six.text_type(value)
- if isinstance(value, six.text_type):
- value = value.encode('ascii')
- # force conversion to bytes in case chunk is a subclass
- return bytes(value)
- return b''.join(make_bytes(e) for e in self._container)
- return b''.join(force_bytes(e, self._charset) for e in self._container)
+ return b''.join(self.make_bytes(e) for e in self._container)
@content.setter
def content(self, value):
if hasattr(value, '__iter__') and not isinstance(value, (bytes, six.string_types)):
self._container = value
self._base_content_is_iter = True
+ if hasattr(value, 'close'):
+ self._closable_objects.append(value)
else:
self._container = [value]
self._base_content_is_iter = False
@@ -727,25 +781,85 @@ def __next__(self):
next = __next__ # Python 2 compatibility
- def close(self):
- if hasattr(self._container, 'close'):
- self._container.close()
-
- # The remaining methods partially implement the file-like object interface.
- # See http://docs.python.org/lib/bltin-file-objects.html
def write(self, content):
if self._base_content_is_iter:
- raise Exception("This %s instance is not writable" % self.__class__)
+ raise Exception("This %s instance is not writable" % self.__class__.__name__)
self._container.append(content)
- def flush(self):
- pass
-
def tell(self):
if self._base_content_is_iter:
- raise Exception("This %s instance cannot tell its position" % self.__class__)
+ raise Exception("This %s instance cannot tell its position" % self.__class__.__name__)
return sum([len(chunk) for chunk in self])
+class StreamingHttpResponse(HttpResponseBase):
+ """
+ A streaming HTTP response class with an iterator as content.
+
+ This should only be iterated once, when the response is streamed to the
+ client. However, it can be appended to or replaced with a new iterator
+ that wraps the original content (or yields entirely new content).
+ """
+
+ streaming = True
+
+ def __init__(self, streaming_content=(), *args, **kwargs):
+ super(StreamingHttpResponse, self).__init__(*args, **kwargs)
+ # `streaming_content` should be an iterable of bytestrings.
+ # See the `streaming_content` property methods.
+ self.streaming_content = streaming_content
+
+ @property
+ def content(self):
+ raise AttributeError("This %s instance has no `content` attribute. "
+ "Use `streaming_content` instead." % self.__class__.__name__)
+
+ @property
+ def streaming_content(self):
+ return self._iterator
+
+ @streaming_content.setter
+ def streaming_content(self, value):
+ # Ensure we can never iterate on "value" more than once.
+ self._iterator = iter(value)
+ if hasattr(value, 'close'):
+ self._closable_objects.append(value)
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return self.make_bytes(next(self._iterator))
+
+ next = __next__ # Python 2 compatibility
+
+class CompatibleStreamingHttpResponse(StreamingHttpResponse):
+ """
+ This class maintains compatibility with middleware that doesn't know how
+ to handle the content of a streaming response by exposing a `content`
+ attribute that will consume and cache the content iterator when accessed.
+
+ These responses will stream only if no middleware attempts to access the
+ `content` attribute. Otherwise, they will behave like a regular response,
+ and raise a `PendingDeprecationWarning`.
+ """
+ @property
+ def content(self):
+ warnings.warn(
+ 'Accessing the `content` attribute on a streaming response is '
+ 'deprecated. Use the `streaming_content` attribute instead.',
+ PendingDeprecationWarning)
+ content = b''.join(self)
+ self.streaming_content = [content]
+ return content
+
+ @content.setter
+ def content(self, content):
+ warnings.warn(
+ 'Accessing the `content` attribute on a streaming response is '
+ 'deprecated. Use the `streaming_content` attribute instead.',
+ PendingDeprecationWarning)
+ self.streaming_content = [content]
+
class HttpResponseRedirectBase(HttpResponse):
allowed_schemes = ['http', 'https', 'ftp']
View
@@ -26,10 +26,16 @@ def conditional_content_removal(request, response):
responses. Ensures compliance with RFC 2616, section 4.3.
"""
if 100 <= response.status_code < 200 or response.status_code in (204, 304):
- response.content = ''
- response['Content-Length'] = 0
+ if response.streaming:
+ response.streaming_content = []
+ else:
+ response.content = ''
+ response['Content-Length'] = '0'
if request.method == 'HEAD':
- response.content = ''
+ if response.streaming:
+ response.streaming_content = []
+ else:
+ response.content = ''
return response
def fix_IE_for_attach(request, response):
@@ -113,14 +113,18 @@ def process_response(self, request, response):
if settings.USE_ETAGS:
if response.has_header('ETag'):
etag = response['ETag']
+ elif response.streaming:
+ etag = None
else:
etag = '"%s"' % hashlib.md5(response.content).hexdigest()
- if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
- cookies = response.cookies
- response = http.HttpResponseNotModified()
- response.cookies = cookies
- else:
- response['ETag'] = etag
+ if etag is not None:
+ if (200 <= response.status_code < 300
+ and request.META.get('HTTP_IF_NONE_MATCH') == etag):
+ cookies = response.cookies
+ response = http.HttpResponseNotModified()
+ response.cookies = cookies
+ else:
+ response['ETag'] = etag
return response
View
@@ -1,6 +1,6 @@
import re
-from django.utils.text import compress_string
+from django.utils.text import compress_sequence, compress_string
from django.utils.cache import patch_vary_headers
re_accepts_gzip = re.compile(r'\bgzip\b')
@@ -13,7 +13,7 @@ class GZipMiddleware(object):
"""
def process_response(self, request, response):
# It's not worth attempting to compress really short responses.
- if len(response.content) < 200:
+ if not response.streaming and len(response.content) < 200:
return response
patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,15 +32,21 @@ def process_response(self, request, response):
if not re_accepts_gzip.search(ae):
return response
- # Return the compressed content only if it's actually shorter.
- compressed_content = compress_string(response.content)
- if len(compressed_content) >= len(response.content):
- return response
+ if response.streaming:
+ # Delete the `Content-Length` header for streaming content, because
+ # we won't know the compressed size until we stream it.
+ response.streaming_content = compress_sequence(response.streaming_content)
+ del response['Content-Length']
+ else:
+ # Return the compressed content only if it's actually shorter.
+ compressed_content = compress_string(response.content)
+ if len(compressed_content) >= len(response.content):
+ return response
+ response.content = compressed_content
+ response['Content-Length'] = str(len(response.content))
if response.has_header('ETag'):
response['ETag'] = re.sub('"$', ';gzip"', response['ETag'])
-
- response.content = compressed_content
response['Content-Encoding'] = 'gzip'
- response['Content-Length'] = str(len(response.content))
+
return response
@@ -10,7 +10,7 @@ class ConditionalGetMiddleware(object):
"""
def process_response(self, request, response):
response['Date'] = http_date()
- if not response.has_header('Content-Length'):
+ if not response.streaming and not response.has_header('Content-Length'):
response['Content-Length'] = str(len(response.content))
if response.has_header('ETag'):
View
@@ -596,7 +596,9 @@ def assertContains(self, response, text, count=None, status_code=200,
msg_prefix + "Couldn't retrieve content: Response code was %d"
" (expected %d)" % (response.status_code, status_code))
text = force_text(text, encoding=response._charset)
- content = response.content.decode(response._charset)
+ content = b''.join(response).decode(response._charset)
+ # Avoid ResourceWarning about unclosed files.
+ response.close()
if html:
content = assert_and_parse_html(self, content, None,
"Response's content is not valid HTML:")
View
@@ -95,7 +95,8 @@ def get_max_age(response):
pass
def _set_response_etag(response):
- response['ETag'] = '"%s"' % hashlib.md5(response.content).hexdigest()
+ if not response.streaming:
+ response['ETag'] = '"%s"' % hashlib.md5(response.content).hexdigest()
return response
def patch_response_headers(response, cache_timeout=None):
Oops, something went wrong.

0 comments on commit 4b27813

Please sign in to comment.