Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #7581 -- Added streaming responses.

Thanks mrmachine and everyone else involved on this long-standing ticket.
  • Loading branch information...
commit 4b27813198ae31892f1159d437e492f7745761a0 1 parent 300d052
Aymeric Augustin authored October 20, 2012
178  django/http/__init__.py
@@ -528,18 +528,23 @@ def parse_cookie(cookie):
528 528
 class BadHeaderError(ValueError):
529 529
     pass
530 530
 
531  
-class HttpResponse(object):
532  
-    """A basic HTTP response, with content and dictionary-accessed headers."""
  531
+class HttpResponseBase(object):
  532
+    """
  533
+    An HTTP response base class with dictionary-accessed headers.
  534
+
  535
+    This class doesn't handle content. It should not be used directly.
  536
+    Use the HttpResponse and StreamingHttpResponse subclasses instead.
  537
+    """
533 538
 
534 539
     status_code = 200
535 540
 
536  
-    def __init__(self, content='', content_type=None, status=None,
537  
-            mimetype=None):
  541
+    def __init__(self, content_type=None, status=None, mimetype=None):
538 542
         # _headers is a mapping of the lower-case name to the original case of
539 543
         # the header (required for working with legacy systems) and the header
540 544
         # value. Both the name of the header and its value are ASCII strings.
541 545
         self._headers = {}
542 546
         self._charset = settings.DEFAULT_CHARSET
  547
+        self._closable_objects = []
543 548
         if mimetype:
544 549
             warnings.warn("Using mimetype keyword argument is deprecated, use"
545 550
                           " content_type instead", PendingDeprecationWarning)
@@ -547,26 +552,24 @@ def __init__(self, content='', content_type=None, status=None,
547 552
         if not content_type:
548 553
             content_type = "%s; charset=%s" % (settings.DEFAULT_CONTENT_TYPE,
549 554
                     self._charset)
550  
-        # content is a bytestring. See the content property methods.
551  
-        self.content = content
552 555
         self.cookies = SimpleCookie()
553 556
         if status:
554 557
             self.status_code = status
555 558
 
556 559
         self['Content-Type'] = content_type
557 560
 
558  
-    def serialize(self):
559  
-        """Full HTTP message, including headers, as a bytestring."""
  561
+    def serialize_headers(self):
  562
+        """HTTP headers as a bytestring."""
560 563
         headers = [
561 564
             ('%s: %s' % (key, value)).encode('us-ascii')
562 565
             for key, value in self._headers.values()
563 566
         ]
564  
-        return b'\r\n'.join(headers) + b'\r\n\r\n' + self.content
  567
+        return b'\r\n'.join(headers)
565 568
 
566 569
     if six.PY3:
567  
-        __bytes__ = serialize
  570
+        __bytes__ = serialize_headers
568 571
     else:
569  
-        __str__ = serialize
  572
+        __str__ = serialize_headers
570 573
 
571 574
     def _convert_to_charset(self, value, charset, mime_encode=False):
572 575
         """Converts headers key/value to ascii/latin1 native strings.
@@ -690,24 +693,75 @@ def delete_cookie(self, key, path='/', domain=None):
690 693
         self.set_cookie(key, max_age=0, path=path, domain=domain,
691 694
                         expires='Thu, 01-Jan-1970 00:00:00 GMT')
692 695
 
  696
+    # Common methods used by subclasses
  697
+
  698
+    def make_bytes(self, value):
  699
+        """Turn a value into a bytestring encoded in the output charset."""
  700
+        # For backwards compatibility, this method supports values that are
  701
+        # unlikely to occur in real applications. It has grown complex and
  702
+        # should be refactored. It also overlaps __next__. See #18796.
  703
+        if self.has_header('Content-Encoding'):
  704
+            if isinstance(value, int):
  705
+                value = six.text_type(value)
  706
+            if isinstance(value, six.text_type):
  707
+                value = value.encode('ascii')
  708
+            # force conversion to bytes in case chunk is a subclass
  709
+            return bytes(value)
  710
+        else:
  711
+            return force_bytes(value, self._charset)
  712
+
  713
+    # These methods partially implement the file-like object interface.
  714
+    # See http://docs.python.org/lib/bltin-file-objects.html
  715
+
  716
+    # The WSGI server must call this method upon completion of the request.
  717
+    # See http://blog.dscpl.com.au/2012/10/obligations-for-calling-close-on.html
  718
+    def close(self):
  719
+        for closable in self._closable_objects:
  720
+            closable.close()
  721
+
  722
+    def write(self, content):
  723
+        raise Exception("This %s instance is not writable" % self.__class__.__name__)
  724
+
  725
+    def flush(self):
  726
+        pass
  727
+
  728
+    def tell(self):
  729
+        raise Exception("This %s instance cannot tell its position" % self.__class__.__name__)
  730
+
  731
+class HttpResponse(HttpResponseBase):
  732
+    """
  733
+    An HTTP response class with a string as content.
  734
+
  735
+    This content that can be read, appended to or replaced.
  736
+    """
  737
+
  738
+    streaming = False
  739
+
  740
+    def __init__(self, content='', *args, **kwargs):
  741
+        super(HttpResponse, self).__init__(*args, **kwargs)
  742
+        # Content is a bytestring. See the `content` property methods.
  743
+        self.content = content
  744
+
  745
+    def serialize(self):
  746
+        """Full HTTP message, including headers, as a bytestring."""
  747
+        return self.serialize_headers() + b'\r\n\r\n' + self.content
  748
+
  749
+    if six.PY3:
  750
+        __bytes__ = serialize
  751
+    else:
  752
+        __str__ = serialize
  753
+
693 754
     @property
694 755
     def content(self):
695  
-        if self.has_header('Content-Encoding'):
696  
-            def make_bytes(value):
697  
-                if isinstance(value, int):
698  
-                    value = six.text_type(value)
699  
-                if isinstance(value, six.text_type):
700  
-                    value = value.encode('ascii')
701  
-                # force conversion to bytes in case chunk is a subclass
702  
-                return bytes(value)
703  
-            return b''.join(make_bytes(e) for e in self._container)
704  
-        return b''.join(force_bytes(e, self._charset) for e in self._container)
  756
+        return b''.join(self.make_bytes(e) for e in self._container)
705 757
 
706 758
     @content.setter
707 759
     def content(self, value):
708 760
         if hasattr(value, '__iter__') and not isinstance(value, (bytes, six.string_types)):
709 761
             self._container = value
710 762
             self._base_content_is_iter = True
  763
+            if hasattr(value, 'close'):
  764
+                self._closable_objects.append(value)
711 765
         else:
712 766
             self._container = [value]
713 767
             self._base_content_is_iter = False
@@ -727,25 +781,85 @@ def __next__(self):
727 781
 
728 782
     next = __next__             # Python 2 compatibility
729 783
 
730  
-    def close(self):
731  
-        if hasattr(self._container, 'close'):
732  
-            self._container.close()
733  
-
734  
-    # The remaining methods partially implement the file-like object interface.
735  
-    # See http://docs.python.org/lib/bltin-file-objects.html
736 784
     def write(self, content):
737 785
         if self._base_content_is_iter:
738  
-            raise Exception("This %s instance is not writable" % self.__class__)
  786
+            raise Exception("This %s instance is not writable" % self.__class__.__name__)
739 787
         self._container.append(content)
740 788
 
741  
-    def flush(self):
742  
-        pass
743  
-
744 789
     def tell(self):
745 790
         if self._base_content_is_iter:
746  
-            raise Exception("This %s instance cannot tell its position" % self.__class__)
  791
+            raise Exception("This %s instance cannot tell its position" % self.__class__.__name__)
747 792
         return sum([len(chunk) for chunk in self])
748 793
 
  794
+class StreamingHttpResponse(HttpResponseBase):
  795
+    """
  796
+    A streaming HTTP response class with an iterator as content.
  797
+
  798
+    This should only be iterated once, when the response is streamed to the
  799
+    client. However, it can be appended to or replaced with a new iterator
  800
+    that wraps the original content (or yields entirely new content).
  801
+    """
  802
+
  803
+    streaming = True
  804
+
  805
+    def __init__(self, streaming_content=(), *args, **kwargs):
  806
+        super(StreamingHttpResponse, self).__init__(*args, **kwargs)
  807
+        # `streaming_content` should be an iterable of bytestrings.
  808
+        # See the `streaming_content` property methods.
  809
+        self.streaming_content = streaming_content
  810
+
  811
+    @property
  812
+    def content(self):
  813
+        raise AttributeError("This %s instance has no `content` attribute. "
  814
+            "Use `streaming_content` instead." % self.__class__.__name__)
  815
+
  816
+    @property
  817
+    def streaming_content(self):
  818
+        return self._iterator
  819
+
  820
+    @streaming_content.setter
  821
+    def streaming_content(self, value):
  822
+        # Ensure we can never iterate on "value" more than once.
  823
+        self._iterator = iter(value)
  824
+        if hasattr(value, 'close'):
  825
+            self._closable_objects.append(value)
  826
+
  827
+    def __iter__(self):
  828
+        return self
  829
+
  830
+    def __next__(self):
  831
+        return self.make_bytes(next(self._iterator))
  832
+
  833
+    next = __next__             # Python 2 compatibility
  834
+
  835
+class CompatibleStreamingHttpResponse(StreamingHttpResponse):
  836
+    """
  837
+    This class maintains compatibility with middleware that doesn't know how
  838
+    to handle the content of a streaming response by exposing a `content`
  839
+    attribute that will consume and cache the content iterator when accessed.
  840
+
  841
+    These responses will stream only if no middleware attempts to access the
  842
+    `content` attribute. Otherwise, they will behave like a regular response,
  843
+    and raise a `PendingDeprecationWarning`.
  844
+    """
  845
+    @property
  846
+    def content(self):
  847
+        warnings.warn(
  848
+            'Accessing the `content` attribute on a streaming response is '
  849
+            'deprecated. Use the `streaming_content` attribute instead.',
  850
+            PendingDeprecationWarning)
  851
+        content = b''.join(self)
  852
+        self.streaming_content = [content]
  853
+        return content
  854
+
  855
+    @content.setter
  856
+    def content(self, content):
  857
+        warnings.warn(
  858
+            'Accessing the `content` attribute on a streaming response is '
  859
+            'deprecated. Use the `streaming_content` attribute instead.',
  860
+            PendingDeprecationWarning)
  861
+        self.streaming_content = [content]
  862
+
749 863
 class HttpResponseRedirectBase(HttpResponse):
750 864
     allowed_schemes = ['http', 'https', 'ftp']
751 865
 
12  django/http/utils.py
@@ -26,10 +26,16 @@ def conditional_content_removal(request, response):
26 26
     responses. Ensures compliance with RFC 2616, section 4.3.
27 27
     """
28 28
     if 100 <= response.status_code < 200 or response.status_code in (204, 304):
29  
-       response.content = ''
30  
-       response['Content-Length'] = 0
  29
+        if response.streaming:
  30
+            response.streaming_content = []
  31
+        else:
  32
+            response.content = ''
  33
+        response['Content-Length'] = '0'
31 34
     if request.method == 'HEAD':
32  
-        response.content = ''
  35
+        if response.streaming:
  36
+            response.streaming_content = []
  37
+        else:
  38
+            response.content = ''
33 39
     return response
34 40
 
35 41
 def fix_IE_for_attach(request, response):
16  django/middleware/common.py
@@ -113,14 +113,18 @@ def process_response(self, request, response):
113 113
         if settings.USE_ETAGS:
114 114
             if response.has_header('ETag'):
115 115
                 etag = response['ETag']
  116
+            elif response.streaming:
  117
+                etag = None
116 118
             else:
117 119
                 etag = '"%s"' % hashlib.md5(response.content).hexdigest()
118  
-            if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
119  
-                cookies = response.cookies
120  
-                response = http.HttpResponseNotModified()
121  
-                response.cookies = cookies
122  
-            else:
123  
-                response['ETag'] = etag
  120
+            if etag is not None:
  121
+                if (200 <= response.status_code < 300
  122
+                    and request.META.get('HTTP_IF_NONE_MATCH') == etag):
  123
+                    cookies = response.cookies
  124
+                    response = http.HttpResponseNotModified()
  125
+                    response.cookies = cookies
  126
+                else:
  127
+                    response['ETag'] = etag
124 128
 
125 129
         return response
126 130
 
24  django/middleware/gzip.py
... ...
@@ -1,6 +1,6 @@
1 1
 import re
2 2
 
3  
-from django.utils.text import compress_string
  3
+from django.utils.text import compress_sequence, compress_string
4 4
 from django.utils.cache import patch_vary_headers
5 5
 
6 6
 re_accepts_gzip = re.compile(r'\bgzip\b')
@@ -13,7 +13,7 @@ class GZipMiddleware(object):
13 13
     """
14 14
     def process_response(self, request, response):
15 15
         # It's not worth attempting to compress really short responses.
16  
-        if len(response.content) < 200:
  16
+        if not response.streaming and len(response.content) < 200:
17 17
             return response
18 18
 
19 19
         patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,15 +32,21 @@ def process_response(self, request, response):
32 32
         if not re_accepts_gzip.search(ae):
33 33
             return response
34 34
 
35  
-        # Return the compressed content only if it's actually shorter.
36  
-        compressed_content = compress_string(response.content)
37  
-        if len(compressed_content) >= len(response.content):
38  
-            return response
  35
+        if response.streaming:
  36
+            # Delete the `Content-Length` header for streaming content, because
  37
+            # we won't know the compressed size until we stream it.
  38
+            response.streaming_content = compress_sequence(response.streaming_content)
  39
+            del response['Content-Length']
  40
+        else:
  41
+            # Return the compressed content only if it's actually shorter.
  42
+            compressed_content = compress_string(response.content)
  43
+            if len(compressed_content) >= len(response.content):
  44
+                return response
  45
+            response.content = compressed_content
  46
+            response['Content-Length'] = str(len(response.content))
39 47
 
40 48
         if response.has_header('ETag'):
41 49
             response['ETag'] = re.sub('"$', ';gzip"', response['ETag'])
42  
-
43  
-        response.content = compressed_content
44 50
         response['Content-Encoding'] = 'gzip'
45  
-        response['Content-Length'] = str(len(response.content))
  51
+
46 52
         return response
2  django/middleware/http.py
@@ -10,7 +10,7 @@ class ConditionalGetMiddleware(object):
10 10
     """
11 11
     def process_response(self, request, response):
12 12
         response['Date'] = http_date()
13  
-        if not response.has_header('Content-Length'):
  13
+        if not response.streaming and not response.has_header('Content-Length'):
14 14
             response['Content-Length'] = str(len(response.content))
15 15
 
16 16
         if response.has_header('ETag'):
4  django/test/testcases.py
@@ -596,7 +596,9 @@ def assertContains(self, response, text, count=None, status_code=200,
596 596
             msg_prefix + "Couldn't retrieve content: Response code was %d"
597 597
             " (expected %d)" % (response.status_code, status_code))
598 598
         text = force_text(text, encoding=response._charset)
599  
-        content = response.content.decode(response._charset)
  599
+        content = b''.join(response).decode(response._charset)
  600
+        # Avoid ResourceWarning about unclosed files.
  601
+        response.close()
600 602
         if html:
601 603
             content = assert_and_parse_html(self, content, None,
602 604
                 "Response's content is not valid HTML:")
3  django/utils/cache.py
@@ -95,7 +95,8 @@ def get_max_age(response):
95 95
             pass
96 96
 
97 97
 def _set_response_etag(response):
98  
-    response['ETag'] = '"%s"' % hashlib.md5(response.content).hexdigest()
  98
+    if not response.streaming:
  99
+        response['ETag'] = '"%s"' % hashlib.md5(response.content).hexdigest()
99 100
     return response
100 101
 
101 102
 def patch_response_headers(response, cache_timeout=None):
31  django/utils/text.py
@@ -288,6 +288,37 @@ def compress_string(s):
288 288
     zfile.close()
289 289
     return zbuf.getvalue()
290 290
 
  291
+class StreamingBuffer(object):
  292
+    def __init__(self):
  293
+        self.vals = []
  294
+
  295
+    def write(self, val):
  296
+        self.vals.append(val)
  297
+
  298
+    def read(self):
  299
+        ret = b''.join(self.vals)
  300
+        self.vals = []
  301
+        return ret
  302
+
  303
+    def flush(self):
  304
+        return
  305
+
  306
+    def close(self):
  307
+        return
  308
+
  309
+# Like compress_string, but for iterators of strings.
  310
+def compress_sequence(sequence):
  311
+    buf = StreamingBuffer()
  312
+    zfile = GzipFile(mode='wb', compresslevel=6, fileobj=buf)
  313
+    # Output headers...
  314
+    yield buf.read()
  315
+    for item in sequence:
  316
+        zfile.write(item)
  317
+        zfile.flush()
  318
+        yield buf.read()
  319
+    zfile.close()
  320
+    yield buf.read()
  321
+
291 322
 ustring_re = re.compile("([\u0080-\uffff])")
292 323
 
293 324
 def javascript_quote(s, quote_double_quotes=False):
2  django/views/generic/base.py
@@ -99,7 +99,7 @@ def options(self, request, *args, **kwargs):
99 99
         """
100 100
         response = http.HttpResponse()
101 101
         response['Allow'] = ', '.join(self._allowed_methods())
102  
-        response['Content-Length'] = 0
  102
+        response['Content-Length'] = '0'
103 103
         return response
104 104
 
105 105
     def _allowed_methods(self):
6  django/views/static.py
@@ -14,7 +14,8 @@
14 14
 except ImportError:     # Python 2
15 15
     from urllib import unquote
16 16
 
17  
-from django.http import Http404, HttpResponse, HttpResponseRedirect, HttpResponseNotModified
  17
+from django.http import (CompatibleStreamingHttpResponse, Http404,
  18
+    HttpResponse, HttpResponseRedirect, HttpResponseNotModified)
18 19
 from django.template import loader, Template, Context, TemplateDoesNotExist
19 20
 from django.utils.http import http_date, parse_http_date
20 21
 from django.utils.translation import ugettext as _, ugettext_noop
@@ -62,8 +63,7 @@ def serve(request, path, document_root=None, show_indexes=False):
62 63
     if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'),
63 64
                               statobj.st_mtime, statobj.st_size):
64 65
         return HttpResponseNotModified()
65  
-    with open(fullpath, 'rb') as f:
66  
-        response = HttpResponse(f.read(), content_type=mimetype)
  66
+    response = CompatibleStreamingHttpResponse(open(fullpath, 'rb'), content_type=mimetype)
67 67
     response["Last-Modified"] = http_date(statobj.st_mtime)
68 68
     if stat.S_ISREG(statobj.st_mode):
69 69
         response["Content-Length"] = statobj.st_size
87  docs/ref/request-response.txt
@@ -566,13 +566,21 @@ file-like object::
566 566
 Passing iterators
567 567
 ~~~~~~~~~~~~~~~~~
568 568
 
569  
-Finally, you can pass ``HttpResponse`` an iterator rather than passing it
570  
-hard-coded strings. If you use this technique, follow these guidelines:
  569
+Finally, you can pass ``HttpResponse`` an iterator rather than strings. If you
  570
+use this technique, the iterator should return strings.
571 571
 
572  
-* The iterator should return strings.
573  
-* If an :class:`HttpResponse` has been initialized with an iterator as its
574  
-  content, you can't use the :class:`HttpResponse` instance as a file-like
575  
-  object. Doing so will raise ``Exception``.
  572
+.. versionchanged:: 1.5
  573
+
  574
+    Passing an iterator as content to :class:`HttpResponse` creates a
  575
+    streaming response if (and only if) no middleware accesses the
  576
+    :attr:`HttpResponse.content` attribute before the response is returned.
  577
+
  578
+    If you want to guarantee that your response will stream to the client, you
  579
+    should use the new :class:`StreamingHttpResponse` class instead.
  580
+
  581
+If an :class:`HttpResponse` instance has been initialized with an iterator as
  582
+its content, you can't use it as a file-like object. Doing so will raise an
  583
+exception.
576 584
 
577 585
 Setting headers
578 586
 ~~~~~~~~~~~~~~~
@@ -614,6 +622,13 @@ Attributes
614 622
 
615 623
     The `HTTP Status code`_ for the response.
616 624
 
  625
+.. attribute:: HttpResponse.streaming
  626
+
  627
+    This is always ``False``.
  628
+
  629
+    This attribute exists so middleware can treat streaming responses
  630
+    differently from regular responses.
  631
+
617 632
 Methods
618 633
 -------
619 634
 
@@ -781,3 +796,63 @@ types of HTTP responses. Like ``HttpResponse``, these subclasses live in
781 796
     method, Django will treat it as emulating a
782 797
     :class:`~django.template.response.SimpleTemplateResponse`, and the
783 798
     ``render`` method must itself return a valid response object.
  799
+
  800
+StreamingHttpResponse objects
  801
+=============================
  802
+
  803
+.. versionadded:: 1.5
  804
+
  805
+.. class:: StreamingHttpResponse
  806
+
  807
+The :class:`StreamingHttpResponse` class is used to stream a response from
  808
+Django to the browser. You might want to do this if generating the response
  809
+takes too long or uses too much memory. For instance, it's useful for
  810
+generating large CSV files.
  811
+
  812
+.. admonition:: Performance considerations
  813
+
  814
+    Django is designed for short-lived requests. Streaming responses will tie
  815
+    a worker process and keep a database connection idle in transaction for
  816
+    the entire duration of the response. This may result in poor performance.
  817
+
  818
+    Generally speaking, you should perform expensive tasks outside of the
  819
+    request-response cycle, rather than resorting to a streamed response.
  820
+
  821
+The :class:`StreamingHttpResponse` is not a subclass of :class:`HttpResponse`,
  822
+because it features a slightly different API. However, it is almost identical,
  823
+with the following notable differences:
  824
+
  825
+* It should be given an iterator that yields strings as content.
  826
+
  827
+* You cannot access its content, except by iterating the response object
  828
+  itself. This should only occur when the response is returned to the client.
  829
+
  830
+* It has no ``content`` attribute. Instead, it has a
  831
+  :attr:`~StreamingHttpResponse.streaming_content` attribute.
  832
+
  833
+* You cannot use the file-like object ``tell()`` or ``write()`` methods.
  834
+  Doing so will raise an exception.
  835
+
  836
+* Any iterators that have a ``close()`` method and are assigned as content will
  837
+  be closed automatically after the response has been iterated.
  838
+
  839
+:class:`StreamingHttpResponse` should only be used in situations where it is
  840
+absolutely required that the whole content isn't iterated before transferring
  841
+the data to the client. Because the content can't be accessed, many
  842
+middlewares can't function normally. For example the ``ETag`` and ``Content-
  843
+Length`` headers can't be generated for streaming responses.
  844
+
  845
+Attributes
  846
+----------
  847
+
  848
+.. attribute:: StreamingHttpResponse.streaming_content
  849
+
  850
+    An iterator of strings representing the content.
  851
+
  852
+.. attribute:: HttpResponse.status_code
  853
+
  854
+    The `HTTP Status code`_ for the response.
  855
+
  856
+.. attribute:: HttpResponse.streaming
  857
+
  858
+    This is always ``True``.
18  docs/releases/1.5.txt
@@ -84,6 +84,24 @@ For one-to-one relationships, both sides can be cached. For many-to-one
84 84
 relationships, only the single side of the relationship can be cached. This
85 85
 is particularly helpful in combination with ``prefetch_related``.
86 86
 
  87
+Explicit support for streaming responses
  88
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  89
+
  90
+Before Django 1.5, it was possible to create a streaming response by passing
  91
+an iterator to :class:`~django.http.HttpResponse`. But this was unreliable:
  92
+any middleware that accessed the :attr:`~django.http.HttpResponse.content`
  93
+attribute would consume the iterator prematurely.
  94
+
  95
+You can now explicitly generate a streaming response with the new
  96
+:class:`~django.http.StreamingHttpResponse` class. This class exposes a
  97
+:class:`~django.http.StreamingHttpResponse.streaming_content` attribute which
  98
+is an iterator.
  99
+
  100
+Since :class:`~django.http.StreamingHttpResponse` does not have a ``content``
  101
+attribute, middleware that need access to the response content must test for
  102
+streaming responses and behave accordingly. See :ref:`response-middleware` for
  103
+more information.
  104
+
87 105
 ``{% verbatim %}`` template tag
88 106
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
89 107
 
17  docs/topics/http/middleware.txt
@@ -164,6 +164,23 @@ an earlier middleware method returned an :class:`~django.http.HttpResponse`
164 164
 classes are applied in reverse order, from the bottom up. This means classes
165 165
 defined at the end of :setting:`MIDDLEWARE_CLASSES` will be run first.
166 166
 
  167
+.. versionchanged:: 1.5
  168
+    ``response`` may also be an :class:`~django.http.StreamingHttpResponse`
  169
+    object.
  170
+
  171
+Unlike :class:`~django.http.HttpResponse`,
  172
+:class:`~django.http.StreamingHttpResponse` does not have a ``content``
  173
+attribute. As a result, middleware can no longer assume that all responses
  174
+will have a ``content`` attribute. If they need access to the content, they
  175
+must test for streaming responses and adjust their behavior accordingly::
  176
+
  177
+    if response.streaming:
  178
+        response.streaming_content = wrap_streaming_content(response.streaming_content)
  179
+    else:
  180
+        response.content = wrap_content(response.content)
  181
+
  182
+``streaming_content`` should be assumed to be too large to hold in memory.
  183
+Middleware may wrap it in a new generator, but must not consume it.
167 184
 
168 185
 .. _exception-middleware:
169 186
 
26  tests/regressiontests/cache/tests.py
@@ -19,7 +19,8 @@
19 19
 from django.core.cache.backends.base import (CacheKeyWarning,
20 20
     InvalidCacheBackendError)
21 21
 from django.db import router
22  
-from django.http import HttpResponse, HttpRequest, QueryDict
  22
+from django.http import (HttpResponse, HttpRequest, StreamingHttpResponse,
  23
+    QueryDict)
23 24
 from django.middleware.cache import (FetchFromCacheMiddleware,
24 25
     UpdateCacheMiddleware, CacheMiddleware)
25 26
 from django.template import Template
@@ -1416,6 +1417,29 @@ def set_cache(request, lang, msg):
1416 1417
         # reset the language
1417 1418
         translation.deactivate()
1418 1419
 
  1420
+    @override_settings(
  1421
+            CACHE_MIDDLEWARE_KEY_PREFIX="test",
  1422
+            CACHE_MIDDLEWARE_SECONDS=60,
  1423
+            USE_ETAGS=True,
  1424
+    )
  1425
+    def test_middleware_with_streaming_response(self):
  1426
+        # cache with non empty request.GET
  1427
+        request = self._get_request_cache(query_string='foo=baz&other=true')
  1428
+
  1429
+        # first access, cache must return None
  1430
+        get_cache_data = FetchFromCacheMiddleware().process_request(request)
  1431
+        self.assertEqual(get_cache_data, None)
  1432
+
  1433
+        # pass streaming response through UpdateCacheMiddleware.
  1434
+        content = 'Check for cache with QUERY_STRING and streaming content'
  1435
+        response = StreamingHttpResponse(content)
  1436
+        UpdateCacheMiddleware().process_response(request, response)
  1437
+
  1438
+        # second access, cache must still return None, because we can't cache
  1439
+        # streaming response.
  1440
+        get_cache_data = FetchFromCacheMiddleware().process_request(request)
  1441
+        self.assertEqual(get_cache_data, None)
  1442
+
1419 1443
 
1420 1444
 @override_settings(
1421 1445
         CACHES={
1  tests/regressiontests/httpwrappers/abc.txt
... ...
@@ -0,0 +1 @@
  1
+random content
111  tests/regressiontests/httpwrappers/tests.py
@@ -2,12 +2,13 @@
2 2
 from __future__ import unicode_literals
3 3
 
4 4
 import copy
  5
+import os
5 6
 import pickle
6 7
 
7 8
 from django.core.exceptions import SuspiciousOperation
8 9
 from django.http import (QueryDict, HttpResponse, HttpResponseRedirect,
9 10
                          HttpResponsePermanentRedirect, HttpResponseNotAllowed,
10  
-                         HttpResponseNotModified,
  11
+                         HttpResponseNotModified, StreamingHttpResponse,
11 12
                          SimpleCookie, BadHeaderError,
12 13
                          parse_cookie)
13 14
 from django.test import TestCase
@@ -351,7 +352,6 @@ def test_unsafe_redirect(self):
351 352
             self.assertRaises(SuspiciousOperation,
352 353
                               HttpResponsePermanentRedirect, url)
353 354
 
354  
-
355 355
 class HttpResponseSubclassesTests(TestCase):
356 356
     def test_redirect(self):
357 357
         response = HttpResponseRedirect('/redirected/')
@@ -379,6 +379,113 @@ def test_not_allowed(self):
379 379
             content_type='text/html')
380 380
         self.assertContains(response, 'Only the GET method is allowed', status_code=405)
381 381
 
  382
+class StreamingHttpResponseTests(TestCase):
  383
+    def test_streaming_response(self):
  384
+        r = StreamingHttpResponse(iter(['hello', 'world']))
  385
+
  386
+        # iterating over the response itself yields bytestring chunks.
  387
+        chunks = list(r)
  388
+        self.assertEqual(chunks, [b'hello', b'world'])
  389
+        for chunk in chunks:
  390
+            self.assertIsInstance(chunk, six.binary_type)
  391
+
  392
+        # and the response can only be iterated once.
  393
+        self.assertEqual(list(r), [])
  394
+
  395
+        # even when a sequence that can be iterated many times, like a list,
  396
+        # is given as content.
  397
+        r = StreamingHttpResponse(['abc', 'def'])
  398
+        self.assertEqual(list(r), [b'abc', b'def'])
  399
+        self.assertEqual(list(r), [])
  400
+
  401
+        # streaming responses don't have a `content` attribute.
  402
+        self.assertFalse(hasattr(r, 'content'))
  403
+
  404
+        # and you can't accidentally assign to a `content` attribute.
  405
+        with self.assertRaises(AttributeError):
  406
+            r.content = 'xyz'
  407
+
  408
+        # but they do have a `streaming_content` attribute.
  409
+        self.assertTrue(hasattr(r, 'streaming_content'))
  410
+
  411
+        # that exists so we can check if a response is streaming, and wrap or
  412
+        # replace the content iterator.
  413
+        r.streaming_content = iter(['abc', 'def'])
  414
+        r.streaming_content = (chunk.upper() for chunk in r.streaming_content)
  415
+        self.assertEqual(list(r), [b'ABC', b'DEF'])
  416
+
  417
+        # coercing a streaming response to bytes doesn't return a complete HTTP
  418
+        # message like a regular response does. it only gives us the headers.
  419
+        r = StreamingHttpResponse(iter(['hello', 'world']))
  420
+        self.assertEqual(
  421
+            six.binary_type(r), b'Content-Type: text/html; charset=utf-8')
  422
+
  423
+        # and this won't consume its content.
  424
+        self.assertEqual(list(r), [b'hello', b'world'])
  425
+
  426
+        # additional content cannot be written to the response.
  427
+        r = StreamingHttpResponse(iter(['hello', 'world']))
  428
+        with self.assertRaises(Exception):
  429
+            r.write('!')
  430
+
  431
+        # and we can't tell the current position.
  432
+        with self.assertRaises(Exception):
  433
+            r.tell()
  434
+
  435
+class FileCloseTests(TestCase):
  436
+    def test_response(self):
  437
+        filename = os.path.join(os.path.dirname(__file__), 'abc.txt')
  438
+
  439
+        # file isn't closed until we close the response.
  440
+        file1 = open(filename)
  441
+        r = HttpResponse(file1)
  442
+        self.assertFalse(file1.closed)
  443
+        r.close()
  444
+        self.assertTrue(file1.closed)
  445
+
  446
+        # don't automatically close file when we finish iterating the response.
  447
+        file1 = open(filename)
  448
+        r = HttpResponse(file1)
  449
+        self.assertFalse(file1.closed)
  450
+        list(r)
  451
+        self.assertFalse(file1.closed)
  452
+        r.close()
  453
+        self.assertTrue(file1.closed)
  454
+
  455
+        # when multiple file are assigned as content, make sure they are all
  456
+        # closed with the response.
  457
+        file1 = open(filename)
  458
+        file2 = open(filename)
  459
+        r = HttpResponse(file1)
  460
+        r.content = file2
  461
+        self.assertFalse(file1.closed)
  462
+        self.assertFalse(file2.closed)
  463
+        r.close()
  464
+        self.assertTrue(file1.closed)
  465
+        self.assertTrue(file2.closed)
  466
+
  467
+    def test_streaming_response(self):
  468
+        filename = os.path.join(os.path.dirname(__file__), 'abc.txt')
  469
+
  470
+        # file isn't closed until we close the response.
  471
+        file1 = open(filename)
  472
+        r = StreamingHttpResponse(file1)
  473
+        self.assertFalse(file1.closed)
  474
+        r.close()
  475
+        self.assertTrue(file1.closed)
  476
+
  477
+        # when multiple file are assigned as content, make sure they are all
  478
+        # closed with the response.
  479
+        file1 = open(filename)
  480
+        file2 = open(filename)
  481
+        r = StreamingHttpResponse(file1)
  482
+        r.streaming_content = file2
  483
+        self.assertFalse(file1.closed)
  484
+        self.assertFalse(file2.closed)
  485
+        r.close()
  486
+        self.assertTrue(file1.closed)
  487
+        self.assertTrue(file2.closed)
  488
+
382 489
 class CookieTests(unittest.TestCase):
383 490
     def test_encode(self):
384 491
         """
43  tests/regressiontests/middleware/tests.py
@@ -8,7 +8,7 @@
8 8
 from django.conf import settings
9 9
 from django.core import mail
10 10
 from django.http import HttpRequest
11  
-from django.http import HttpResponse
  11
+from django.http import HttpResponse, StreamingHttpResponse
12 12
 from django.middleware.clickjacking import XFrameOptionsMiddleware
13 13
 from django.middleware.common import CommonMiddleware
14 14
 from django.middleware.http import ConditionalGetMiddleware
@@ -322,6 +322,12 @@ def test_content_length_header_added(self):
322 322
         self.assertTrue('Content-Length' in self.resp)
323 323
         self.assertEqual(int(self.resp['Content-Length']), content_length)
324 324
 
  325
+    def test_content_length_header_not_added(self):
  326
+        resp = StreamingHttpResponse('content')
  327
+        self.assertFalse('Content-Length' in resp)
  328
+        resp = ConditionalGetMiddleware().process_response(self.req, resp)
  329
+        self.assertFalse('Content-Length' in resp)
  330
+
325 331
     def test_content_length_header_not_changed(self):
326 332
         bad_content_length = len(self.resp.content) + 10
327 333
         self.resp['Content-Length'] = bad_content_length
@@ -351,6 +357,29 @@ def test_if_none_match_and_different_etag(self):
351 357
         self.resp = ConditionalGetMiddleware().process_response(self.req, self.resp)
352 358
         self.assertEqual(self.resp.status_code, 200)
353 359
 
  360
+    @override_settings(USE_ETAGS=True)
  361
+    def test_etag(self):
  362
+        req = HttpRequest()
  363
+        res = HttpResponse('content')
  364
+        self.assertTrue(
  365
+            CommonMiddleware().process_response(req, res).has_header('ETag'))
  366
+
  367
+    @override_settings(USE_ETAGS=True)
  368
+    def test_etag_streaming_response(self):
  369
+        req = HttpRequest()
  370
+        res = StreamingHttpResponse(['content'])
  371
+        res['ETag'] = 'tomatoes'
  372
+        self.assertEqual(
  373
+            CommonMiddleware().process_response(req, res).get('ETag'),
  374
+            'tomatoes')
  375
+
  376
+    @override_settings(USE_ETAGS=True)
  377
+    def test_no_etag_streaming_response(self):
  378
+        req = HttpRequest()
  379
+        res = StreamingHttpResponse(['content'])
  380
+        self.assertFalse(
  381
+            CommonMiddleware().process_response(req, res).has_header('ETag'))
  382
+
354 383
     # Tests for the Last-Modified header
355 384
 
356 385
     def test_if_modified_since_and_no_last_modified(self):
@@ -511,6 +540,7 @@ class GZipMiddlewareTest(TestCase):
511 540
     short_string = b"This string is too short to be worth compressing."
512 541
     compressible_string = b'a' * 500
513 542
     uncompressible_string = b''.join(six.int2byte(random.randint(0, 255)) for _ in xrange(500))
  543
+    sequence = [b'a' * 500, b'b' * 200, b'a' * 300]
514 544
 
515 545
     def setUp(self):
516 546
         self.req = HttpRequest()
@@ -525,6 +555,8 @@ def setUp(self):
525 555
         self.resp.status_code = 200
526 556
         self.resp.content = self.compressible_string
527 557
         self.resp['Content-Type'] = 'text/html; charset=UTF-8'
  558
+        self.stream_resp = StreamingHttpResponse(self.sequence)
  559
+        self.stream_resp['Content-Type'] = 'text/html; charset=UTF-8'
528 560
 
529 561
     @staticmethod
530 562
     def decompress(gzipped_string):
@@ -539,6 +571,15 @@ def test_compress_response(self):
539 571
         self.assertEqual(r.get('Content-Encoding'), 'gzip')
540 572
         self.assertEqual(r.get('Content-Length'), str(len(r.content)))
541 573
 
  574
+    def test_compress_streaming_response(self):
  575
+        """
  576
+        Tests that compression is performed on responses with streaming content.
  577
+        """
  578
+        r = GZipMiddleware().process_response(self.req, self.stream_resp)
  579
+        self.assertEqual(self.decompress(b''.join(r)), b''.join(self.sequence))
  580
+        self.assertEqual(r.get('Content-Encoding'), 'gzip')
  581
+        self.assertFalse(r.has_header('Content-Length'))
  582
+
542 583
     def test_compress_non_200_response(self):
543 584
         """
544 585
         Tests that compression is performed on responses with a status other than 200.
27  tests/regressiontests/views/tests/static.py
@@ -31,28 +31,35 @@ def test_serve(self):
31 31
         media_files = ['file.txt', 'file.txt.gz']
32 32
         for filename in media_files:
33 33
             response = self.client.get('/views/%s/%s' % (self.prefix, filename))
  34
+            response_content = b''.join(response)
  35
+            response.close()
34 36
             file_path = path.join(media_dir, filename)
35 37
             with open(file_path, 'rb') as fp:
36  
-                self.assertEqual(fp.read(), response.content)
37  
-            self.assertEqual(len(response.content), int(response['Content-Length']))
  38
+                self.assertEqual(fp.read(), response_content)
  39
+            self.assertEqual(len(response_content), int(response['Content-Length']))
38 40
             self.assertEqual(mimetypes.guess_type(file_path)[1], response.get('Content-Encoding', None))
39 41
 
40 42
     def test_unknown_mime_type(self):
41 43
         response = self.client.get('/views/%s/file.unknown' % self.prefix)
  44
+        response.close()
42 45
         self.assertEqual('application/octet-stream', response['Content-Type'])
43 46
 
44 47
     def test_copes_with_empty_path_component(self):
45 48
         file_name = 'file.txt'
46 49
         response = self.client.get('/views/%s//%s' % (self.prefix, file_name))
  50
+        response_content = b''.join(response)
  51
+        response.close()
47 52
         with open(path.join(media_dir, file_name), 'rb') as fp:
48  
-            self.assertEqual(fp.read(), response.content)
  53
+            self.assertEqual(fp.read(), response_content)
49 54
 
50 55
     def test_is_modified_since(self):
51 56
         file_name = 'file.txt'
52 57
         response = self.client.get('/views/%s/%s' % (self.prefix, file_name),
53 58
             HTTP_IF_MODIFIED_SINCE='Thu, 1 Jan 1970 00:00:00 GMT')
  59
+        response_content = b''.join(response)
  60
+        response.close()
54 61
         with open(path.join(media_dir, file_name), 'rb') as fp:
55  
-            self.assertEqual(fp.read(), response.content)
  62
+            self.assertEqual(fp.read(), response_content)
56 63
 
57 64
     def test_not_modified_since(self):
58 65
         file_name = 'file.txt'
@@ -74,9 +81,11 @@ def test_invalid_if_modified_since(self):
74 81
         invalid_date = 'Mon, 28 May 999999999999 28:25:26 GMT'
75 82
         response = self.client.get('/views/%s/%s' % (self.prefix, file_name),
76 83
                                    HTTP_IF_MODIFIED_SINCE=invalid_date)
  84
+        response_content = b''.join(response)
  85
+        response.close()
77 86
         with open(path.join(media_dir, file_name), 'rb') as fp:
78  
-            self.assertEqual(fp.read(), response.content)
79  
-        self.assertEqual(len(response.content),
  87
+            self.assertEqual(fp.read(), response_content)
  88
+        self.assertEqual(len(response_content),
80 89
                           int(response['Content-Length']))
81 90
 
82 91
     def test_invalid_if_modified_since2(self):
@@ -89,9 +98,11 @@ def test_invalid_if_modified_since2(self):
89 98
         invalid_date = ': 1291108438, Wed, 20 Oct 2010 14:05:00 GMT'
90 99
         response = self.client.get('/views/%s/%s' % (self.prefix, file_name),
91 100
                                    HTTP_IF_MODIFIED_SINCE=invalid_date)
  101
+        response_content = b''.join(response)
  102
+        response.close()
92 103
         with open(path.join(media_dir, file_name), 'rb') as fp:
93  
-            self.assertEqual(fp.read(), response.content)
94  
-        self.assertEqual(len(response.content),
  104
+            self.assertEqual(fp.read(), response_content)
  105
+        self.assertEqual(len(response_content),
95 106
                           int(response['Content-Length']))
96 107
 
97 108
 

0 notes on commit 4b27813

Please sign in to comment.
Something went wrong with that request. Please try again.