Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

[soc2009/http-wsgi-improvements] Adds http.HttpResponseStreaming, wit…

…h docs, tests, and support in four built-in middleware classes. Refs #7581.

git-svn-id: http://code.djangoproject.com/svn/django/branches/soc2009/http-wsgi-improvements@11449 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit b15984b179953e93c74f47d06761c9a5c22b0716 1 parent c2d80a5
Christopher Cahoon authored August 13, 2009
7  django/contrib/csrf/middleware.py
@@ -64,6 +64,8 @@ class CsrfResponseMiddleware(object):
64 64
     csrfmiddlewaretoken if the response/request have an active
65 65
     session.
66 66
     """
  67
+    streaming_safe = True
  68
+
67 69
     def process_response(self, request, response):
68 70
         if getattr(response, 'csrf_exempt', False):
69 71
             return response
@@ -102,6 +104,11 @@ def add_csrf_field(match):
102 104
 
103 105
             # Modify any POST forms
104 106
             response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
  107
+            # Handle streaming responses
  108
+            if getattr(response, "content_generator", False):
  109
+                response.content = (_POST_FORM_RE.sub(add_csrf_field, chunk) for chunk in response.content_generator)
  110
+            else:
  111
+                response.content = _POST_FORM_RE.sub(add_csrf_field, response.content)
105 112
         return response
106 113
 
107 114
 class CsrfMiddleware(CsrfViewMiddleware, CsrfResponseMiddleware):
6  django/core/handlers/base.py
@@ -74,9 +74,13 @@ def process_request(self, request_env):
74 74
                 response = self.get_response(request)
75 75
 
76 76
                 # Apply response middleware
  77
+                streaming = getattr(response, "content_generator", False)
  78
+                streaming_safe = lambda x: getattr(x.im_self, "streaming_safe", False)
77 79
                 if not isinstance(response, http.HttpResponseSendFile):
78 80
                     for middleware_method in self._response_middleware:
79  
-                        response = middleware_method(request, response)
  81
+                        if not streaming or streaming_safe(middleware_method):
  82
+                            print middleware_method
  83
+                            response = middleware_method(request, response)
80 84
                     response = self.apply_response_fixes(request, response)
81 85
         finally:
82 86
             signals.request_finished.send(sender=self.__class__)
41  django/http/__init__.py
@@ -436,6 +436,47 @@ def flush(self):
436 436
     def tell(self):
437 437
         return sum([len(chunk) for chunk in self._container])
438 438
 
  439
+class HttpResponseStreaming(HttpResponse):
  440
+    """
  441
+        This class behaves the same as HttpResponse, except that the content
  442
+        attribute is an unconsumed generator or iterator.
  443
+    """
  444
+    def __init__(self, content='', mimetype=None, status=None,
  445
+                 content_type=None, request=None):
  446
+        super(HttpResponseStreaming, self).__init__('', mimetype,
  447
+            status, content_type, request)
  448
+
  449
+        self._container = content
  450
+        self._is_string = False
  451
+
  452
+    def _consume_content(self):
  453
+        if not self._is_string:
  454
+            content = self._container
  455
+            self._container = [''.join(content)]
  456
+            if hasattr(content, 'close'):
  457
+                content.close()
  458
+            self._is_string = True
  459
+
  460
+    def _get_content(self):
  461
+        self._consume_content()
  462
+        return super(HttpResponseStreaming, self)._get_content()
  463
+
  464
+    def _set_content(self, value):
  465
+        if not isinstance(value, basestring) and hasattr(value, "__iter__"):
  466
+            self._container = value
  467
+            self._is_string = False
  468
+        else:
  469
+            self._container = [value]
  470
+            self._is_string = True
  471
+
  472
+    content = property(_get_content, _set_content)
  473
+
  474
+    def _get_content_generator(self):
  475
+        if not self._is_string:
  476
+            return self._container
  477
+
  478
+    content_generator = property(_get_content_generator)
  479
+
439 480
 class HttpResponseSendFile(HttpResponse): 
440 481
     sendfile_fh = None
441 482
 
16  django/middleware/common.py
@@ -27,6 +27,7 @@ class CommonMiddleware(object):
27 27
           the entire page content and Not Modified responses will be returned
28 28
           appropriately.
29 29
     """
  30
+    streaming_safe = True
30 31
 
31 32
     def process_request(self, request):
32 33
         """
@@ -100,14 +101,15 @@ def process_response(self, request, response):
100 101
         if settings.USE_ETAGS:
101 102
             if response.has_header('ETag'):
102 103
                 etag = response['ETag']
103  
-            else:
  104
+            # Do not consume the content of HttpResponseStreaming
  105
+            elif not getattr(response, "content_generator", False):
104 106
                 etag = '"%s"' % md5_constructor(response.content).hexdigest()
105  
-            if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
106  
-                cookies = response.cookies
107  
-                response = http.HttpResponseNotModified()
108  
-                response.cookies = cookies
109  
-            else:
110  
-                response['ETag'] = etag
  107
+                if response.status_code >= 200 and response.status_code < 300 and request.META.get('HTTP_IF_NONE_MATCH') == etag:
  108
+                    cookies = response.cookies
  109
+                    response = http.HttpResponseNotModified()
  110
+                    response.cookies = cookies
  111
+                else:
  112
+                    response['ETag'] = etag
111 113
 
112 114
         return response
113 115
 
20  django/middleware/gzip.py
... ...
@@ -1,6 +1,6 @@
1 1
 import re
2 2
 
3  
-from django.utils.text import compress_string
  3
+from django.utils.text import compress_sequence, compress_string
4 4
 from django.utils.cache import patch_vary_headers
5 5
 
6 6
 re_accepts_gzip = re.compile(r'\bgzip\b')
@@ -11,9 +11,15 @@ class GZipMiddleware(object):
11 11
     It sets the Vary header accordingly, so that caches will base their storage
12 12
     on the Accept-Encoding header.
13 13
     """
14  
-    def process_response(self, request, response):
  14
+    streaming_safe = True
  15
+
  16
+    def process_response(self, request, response):         
  17
+        # Do not consume the content of HttpResponseStreaming responses just to
  18
+        # check content length
  19
+        streaming = getattr(response, "content_generator", False)
  20
+
15 21
         # It's not worth compressing non-OK or really short responses.
16  
-        if response.status_code != 200 or len(response.content) < 200:
  22
+        if response.status_code != 200 or (not streaming and len(response.content) < 200):
17 23
             return response
18 24
 
19 25
         patch_vary_headers(response, ('Accept-Encoding',))
@@ -32,7 +38,11 @@ def process_response(self, request, response):
32 38
         if not re_accepts_gzip.search(ae):
33 39
             return response
34 40
 
35  
-        response.content = compress_string(response.content)
  41
+        if streaming:
  42
+            response.content = compress_sequence(response.content_generator)
  43
+            del response['Content-Length']
  44
+        else:
  45
+            response.content = compress_string(response.content)
  46
+            response['Content-Length'] = str(len(response.content))
36 47
         response['Content-Encoding'] = 'gzip'
37  
-        response['Content-Length'] = str(len(response.content))
38 48
         return response
5  django/middleware/http.py
@@ -8,9 +8,12 @@ class ConditionalGetMiddleware(object):
8 8
 
9 9
     Also sets the Date and Content-Length response-headers.
10 10
     """
  11
+    streaming_safe = True
  12
+
11 13
     def process_response(self, request, response):
12 14
         response['Date'] = http_date()
13  
-        if not response.has_header('Content-Length'):
  15
+        streaming = getattr(response, "content_generator", False)
  16
+        if not response.has_header('Content-Length') and not streaming:
14 17
             response['Content-Length'] = str(len(response.content))
15 18
 
16 19
         if response.has_header('ETag'):
18  django/utils/text.py
@@ -176,6 +176,24 @@ def compress_string(s):
176 176
     zfile.close()
177 177
     return zbuf.getvalue()
178 178
 
  179
+# WARNING - be aware that compress_sequence does not achieve the same
  180
+# level of compression as compress_string
  181
+def compress_sequence(sequence):
  182
+    import cStringIO, gzip
  183
+    zbuf = cStringIO.StringIO()
  184
+    zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
  185
+    yield zbuf.getvalue()
  186
+    for item in sequence:
  187
+        position = zbuf.tell()
  188
+        zfile.write(item)
  189
+        zfile.flush()
  190
+        zbuf.seek(position)
  191
+        yield zbuf.read()
  192
+    position = zbuf.tell()
  193
+    zfile.close()
  194
+    zbuf.seek(position)
  195
+    yield zbuf.read()
  196
+
179 197
 ustring_re = re.compile(u"([\u0080-\uffff])")
180 198
 
181 199
 def javascript_quote(s, quote_double_quotes=False):
29  docs/ref/request-response.txt
@@ -597,6 +597,35 @@ live in :mod:`django.http`.
597 597
 
598 598
     **Note:** Response middleware is bypassed by HttpResponseSendFile.
599 599
 
  600
+.. class:: HttpResponseStreaming
  601
+
  602
+    .. versionadded:: 1.1
  603
+
  604
+    A special response class that does not consume generators before returning
  605
+    the response. To do this, it bypasses middleware that is not useful for
  606
+    chunked responses, and is treated specially by middleware that is useful.
  607
+
  608
+    It is primarily useful for sending large responses that would cause
  609
+    timeouts if sent with a normal HttpResponse.
  610
+
  611
+    **Note:** Of the built-in response middleware, this class works correctly with:
  612
+
  613
+    * :class:`django.middleware.common.CommonMiddleware`
  614
+
  615
+    * :class:`django.middleware.gzip.GZipMiddleware`
  616
+
  617
+    * :class:`django.middleware.http.ConditionalGetMiddleware`
  618
+
  619
+    * :class:`django.contrib.csrf.middleware.CsrfMiddleware`
  620
+
  621
+    Developers of third-party middleware who wish to make it work with this class
  622
+    should note that any time they access :class:`HttpResponseStreaming.content`, it will
  623
+    break the functionality of this class. Instead, replace :attr:`HttpResponseStreaming.content`
  624
+    by wrapping the value of :attr:`HttpResponseStreaming.content_generator`. :class:`django.middleware.gzip.GZipMiddleware`
  625
+    is a good example to follow. To inform the handler to send :class:`HttpResponseStreaming`
  626
+    responses through your middleware, add the class attribute ``streaming_safe = True``
  627
+    to your middleware class.
  628
+
600 629
 .. class:: HttpResponseRedirect
601 630
 
602 631
     The constructor takes a single argument -- the path to redirect to. This
0  tests/regressiontests/response_streaming/__init__.py
No changes.
0  tests/regressiontests/response_streaming/models.py
No changes.
32  tests/regressiontests/response_streaming/tests.py
... ...
@@ -0,0 +1,32 @@
  1
+import urllib, os
  2
+
  3
+from django.test import TestCase
  4
+from django.conf import settings
  5
+from django.core.files import temp as tempfile
  6
+
  7
+def x():
  8
+    for i in range(0, 10):
  9
+        yield unicode(i) + u'\n'
  10
+
  11
+class ResponseStreamingTests(TestCase):
  12
+    def test_streaming(self):
  13
+        response = self.client.get('/streaming/stream_file/')
  14
+
  15
+        self.assertEqual(response.status_code, 200)
  16
+        self.assertEqual(response['Content-Disposition'],
  17
+                'attachment; filename=test.csv')
  18
+        self.assertEqual(response['Content-Type'], 'text/csv')
  19
+        self.assertTrue(not response._is_string)
  20
+        self.assertEqual("".join(iter(response)), "".join(x()))
  21
+        self.assertTrue(not response._is_string)
  22
+
  23
+    def test_bad_streaming(self):
  24
+        response = self.client.get('/streaming/stream_file/')
  25
+
  26
+        self.assertEqual(response.status_code, 200)
  27
+        self.assertEqual(response['Content-Disposition'],
  28
+                'attachment; filename=test.csv')
  29
+        self.assertEqual(response['Content-Type'], 'text/csv')
  30
+        self.assertTrue(not response._is_string)
  31
+        self.assertEqual(response.content, "".join(x()))
  32
+        self.assertTrue(response._is_string)
7  tests/regressiontests/response_streaming/urls.py
... ...
@@ -0,0 +1,7 @@
  1
+from django.conf.urls.defaults import patterns
  2
+
  3
+import views
  4
+
  5
+urlpatterns = patterns('',
  6
+    (r'^stream_file/$', views.test_streaming),
  7
+)
13  tests/regressiontests/response_streaming/views.py
... ...
@@ -0,0 +1,13 @@
  1
+import urllib
  2
+
  3
+from django.http import HttpResponseStreaming
  4
+from time import sleep
  5
+
  6
+def x():
  7
+    for i in range(0, 10):
  8
+        yield unicode(i) + u'\n'
  9
+
  10
+def test_streaming(request):
  11
+    response = HttpResponseStreaming(content=x(), mimetype='text/csv')
  12
+    response['Content-Disposition'] = 'attachment; filename=test.csv'
  13
+    return response
3  tests/urls.py
@@ -36,6 +36,9 @@
36 36
     # HttpResponseSendfile tests 
37 37
  	(r'^sendfile/', include('regressiontests.sendfile.urls')), 
38 38
 
  39
+    # HttpResponseStreaming tests
  40
+    (r'^streaming/', include('regressiontests.response_streaming.urls')),
  41
+
39 42
     # conditional get views
40 43
     (r'condition/', include('regressiontests.conditional_processing.urls')),
41 44
 )

0 notes on commit b15984b

Please sign in to comment.
Something went wrong with that request. Please try again.