Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #18972 -- Refactored bundled wsgi server's chunking algorithm.

Thanks to amosonn at yahoo.com for the report, @doda for the initial patch and
@datagrok for the revamped logic and test case.
  • Loading branch information...
commit a7960bcb3575fd9fcd5180986ddcdba1caa46dd5 1 parent 80e68ee
Matthew Wood authored March 18, 2013 charettes committed March 20, 2013
26  django/core/servers/basehttp.py
@@ -9,7 +9,7 @@
9 9
 
10 10
 from __future__ import unicode_literals
11 11
 
12  
-import os
  12
+from io import BytesIO
13 13
 import socket
14 14
 import sys
15 15
 import traceback
@@ -26,7 +26,13 @@
26 26
 from django.utils.module_loading import import_by_path
27 27
 from django.utils import six
28 28
 
29  
-__all__ = ['WSGIServer', 'WSGIRequestHandler']
  29
+__all__ = ('WSGIServer', 'WSGIRequestHandler', 'MAX_SOCKET_CHUNK_SIZE')
  30
+
  31
+
  32
+# If data is too large, socket will choke, so write chunks no larger than 32MB
  33
+# at a time. The rationale behind the 32MB can be found on Django's Trac:
  34
+# https://code.djangoproject.com/ticket/5596#comment:4
  35
+MAX_SOCKET_CHUNK_SIZE = 32 * 1024 * 1024  # 32 MB
30 36
 
31 37
 
32 38
 def get_internal_wsgi_application():
@@ -78,19 +84,9 @@ def write(self, data):
78 84
             self.bytes_sent += len(data)
79 85
 
80 86
         # XXX check Content-Length and truncate if too many bytes written?
81  
-
82  
-        # If data is too large, socket will choke, so write chunks no larger
83  
-        # than 32MB at a time.
84  
-        length = len(data)
85  
-        if length > 33554432:
86  
-            offset = 0
87  
-            while offset < length:
88  
-                chunk_size = min(33554432, length)
89  
-                self._write(data[offset:offset+chunk_size])
90  
-                self._flush()
91  
-                offset += chunk_size
92  
-        else:
93  
-            self._write(data)
  87
+        data = BytesIO(data)
  88
+        for chunk in iter(lambda: data.read(MAX_SOCKET_CHUNK_SIZE), b''):
  89
+            self._write(chunk)
94 90
             self._flush()
95 91
 
96 92
     def error_output(self, environ, start_response):
63  tests/builtin_server/tests.py
@@ -2,22 +2,18 @@
2 2
 
3 3
 from io import BytesIO
4 4
 
5  
-from django.core.servers.basehttp import ServerHandler
  5
+from django.core.servers.basehttp import ServerHandler, MAX_SOCKET_CHUNK_SIZE
6 6
 from django.utils.unittest import TestCase
7 7
 
8  
-#
9  
-# Tests for #9659: wsgi.file_wrapper in the builtin server.
10  
-# We need to mock a couple of handlers and keep track of what
11  
-# gets called when using a couple kinds of WSGI apps.
12  
-#
13 8
 
14 9
 class DummyHandler(object):
15  
-    def log_request(*args, **kwargs):
  10
+    def log_request(self, *args, **kwargs):
16 11
         pass
17 12
 
  13
+
18 14
 class FileWrapperHandler(ServerHandler):
19 15
     def __init__(self, *args, **kwargs):
20  
-        ServerHandler.__init__(self, *args, **kwargs)
  16
+        super(FileWrapperHandler, self).__init__(*args, **kwargs)
21 17
         self.request_handler = DummyHandler()
22 18
         self._used_sendfile = False
23 19
 
@@ -25,17 +21,24 @@ def sendfile(self):
25 21
         self._used_sendfile = True
26 22
         return True
27 23
 
  24
+
28 25
 def wsgi_app(environ, start_response):
29 26
     start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
30 27
     return [b'Hello World!']
31 28
 
  29
+
32 30
 def wsgi_app_file_wrapper(environ, start_response):
33 31
     start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
34 32
     return environ['wsgi.file_wrapper'](BytesIO(b'foo'))
35 33
 
  34
+
36 35
 class WSGIFileWrapperTests(TestCase):
37 36
     """
38 37
     Test that the wsgi.file_wrapper works for the builting server.
  38
+
  39
+    Tests for #9659: wsgi.file_wrapper in the builtin server.
  40
+    We need to mock a couple of handlers and keep track of what
  41
+    gets called when using a couple kinds of WSGI apps.
39 42
     """
40 43
 
41 44
     def test_file_wrapper_uses_sendfile(self):
@@ -53,3 +56,47 @@ def test_file_wrapper_no_sendfile(self):
53 56
         self.assertFalse(handler._used_sendfile)
54 57
         self.assertEqual(handler.stdout.getvalue().splitlines()[-1], b'Hello World!')
55 58
         self.assertEqual(handler.stderr.getvalue(), b'')
  59
+
  60
+
  61
+class WriteChunkCounterHandler(ServerHandler):
  62
+    """
  63
+    Server handler that counts the number of chunks written after headers were
  64
+    sent. Used to make sure large response body chunking works properly.
  65
+    """
  66
+
  67
+    def __init__(self, *args, **kwargs):
  68
+        super(WriteChunkCounterHandler, self).__init__(*args, **kwargs)
  69
+        self.request_handler = DummyHandler()
  70
+        self.headers_written = False
  71
+        self.write_chunk_counter = 0
  72
+
  73
+    def send_headers(self):
  74
+        super(WriteChunkCounterHandler, self).send_headers()
  75
+        self.headers_written = True
  76
+
  77
+    def _write(self, data):
  78
+        if self.headers_written:
  79
+            self.write_chunk_counter += 1
  80
+        self.stdout.write(data)
  81
+
  82
+
  83
+def send_big_data_app(environ, start_response):
  84
+    start_response(str('200 OK'), [(str('Content-Type'), str('text/plain'))])
  85
+    # Return a blob of data that is 1.5 times the maximum chunk size.
  86
+    return [b'x' * (MAX_SOCKET_CHUNK_SIZE + MAX_SOCKET_CHUNK_SIZE // 2)]
  87
+
  88
+
  89
+class ServerHandlerChunksProperly(TestCase):
  90
+    """
  91
+    Test that the ServerHandler chunks data properly.
  92
+
  93
+    Tests for #18972: The logic that performs the math to break data into
  94
+    32MB (MAX_SOCKET_CHUNK_SIZE) chunks was flawed, BUT it didn't actually
  95
+    cause any problems.
  96
+    """
  97
+
  98
+    def test_chunked_data(self):
  99
+        env = {'SERVER_PROTOCOL': 'HTTP/1.0'}
  100
+        handler = WriteChunkCounterHandler(None, BytesIO(), BytesIO(), env)
  101
+        handler.run(send_big_data_app)
  102
+        self.assertEqual(handler.write_chunk_counter, 2)

0 notes on commit a7960bc

Please sign in to comment.
Something went wrong with that request. Please try again.