Modified Stream for more intelligent handling of buffer overflow, giving...

pants/http/client.py

@@ -152,10 +152,6 @@ def __init__(self, client, *args, **kwargs):
         Client.__init__(self, *args, **kwargs)
         self.client = client
 
-        # Increase the buffer size, hopefully preventing a buffer overflow
-        # from taking place.
-        self._recv_buffer_size_limit = 10 * (2 ** 20)
-
         # This should be true when connected to certain proxies.
         self.need_full_url = False
 

pants/stream.py

@@ -74,7 +74,6 @@ def __init__(self, **kwargs):
         # I/O attributes
         self.read_delimiter = None
         self._recv_buffer = ""
-        self._recv_buffer_size_limit = 2 ** 16  # 64kb
         self._send_buffer = []
 
         # Channel state
@@ -94,6 +93,44 @@ def __init__(self, **kwargs):
         elif kwargs.get("ssl_options", None) is not None:
             self.startSSL(kwargs["ssl_options"])
 
+    ##### Properties ##########################################################
+
+    @property
+    def read_delimiter(self):
+        """ TODO: Document this! """
+        return self._read_delimiter
+
+    @read_delimiter.setter
+    def read_delimiter(self, value):
+        if value is None or isinstance(value, basestring):
+            self._read_delimiter = value
+            self._recv_buffer_size_limit = self._buffer_size
+
+        elif isinstance(value, (int, long)):
+            self._read_delimiter = value
+            self._recv_buffer_size_limit = max(self._buffer_size, value)
+
+        else:
+            raise TypeError(
+                    "read_delimiter must be None, a string, an int, or a long"
+                    )
+
+    _buffer_size = 2 ** 16  # 64kb
+
+    @property
+    def buffer_size(self):
+        return self._buffer_size
+
+    @buffer_size.setter
+    def buffer_size(self, value):
+        if not isinstance(value, (long, int)):
+            raise TypeError("buffer_size must be an int or a long")
+        self._buffer_size = value
+        if isinstance(self._read_delimiter, (int, long)):
+            self._recv_buffer_size_limit = max(value, self._read_delimiter)
+        else:
+            self._recv_buffer_size_limit = value
+
     ##### Control Methods #####################################################
 
     def startSSL(self, ssl_options={}):
@@ -389,11 +426,17 @@ def _handle_read_event(self):
                 self._recv_buffer += data
 
                 if len(self._recv_buffer) > self._recv_buffer_size_limit:
-                    e = StreamBufferOverflow(
-                            "Buffer length exceeded upper limit on %r." % self
-                        )
-                    self._safely_call(self.on_overflow_error, e)
-                    return
+                    # Try processing the buffer to reduce its length.
+                    self._process_recv_buffer()
+
+                    # If the buffer's still too long, overflow error.
+                    if len(self._recv_buffer) > self._recv_buffer_size_limit:
+                        e = StreamBufferOverflow(
+                                "Buffer length exceeded upper limit on %r." %
+                                    self
+                            )
+                        self._safely_call(self.on_overflow_error, e)
+                        return
 
         self._process_recv_buffer()