Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Modified Stream for more intelligent handling of buffer overflow, giving... #32

Merged
merged 1 commit into from

2 participants

@stendec
Collaborator

... the application logic a chance to empty the buffer before just giving up, and setting the maximum buffer size to read_delimiter if read_delimiter is larger than buffer_size, a new property used for setting the maximum buffer size.

(Also, I wanted to try making a pull request.)

@stendec stendec Modified Stream for more intelligent handling of buffer overflow, giv…
…ing the application logic a chance to empty the buffer before just giving up, and setting the maximum buffer size to read_delimiter if read_delimiter is larger than buffer_size, a new property used for setting the maximum buffer size.
7c75ec4
@ecdavis ecdavis merged commit 9a4266f into from
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 6, 2012
  1. @stendec

    Modified Stream for more intelligent handling of buffer overflow, giv…

    stendec authored
    …ing the application logic a chance to empty the buffer before just giving up, and setting the maximum buffer size to read_delimiter if read_delimiter is larger than buffer_size, a new property used for setting the maximum buffer size.
This page is out of date. Refresh to see the latest.
Showing with 49 additions and 10 deletions.
  1. +0 −4 pants/http/client.py
  2. +49 −6 pants/stream.py
View
4 pants/http/client.py
@@ -152,10 +152,6 @@ def __init__(self, client, *args, **kwargs):
Client.__init__(self, *args, **kwargs)
self.client = client
- # Increase the buffer size, hopefully preventing a buffer overflow
- # from taking place.
- self._recv_buffer_size_limit = 10 * (2 ** 20)
-
# This should be true when connected to certain proxies.
self.need_full_url = False
View
55 pants/stream.py
@@ -74,7 +74,6 @@ def __init__(self, **kwargs):
# I/O attributes
self.read_delimiter = None
self._recv_buffer = ""
- self._recv_buffer_size_limit = 2 ** 16 # 64kb
self._send_buffer = []
# Channel state
@@ -94,6 +93,44 @@ def __init__(self, **kwargs):
elif kwargs.get("ssl_options", None) is not None:
self.startSSL(kwargs["ssl_options"])
+ ##### Properties ##########################################################
+
+ @property
+ def read_delimiter(self):
+ """ TODO: Document this! """
+ return self._read_delimiter
+
+ @read_delimiter.setter
+ def read_delimiter(self, value):
+ if value is None or isinstance(value, basestring):
+ self._read_delimiter = value
+ self._recv_buffer_size_limit = self._buffer_size
+
+ elif isinstance(value, (int, long)):
+ self._read_delimiter = value
+ self._recv_buffer_size_limit = max(self._buffer_size, value)
+
+ else:
+ raise TypeError(
+ "read_delimiter must be None, a string, an int, or a long"
+ )
+
+ _buffer_size = 2 ** 16 # 64kb
+
+ @property
+ def buffer_size(self):
+ return self._buffer_size
+
+ @buffer_size.setter
+ def buffer_size(self, value):
+ if not isinstance(value, (long, int)):
+ raise TypeError("buffer_size must be an int or a long")
+ self._buffer_size = value
+ if isinstance(self._read_delimiter, (int, long)):
+ self._recv_buffer_size_limit = max(value, self._read_delimiter)
+ else:
+ self._recv_buffer_size_limit = value
+
##### Control Methods #####################################################
def startSSL(self, ssl_options={}):
@@ -389,11 +426,17 @@ def _handle_read_event(self):
self._recv_buffer += data
if len(self._recv_buffer) > self._recv_buffer_size_limit:
- e = StreamBufferOverflow(
- "Buffer length exceeded upper limit on %r." % self
- )
- self._safely_call(self.on_overflow_error, e)
- return
+ # Try processing the buffer to reduce its length.
+ self._process_recv_buffer()
+
+ # If the buffer's still too long, overflow error.
+ if len(self._recv_buffer) > self._recv_buffer_size_limit:
+ e = StreamBufferOverflow(
+ "Buffer length exceeded upper limit on %r." %
+ self
+ )
+ self._safely_call(self.on_overflow_error, e)
+ return
self._process_recv_buffer()
Something went wrong with that request. Please try again.