Skip to content
Browse files

Added ProxyServer.read_all() and Server.read_all() so we don't read in

full buffers before we read them out. Closed Issue #39. Seems to
improve performance greatly.
  • Loading branch information...
1 parent 13bc318 commit c40d50d75e7cbc8fd50f1b97598dca3ac43e0234 @von committed Dec 10, 2011
Showing with 55 additions and 9 deletions.
  1. +28 −9 perproxy/ProxyServer.py
  2. +27 −0 perproxy/Server.py
View
37 perproxy/ProxyServer.py
@@ -227,15 +227,16 @@ def pass_through(self, server):
continue
for s in read_ready:
instance = instances[s]
- self.logger.debug("Reading from %s" % instance)
+ out = peer[instance]
+ self.logger.debug("Reading from %s, writing to %s" % (instance, out))
try:
- data = instance.recvall()
+ data_len = instance.read_all(out.sendall)
except IOError as e:
self.logger.error("Error reading from %s: %s" % (instance,
str(e)))
done = True
break
- if data is None:
+ if data_len is None:
# M2Crypto.SSL.Connection returns None sometimes.
# This does not indicate an EOF. Be done if we
# see a lot of None reads in a row, might be causing
@@ -249,15 +250,11 @@ def pass_through(self, server):
done = True
break
none_read_count = 0
- if len(data) == 0:
+ if data_len == 0:
self.logger.info("Got EOF from %s" % instance)
done = True
break
- out = peer[instance]
- self.logger.debug("Writing %s bytes to %s" % (len(data),
- out))
- out.sendall(data)
-
+ self.logger.debug("Read %d bytes from %s and wrote to %s" % (data_len, instance, out))
self.logger.info("Pass through done.")
def handle_server_error(self, server_error):
@@ -344,6 +341,28 @@ def recvall(self, buflen=8192):
chunks.append(data)
return "".join(chunks)
+ def read_all(self, callback, buflen=8192):
+ """Read buflen buffers, calling callback for each.
+
+ Returns total number of bytes read. 0 on EOF. None if None returned from read."""
+ total_read = 0
+ while True:
+ try:
+ data = self.request.recv(buflen)
+ except Exception as e:
+ self.logger.warning("Got error reading: %s" % str(e))
+ return 0 # Treat as EOF
+ if data is None:
+ if total_read == 0:
+ # Indicate a sole None read
+ total_read = None
+ break
+ total_read += len(data)
+ if len(data) == 0:
+ break
+ callback(data)
+ return total_read
+
def __str__(self):
return "client at %s:%s" % (self.client_address[0],
self.client_address[1])
View
27 perproxy/Server.py
@@ -92,5 +92,32 @@ def recvall(self, buflen=8192):
return None
return "".join(chunks)
+ def read_all(self, callback, buflen=8192):
+ """Read buflen buffers, calling callback for each.
+
+ Returns total number of bytes read. 0 on EOF. None if None returned from read.
+
+ M2Crypto.SSL.Connection seems to randomly return None
+ sometimes after claiming it has data to read. This does not indicate
+ EOF. In this case, this function will return None and the caller
+ should not treat as an EOF."""
+ total_read = 0
+ while True:
+ try:
+ data = self.sock.recv(buflen)
+ except Exception as e:
+ self.logger.warning("Got error reading: %s" % str(e))
+ return 0 # Treat as EOF
+ if data is None:
+ if total_read == 0:
+ # Indicate a sole None read
+ total_read = None
+ break
+ total_read += len(data)
+ if len(data) == 0:
+ break
+ callback(data)
+ return total_read
+
def __str__(self):
return "server at %s:%s" % (self.hostname, self.port)

0 comments on commit c40d50d

Please sign in to comment.
Something went wrong with that request. Please try again.