Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

HTTPServer keepalive timeout #362

Closed
wants to merge 3 commits into from

2 participants

@flodiebold

I didn't want to make a new branch and pull request for each of these, so I'll just put them all together (unless you want separate branches):

468d37c adds an optional timeout to keep-alive connections in the HTTP server, via the keyword argument connection_timeout in the HTTPServer constructor. Without this, when the client's connection dropped, the connections would sometimes never get closed, which led to our server accumulating open sockets and eventually running out of file descriptors.

96f057c fixes the IOStream close callback sometimes never getting called if it was delayed due to pending callbacks.

690081f fixes a bug in the draft-10 WebSocket implementation: The size of messages of length 126 was not encoded properly.

@bdarnell
Owner

Cool, thanks for the fixes. The iostream and websocket fixes look good, so I'll go ahead and cherry-pick them.

The keepalive timeout change needs a little work. First, a nit: use None instead of -1 for unset timeouts. Second, I think it might be best to have separate timeouts for separate phases of the connection: idle, uploading requests, application processing (this one should maybe be handled in Application rather than HTTPServer), downloading responses. It doesn't necessarily make sense to use the same timeout value for all of these (although maybe a simple end-to-end timeout is worth it for simplicity). The most important of these is probably for the idle phase, since otherwise we let clients easily and accidentally keep connections open forever.

Are you sure you've seen the HTTPServer hold on to an idle connection after the client has closed it (as opposed to clients just keeping connections open for a long time)? The IOStream should detect the close and shut itself down, and if it's not that's a bug (was this how you found the IOStream bug you fixed here?). The keepalive portion of HTTPServer actually hasn't been tested very much in practice since most large-scale uses of tornado are behind nginx which doesn't reuse its client connections.

@flodiebold

Yes -- our server has actually run out of file descriptors several times, after running about two weeks (and it isn't that frequented); with the timeout, the number of used file descriptors is now stable. This may very well relate to some problem in the server configuration; although, if TCP keepalive is disabled and the server has no reason to send something over the connection, there would be no reason for it to notice if the client's connection suddenly dropped, or am I missing something here?
This problem existed already before the IOStream bug was introduced.

I'll look into having separate timeouts.

@bdarnell
Owner

The master branch now has header_timeout and body_timeout arguments; the body_timeout can be overridden in prepare() for handlers in streaming mode.

@bdarnell bdarnell closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Sep 18, 2011
  1. @flodiebold
  2. @flodiebold

    Fix the IOStream close callback not getting called if there are pendi…

    flodiebold authored
    …ng callbacks.
    
    _maybe_add_error_listener only does anything if _state is None.
  3. @flodiebold
This page is out of date. Refresh to see the latest.
View
38 tornado/httpserver.py
@@ -76,6 +76,9 @@ def handle_request(request):
ensure the connection is closed on every request no matter what HTTP
version the client is using.
+ If connection_timeout is set, HTTP keep-alive connections will be closed
+ after that many seconds of inactivity.
+
If ``xheaders`` is ``True``, we support the ``X-Real-Ip`` and ``X-Scheme``
headers, which override the remote IP and HTTP scheme for all requests.
These headers are useful when running Tornado behind a reverse proxy or
@@ -132,16 +135,18 @@ def handle_request(request):
"""
def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
- xheaders=False, ssl_options=None, **kwargs):
+ xheaders=False, ssl_options=None, connection_timeout=-1,
+ **kwargs):
self.request_callback = request_callback
self.no_keep_alive = no_keep_alive
+ self.connection_timeout = connection_timeout
self.xheaders = xheaders
TCPServer.__init__(self, io_loop=io_loop, ssl_options=ssl_options,
**kwargs)
def handle_stream(self, stream, address):
HTTPConnection(stream, address, self.request_callback,
- self.no_keep_alive, self.xheaders)
+ self.no_keep_alive, self.xheaders, self.connection_timeout)
class _BadRequestException(Exception):
"""Exception class for malformed HTTP requests."""
@@ -154,7 +159,7 @@ class HTTPConnection(object):
until the HTTP conection is closed.
"""
def __init__(self, stream, address, request_callback, no_keep_alive=False,
- xheaders=False):
+ xheaders=False, connection_timeout=-1):
self.stream = stream
if self.stream.socket.family not in (socket.AF_INET, socket.AF_INET6):
# Unix (or other) socket; fake the remote address
@@ -162,6 +167,7 @@ def __init__(self, stream, address, request_callback, no_keep_alive=False,
self.address = address
self.request_callback = request_callback
self.no_keep_alive = no_keep_alive
+ self.connection_timeout = connection_timeout
self.xheaders = xheaders
self._request = None
self._request_finished = False
@@ -170,6 +176,28 @@ def __init__(self, stream, address, request_callback, no_keep_alive=False,
self._header_callback = stack_context.wrap(self._on_headers)
self.stream.read_until(b("\r\n\r\n"), self._header_callback)
self._write_callback = None
+ self._timeout_handle = None
+ self.reset_connection_timeout()
+
+ def reset_connection_timeout(self):
+ if self.connection_timeout == -1:
+ return
+ self.remove_connection_timeout()
+ self._timeout_handle = self.stream.io_loop.add_timeout(
+ time.time() + self.connection_timeout, self._handle_timeout)
+
+ def remove_connection_timeout(self):
+ if self._timeout_handle:
+ self.stream.io_loop.remove_timeout(self._timeout_handle)
+
+ def _handle_timeout(self):
+ if self.stream.closed():
+ return
+
+ if self.stream.writing():
+ self.reset_connection_timeout()
+ else:
+ self.stream.close()
def write(self, chunk, callback=None):
"""Writes a chunk of output to the stream."""
@@ -210,10 +238,13 @@ def _finish_request(self):
if disconnect:
self.stream.close()
return
+ else:
+ self.reset_connection_timeout()
self.stream.read_until(b("\r\n\r\n"), self._header_callback)
def _on_headers(self, data):
try:
+ self.reset_connection_timeout()
data = native_str(data.decode('latin1'))
eol = data.find("\r\n")
start_line = data[:eol]
@@ -246,6 +277,7 @@ def _on_headers(self, data):
return
def _on_request_body(self, data):
+ self.reset_connection_timeout()
self._request.body = data
content_type = self._request.headers.get("Content-Type", "")
if self._request.method in ("POST", "PUT"):
View
1  tornado/iostream.py
@@ -224,6 +224,7 @@ def close(self):
self._consume(self._read_buffer_size))
if self._state is not None:
self.io_loop.remove_handler(self.socket.fileno())
+ self._state = None
self.socket.close()
self.socket = None
if self._close_callback and self._pending_callbacks == 0:
View
4 tornado/websocket.py
@@ -79,6 +79,8 @@ def _execute(self, transforms, *args, **kwargs):
self.open_args = args
self.open_kwargs = kwargs
+ self.request.connection.remove_connection_timeout()
+
if (self.request.headers.get("Sec-WebSocket-Version") == "8" or
self.request.headers.get("Sec-WebSocket-Version") == "7"):
self.ws_connection = WebSocketProtocol8(self)
@@ -390,7 +392,7 @@ def _write_frame(self, fin, opcode, data):
finbit = 0
frame = struct.pack("B", finbit | opcode)
l = len(data)
- if l <= 126:
+ if l < 126:
frame += struct.pack("B", l)
elif l <= 0xFFFF:
frame += struct.pack("!BH", 126, l)
Something went wrong with that request. Please try again.