Permalink
Browse files

Add sendfile() optimization for static file serving

git-svn-id: http://chiral.j4cbo.com/svn/trunk@33 a827fe29-2235-0410-a7a9-cacdfe24a5d3
  • Loading branch information...
jacobdp
jacobdp committed Jul 23, 2007
1 parent 8e60170 commit cdbf2dc4576106bc790ea211d16a9e62a3c94b22
Showing with 383 additions and 20 deletions.
  1. +40 −1 chiral/http/wsgihttpd.py
  2. +57 −8 chiral/inet/tcp.py
  3. +16 −11 chiral/web/servers.py
  4. +244 −0 py-sendfile-1.2.2/sendfilemodule.c
  5. +9 −0 py-sendfile-1.2.2/setup.py
  6. +17 −0 py-sendfile-1.2.2/test.py
View
@@ -49,6 +49,14 @@ def render_headers(self, no_content = False):
"\r\n".join(("%s: %s" % (k, v)) for k, v in self.headers.iteritems())
)
class WSGIFileWrapper(object):
def __init__(self, filelike, blocksize = 4096):
self.filelike = filelike
self.blocksize = blocksize
def close(self):
if hasattr(self.filelike, "close"):
self.filelike.close()
class HTTPConnection(tcp.TCPConnection):
"""An HTTP connection."""
@@ -76,7 +84,7 @@ def handler(self):
line = yield self.read_line(delimiter="\r\n")
# Ignore blank lines, as suggested by the RFC
if not line:
pass
continue
method, url, protocol = line.split(' ')
except (tcp.ConnectionOverflowException, ValueError):
yield self.send_error("400 Bad Request")
@@ -94,6 +102,7 @@ def handler(self):
'wsgi.url_scheme': 'http',
'wsgi.input': '',
'wsgi.errors': sys.stderr,
'wsgi.file_wrapper': WSGIFileWrapper,
'wsgi.multithread': False,
'wsgi.multiprocess': True,
'wsgi.run_once': False,
@@ -237,6 +246,36 @@ def start_response(status, response_headers, exc_info=None):
except TypeError:
pass
# Handle file wrapper
if isinstance(result, WSGIFileWrapper):
# Use TCP_CORK if available
if hasattr(socket, "TCP_CORK"):
self.client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1)
yield self.sendall(response.render_headers())
res = yield self.sendfile(result.filelike, 0, result.blocksize)
if hasattr(socket, "TCP_CORK"):
self.client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 0)
offset = res
while res:
res = yield self.sendfile(result.filelike, offset, result.blocksize)
offset += res
# Close the file
result.close()
# Close if necessary
if not response.should_keep_alive:
self.close()
break
# And we're now done with this request.
continue
# Iterate through the result chunks provided by the application.
res_iter = iter(result)
headers_sent = False
View
@@ -10,6 +10,12 @@
if sys.version_info[:2] < (2, 5):
raise RuntimeError("chiral.inet.tcp requires Python 2.5 for generator expressions.")
try:
from sendfile import sendfile
_SENDFILE_AVAILABLE = True
except ImportError:
_SENDFILE_AVAILABLE = False
class ConnectionOverflowException(Exception):
"""Indicates that an excessive amount of data was received without line terminators."""
pass
@@ -124,7 +130,7 @@ def read_line(self, max_len = 1024, delimiter = "\n"):
def read_exactly(self, length, read_increment = 4096):
def read_exactly(self, length, read_increment = 32768):
"""
Read and return exactly length bytes.
@@ -224,18 +230,61 @@ def sendall(self, data):
# There's still more data to be sent, so hand things off to the tasklet.
return tasklet.WaitForTasklet(tasklet.Tasklet(self._sendall_tasklet(data)))
def send(self, data, try_now=True):
"""
Send data. Returns a Callback, which fires once some of the data has sent;
the callback returns the amount actually written, which may be less than
all the data given. Use sendall() if all the data must be send.
"""
return self._async_socket_operation(
self.client_sock.send,
reactor.wait_for_writeable,
data,
try_now
)
"""
return self._async_socket_operation(
self.client_sock.send,
reactor.wait_for_writeable,
data,
try_now
)
def sendfile(self, infile, offset, length):
"""
Send up to len bytes of data from infile, starting at offset.
Returns the amount actually written, which may be less than
all the data given. Use sendall() if all the data must be send.
"""
if not _SENDFILE_AVAILABLE:
# No sendfile, so just pass this on to sendall().
data = infile.read(length)
return self.sendall(data)
# sendfile() is available. It takes a number of parameters, so we can't just use
# the _async_socket_operation helper.
callback = tasklet.WaitForCallback("sendfile")
def blocked_operation_handler():
"""Callback for asynchronous operations."""
# Prevent pylint from complaining about "except Exception"
# pylint: disable-msg=W0703
try:
res = sendfile(self.client_sock.fileno(), infile.fileno(), offset, length)
except Exception, exc:
callback.throw(exc)
else:
callback(res[1])
# Attempt the sendfile now; only do a callback if it returns EAGAIN
try:
res = sendfile(self.client_sock.fileno(), infile.fileno(), offset, length)
except OSError, exc:
if exc.errno == errno.EAGAIN:
reactor.wait_for_writeable(self, self.client_sock, blocked_operation_handler)
return callback
else:
raise exc
# sendfile() worked, so we're done.
return tasklet.WaitForNothing(res[1])
def __init__(self, server, sock, addr):
self.server = server
View
@@ -22,32 +22,29 @@ def __call__(self, environ, start_response):
for item in path_components:
if item == "..":
start_response("403 Forbidden", [])
yield "<html><head><title>403 Forbidden</title></head><body><h1>403 Forbidden</h1></body></html>"
return
return [ "<html><head><title>403 Forbidden</title></head><body><h1>403 Forbidden</h1></body></html>" ]
filename = os.path.join(self.path, *path_components)
try:
file_stats = os.stat(filename)
except OSError, exc:
if exc.errno == errno.ENOENT:
start_response("404 Not Found", [])
yield "<html><head><title>404 Not Found</title></head><body><h1>404 Not Found</h1></body></html>"
return
return [ "<html><head><title>404 Not Found</title></head><body><h1>404 Not Found</h1></body></html>" ]
else:
raise exc
if IF_MODIFIED_SINCE.parse(environ) >= file_stats.st_mtime:
start_response("304 Not Modified", [])
return
return [ "" ]
try:
rfile = file(filename)
except IOError, exc:
# We don't support directory listings.
if exc.errno in (errno.EACCES, errno.EISDIR):
start_response("403 Forbidden", [])
yield "<html><head><title>403 Forbidden</title></head><body><h1>403 Forbidden</h1></body></html>"
return
return [ "<html><head><title>403 Forbidden</title></head><body><h1>403 Forbidden</h1></body></html>" ]
else:
raise exc
@@ -66,7 +63,15 @@ def __call__(self, environ, start_response):
# Read the file and send chunks out.
start_response('200 OK', response_tuples)
while True:
chunk = rfile.read(4096)
if chunk == '': break
yield chunk
# Use file_wrapper if possible.
if 'wsgi.file_wrapper' in environ:
return environ['wsgi.file_wrapper'](rfile)
else:
return iter(lambda: rfile.read(4096), '')
#return rfile
#while True:
# chunk = rfile.read(4096)
# if chunk == '': break
# yield chunk
Oops, something went wrong.

0 comments on commit cdbf2dc

Please sign in to comment.