From 1667e0b7f800f4fe41ad49c17f7151c6a6058c92 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 28 Sep 2011 23:26:58 +1300 Subject: [PATCH 01/53] handle HTTP 303 redirects correctly see http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 In response to a 303 the client SHOULD make a GET to the Location, was using the original method. --- tornado/simple_httpclient.py | 7 ++++++- tornado/test/simple_httpclient_test.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index a98eb54484..79a5960c31 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -350,12 +350,17 @@ def _on_body(self, data): self.request) if (self.request.follow_redirects and self.request.max_redirects > 0 and - self.code in (301, 302)): + self.code in (301, 302, 303)): new_request = copy.copy(self.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) new_request.max_redirects -= 1 del new_request.headers["Host"] + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + # client SHOULD make a GET request + if self.code == 303: + new_request.method = "GET" + new_request.body = None new_request.original_request = original_request final_callback = self.final_callback self.final_callback = None diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index f6d7235ae5..55d5cf1445 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -44,6 +44,16 @@ def get(self): self.set_header("Content-Length", self.get_argument("value")) self.write("ok") +class PRGPostHandler(RequestHandler): + def post(self): + self.set_header("Location", "/prg_get") + self.set_status(303) + +class PRGGetHandler(RequestHandler): + def get(self): + self.write("ok") + + class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase): def get_app(self): # callable objects to finish pending /trigger requests @@ -56,6 +66,8 @@ def get_app(self): url("/hang", HangHandler), url("/hello", HelloWorldHandler), url("/content_length", ContentLengthHandler), + url("/prg_post", PRGPostHandler), + url("/prg_get", PRGGetHandler), ], gzip=True) def test_singleton(self): @@ -134,6 +146,14 @@ def test_max_redirects(self): self.assertTrue(response.effective_url.endswith("/countdown/2")) self.assertTrue(response.headers["Location"].endswith("/countdown/1")) + def test_303_redirect(self): + response = self.fetch("/prg_post", method="POST", body="") + self.assertEqual(200, response.code) + self.assertTrue(response.request.url.endswith("/prg_post")) + self.assertTrue(response.effective_url.endswith("/prg_get")) + #request is the original request, is a POST still + self.assertEqual("POST", response.request.method) + def test_request_timeout(self): response = self.fetch('/hang', request_timeout=0.1) self.assertEqual(response.code, 599) From 6cd8e08fe3092b3ccfa167b0560c42c4348989bd Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Sun, 2 Oct 2011 21:07:02 +1300 Subject: [PATCH 02/53] remove content headers for 303 redirect when doing a GET request for a 303 redirect clear the Content-Length and Content-Type headers, they are set during the initial POST. --- tornado/simple_httpclient.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 79a5960c31..5bf3d1585c 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -361,6 +361,11 @@ def _on_body(self, data): if self.code == 303: new_request.method = "GET" new_request.body = None + for h in ["Content-Length", "Content-Type"]: + try: + del self.request.headers[h] + except KeyError: + pass new_request.original_request = original_request final_callback = self.final_callback self.final_callback = None From 0ea54f983629f88730a4c6810beba18d7219c40d Mon Sep 17 00:00:00 2001 From: David Galeano Date: Tue, 4 Oct 2011 13:09:15 +0100 Subject: [PATCH 03/53] Added support for subprotocols. --- tornado/websocket.py | 57 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 3a17f93e10..a69a5e8a7e 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -6,8 +6,8 @@ .. warning:: The WebSocket protocol is still in development. This module currently - implements the "hixie-76" and "hybi-10" versions of the protocol. - See this `browser compatibility table + implements the "hixie-76" and "hybi-10" versions of the protocol. + See this `browser compatibility table `_ on Wikipedia. """ # Author: Jacob Kristhammar, 2010 @@ -83,13 +83,13 @@ def _execute(self, transforms, *args, **kwargs): self.request.headers.get("Sec-WebSocket-Version") == "7"): self.ws_connection = WebSocketProtocol8(self) self.ws_connection.accept_connection() - + elif self.request.headers.get("Sec-WebSocket-Version"): self.stream.write(tornado.escape.utf8( "HTTP/1.1 426 Upgrade Required\r\n" "Sec-WebSocket-Version: 8\r\n\r\n")) self.stream.close() - + else: self.ws_connection = WebSocketProtocol76(self) self.ws_connection.accept_connection() @@ -98,6 +98,10 @@ def write_message(self, message): """Sends the given message to the client of this Web Socket.""" self.ws_connection.write_message(message) + def validate_subprotocol(self, subprotocols): + """Invoked when a new WebSocket requests specific subprotocols.""" + return None + def open(self, *args, **kwargs): """Invoked when a new WebSocket is opened.""" pass @@ -199,6 +203,18 @@ def accept_connection(self): logging.debug("Malformed WebSocket request received") self._abort() return + + subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", None) + if subprotocols: + subprotocol = self.handler.validate_subprotocol(subprotocols) + if not subprotocol: + logging.debug("Subprotocol rejected by handler.") + self._abort() + return + subprotocol = "Sec-WebSocket-Protocol: %s\r\n" % subprotocol + else: + subprotocol = '' + scheme = "wss" if self.request.protocol == "https" else "ws" # Write the initial headers before attempting to read the challenge. # This is necessary when using proxies (such as HAProxy), which @@ -210,12 +226,15 @@ def accept_connection(self): "Connection: Upgrade\r\n" "Server: TornadoServer/%(version)s\r\n" "Sec-WebSocket-Origin: %(origin)s\r\n" - "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n\r\n" % (dict( + "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n" + "%(subprotocol)s" + "\r\n" % (dict( version=tornado.version, origin=self.request.headers["Origin"], scheme=scheme, host=self.request.host, - uri=self.request.uri)))) + uri=self.request.uri, + subprotocol=subprotocol)))) self.stream.read_bytes(8, self._handle_challenge) def challenge_response(self, challenge): @@ -350,7 +369,7 @@ def accept_connection(self): logging.debug("Malformed WebSocket request received") self._abort() return - + def _handle_websocket_headers(self): """Verifies all invariant- and required headers @@ -374,11 +393,24 @@ def _challenge_response(self): return tornado.escape.native_str(base64.b64encode(sha1.digest())) def _accept_connection(self): + subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", None) + if subprotocols: + subprotocol = self.handler.validate_subprotocol(subprotocols) + if not subprotocol: + logging.debug("Subprotocol rejected by handler.") + self._abort() + return + subprotocol = "Sec-WebSocket-Protocol: %s\r\n" % subprotocol + else: + subprotocol = '' + self.stream.write(tornado.escape.utf8( "HTTP/1.1 101 Switching Protocols\r\n" "Upgrade: websocket\r\n" "Connection: Upgrade\r\n" - "Sec-WebSocket-Accept: %s\r\n\r\n" % self._challenge_response())) + "Sec-WebSocket-Accept: %s\r\n" + "%s" + "\r\n" % (self._challenge_response(), subprotocol))) self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) self._receive_frame() @@ -434,7 +466,7 @@ def _on_frame_start(self, data): def _on_frame_length_16(self, data): self._frame_length = struct.unpack("!H", data)[0]; self.stream.read_bytes(4, self._on_masking_key); - + def _on_frame_length_64(self, data): self._frame_length = struct.unpack("!Q", data)[0]; self.stream.read_bytes(4, self._on_masking_key); @@ -466,11 +498,11 @@ def _on_frame_data(self, data): if not self.client_terminated: self._receive_frame() - + def _handle_message(self, opcode, data): if self.client_terminated: return - + if opcode == 0x1: # UTF-8 data self.async_callback(self.handler.on_message)(data.decode("utf-8", "replace")) @@ -491,10 +523,9 @@ def _handle_message(self, opcode, data): pass else: self._abort() - + def close(self): """Closes the WebSocket connection.""" self._write_frame(True, 0x8, b("")) self._started_closing_handshake = True self._waiting = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 5, self._abort) - From 95b904236d770a3517f62a0c3801c95167097461 Mon Sep 17 00:00:00 2001 From: Guto Maia Date: Fri, 7 Oct 2011 16:21:10 -0300 Subject: [PATCH 04/53] allows the TwitterMixin define the callback_uri, that solves issue #377 --- tornado/auth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tornado/auth.py b/tornado/auth.py index b9f7f57da1..5701bee400 100644 --- a/tornado/auth.py +++ b/tornado/auth.py @@ -451,14 +451,14 @@ def _on_auth(self, user): _OAUTH_NO_CALLBACKS = False - def authenticate_redirect(self): + def authenticate_redirect(self, callback_uri = None): """Just like authorize_redirect(), but auto-redirects if authorized. This is generally the right interface to use if you are using Twitter for single-sign on. """ http = httpclient.AsyncHTTPClient() - http.fetch(self._oauth_request_token_url(), self.async_callback( + http.fetch(self._oauth_request_token_url(callback_uri = callback_uri), self.async_callback( self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None)) def twitter_request(self, path, callback, access_token=None, From 45752bfdf96724e88d616f626132461e08e02a33 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 24 Oct 2011 23:38:54 +1300 Subject: [PATCH 05/53] Clear Content-Encoding and Transfer-Encoding Also better test names. --- tornado/simple_httpclient.py | 3 ++- tornado/test/simple_httpclient_test.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 5bf3d1585c..00978a6cd6 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -361,7 +361,8 @@ def _on_body(self, data): if self.code == 303: new_request.method = "GET" new_request.body = None - for h in ["Content-Length", "Content-Type"]: + for h in ["Content-Length", "Content-Type", + "Content-Encoding", "Transfer-Encoding"]: try: del self.request.headers[h] except KeyError: diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index 55d5cf1445..a87fb03be2 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -44,12 +44,12 @@ def get(self): self.set_header("Content-Length", self.get_argument("value")) self.write("ok") -class PRGPostHandler(RequestHandler): +class SeeOther303PostHandler(RequestHandler): def post(self): - self.set_header("Location", "/prg_get") + self.set_header("Location", "/303_get") self.set_status(303) -class PRGGetHandler(RequestHandler): +class SeeOther303GetHandler(RequestHandler): def get(self): self.write("ok") @@ -66,8 +66,8 @@ def get_app(self): url("/hang", HangHandler), url("/hello", HelloWorldHandler), url("/content_length", ContentLengthHandler), - url("/prg_post", PRGPostHandler), - url("/prg_get", PRGGetHandler), + url("/303_post", SeeOther303PostHandler), + url("/303_get", SeeOther303GetHandler), ], gzip=True) def test_singleton(self): @@ -147,10 +147,10 @@ def test_max_redirects(self): self.assertTrue(response.headers["Location"].endswith("/countdown/1")) def test_303_redirect(self): - response = self.fetch("/prg_post", method="POST", body="") + response = self.fetch("/303_post", method="POST", body="") self.assertEqual(200, response.code) - self.assertTrue(response.request.url.endswith("/prg_post")) - self.assertTrue(response.effective_url.endswith("/prg_get")) + self.assertTrue(response.request.url.endswith("/303_post")) + self.assertTrue(response.effective_url.endswith("/303_get")) #request is the original request, is a POST still self.assertEqual("POST", response.request.method) From 60b3e66232ec6b2021dbc68b1ef42f8443162c68 Mon Sep 17 00:00:00 2001 From: Luca Wehrstedt Date: Sun, 30 Oct 2011 01:15:19 +0300 Subject: [PATCH 06/53] Add Content-Length header on HEAD requests for StaticFileHandler. --- tornado/web.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tornado/web.py b/tornado/web.py index e87f330545..78267b3aeb 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1484,6 +1484,7 @@ def get(self, path, include_body=True): return if not include_body: + self.set_header("Content-Length", os.path.getsize(abspath)) return file = open(abspath, "rb") try: From 848244ad0449a1906004ee0d7ccda3162d07e929 Mon Sep 17 00:00:00 2001 From: Luca Wehrstedt Date: Sun, 30 Oct 2011 17:37:29 +0100 Subject: [PATCH 07/53] Add Etag header on HEAD requests for StaticFileHandler. --- tornado/web.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tornado/web.py b/tornado/web.py index 78267b3aeb..86447bdc43 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1485,6 +1485,10 @@ def get(self, path, include_body=True): if not include_body: self.set_header("Content-Length", os.path.getsize(abspath)) + with open(abspath, "rb") as file: + hasher = hashlib.sha1() + hasher.update(file.read()) + self.set_header("Etag", '"%s"' % hasher.hexdigest()) return file = open(abspath, "rb") try: From 9484eb08cd9944f09f2dd4d961c466c499d8bb01 Mon Sep 17 00:00:00 2001 From: Birk Nilson Date: Wed, 30 Nov 2011 20:51:33 +0000 Subject: [PATCH 08/53] Allow override of include_host in static_url. Thereby supporting the generation of absolute URLs even though the handler - by default - would have generated a relative URL for instance. The reverse scenario is of course supported also. --- tornado/test/web_test.py | 33 ++++++++++++++++++++++++++++++++- tornado/web.py | 13 ++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index c5ae99a377..69bff6c681 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -528,8 +528,30 @@ class AbsoluteStaticUrlHandler(RequestHandler): def get(self, path): self.write(self.static_url(path)) + class OverrideStaticUrlHandler(RequestHandler): + def get(self, path): + do_include = bool(self.get_argument("include_host")) + self.include_host = not do_include + + regular_url = self.static_url(path) + override_url = self.static_url(path, include_host=do_include) + if override_url == regular_url: + return self.write(str(False)) + + protocol = self.request.protocol + "://" + protocol_length = len(protocol) + check_regular = regular_url.find(protocol, 0, protocol_length) + check_override = override_url.find(protocol, 0, protocol_length) + + if do_include: + result = (check_override == 0 and check_regular == -1) + else: + result = (check_override == -1 and check_regular == 0) + self.write(str(result)) + return Application([('/static_url/(.*)', StaticUrlHandler), - ('/abs_static_url/(.*)', AbsoluteStaticUrlHandler)], + ('/abs_static_url/(.*)', AbsoluteStaticUrlHandler), + ('/override_static_url/(.*)', OverrideStaticUrlHandler)], static_path=os.path.join(os.path.dirname(__file__), 'static')) def test_static_files(self): @@ -548,6 +570,15 @@ def test_absolute_static_url(self): self.assertEqual(response.body, utf8(self.get_url("/") + "static/robots.txt?v=f71d2")) + def test_include_host_override(self): + self._trigger_include_host_check(False) + self._trigger_include_host_check(True) + + def _trigger_include_host_check(self, include_host): + path = "/override_static_url/robots.txt?include_host=%s" + response = self.fetch(path % int(include_host)) + self.assertEqual(response.body, utf8(str(True))) + class CustomStaticFileTest(AsyncHTTPTestCase, LogTrapTestCase): def get_app(self): class MyStaticFileHandler(StaticFileHandler): diff --git a/tornado/web.py b/tornado/web.py index c7801a4fef..7ff1326dbe 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -862,7 +862,7 @@ def xsrf_form_html(self): return '' - def static_url(self, path): + def static_url(self, path, include_host=None): """Returns a static URL for the given relative static file path. This method requires you set the 'static_path' setting in your @@ -877,12 +877,19 @@ def static_url(self, path): If this handler has a "include_host" attribute, we include the full host for every static URL, including the "http://". Set this attribute for handlers whose output needs non-relative static - path names. + path names. However, in case the "include_host" argument to this + method is given a value other than None it will override the + attribute value when determening whether to generate a relative + or absolute URL. """ self.require_setting("static_path", "static_url") static_handler_class = self.settings.get( "static_handler_class", StaticFileHandler) - if getattr(self, "include_host", False): + + if include_host is None: + include_host = getattr(self, "include_host", False) + + if include_host: base = self.request.protocol + "://" + self.request.host else: base = "" From 2468216f5918c16c055756f52bf95ac7b166c92e Mon Sep 17 00:00:00 2001 From: Birk Nilson Date: Fri, 2 Dec 2011 12:28:03 +0000 Subject: [PATCH 09/53] Separate generation of static URLs and versioning in order ease customization of how versioning is handled in subclasses. --- tornado/web.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index c7801a4fef..906b012845 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1528,6 +1528,21 @@ def make_static_url(cls, settings, path): is the static path being requested. The url returned should be relative to the current host. """ + static_url_prefix = settings.get('static_url_prefix', '/static/') + version_hash = cls.get_version(settings, path) + if version_hash: + return static_url_prefix + path + "?v=" + version_hash + return static_url_prefix + path + + @classmethod + def get_version(cls, settings, path): + """Generate the version string to be appended as a query string + to the static URL - allowing aggressive caching. + + ``settings`` is the `Application.settings` dictionary and ```path`` + is the relative location of the requested asset on the filesystem. + """ + hsh = None abs_path = os.path.join(settings["static_path"], path) with cls._lock: hashes = cls._static_hashes @@ -1539,12 +1554,8 @@ def make_static_url(cls, settings, path): except Exception: logging.error("Could not open static file %r", path) hashes[abs_path] = None - hsh = hashes.get(abs_path) - static_url_prefix = settings.get('static_url_prefix', '/static/') - if hsh: - return static_url_prefix + path + "?v=" + hsh[:5] - else: - return static_url_prefix + path + hsh = hashes.get(abs_path)[:5] + return hsh class FallbackHandler(RequestHandler): From 782c4deffff72a8d9bcec20ae16128380ce0ace4 Mon Sep 17 00:00:00 2001 From: Birk Nilson Date: Fri, 2 Dec 2011 14:25:45 +0000 Subject: [PATCH 10/53] Fixed issue when slicing static_url version string when none exists. --- tornado/web.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index 906b012845..d6a82d5640 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1542,7 +1542,6 @@ def get_version(cls, settings, path): ``settings`` is the `Application.settings` dictionary and ```path`` is the relative location of the requested asset on the filesystem. """ - hsh = None abs_path = os.path.join(settings["static_path"], path) with cls._lock: hashes = cls._static_hashes @@ -1554,8 +1553,10 @@ def get_version(cls, settings, path): except Exception: logging.error("Could not open static file %r", path) hashes[abs_path] = None - hsh = hashes.get(abs_path)[:5] - return hsh + hsh = hashes.get(abs_path) + if hsh: + return hsh[:5] + return None class FallbackHandler(RequestHandler): From 59812edca7fec856a737d99363274ff650356988 Mon Sep 17 00:00:00 2001 From: Birk Nilson Date: Fri, 2 Dec 2011 14:28:31 +0000 Subject: [PATCH 11/53] Add ability to parse static path before using it as though it was a relative filesystem path. Thereby enabling developers to add the versioning string as a component in the path rather than a query string. Which is required when working with CloudFront for instance. --- tornado/test/web_test.py | 18 +++++++++++++++--- tornado/web.py | 5 +++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/tornado/test/web_test.py b/tornado/test/web_test.py index c5ae99a377..bef6f529ad 100644 --- a/tornado/test/web_test.py +++ b/tornado/test/web_test.py @@ -552,12 +552,24 @@ class CustomStaticFileTest(AsyncHTTPTestCase, LogTrapTestCase): def get_app(self): class MyStaticFileHandler(StaticFileHandler): def get(self, path): + path = self.parse_url_path(path) assert path == "foo.txt" self.write("bar") @classmethod def make_static_url(cls, settings, path): - return "/static/%s?v=42" % path + version_hash = cls.get_version(settings, path) + extension_index = path.rindex('.') + before_version = path[:extension_index] + after_version = path[(extension_index + 1):] + return '/static/%s.%s.%s' % (before_version, 42, after_version) + + @classmethod + def parse_url_path(cls, url_path): + extension_index = url_path.rindex('.') + version_index = url_path.rindex('.', 0, extension_index) + return '%s%s' % (url_path[:version_index], + url_path[extension_index:]) class StaticUrlHandler(RequestHandler): def get(self, path): @@ -568,9 +580,9 @@ def get(self, path): static_handler_class=MyStaticFileHandler) def test_serve(self): - response = self.fetch("/static/foo.txt") + response = self.fetch("/static/foo.42.txt") self.assertEqual(response.body, b("bar")) def test_static_url(self): response = self.fetch("/static_url/foo.txt") - self.assertEqual(response.body, b("/static/foo.txt?v=42")) + self.assertEqual(response.body, b("/static/foo.42.txt")) diff --git a/tornado/web.py b/tornado/web.py index d6a82d5640..5a25acdf67 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1446,6 +1446,7 @@ def head(self, path): def get(self, path, include_body=True): if os.path.sep != "/": path = path.replace("/", os.path.sep) + path = self.parse_url_path(path) abspath = os.path.abspath(os.path.join(self.root, path)) # os.path.abspath strips a trailing / # it needs to be temporarily added back for requests to root/ @@ -1558,6 +1559,10 @@ def get_version(cls, settings, path): return hsh[:5] return None + @classmethod + def parse_url_path(cls, url_path): + return url_path + class FallbackHandler(RequestHandler): """A RequestHandler that wraps another HTTP server callback. From 70bb948f77e022958dc49140abad9d49824aa9f1 Mon Sep 17 00:00:00 2001 From: "Serge S. Koval" Date: Thu, 8 Dec 2011 14:42:37 +0200 Subject: [PATCH 12/53] Additional checks for WebSocket protocol handshake. --- tornado/websocket.py | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index ecc80ac436..0d51d8203a 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -35,7 +35,7 @@ class WebSocketHandler(tornado.web.RequestHandler): http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17 The older protocol versions specified at http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 - and + and http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76. are also supported. @@ -82,19 +82,46 @@ def _execute(self, transforms, *args, **kwargs): self.open_args = args self.open_kwargs = kwargs + # Websocket requires GET method + if self.request.method != 'GET': + self.stream.write(tornado.escape.utf8( + "HTTP/1.1 405 Method Not Allowed\r\n\r\n" + )) + self.stream.close() + return + + # Upgrade header should be present and should be equal to WebSocket + if self.request.headers.get("Upgrade", "").lower() != 'websocket': + self.stream.write(tornado.escape.utf8( + "HTTP/1.1 400 Bad Request\r\n\r\n" + "Can \"Upgrade\" only to \"WebSocket\"." + )) + self.stream.close() + return + + # Connection header should be upgrade. Some proxy servers/load balancers + # might mess with it. + if self.request.headers.get("Connection", "").lower() != 'upgrade': + self.stream.write(tornado.escape.utf8( + "HTTP/1.1 400 Bad Request\r\n\r\n" + "\"Connection\" must be \"Upgrade\"." + )) + self.stream.close() + return + # The difference between version 8 and 13 is that in 8 the # client sends a "Sec-Websocket-Origin" header and in 13 it's # simply "Origin". if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): self.ws_connection = WebSocketProtocol8(self) self.ws_connection.accept_connection() - + elif self.request.headers.get("Sec-WebSocket-Version"): self.stream.write(tornado.escape.utf8( "HTTP/1.1 426 Upgrade Required\r\n" "Sec-WebSocket-Version: 8\r\n\r\n")) self.stream.close() - + else: self.ws_connection = WebSocketProtocol76(self) self.ws_connection.accept_connection() @@ -355,7 +382,7 @@ def accept_connection(self): logging.debug("Malformed WebSocket request received") self._abort() return - + def _handle_websocket_headers(self): """Verifies all invariant- and required headers @@ -439,7 +466,7 @@ def _on_frame_start(self, data): def _on_frame_length_16(self, data): self._frame_length = struct.unpack("!H", data)[0]; self.stream.read_bytes(4, self._on_masking_key); - + def _on_frame_length_64(self, data): self._frame_length = struct.unpack("!Q", data)[0]; self.stream.read_bytes(4, self._on_masking_key); @@ -471,11 +498,11 @@ def _on_frame_data(self, data): if not self.client_terminated: self._receive_frame() - + def _handle_message(self, opcode, data): if self.client_terminated: return - + if opcode == 0x1: # UTF-8 data self.async_callback(self.handler.on_message)(data.decode("utf-8", "replace")) @@ -496,7 +523,7 @@ def _handle_message(self, opcode, data): pass else: self._abort() - + def close(self): """Closes the WebSocket connection.""" self._write_frame(True, 0x8, b("")) From 0b5e73b054d718f78aa321d261279832c8cb1ec0 Mon Sep 17 00:00:00 2001 From: "Serge S. Koval" Date: Thu, 8 Dec 2011 14:48:38 +0200 Subject: [PATCH 13/53] Connection header must include "Upgrade", but should not be equal to. --- tornado/websocket.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 0d51d8203a..60f1e49710 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -82,7 +82,7 @@ def _execute(self, transforms, *args, **kwargs): self.open_args = args self.open_kwargs = kwargs - # Websocket requires GET method + # Websocket only supports GET method if self.request.method != 'GET': self.stream.write(tornado.escape.utf8( "HTTP/1.1 405 Method Not Allowed\r\n\r\n" @@ -101,7 +101,7 @@ def _execute(self, transforms, *args, **kwargs): # Connection header should be upgrade. Some proxy servers/load balancers # might mess with it. - if self.request.headers.get("Connection", "").lower() != 'upgrade': + if self.request.headers.get("Connection", "").lower().find('upgrade') == -1: self.stream.write(tornado.escape.utf8( "HTTP/1.1 400 Bad Request\r\n\r\n" "\"Connection\" must be \"Upgrade\"." From 93465a1b034d2a3a758d44baa1e187442ed84875 Mon Sep 17 00:00:00 2001 From: "Serge S. Koval" Date: Sat, 10 Dec 2011 11:56:45 +0200 Subject: [PATCH 14/53] Removed unnecessary checks from appropriate websocket protocol implementations. --- tornado/websocket.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 60f1e49710..a836c4bcfd 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -101,7 +101,9 @@ def _execute(self, transforms, *args, **kwargs): # Connection header should be upgrade. Some proxy servers/load balancers # might mess with it. - if self.request.headers.get("Connection", "").lower().find('upgrade') == -1: + headers = self.request.headers + connection = map(lambda s: s.strip().lower(), headers.get("Connection", "").split(",")) + if 'upgrade' not in connection: self.stream.write(tornado.escape.utf8( "HTTP/1.1 400 Bad Request\r\n\r\n" "\"Connection\" must be \"Upgrade\"." @@ -231,6 +233,7 @@ def accept_connection(self): logging.debug("Malformed WebSocket request received") self._abort() return + scheme = "wss" if self.request.protocol == "https" else "ws" # Write the initial headers before attempting to read the challenge. # This is necessary when using proxies (such as HAProxy), which @@ -285,12 +288,9 @@ def _handle_websocket_headers(self): If a header is missing or have an incorrect value ValueError will be raised """ - headers = self.request.headers fields = ("Origin", "Host", "Sec-Websocket-Key1", "Sec-Websocket-Key2") - if headers.get("Upgrade", '').lower() != "websocket" or \ - headers.get("Connection", '').lower() != "upgrade" or \ - not all(map(lambda f: self.request.headers.get(f), fields)): + if not all(map(lambda f: self.request.headers.get(f), fields)): raise ValueError("Missing/Invalid WebSocket headers") def _calculate_part(self, key): @@ -389,13 +389,8 @@ def _handle_websocket_headers(self): If a header is missing or have an incorrect value ValueError will be raised """ - headers = self.request.headers fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version") - connection = map(lambda s: s.strip().lower(), headers.get("Connection", '').split(",")) - if (self.request.method != "GET" or - headers.get("Upgrade", '').lower() != "websocket" or - "upgrade" not in connection or - not all(map(lambda f: self.request.headers.get(f), fields))): + if (not all(map(lambda f: self.request.headers.get(f), fields))): raise ValueError("Missing/Invalid WebSocket headers") def _challenge_response(self): From 3537a66ef1580a98ea3c4b01dc7f524b4cc5bbd2 Mon Sep 17 00:00:00 2001 From: "Serge S. Koval" Date: Sat, 10 Dec 2011 11:58:16 +0200 Subject: [PATCH 15/53] Minor cleanup. --- tornado/websocket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index a836c4bcfd..ce2af86000 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -390,7 +390,7 @@ def _handle_websocket_headers(self): raised """ fields = ("Host", "Sec-Websocket-Key", "Sec-Websocket-Version") - if (not all(map(lambda f: self.request.headers.get(f), fields))): + if not all(map(lambda f: self.request.headers.get(f), fields)): raise ValueError("Missing/Invalid WebSocket headers") def _challenge_response(self): From 756fadac7cd3bb669cc2552a1ef165f2719ba804 Mon Sep 17 00:00:00 2001 From: Jacob Sondergaard Date: Thu, 5 Jan 2012 12:17:01 +0100 Subject: [PATCH 16/53] Set the request cookies property to an empty dict if cookie parsing fails This commit reverses the bug introduced in commit 4a4d871, leaving an undefined cookies property on failure to parse a request cookie. The bug results in a 'NoneType is not iterable' error when calling RequestHandler.get_cookie() after a cookie parsing exception. --- tornado/httpserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tornado/httpserver.py b/tornado/httpserver.py index e692ba8a9c..3e985ba6c7 100644 --- a/tornado/httpserver.py +++ b/tornado/httpserver.py @@ -407,7 +407,7 @@ def cookies(self): self._cookies.load( native_str(self.headers["Cookie"])) except Exception: - self._cookies = None + self._cookies = {} return self._cookies def write(self, chunk, callback=None): From 33cd3456d5c773d4ee53a2d5ce71ce62f6b4209f Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 7 Jan 2012 17:31:56 -0800 Subject: [PATCH 17/53] Add an explicit binary option to WebSocketHandler.write_message. Switching automatically based on python's bytes vs unicode types is error-prone in python2 where e.g. json_encode returns bytes. Closes #429. --- maint/test/websocket/server.py | 3 ++- tornado/websocket.py | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/maint/test/websocket/server.py b/maint/test/websocket/server.py index 276343ae16..83e2da6191 100644 --- a/maint/test/websocket/server.py +++ b/maint/test/websocket/server.py @@ -2,6 +2,7 @@ from tornado.ioloop import IOLoop from tornado.options import define, options, parse_command_line +from tornado.util import bytes_type from tornado.websocket import WebSocketHandler from tornado.web import Application @@ -9,7 +10,7 @@ class EchoHandler(WebSocketHandler): def on_message(self, message): - self.write_message(message) + self.write_message(message, binary=isinstance(message, bytes_type)) if __name__ == '__main__': parse_command_line() diff --git a/tornado/websocket.py b/tornado/websocket.py index ce8e41dd6c..4709ea8240 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -100,9 +100,11 @@ def _execute(self, transforms, *args, **kwargs): self.ws_connection = WebSocketProtocol76(self) self.ws_connection.accept_connection() - def write_message(self, message): + def write_message(self, message, binary=False): """Sends the given message to the client of this Web Socket.""" - self.ws_connection.write_message(message) + if isinstance(message, dict): + message = tornado.escape.json_encode(message) + self.ws_connection.write_message(message, binary=binary) def open(self, *args, **kwargs): """Invoked when a new WebSocket is opened.""" @@ -312,10 +314,11 @@ def _on_length_indicator(self, byte): self.client_terminated = True self.close() - def write_message(self, message): + def write_message(self, message, binary=False): """Sends the given message to the client of this Web Socket.""" - if isinstance(message, dict): - message = tornado.escape.json_encode(message) + if binary: + raise ValueError( + "Binary messages not supported by this version of websockets") if isinstance(message, unicode): message = message.encode("utf-8") assert isinstance(message, bytes_type) @@ -405,15 +408,13 @@ def _write_frame(self, fin, opcode, data): frame += data self.stream.write(frame) - def write_message(self, message): + def write_message(self, message, binary=False): """Sends the given message to the client of this Web Socket.""" - if isinstance(message, dict): - message = tornado.escape.json_encode(message) - if isinstance(message, unicode): - opcode = 0x1 - message = message.encode("utf-8") - else: + if binary: opcode = 0x2 + else: + opcode = 0x1 + message = tornado.escape.utf8(message) assert isinstance(message, bytes_type) self._write_frame(True, opcode, message) From ef4f2bb0e1b6c051703fc8badfde5cd1220fa771 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 7 Jan 2012 23:24:30 -0800 Subject: [PATCH 18/53] Disable websocket draft76 protocol by default. --- tornado/websocket.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 4709ea8240..785ca95abb 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -86,19 +86,19 @@ def _execute(self, transforms, *args, **kwargs): # The difference between version 8 and 13 is that in 8 the # client sends a "Sec-Websocket-Origin" header and in 13 it's # simply "Origin". + logging.info('starting websocket') if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): self.ws_connection = WebSocketProtocol8(self) self.ws_connection.accept_connection() - - elif self.request.headers.get("Sec-WebSocket-Version"): + elif (self.allow_draft76() and + "Sec-WebSocket-Version" not in self.request.headers): + self.ws_connection = WebSocketProtocol76(self) + self.ws_connection.accept_connection() + else: self.stream.write(tornado.escape.utf8( "HTTP/1.1 426 Upgrade Required\r\n" "Sec-WebSocket-Version: 8\r\n\r\n")) self.stream.close() - - else: - self.ws_connection = WebSocketProtocol76(self) - self.ws_connection.accept_connection() def write_message(self, message, binary=False): """Sends the given message to the client of this Web Socket.""" @@ -113,7 +113,7 @@ def open(self, *args, **kwargs): def on_message(self, message): """Handle incoming messages on the WebSocket - This method must be overloaded + This method must be overridden. """ raise NotImplementedError @@ -128,6 +128,21 @@ def close(self): """ self.ws_connection.close() + def allow_draft76(self): + """Override to enable support for the older "draft76" protocol. + + The draft76 version of the websocket protocol is disabled by + default due to security concerns, but it can be enabled by + overriding this method to return True. + + Connections using the draft76 protocol do not support the + ``binary=True`` flag to `write_message`. + + Support for the draft76 protocol is deprecated and will be + removed in a future version of Tornado. + """ + return False + def async_callback(self, callback, *args, **kwargs): """Wrap callbacks with this if they are used on asynchronous requests. From 0a949c07be95a5c1b453f3e329d8a68a1280e873 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 8 Jan 2012 00:01:18 -0800 Subject: [PATCH 19/53] Update docs for recent websocket changes --- tornado/websocket.py | 42 ++++++++++++++++++-------------- website/sphinx/conf.py | 5 ++++ website/sphinx/releases/next.rst | 16 ++++++++++-- website/sphinx/websocket.rst | 4 ++- 4 files changed, 46 insertions(+), 21 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 785ca95abb..8fcca20123 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -5,11 +5,16 @@ .. warning:: - The WebSocket protocol is still in development. This module - currently implements the "hixie-76", "hybi-10", and "hybi-17" - versions of the protocol. See this `browser compatibility table - `_ on - Wikipedia. + The WebSocket protocol was recently finalized as `RFC 6455 + `_ and is not yet supported in + all browsers. Refer to http://caniuse.com/websockets for details + on compatibility. In addition, during development the protocol + went through several incompatible versions, and some browsers only + support older versions. By default this module only supports the + latest version of the protocol, but optional support for an older + version (known as "draft 76" or "hixie-76") can be enabled by + overriding `WebSocketHandler.allow_draft76` (see that method's + documentation for caveats). """ # Author: Jacob Kristhammar, 2010 @@ -32,13 +37,8 @@ class WebSocketHandler(tornado.web.RequestHandler): open and on_close to handle opened and closed connections. See http://dev.w3.org/html5/websockets/ for details on the - JavaScript interface. This implement the protocol as specified at - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-17 - The older protocol versions specified at - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 - and - http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76. - are also supported. + JavaScript interface. The protocol is specified at + http://tools.ietf.org/html/rfc6455. Here is an example Web Socket handler that echos back all received messages back to the client:: @@ -88,7 +88,7 @@ def _execute(self, transforms, *args, **kwargs): # simply "Origin". logging.info('starting websocket') if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): - self.ws_connection = WebSocketProtocol8(self) + self.ws_connection = WebSocketProtocol13(self) self.ws_connection.accept_connection() elif (self.allow_draft76() and "Sec-WebSocket-Version" not in self.request.headers): @@ -101,7 +101,13 @@ def _execute(self, transforms, *args, **kwargs): self.stream.close() def write_message(self, message, binary=False): - """Sends the given message to the client of this Web Socket.""" + """Sends the given message to the client of this Web Socket. + + The message may be either a string or a dict (which will be + encoded as json). If the ``binary`` argument is false, the + message will be sent as utf8; in binary mode any byte string + is allowed. + """ if isinstance(message, dict): message = tornado.escape.json_encode(message) self.ws_connection.write_message(message, binary=binary) @@ -350,11 +356,11 @@ def close(self): time.time() + 5, self._abort) -class WebSocketProtocol8(WebSocketProtocol): - """Implementation of the WebSocket protocol, version 8 (draft version 10). +class WebSocketProtocol13(WebSocketProtocol): + """Implementation of the WebSocket protocol from RFC 6455. - Compare - http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 + This class supports versions 7 and 8 of the protocol in addition to the + final version 13. """ def __init__(self, handler): WebSocketProtocol.__init__(self, handler) diff --git a/website/sphinx/conf.py b/website/sphinx/conf.py index 036fb411a5..84f833e64f 100644 --- a/website/sphinx/conf.py +++ b/website/sphinx/conf.py @@ -36,6 +36,11 @@ "OutputTransform", "TemplateModule", "url", + + # tornado.websocket + "WebSocketProtocol", + "WebSocketProtocol13", + "WebSocketProtocol76", ] coverage_ignore_functions = [ diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index b2b9db3a21..9ddd8da2f7 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -11,6 +11,9 @@ Backwards-incompatible changes processes exit cleanly rather than returning ``None``. The old behavior was surprising and inconsistent with most of the documented examples of this function (which did not check the return value). +* `tornado.websocket` no longer supports the older "draft 76" version + of the websocket protocol by default, although this version can + be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. ``IOLoop`` and ``IOStream`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -33,13 +36,22 @@ Backwards-incompatible changes ``{% comment %}`` directive, these can wrap other template directives). * Template directives may now span multiple lines. +``tornado.websocket`` +~~~~~~~~~~~~~~~~~~~~~ + +* Updated to support the latest version of the protocol, as finalized + in RFC 6455. +* `tornado.websocket` no longer supports the older "draft 76" version + of the websocket protocol by default, although this version can + be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. +* `WebSocketHandler.write_message` now accepts a ``binary`` argument + to send binary messages. + Other modules ~~~~~~~~~~~~~ * `SimpleAsyncHTTPClient` no longer hangs on ``HEAD`` requests, responses with no content, or empty ``POST``/``PUT`` response bodies. -* `tornado.websocket` has been updated to support the latest protocol - (as finalized in RFC 6455). * `tornado.platform.twisted` compatibility has been improved. However, only Twisted version 11.0.0 is supported (and not 11.1.0). * `tornado.web` now behaves better when given malformed ``Cookie`` headers diff --git a/website/sphinx/websocket.rst b/website/sphinx/websocket.rst index c4bd0ed5a5..7da05071f3 100644 --- a/website/sphinx/websocket.rst +++ b/website/sphinx/websocket.rst @@ -2,4 +2,6 @@ ==================================================================== .. automodule:: tornado.websocket - :members: + + .. autoclass:: WebSocketHandler + :members: From 98f77972ae67554d85b0b9cf58c9e032c619788c Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 8 Jan 2012 13:49:30 -0800 Subject: [PATCH 20/53] Make it possible to run the test suite with --httpclient=CurlAsyncHTTPClient --- tornado/curl_httpclient.py | 2 +- tornado/httpclient.py | 6 ++++-- tornado/test/httpserver_test.py | 4 +++- tornado/test/process_test.py | 7 ++++++- tornado/test/simple_httpclient_test.py | 4 ++++ tox.ini | 13 ++++++++++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index b8fd40527a..53824c84b2 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -271,7 +271,7 @@ def _curl_create(max_simultaneous_connections=None): def _curl_setup_request(curl, request, buffer, headers): - curl.setopt(pycurl.URL, request.url) + curl.setopt(pycurl.URL, utf8(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, diff --git a/tornado/httpclient.py b/tornado/httpclient.py index a6ad4fa139..1280a47d64 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -54,9 +54,11 @@ class HTTPClient(object): except httpclient.HTTPError, e: print "Error:", e """ - def __init__(self): + def __init__(self, async_client_class=None): self._io_loop = IOLoop() - self._async_client = AsyncHTTPClient(self._io_loop) + if async_client_class is None: + async_client_class = AsyncHTTPClient + self._async_client = async_client_class(self._io_loop) self._response = None self._closed = False diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 0363485224..5eb4620830 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -118,7 +118,8 @@ def get_app(self): return Application(self.get_handlers()) def raw_fetch(self, headers, body): - conn = RawRequestHTTPConnection(self.io_loop, self.http_client, + client = SimpleAsyncHTTPClient(self.io_loop) + conn = RawRequestHTTPConnection(self.io_loop, client, httpclient.HTTPRequest(self.get_url("/")), None, self.stop, 1024*1024) @@ -127,6 +128,7 @@ def raw_fetch(self, headers, body): [utf8("Content-Length: %d\r\n" % len(body))]) + b("\r\n") + body) response = self.wait() + client.close() response.rethrow() return response diff --git a/tornado/test/process_test.py b/tornado/test/process_test.py index 91f75b0ab6..de9ae523b4 100644 --- a/tornado/test/process_test.py +++ b/tornado/test/process_test.py @@ -9,6 +9,7 @@ from tornado.ioloop import IOLoop from tornado.netutil import bind_sockets from tornado.process import fork_processes, task_id +from tornado.simple_httpclient import SimpleAsyncHTTPClient from tornado.testing import LogTrapTestCase, get_unused_port from tornado.web import RequestHandler, Application @@ -72,7 +73,11 @@ def get_url(path): signal.alarm(5) self.assertEqual(id, task_id()) for sock in sockets: sock.close() - client = HTTPClient() + # Always use SimpleAsyncHTTPClient here; the curl + # version appears to get confused sometimes if the + # connection gets closed before it's had a chance to + # switch from writing mode to reading mode. + client = HTTPClient(SimpleAsyncHTTPClient) def fetch(url, fail_ok=False): try: diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index 7b7eb19e72..a65c8c687a 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -55,6 +55,10 @@ def get(self): self.set_status(204) class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase): + def setUp(self): + super(SimpleHTTPClientTestCase, self).setUp() + self.http_client = SimpleAsyncHTTPClient(self.io_loop) + def get_app(self): # callable objects to finish pending /trigger requests self.triggers = collections.deque() diff --git a/tox.ini b/tox.ini index 96e2427639..30a78088d1 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ [tox] # "-full" variants include optional dependencies, to ensure # that things work both in a bare install and with all the extras. -envlist = py27-full, py25-full, py32, pypy, py25, py26, py26-full, py27 +envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27 [testenv] commands = python -m tornado.test.runtests {posargs:} @@ -52,6 +52,17 @@ deps = pycurl twisted==11.0.0 +[testenv:py27-curl] +# Same as py27-full, but runs the tests with curl_httpclient by default. +# Note that httpclient_test is always run with both client implementations; +# this flag controls which client all the other tests use. +basepython = python2.7 +deps = + MySQL-python + pycurl + twisted==11.0.0 +commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient {posargs:} + # No pypy-full yet: pycurl doesn't build with pypy, and installing # twisted under pypy takes a *very* long time. MySQL-python builds with # pypy, but doesn't work. From 17052f33c9e467dfc7d3b755146d09012c881046 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 9 Jan 2012 10:33:01 -0800 Subject: [PATCH 21/53] Make SSLIOStream compatible with SSLv3- and TLSv1-only servers. Due to some implementation detail the default SSLv23 mode allows reads before the handshake has completed, but the other modes do not. Closes #431. --- tornado/httpclient.py | 4 +++- tornado/iostream.py | 5 +++++ tornado/test/httpserver_test.py | 35 +++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/tornado/httpclient.py b/tornado/httpclient.py index 1280a47d64..354d907c37 100644 --- a/tornado/httpclient.py +++ b/tornado/httpclient.py @@ -393,12 +393,14 @@ def main(): define("print_headers", type=bool, default=False) define("print_body", type=bool, default=True) define("follow_redirects", type=bool, default=True) + define("validate_cert", type=bool, default=True) args = parse_command_line() client = HTTPClient() for arg in args: try: response = client.fetch(arg, - follow_redirects=options.follow_redirects + follow_redirects=options.follow_redirects, + validate_cert=options.validate_cert, ) except HTTPError, e: if e.response is not None: diff --git a/tornado/iostream.py b/tornado/iostream.py index 460f7f349f..db7895f0ed 100644 --- a/tornado/iostream.py +++ b/tornado/iostream.py @@ -657,6 +657,11 @@ def _handle_connect(self): def _read_from_socket(self): + if self._ssl_accepting: + # If the handshake hasn't finished yet, there can't be anything + # to read (attempting to read may or may not raise an exception + # depending on the SSL version) + return None try: # SSLSocket objects have both a read() and recv() method, # while regular sockets only have recv(). diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 5eb4620830..1f0fb23c2b 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -40,9 +40,12 @@ def get(self): def post(self): self.finish("Got %d bytes in POST" % len(self.request.body)) -class SSLTest(AsyncHTTPTestCase, LogTrapTestCase): +class BaseSSLTest(AsyncHTTPTestCase, LogTrapTestCase): + def get_ssl_version(self): + raise NotImplementedError() + def setUp(self): - super(SSLTest, self).setUp() + super(BaseSSLTest, self).setUp() # Replace the client defined in the parent class. # Some versions of libcurl have deadlock bugs with ssl, # so always run these tests with SimpleAsyncHTTPClient. @@ -59,7 +62,8 @@ def get_httpserver_options(self): test_dir = os.path.dirname(__file__) return dict(ssl_options=dict( certfile=os.path.join(test_dir, 'test.crt'), - keyfile=os.path.join(test_dir, 'test.key'))) + keyfile=os.path.join(test_dir, 'test.key'), + ssl_version=self.get_ssl_version())) def fetch(self, path, **kwargs): self.http_client.fetch(self.get_url(path).replace('http', 'https'), @@ -68,6 +72,7 @@ def fetch(self, path, **kwargs): **kwargs) return self.wait() +class SSLTestMixin(object): def test_ssl(self): response = self.fetch('/') self.assertEqual(response.body, b("Hello world")) @@ -88,8 +93,30 @@ def test_non_ssl_request(self): response = self.wait() self.assertEqual(response.code, 599) +# Python's SSL implementation differs significantly between versions. +# For example, SSLv3 and TLSv1 throw an exception if you try to read +# from the socket before the handshake is complete, but the default +# of SSLv23 allows it. +class SSLv23Test(BaseSSLTest, SSLTestMixin): + def get_ssl_version(self): return ssl.PROTOCOL_SSLv23 +class SSLv3Test(BaseSSLTest, SSLTestMixin): + def get_ssl_version(self): return ssl.PROTOCOL_SSLv3 +class TLSv1Test(BaseSSLTest, SSLTestMixin): + def get_ssl_version(self): return ssl.PROTOCOL_TLSv1 + if ssl is None: - del SSLTest + del BaseSSLTest + del SSLv23Test + del SSLv3Test + del TLSv1Test +elif getattr(ssl, 'OPENSSL_VERSION_INFO', (0,0)) < (1,0): + # In pre-1.0 versions of openssl, SSLv23 clients always send SSLv2 + # ClientHello messages, which are rejected by SSLv3 and TLSv1 + # servers. Note that while the OPENSSL_VERSION_INFO was formally + # introduced in python3.2, it was present but undocumented in + # python 2.7 + del SSLv3Test + del TLSv1Test class MultipartTestHandler(RequestHandler): def post(self): From dbe1c9bc84e48da50cee13124efb8b8a86870e7b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 9 Jan 2012 11:59:24 -0800 Subject: [PATCH 22/53] Disable SSLv2 in simple_httpclient. Contrary to the python documentation, SSLv2 is enabled by default unless openssl version 1.0 or newer is used; older versions appear to still be in use (e.g. in Ubuntu 10.04 LTS and Mac OS X Lion) --- tornado/simple_httpclient.py | 21 +++++++++++++++++++++ tornado/test/httpserver_test.py | 11 +++++++++++ 2 files changed, 32 insertions(+) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index ce2762f1d3..1f90bef93b 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -17,6 +17,7 @@ import os.path import re import socket +import sys import time import urlparse import zlib @@ -185,6 +186,26 @@ def __init__(self, io_loop, client, request, release_callback, ssl_options["keyfile"] = request.client_key if request.client_cert is not None: ssl_options["certfile"] = request.client_cert + + # SSL interoperability is tricky. We want to disable + # SSLv2 for security reasons; it wasn't disabled by default + # until openssl 1.0. The best way to do this is to use + # the SSL_OP_NO_SSLv2, but that wasn't exposed to python + # until 3.2. Python 2.7 adds the ciphers argument, which + # can also be used to disable SSLv2. As a last resort + # on python 2.6, we set ssl_version to SSLv3. This is + # more narrow than we'd like since it also breaks + # compatibility with servers configured for TLSv1 only, + # but nearly all servers support SSLv3: + # http://blog.ivanristic.com/2011/09/ssl-survey-protocol-support.html + if sys.version_info >= (2,7): + ssl_options["ciphers"] = "DEFAULT:!SSLv2" + else: + # This is really only necessary for pre-1.0 versions + # of openssl, but python 2.6 doesn't expose version + # information. + ssl_options["ssl_version"] = ssl.PROTOCOL_SSLv3 + self.stream = SSLIOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options, diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 1f0fb23c2b..a55ca92fc8 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -104,11 +104,22 @@ def get_ssl_version(self): return ssl.PROTOCOL_SSLv3 class TLSv1Test(BaseSSLTest, SSLTestMixin): def get_ssl_version(self): return ssl.PROTOCOL_TLSv1 +class SSLv2Test(BaseSSLTest): + def get_ssl_version(self): return ssl.PROTOCOL_SSLv2 + + def test_sslv2_fail(self): + # This is really more of a client test, but run it here since + # we've got all the other ssl version tests here. + # Clients should have SSLv2 disabled by default. + response = self.fetch('/') + self.assertEqual(response.code, 599) + if ssl is None: del BaseSSLTest del SSLv23Test del SSLv3Test del TLSv1Test + del SSLv2Test elif getattr(ssl, 'OPENSSL_VERSION_INFO', (0,0)) < (1,0): # In pre-1.0 versions of openssl, SSLv23 clients always send SSLv2 # ClientHello messages, which are rejected by SSLv3 and TLSv1 From f62e915404e58c273127993cbfcbf8f1db8d60e7 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 9 Jan 2012 23:50:43 -0800 Subject: [PATCH 23/53] Release notes for the last two ssl changes --- website/sphinx/releases/next.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 9ddd8da2f7..6144200138 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -4,6 +4,13 @@ What's new in the next release of Tornado In progress ----------- +Security fixes +~~~~~~~~~~~~~~ + +* `tornado.simple_httpclient` now disables SSLv2 in all cases. Previously + SSLv2 would be allowed if the Python interpreter was linked against a + pre-1.0 version of OpenSSL. + Backwards-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -11,6 +18,9 @@ Backwards-incompatible changes processes exit cleanly rather than returning ``None``. The old behavior was surprising and inconsistent with most of the documented examples of this function (which did not check the return value). +* On Python 2.6, `tornado.simple_httpclient` only supports SSLv3. This + is because Python 2.6 does not expose a way to support both SSLv3 and TLSv1 + without also supporting the insecure SSLv2. * `tornado.websocket` no longer supports the older "draft 76" version of the websocket protocol by default, although this version can be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. @@ -23,6 +33,8 @@ Backwards-incompatible changes when there is a lot of buffered data, which improves peformance of `SimpleAsyncHTTPClient` when downloading files with lots of chunks. +* `SSLIOStream` now works correctly when ``ssl_version`` is set to + a value other than ``SSLv23``. * Idle ``IOLoops`` no longer wake up several times a second. * `tornado.ioloop.PeriodicCallback` no longer triggers duplicate callbacks when stopped and started repeatedly. From 5d240e492480bb3fd3ee3896ed60722370f79f0f Mon Sep 17 00:00:00 2001 From: Paul Kienzle Date: Tue, 10 Jan 2012 12:41:43 -0500 Subject: [PATCH 24/53] make sure addresses returned from getaddrinfo are unique --- tornado/netutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tornado/netutil.py b/tornado/netutil.py index cfff0ba93a..1e1bcbf917 100644 --- a/tornado/netutil.py +++ b/tornado/netutil.py @@ -244,8 +244,8 @@ def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128): # exist on some platforms (specifically WinXP, although # newer versions of windows have it) flags |= socket.AI_ADDRCONFIG - for res in socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, - 0, flags): + for res in set(socket.getaddrinfo(address, port, family, socket.SOCK_STREAM, + 0, flags)): af, socktype, proto, canonname, sockaddr = res sock = socket.socket(af, socktype, proto) set_close_exec(sock.fileno()) From b8939293f39f21abe4f37fd98cbe0cefdad7ca4d Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Tue, 10 Jan 2012 10:24:10 -0800 Subject: [PATCH 25/53] Some python/openssl builds don't have SSLv2 compiled in, so skip the test in this case. --- tornado/test/httpserver_test.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index a55ca92fc8..9a5c201ee5 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -104,22 +104,31 @@ def get_ssl_version(self): return ssl.PROTOCOL_SSLv3 class TLSv1Test(BaseSSLTest, SSLTestMixin): def get_ssl_version(self): return ssl.PROTOCOL_TLSv1 -class SSLv2Test(BaseSSLTest): - def get_ssl_version(self): return ssl.PROTOCOL_SSLv2 - - def test_sslv2_fail(self): - # This is really more of a client test, but run it here since - # we've got all the other ssl version tests here. - # Clients should have SSLv2 disabled by default. - response = self.fetch('/') - self.assertEqual(response.code, 599) +if hasattr(ssl, 'PROTOCOL_SSLv2'): + class SSLv2Test(BaseSSLTest): + def get_ssl_version(self): return ssl.PROTOCOL_SSLv2 + + def test_sslv2_fail(self): + # This is really more of a client test, but run it here since + # we've got all the other ssl version tests here. + # Clients should have SSLv2 disabled by default. + try: + response = self.fetch('/') + except ssl.SSLError: + # In some python/ssl builds the PROTOCOL_SSLv2 constant + # exists but SSLv2 support is still compiled out, which + # would result in an SSLError here (details vary depending + # on python version). The important thing is that + # SSLv2 request's don't succeed, so we can just ignore + # the errors here. + return + self.assertEqual(response.code, 599) if ssl is None: del BaseSSLTest del SSLv23Test del SSLv3Test del TLSv1Test - del SSLv2Test elif getattr(ssl, 'OPENSSL_VERSION_INFO', (0,0)) < (1,0): # In pre-1.0 versions of openssl, SSLv23 clients always send SSLv2 # ClientHello messages, which are rejected by SSLv3 and TLSv1 From 26a06a0b167773a5e6dae237e90d31741140b4cc Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Tue, 10 Jan 2012 10:26:15 -0800 Subject: [PATCH 26/53] Add a new(er) ubuntu vm setup to test missing SSLv2 --- maint/vm/ubuntu11.04/Vagrantfile | 8 +++++ maint/vm/ubuntu11.04/setup.sh | 56 ++++++++++++++++++++++++++++++++ maint/vm/ubuntu11.04/tox.ini | 32 ++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 maint/vm/ubuntu11.04/Vagrantfile create mode 100644 maint/vm/ubuntu11.04/setup.sh create mode 100644 maint/vm/ubuntu11.04/tox.ini diff --git a/maint/vm/ubuntu11.04/Vagrantfile b/maint/vm/ubuntu11.04/Vagrantfile new file mode 100644 index 0000000000..00a1938873 --- /dev/null +++ b/maint/vm/ubuntu11.04/Vagrantfile @@ -0,0 +1,8 @@ +Vagrant::Config.run do |config| + config.vm.box = "ubuntu11.04" + + config.vm.network "172.19.1.4" + config.vm.share_folder("tornado", "/tornado", "../../..", :nfs=> true) + + config.vm.provision :shell, :path => "setup.sh" +end \ No newline at end of file diff --git a/maint/vm/ubuntu11.04/setup.sh b/maint/vm/ubuntu11.04/setup.sh new file mode 100644 index 0000000000..d5a30f6543 --- /dev/null +++ b/maint/vm/ubuntu11.04/setup.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +set -e + +# Ubuntu 10.10+ do some extra permissions checks for hard links. +# Vagrant's nfs shared folders come through with funny uids, but +# attempts to access them still work despite the visible permissions +# being incorrect. +sysctl -w kernel.yama.protected_nonaccess_hardlinks=0 + +apt-get update + +# libcurl4-gnutls-dev is the default if you ask for libcurl4-dev, but it +# has bugs that make our tests deadlock (the relevant tests detect this and +# disable themselves, but it means that to get full coverage we have to use +# the openssl version). +# The oddly-named python-software-properties includes add-apt-repository. +APT_PACKAGES=" +python-pip +python-dev +libmysqlclient-dev +libcurl4-openssl-dev +python-software-properties +" + +apt-get -y install $APT_PACKAGES + + +# Ubuntu 11.04 has python 2.7 as default; install more from here. +# The most important thing is to have both 2.5 and a later version so we +# test with both tornado.epoll and 2.6+ stdlib's select.epoll. +add-apt-repository ppa:fkrull/deadsnakes +apt-get update + +DEADSNAKES_PACKAGES=" +python2.5 +python2.5-dev +python2.6 +python2.6-dev +python3.2 +python3.2-dev +" +apt-get -y install $DEADSNAKES_PACKAGES + + +PIP_PACKAGES=" +virtualenv +tox +MySQL-python +pycurl +twisted +" + +pip install $PIP_PACKAGES + +/tornado/maint/vm/shared-setup.sh diff --git a/maint/vm/ubuntu11.04/tox.ini b/maint/vm/ubuntu11.04/tox.ini new file mode 100644 index 0000000000..87841ac881 --- /dev/null +++ b/maint/vm/ubuntu11.04/tox.ini @@ -0,0 +1,32 @@ +[tox] +envlist = py27-full, py25-full, py32, py25, py26, py26-full, py27 +setupdir=/tornado +toxworkdir=/home/vagrant/tox-tornado + +[testenv] +commands = python -m tornado.test.runtests {posargs:} + +[testenv:py25] +basepython = python2.5 +deps = simplejson + +[testenv:py25-full] +basepython = python2.5 +deps = + MySQL-python + pycurl + simplejson + twisted==11.0.0 + +[testenv:py26-full] +deps = + MySQL-python + pycurl + twisted==11.0.0 + +[testenv:py27-full] +basepython = python2.7 +deps = + MySQL-python + pycurl + twisted==11.0.0 From e552346a49dd3b357eaf1289d724b79e2f897776 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 13 Jan 2012 22:42:16 -0800 Subject: [PATCH 27/53] The Connection request header should be case-insensitive. --- tornado/httpserver.py | 4 +++- website/sphinx/releases/next.rst | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tornado/httpserver.py b/tornado/httpserver.py index e692ba8a9c..efde98c37e 100644 --- a/tornado/httpserver.py +++ b/tornado/httpserver.py @@ -205,11 +205,13 @@ def _finish_request(self): disconnect = True else: connection_header = self._request.headers.get("Connection") + if connection_header is not None: + connection_header = connection_header.lower() if self._request.supports_http_1_1(): disconnect = connection_header == "close" elif ("Content-Length" in self._request.headers or self._request.method in ("HEAD", "GET")): - disconnect = connection_header != "Keep-Alive" + disconnect = connection_header != "keep-alive" else: disconnect = True self._request = None diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 6144200138..c62c2e4a2f 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -25,6 +25,7 @@ Backwards-incompatible changes of the websocket protocol by default, although this version can be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. + ``IOLoop`` and ``IOStream`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -76,5 +77,7 @@ Other modules even when `os.urandom` is not implemented. * `HTTPServer` with ``xheaders=True`` will no longer accept ``X-Real-IP`` headers that don't look like valid IP addresses. +* `HTTPServer` now treats the ``Connection`` request header as + case-insensitive. * Exception handling in `tornado.gen` has been improved. It is now possible to catch exceptions thrown by a ``Task``. From fea44c7b4eee035ef8501e5d5490e3702a508ba6 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 13 Jan 2012 23:06:49 -0800 Subject: [PATCH 28/53] Release note update --- website/sphinx/releases/next.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index c62c2e4a2f..c587ad1ffc 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -81,3 +81,5 @@ Other modules case-insensitive. * Exception handling in `tornado.gen` has been improved. It is now possible to catch exceptions thrown by a ``Task``. +* `tornado.netutil.bind_sockets` now works when ``getaddrinfo`` returns + duplicate addresses. From 4457f2b9f8a165d9220c060e79a6299db6b7b15b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 13 Jan 2012 23:07:37 -0800 Subject: [PATCH 29/53] Add a fourth release status field to tornado.version_info. Closes #432. --- tornado/__init__.py | 9 ++++++++- website/sphinx/releases/next.rst | 2 ++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tornado/__init__.py b/tornado/__init__.py index ea48b0f02a..d95c016cc2 100644 --- a/tornado/__init__.py +++ b/tornado/__init__.py @@ -16,5 +16,12 @@ """The Tornado web server and tools.""" +# version is a human-readable version number. + +# version_info is a four-tuple for programmatic comparison. The first +# three numbers are the components of the version number. The fourth +# is zero for an official release, positive for a development branch, +# or negative for a release candidate (after the base version number +# has been incremented) version = "2.1.1git" -version_info = (2, 1, 1) +version_info = (2, 1, 1, 1) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index c587ad1ffc..ceefc741c9 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -83,3 +83,5 @@ Other modules to catch exceptions thrown by a ``Task``. * `tornado.netutil.bind_sockets` now works when ``getaddrinfo`` returns duplicate addresses. +* `tornado.version_info` is now a four-tuple so official releases can be + distinguished from development branches. From 36226505bae80160add2acc67cc516bff79559f6 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Fri, 13 Jan 2012 23:12:39 -0800 Subject: [PATCH 30/53] Support non-integer timeouts for curl_httpclient. Closes #399. --- tornado/curl_httpclient.py | 4 ++-- website/sphinx/releases/next.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index 53824c84b2..8dfcb4a834 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -307,8 +307,8 @@ def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) - curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout)) - curl.setopt(pycurl.TIMEOUT, int(request.request_timeout)) + curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) + curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) else: diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index ceefc741c9..aec6654b8c 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -85,3 +85,4 @@ Other modules duplicate addresses. * `tornado.version_info` is now a four-tuple so official releases can be distinguished from development branches. +* `tornado.curl_httpclient` now accepts non-integer timeouts. From af940f4e0bfd458230a4f6fdc1205e6bd019458b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 16 Jan 2012 23:23:21 -0800 Subject: [PATCH 31/53] Major update to tornado.platform.twisted. Significantly improved compatibility (most important changes are in TornadoReactor._invoke_callback) and expanded test coverage. --- tornado/platform/twisted.py | 169 +++++++++++---------- tornado/test/twisted_test.py | 242 ++++++++++++++++++++++++++----- tox.ini | 8 +- website/sphinx/releases/next.rst | 4 +- website/sphinx/twisted.rst | 40 ++++- 5 files changed, 335 insertions(+), 128 deletions(-) diff --git a/tornado/platform/twisted.py b/tornado/platform/twisted.py index 25fe36f351..5d406d3471 100644 --- a/tornado/platform/twisted.py +++ b/tornado/platform/twisted.py @@ -13,14 +13,35 @@ # License for the specific language governing permissions and limitations # under the License. -""" -A twisted-style reactor for the Tornado IOLoop. +# Note: This module's docs are not currently extracted automatically, +# so changes must be made manually to twisted.rst +# TODO: refactor doc build process to use an appropriate virtualenv +"""A Twisted reactor built on the Tornado IOLoop. + +This module lets you run applications and libraries written for +Twisted in a Tornado application. To use it, simply call `install` at +the beginning of the application:: + + import tornado.platform.twisted + tornado.platform.twisted.install() + from twisted.internet import reactor + +When the app is ready to start, call `IOLoop.instance().start()` +instead of `reactor.run()`. This will allow you to use a mixture of +Twisted and Tornado code in the same process. + +It is also possible to create a non-global reactor by calling +`tornado.platform.twisted.TornadoReactor(io_loop)`. However, if +the `IOLoop` and reactor are to be short-lived (such as those used in +unit tests), additional cleanup may be required. Specifically, it is +recommended to call:: + + reactor.fireSystemEvent('shutdown') + reactor.disconnectAll() -To use it, add the following to your twisted application: +before closing the `IOLoop`. -import tornado.platform.twisted -tornado.platform.twisted.install() -from twisted.internet import reactor +This module has been tested with Twisted versions 11.0.0 and 11.1.0. """ from __future__ import with_statement, absolute_import @@ -32,7 +53,7 @@ from twisted.internet.posixbase import PosixReactorBase from twisted.internet.interfaces import \ IReactorFDSet, IDelayedCall, IReactorTime -from twisted.python import failure +from twisted.python import failure, log from twisted.internet import error from zope.interface import implements @@ -44,9 +65,7 @@ class TornadoDelayedCall(object): - """ - DelayedCall object for Tornado. - """ + """DelayedCall object for Tornado.""" implements(IDelayedCall) def __init__(self, reactor, seconds, f, *args, **kw): @@ -89,8 +108,14 @@ def active(self): return self._active class TornadoReactor(PosixReactorBase): - """ - Twisted style reactor for Tornado. + """Twisted reactor built on the Tornado IOLoop. + + Since it is intented to be used in applications where the top-level + event loop is ``io_loop.start()`` rather than ``reactor.run()``, + it is implemented a little differently than other Twisted reactors. + We override `mainLoop` instead of `doIteration` and must implement + timed call functionality on top of `IOLoop.add_timeout` rather than + using the implementation in `PosixReactorBase`. """ implements(IReactorTime, IReactorFDSet) @@ -98,14 +123,20 @@ def __init__(self, io_loop=None): if not io_loop: io_loop = tornado.ioloop.IOLoop.instance() self._io_loop = io_loop - self._readers = {} - self._writers = {} + self._readers = {} # map of reader objects to fd + self._writers = {} # map of writer objects to fd self._fds = {} # a map of fd to a (reader, writer) tuple self._delayedCalls = {} - self._running = False - self._closed = False PosixReactorBase.__init__(self) + # IOLoop.start() bypasses some of the reactor initialization. + # Fire off the necessary events if they weren't already triggered + # by reactor.run(). + def start_if_necessary(): + if not self._started: + self.fireSystemEvent('startup') + self._io_loop.add_callback(start_if_necessary) + # IReactorTime def seconds(self): return time.time() @@ -124,9 +155,7 @@ def _removeDelayedCall(self, dc): # IReactorThreads def callFromThread(self, f, *args, **kw): - """ - See L{twisted.internet.interfaces.IReactorThreads.callFromThread}. - """ + """See `twisted.internet.interfaces.IReactorThreads.callFromThread`""" assert callable(f), "%s is not callable" % f p = functools.partial(f, *args, **kw) self._io_loop.add_callback(p) @@ -142,25 +171,36 @@ def wakeUp(self): # IReactorFDSet def _invoke_callback(self, fd, events): (reader, writer) = self._fds[fd] - if events & IOLoop.READ and reader: - reader.doRead() - if events & IOLoop.WRITE and writer: - writer.doWrite() - if events & IOLoop.ERROR: - if reader: - reader.readConnectionLost(failure.Failure(error.ConnectionLost())) - if writer: - writer.connectionLost(failure.Failure(error.ConnectionLost())) + if reader: + err = None + if reader.fileno() == -1: + err = error.ConnectionLost() + elif events & IOLoop.READ: + err = log.callWithLogger(reader, reader.doRead) + if err is None and events & IOLoop.ERROR: + err = error.ConnectionLost() + if err is not None: + self.removeReader(reader) + reader.readConnectionLost(failure.Failure(err)) + if writer: + err = None + if writer.fileno() == -1: + err = error.ConnectionLost() + elif events & IOLoop.WRITE: + err = log.callWithLogger(writer, writer.doWrite) + if err is None and events & IOLoop.ERROR: + err = error.ConnectionLost() + if err is not None: + self.removeWriter(writer) + writer.writeConnectionLost(failure.Failure(err)) def addReader(self, reader): - """ - Add a FileDescriptor for notification of data available to read. - """ + """Add a FileDescriptor for notification of data available to read.""" if reader in self._readers: # Don't add the reader if it's already there return - self._readers[reader] = True fd = reader.fileno() + self._readers[reader] = fd if fd in self._fds: (_, writer) = self._fds[fd] self._fds[fd] = (reader, writer) @@ -175,13 +215,11 @@ def addReader(self, reader): IOLoop.READ) def addWriter(self, writer): - """ - Add a FileDescriptor for notification of data available to write. - """ + """Add a FileDescriptor for notification of data available to write.""" if writer in self._writers: return - self._writers[writer] = True fd = writer.fileno() + self._writers[writer] = fd if fd in self._fds: (reader, _) = self._fds[fd] self._fds[fd] = (reader, writer) @@ -196,13 +234,9 @@ def addWriter(self, writer): IOLoop.WRITE) def removeReader(self, reader): - """ - Remove a Selectable for notification of data available to read. - """ - fd = reader.fileno() + """Remove a Selectable for notification of data available to read.""" if reader in self._readers: - del self._readers[reader] - if self._closed: return + fd = self._readers.pop(reader) (_, writer) = self._fds[fd] if writer: # We have a writer so we need to update the IOLoop for @@ -217,13 +251,9 @@ def removeReader(self, reader): self._io_loop.remove_handler(fd) def removeWriter(self, writer): - """ - Remove a Selectable for notification of data available to write. - """ - fd = writer.fileno() + """Remove a Selectable for notification of data available to write.""" if writer in self._writers: - del self._writers[writer] - if self._closed: return + fd = self._writers.pop(writer) (reader, _) = self._fds[fd] if reader: # We have a reader so we need to update the IOLoop for @@ -246,47 +276,30 @@ def getReaders(self): def getWriters(self): return self._writers.keys() + # The following functions are mainly used in twisted-style test cases; + # it is expected that most users of the TornadoReactor will call + # IOLoop.start() instead of Reactor.run(). def stop(self): - """ - Implement L{IReactorCore.stop}. - """ - self._running = False PosixReactorBase.stop(self) - self.runUntilCurrent() - try: - self._io_loop.stop() - self._io_loop.close() - except: - # Ignore any exceptions thrown by IOLoop - pass - self._closed = True + self._io_loop.stop() def crash(self): - if not self._running: - return - self._running = False PosixReactorBase.crash(self) - self.runUntilCurrent() - try: - self._io_loop.stop() - self._io_loop.close() - except: - # Ignore any exceptions thrown by IOLoop - pass - self._closed = True + self._io_loop.stop() def doIteration(self, delay): raise NotImplementedError("doIteration") def mainLoop(self): - self._running = True self._io_loop.start() + if self._stopped: + self.fireSystemEvent("shutdown") class _TestReactor(TornadoReactor): """Subclass of TornadoReactor for use in unittests. This can't go in the test.py file because of import-order dependencies - with the twisted reactor test builder. + with the Twisted reactor test builder. """ def __init__(self): # always use a new ioloop @@ -299,12 +312,16 @@ def listenTCP(self, port, factory, backlog=50, interface=''): return super(_TestReactor, self).listenTCP( port, factory, backlog=backlog, interface=interface) + def listenUDP(self, port, protocol, interface='', maxPacketSize=8192): + if not interface: + interface = '127.0.0.1' + return super(_TestReactor, self).listenUDP( + port, protocol, interface=interface, maxPacketSize=maxPacketSize) + def install(io_loop=None): - """ - Install the Tornado reactor. - """ + """Install this package as the default Twisted reactor.""" if not io_loop: io_loop = tornado.ioloop.IOLoop.instance() reactor = TornadoReactor(io_loop) diff --git a/tornado/test/twisted_test.py b/tornado/test/twisted_test.py index 56312458fa..ba53c78976 100644 --- a/tornado/test/twisted_test.py +++ b/tornado/test/twisted_test.py @@ -25,7 +25,13 @@ try: import fcntl import twisted + from twisted.internet.defer import Deferred from twisted.internet.interfaces import IReadDescriptor, IWriteDescriptor + from twisted.internet.protocol import Protocol + from twisted.web.client import Agent + from twisted.web.resource import Resource + from twisted.web.server import Site + from twisted.python import log from tornado.platform.twisted import TornadoReactor from zope.interface import implements except ImportError: @@ -34,14 +40,22 @@ IReadDescriptor = IWriteDescriptor = None def implements(f): pass +from tornado.httpclient import AsyncHTTPClient from tornado.ioloop import IOLoop from tornado.platform.auto import set_close_exec +from tornado.testing import get_unused_port from tornado.util import import_object +from tornado.web import RequestHandler, Application -class ReactorWhenRunningTest(unittest.TestCase): +class ReactorTestCase(unittest.TestCase): def setUp(self): - self._reactor = TornadoReactor(IOLoop()) + self._io_loop = IOLoop() + self._reactor = TornadoReactor(self._io_loop) + def tearDown(self): + self._io_loop.close(all_fds=True) + +class ReactorWhenRunningTest(ReactorTestCase): def test_whenRunning(self): self._whenRunningCalled = False self._anotherWhenRunningCalled = False @@ -58,10 +72,7 @@ def whenRunningCallback(self): def anotherWhenRunningCallback(self): self._anotherWhenRunningCalled = True -class ReactorCallLaterTest(unittest.TestCase): - def setUp(self): - self._reactor = TornadoReactor(IOLoop()) - +class ReactorCallLaterTest(ReactorTestCase): def test_callLater(self): self._laterCalled = False self._now = self._reactor.seconds() @@ -78,10 +89,7 @@ def callLaterCallback(self): self._called = self._reactor.seconds() self._reactor.stop() -class ReactorTwoCallLaterTest(unittest.TestCase): - def setUp(self): - self._reactor = TornadoReactor(IOLoop()) - +class ReactorTwoCallLaterTest(ReactorTestCase): def test_callLater(self): self._later1Called = False self._later2Called = False @@ -108,13 +116,14 @@ def callLaterCallback2(self): self._called2 = self._reactor.seconds() self._reactor.stop() -class ReactorCallFromThreadTest(unittest.TestCase): +class ReactorCallFromThreadTest(ReactorTestCase): def setUp(self): - self._reactor = TornadoReactor(IOLoop()) + super(ReactorCallFromThreadTest, self).setUp() self._mainThread = thread.get_ident() def tearDown(self): self._thread.join() + super(ReactorCallFromThreadTest, self).tearDown() def _newThreadRun(self): self.assertNotEqual(self._mainThread, thread.get_ident()) @@ -134,9 +143,9 @@ def testCallFromThread(self): self._reactor.callWhenRunning(self._whenRunningCallback) self._reactor.run() -class ReactorCallInThread(unittest.TestCase): +class ReactorCallInThread(ReactorTestCase): def setUp(self): - self._reactor = TornadoReactor(IOLoop()) + super(ReactorCallInThread, self).setUp() self._mainThread = thread.get_ident() def _fnCalledInThread(self, *args, **kwargs): @@ -192,13 +201,13 @@ def connectionLost(self, reason): def doWrite(self): self._callback(self._fd) -class ReactorReaderWriterTest(unittest.TestCase): +class ReactorReaderWriterTest(ReactorTestCase): def _set_nonblocking(self, fd): flags = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) def setUp(self): - self._reactor = TornadoReactor(IOLoop()) + super(ReactorReaderWriterTest, self).setUp() r, w = os.pipe() self._set_nonblocking(r) self._set_nonblocking(w) @@ -207,6 +216,11 @@ def setUp(self): self._p1 = os.fdopen(r, "rb", 0) self._p2 = os.fdopen(w, "wb", 0) + def tearDown(self): + super(ReactorReaderWriterTest, self).tearDown() + self._p1.close() + self._p2.close() + def _testReadWrite(self): """ In this test the writer writes an 'x' to its fd. The reader @@ -267,6 +281,106 @@ def testNoWriter(self): self._reactor.callWhenRunning(self._testNoWriter) self._reactor.run() +# Test various combinations of twisted and tornado http servers, +# http clients, and event loop interfaces. +class CompatibilityTests(unittest.TestCase): + def setUp(self): + self.io_loop = IOLoop() + self.reactor = TornadoReactor(self.io_loop) + + def tearDown(self): + self.reactor.disconnectAll() + self.io_loop.close(all_fds=True) + + def start_twisted_server(self): + class HelloResource(Resource): + isLeaf = True + def render_GET(self, request): + return "Hello from twisted!" + site = Site(HelloResource()) + self.twisted_port = get_unused_port() + self.reactor.listenTCP(self.twisted_port, site, interface='127.0.0.1') + + def start_tornado_server(self): + class HelloHandler(RequestHandler): + def get(self): + self.write("Hello from tornado!") + app = Application([('/', HelloHandler)], + log_function=lambda x: None) + self.tornado_port = get_unused_port() + app.listen(self.tornado_port, address='127.0.0.1', io_loop=self.io_loop) + + def run_ioloop(self): + self.stop_loop = self.io_loop.stop + self.io_loop.start() + self.reactor.fireSystemEvent('shutdown') + + def run_reactor(self): + self.stop_loop = self.reactor.stop + self.stop = self.reactor.stop + self.reactor.run() + + def tornado_fetch(self, url, runner): + responses = [] + client = AsyncHTTPClient(self.io_loop) + def callback(response): + responses.append(response) + self.stop_loop() + client.fetch(url, callback=callback) + runner() + self.assertEqual(len(responses), 1) + responses[0].rethrow() + return responses[0] + + def twisted_fetch(self, url, runner): + # http://twistedmatrix.com/documents/current/web/howto/client.html + chunks = [] + client = Agent(self.reactor) + d = client.request('GET', url) + class Accumulator(Protocol): + def __init__(self, finished): + self.finished = finished + def dataReceived(self, data): + chunks.append(data) + def connectionLost(self, reason): + self.finished.callback(None) + def callback(response): + finished = Deferred() + response.deliverBody(Accumulator(finished)) + return finished + d.addCallback(callback) + def shutdown(ignored): + self.stop_loop() + d.addBoth(shutdown) + runner() + self.assertTrue(chunks) + return ''.join(chunks) + + def testTwistedServerTornadoClientIOLoop(self): + self.start_twisted_server() + response = self.tornado_fetch( + 'http://localhost:%d' % self.twisted_port, self.run_ioloop) + self.assertEqual(response.body, 'Hello from twisted!') + + def testTwistedServerTornadoClientReactor(self): + self.start_twisted_server() + response = self.tornado_fetch( + 'http://localhost:%d' % self.twisted_port, self.run_reactor) + self.assertEqual(response.body, 'Hello from twisted!') + + def testTornadoServerTwistedClientIOLoop(self): + self.start_tornado_server() + response = self.twisted_fetch( + 'http://localhost:%d' % self.tornado_port, self.run_ioloop) + self.assertEqual(response, 'Hello from tornado!') + + def testTornadoServerTwistedClientReactor(self): + self.start_tornado_server() + response = self.twisted_fetch( + 'http://localhost:%d' % self.tornado_port, self.run_reactor) + self.assertEqual(response, 'Hello from tornado!') + + if twisted is None: del ReactorWhenRunningTest del ReactorCallLaterTest @@ -274,39 +388,87 @@ def testNoWriter(self): del ReactorCallFromThreadTest del ReactorCallInThread del ReactorReaderWriterTest + del CompatibilityTests else: # Import and run as much of twisted's test suite as possible. # This is unfortunately rather dependent on implementation details, # but there doesn't appear to be a clean all-in-one conformance test # suite for reactors. + # # This is a list of all test suites using the ReactorBuilder - # available in Twisted 11.0.0. Tests that do not currently pass - # with the TornadoReactor are commented out. - twisted_tests = [ - 'twisted.internet.test.test_core.ObjectModelIntegrationTest', - #'twisted.internet.test.test_core.SystemEventTestsBuilder', - 'twisted.internet.test.test_fdset.ReactorFDSetTestsBuilder', - #'twisted.internet.test.test_process.ProcessTestsBuilder', - #'twisted.internet.test.test_process.PTYProcessTestsBuilder', - #'twisted.internet.test.test_tcp.TCPClientTestsBuilder', - 'twisted.internet.test.test_tcp.TCPPortTestsBuilder', - 'twisted.internet.test.test_tcp.TCPConnectionTestsBuilder', - 'twisted.internet.test.test_threads.ThreadTestsBuilder', - 'twisted.internet.test.test_time.TimeTestsBuilder', - #'twisted.internet.test.test_tls.SSLClientTestsMixin', - 'twisted.internet.test.test_udp.UDPServerTestsBuilder', - #'twisted.internet.test.test_unix.UNIXTestsBuilder', - #'twisted.internet.test.test_unix.UNIXDatagramTestsBuilder', - ] - for test_name in twisted_tests: + # available in Twisted 11.0.0 and 11.1.0 (and a blacklist of + # specific test methods to be disabled). + twisted_tests = { + 'twisted.internet.test.test_core.ObjectModelIntegrationTest': [], + 'twisted.internet.test.test_core.SystemEventTestsBuilder': [ + 'test_iterate', # deliberately not supported + ], + 'twisted.internet.test.test_fdset.ReactorFDSetTestsBuilder': [ + "test_lostFileDescriptor", # incompatible with epoll and kqueue + ], + 'twisted.internet.test.test_process.ProcessTestsBuilder': [ + # Doesn't work on python 2.5 + 'test_systemCallUninterruptedByChildExit', + # Doesn't clean up its temp files + 'test_shebang', + ], + 'twisted.internet.test.test_process.PTYProcessTestsBuilder': [ + 'test_systemCallUninterruptedByChildExit', + ], + 'twisted.internet.test.test_tcp.TCPClientTestsBuilder': [], + 'twisted.internet.test.test_tcp.TCPPortTestsBuilder': [], + 'twisted.internet.test.test_tcp.TCPConnectionTestsBuilder': [], + 'twisted.internet.test.test_tcp.WriteSequenceTests': [], + 'twisted.internet.test.test_tcp.AbortConnectionTestCase': [], + 'twisted.internet.test.test_threads.ThreadTestsBuilder': [], + 'twisted.internet.test.test_time.TimeTestsBuilder': [], + # Extra third-party dependencies (pyOpenSSL) + #'twisted.internet.test.test_tls.SSLClientTestsMixin': [], + 'twisted.internet.test.test_udp.UDPServerTestsBuilder': [], + 'twisted.internet.test.test_unix.UNIXTestsBuilder': [ + # Platform-specific. These tests would be skipped automatically + # if we were running twisted's own test runner. + 'test_connectToLinuxAbstractNamespace', + 'test_listenOnLinuxAbstractNamespace', + ], + 'twisted.internet.test.test_unix.UNIXDatagramTestsBuilder': [ + 'test_listenOnLinuxAbstractNamespace', + ], + 'twisted.internet.test.test_unix.UNIXPortTestsBuilder': [], + } + for test_name, blacklist in twisted_tests.iteritems(): try: - test = import_object(test_name) + test_class = import_object(test_name) except (ImportError, AttributeError): continue - class TornadoTest(test): - _reactors = ["tornado.platform.twisted._TestReactor"] - TornadoTest.__name__ = test.__name__ - globals().update(TornadoTest.makeTestCaseClasses()) + for test_func in blacklist: + if hasattr(test_class, test_func): + # The test_func may be defined in a mixin, so clobber + # it instead of delattr() + setattr(test_class, test_func, lambda self: None) + def make_test_subclass(test_class): + class TornadoTest(test_class): + _reactors = ["tornado.platform.twisted._TestReactor"] + def unbuildReactor(self, reactor): + test_class.unbuildReactor(self, reactor) + # Clean up file descriptors (especially epoll/kqueue + # objects) eagerly instead of leaving them for the + # GC. Unfortunately we can't do this in reactor.stop + # since twisted expects to be able to unregister + # connections in a post-shutdown hook. + reactor._io_loop.close(all_fds=True) + TornadoTest.__name__ = test_class.__name__ + return TornadoTest + test_subclass = make_test_subclass(test_class) + globals().update(test_subclass.makeTestCaseClasses()) + + # Since we're not using twisted's test runner, it's tricky to get + # logging set up well. Most of the time it's easiest to just + # leave it turned off, but while working on these tests you may want + # to uncomment one of the other lines instead. + log.defaultObserver.stop() + #import sys; log.startLogging(sys.stderr, setStdout=0) + #log.startLoggingWithObserver(log.PythonLoggingObserver().emit, setStdout=0) if __name__ == "__main__": unittest.main() diff --git a/tox.ini b/tox.ini index 30a78088d1..88c8c37575 100644 --- a/tox.ini +++ b/tox.ini @@ -36,8 +36,10 @@ deps = MySQL-python pycurl simplejson - twisted==11.0.0 + twisted>=11.1.0 +# py26-full deliberately runs an older version of twisted to ensure +# we're still compatible with the oldest version we support. [testenv:py26-full] basepython = python2.6 deps = @@ -50,7 +52,7 @@ basepython = python2.7 deps = MySQL-python pycurl - twisted==11.0.0 + twisted>=11.1.0 [testenv:py27-curl] # Same as py27-full, but runs the tests with curl_httpclient by default. @@ -60,7 +62,7 @@ basepython = python2.7 deps = MySQL-python pycurl - twisted==11.0.0 + twisted>=11.0.0 commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.CurlAsyncHTTPClient {posargs:} # No pypy-full yet: pycurl doesn't build with pypy, and installing diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index aec6654b8c..6fbbf1eef2 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -65,8 +65,8 @@ Other modules * `SimpleAsyncHTTPClient` no longer hangs on ``HEAD`` requests, responses with no content, or empty ``POST``/``PUT`` response bodies. -* `tornado.platform.twisted` compatibility has been improved. However, - only Twisted version 11.0.0 is supported (and not 11.1.0). +* `tornado.platform.twisted` compatibility has been significantly improved. + Twisted version 11.1.0 is now supported in addition to 11.0.0. * `tornado.web` now behaves better when given malformed ``Cookie`` headers * `RequestHandler.redirect` now has a ``status`` argument to send status codes other than 301 and 302. diff --git a/website/sphinx/twisted.rst b/website/sphinx/twisted.rst index 5b107d46c0..6a4d85416a 100644 --- a/website/sphinx/twisted.rst +++ b/website/sphinx/twisted.rst @@ -3,17 +3,43 @@ .. module:: tornado.platform.twisted -This module contains an implementation of the Twisted Reactor built -on the Tornado IOLoop. This lets you run applications and libraries -written for Twisted in a Tornado application. To use it, simply call -`install` at the beginnging of the application:: +This module contains a Twisted reactor build on the Tornado IOLoop, +which lets you run applications and libraries written for Twisted in a +Tornado application. To use it, simply call `install` at the +beginning of the application:: import tornado.platform.twisted tornado.platform.twisted.install() - from twisted.internet import reactor - ... + +When the app is ready to start, call `IOLoop.instance().start()` +instead of `reactor.run()`. This will allow you to use a mixture of +Twisted and Tornado code in the same process. + +It is also possible to create a non-global reactor by calling +`tornado.platform.twisted.TornadoReactor(io_loop)`. However, if +the `IOLoop` and reactor are to be short-lived (such as those used in +unit tests), additional cleanup may be required. Specifically, it is +recommended to call:: + + reactor.fireSystemEvent('shutdown') + reactor.disconnectAll() + +before closing the `IOLoop`. + +This module has been tested with Twisted versions 11.0.0 and 11.1.0. .. function:: install(io_loop=None) - Installs this package as the default Twisted reactor. +Install this package as the default Twisted reactor. + +.. class:: TornadoReactor(io_loop=None) + +Twisted reactor built on the Tornado IOLoop. + +Since it is intented to be used in applications where the top-level +event loop is ``io_loop.start()`` rather than ``reactor.run()``, +it is implemented a little differently than other Twisted reactors. +We override `mainLoop` instead of `doIteration` and must implement +timed call functionality on top of `IOLoop.add_timeout` rather than +using the implementation in `PosixReactorBase`. From 6dfc1b75d2a7af58add29361c5c9939976c9e1bf Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 18 Jan 2012 00:43:38 -0800 Subject: [PATCH 32/53] Add RequestHandler.on_finish method. Closes #367. --- tornado/web.py | 24 +++++++++++++++++++----- website/sphinx/overview.rst | 3 +++ website/sphinx/releases/next.rst | 12 +++++++++--- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index b403afe24b..3ebab8cac5 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -171,18 +171,31 @@ def options(self, *args, **kwargs): raise HTTPError(405) def prepare(self): - """Called before the actual handler method. + """Called at the beginning of a request before `get`/`post`/etc. - Useful to override in a handler if you want a common bottleneck for - all of your requests. + Override this method to perform common initialization regardless + of the request method. + """ + pass + + def on_finish(self): + """Called after the end of a request. + + Override this method to perform cleanup, logging, etc. + This method is a counterpart to `prepare`. ``on_finish`` may + not produce any output, as it is called after the response + has been sent to the client. """ pass def on_connection_close(self): """Called in async handlers if the client closed the connection. - You may override this to clean up resources associated with - long-lived connections. + Override this to clean up resources associated with + long-lived connections. Note that this method is called only if + the connection was closed during asynchronous processing; if you + need to do cleanup after every request override `on_finish` + instead. Proxies may keep a connection open for a time (perhaps indefinitely) after the client has gone away, so this method @@ -656,6 +669,7 @@ def finish(self, chunk=None): self.request.finish() self._log() self._finished = True + self.on_finish() def send_error(self, status_code=500, **kwargs): """Sends the given HTTP error code to the browser. diff --git a/website/sphinx/overview.rst b/website/sphinx/overview.rst index c1dad40f43..29c88ee745 100644 --- a/website/sphinx/overview.rst +++ b/website/sphinx/overview.rst @@ -140,6 +140,9 @@ place: 4. One of the HTTP methods is called: ``get()``, ``post()``, ``put()``, etc. If the URL regular expression contains capturing groups, they are passed as arguments to this method. +5. When the request is finished, ``on_finish()`` is called. For synchronous + handlers this is immediately after ``get()`` (etc) return; for + asynchronous handlers it is after the call to ``finish()``. Here is an example demonstrating the ``initialize()`` method: diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 6fbbf1eef2..d1d0802faa 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -49,6 +49,15 @@ Backwards-incompatible changes ``{% comment %}`` directive, these can wrap other template directives). * Template directives may now span multiple lines. +``tornado.web`` +~~~~~~~~~~~~~~~ + +* Now behaves better when given malformed ``Cookie`` headers +* `RequestHandler.redirect` now has a ``status`` argument to send + status codes other than 301 and 302. +* New method `RequestHandler.on_finish` may be overridden for post-request + processing (as a counterpart to `RequestHandler.prepare`) + ``tornado.websocket`` ~~~~~~~~~~~~~~~~~~~~~ @@ -67,9 +76,6 @@ Other modules responses with no content, or empty ``POST``/``PUT`` response bodies. * `tornado.platform.twisted` compatibility has been significantly improved. Twisted version 11.1.0 is now supported in addition to 11.0.0. -* `tornado.web` now behaves better when given malformed ``Cookie`` headers -* `RequestHandler.redirect` now has a ``status`` argument to send - status codes other than 301 and 302. * `tornado.testing.main` supports a new flag ``--exception_on_interrupt``, which can be set to false to make ``Ctrl-C`` kill the process more reliably (at the expense of stack traces when it does so). From 76ba2323456df6373b4b2079eb54588d101fa47b Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 18 Jan 2012 00:57:55 -0800 Subject: [PATCH 33/53] Make TwitterMixin.twitter_request accept complete urls, not just partial paths. Closes #418. --- tornado/auth.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tornado/auth.py b/tornado/auth.py index 5701bee400..a716210530 100644 --- a/tornado/auth.py +++ b/tornado/auth.py @@ -499,8 +499,13 @@ def _on_post(self, new_entry): self.finish("Posted a message!") """ + if path.startswith('http:') or path.startswith('https:'): + # Raw urls are useful for e.g. search which doesn't follow the + # usual pattern: http://search.twitter.com/search.json + url = path + else: + url = "http://api.twitter.com/1" + path + ".json" # Add the OAuth resource request signature if we have credentials - url = "http://api.twitter.com/1" + path + ".json" if access_token: all_args = {} all_args.update(args) From 97b4b15cdab3b4c2385470ca613e21390eac7691 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 18 Jan 2012 01:00:06 -0800 Subject: [PATCH 34/53] Release note updates for twitter client changes --- website/sphinx/releases/next.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index d1d0802faa..1fd9e82fce 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -92,3 +92,8 @@ Other modules * `tornado.version_info` is now a four-tuple so official releases can be distinguished from development branches. * `tornado.curl_httpclient` now accepts non-integer timeouts. +* `tornado.auth.TwitterMixin.authenticate_redirect` now takes a + ``callback_uri`` parameter. +* `tornado.auth.TwitterMixin.twitter_request` now accepts both URLs and + partial paths (complete URLs are useful for the search API which follows + different patterns). From 8b668633ad4d3ccfb446a19e7fd1cc76a734e820 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Wed, 18 Jan 2012 01:17:30 -0800 Subject: [PATCH 35/53] Add 307 too and update release notes. --- tornado/simple_httpclient.py | 12 ++++++------ tornado/test/simple_httpclient_test.py | 12 +++++++----- website/sphinx/releases/next.rst | 1 + 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index 5e4d145559..ad5b7e0987 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -53,7 +53,7 @@ class SimpleAsyncHTTPClient(AsyncHTTPClient): Some features found in the curl-based AsyncHTTPClient are not yet supported. In particular, proxies are not supported, connections - are not reused, and callers cannot select the network interface to be + are not reused, and callers cannot select the network interface to be used. Python 2.6 or higher is required for HTTPS support. Users of Python 2.5 @@ -310,7 +310,7 @@ def cleanup(self): yield except Exception, e: logging.warning("uncaught exception", exc_info=True) - self._run_callback(HTTPResponse(self.request, 599, error=e, + self._run_callback(HTTPResponse(self.request, 599, error=e, request_time=time.time() - self.start_time, )) @@ -335,7 +335,7 @@ def _on_headers(self, data): # use them but if they differ it's an error. pieces = re.split(r',\s*', self.headers["Content-Length"]) if any(i != pieces[0] for i in pieces): - raise ValueError("Multiple unequal Content-Lengths: %r" % + raise ValueError("Multiple unequal Content-Lengths: %r" % self.headers["Content-Length"]) self.headers["Content-Length"] = pieces[0] content_length = int(self.headers["Content-Length"]) @@ -380,7 +380,7 @@ def _on_body(self, data): self.request) if (self.request.follow_redirects and self.request.max_redirects > 0 and - self.code in (301, 302, 303)): + self.code in (301, 302, 303, 307)): new_request = copy.copy(self.request) new_request.url = urlparse.urljoin(self.request.url, self.headers["Location"]) @@ -391,8 +391,8 @@ def _on_body(self, data): if self.code == 303: new_request.method = "GET" new_request.body = None - for h in ["Content-Length", "Content-Type", - "Content-Encoding", "Transfer-Encoding"]: + for h in ["Content-Length", "Content-Type", + "Content-Encoding", "Transfer-Encoding"]: try: del self.request.headers[h] except KeyError: diff --git a/tornado/test/simple_httpclient_test.py b/tornado/test/simple_httpclient_test.py index f0f9be0fb1..ebb265b916 100644 --- a/tornado/test/simple_httpclient_test.py +++ b/tornado/test/simple_httpclient_test.py @@ -56,14 +56,16 @@ def get(self): class SeeOther303PostHandler(RequestHandler): def post(self): + assert self.request.body == b("blah") self.set_header("Location", "/303_get") self.set_status(303) -class SeeOther303GetHandler(RequestHandler): +class SeeOther303GetHandler(RequestHandler): def get(self): + assert not self.request.body self.write("ok") - - + + class SimpleHTTPClientTestCase(AsyncHTTPTestCase, LogTrapTestCase): def setUp(self): super(SimpleHTTPClientTestCase, self).setUp() @@ -163,13 +165,13 @@ def test_max_redirects(self): self.assertTrue(response.headers["Location"].endswith("/countdown/1")) def test_303_redirect(self): - response = self.fetch("/303_post", method="POST", body="") + response = self.fetch("/303_post", method="POST", body="blah") self.assertEqual(200, response.code) self.assertTrue(response.request.url.endswith("/303_post")) self.assertTrue(response.effective_url.endswith("/303_get")) #request is the original request, is a POST still self.assertEqual("POST", response.request.method) - + def test_request_timeout(self): response = self.fetch('/hang', request_timeout=0.1) self.assertEqual(response.code, 599) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 1fd9e82fce..907be793ef 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -74,6 +74,7 @@ Other modules * `SimpleAsyncHTTPClient` no longer hangs on ``HEAD`` requests, responses with no content, or empty ``POST``/``PUT`` response bodies. +* `SimpleAsyncHTTPClient` now supports 303 and 307 redirect codes. * `tornado.platform.twisted` compatibility has been significantly improved. Twisted version 11.1.0 is now supported in addition to 11.0.0. * `tornado.testing.main` supports a new flag ``--exception_on_interrupt``, From 35139ff82cdb24bf9978269afe99e6ccff993631 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 14:09:18 -0800 Subject: [PATCH 36/53] Add .coveragerc for test coverage reporting. Closes #439. --- .coveragerc | 18 ++++++++++++++++++ .gitignore | 10 ++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..2db6c2f514 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,18 @@ +# Test coverage configuration. +# Usage: +# pip install coverage +# coverage erase # clears previous data if any +# coverage run -m tornado.test.runtests +# coverage report # prints to stdout +# coverage html # creates ./htmlcov/*.html including annotated source +[run] +branch = true +source = tornado +omit = + tornado/platform/* + tornado/test/* + */_auto2to3* + +[report] +# Ignore missing source files, i.e. fake template-generated "files" +ignore_errors = true diff --git a/.gitignore b/.gitignore index 8571789313..f127f38b56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ *.pyc *.so *~ -build -dist/ +build/ +/dist/ MANIFEST -tornado.egg-info +/tornado.egg-info/ _auto2to3* -.tox +.tox/ .vagrant +/.coverage +/htmlcov/ \ No newline at end of file From 1014d5683e891ccf9537eddf96ae3bc6094aa1ad Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 14:35:21 -0800 Subject: [PATCH 37/53] Check in a requirements file for the tools I use while working on tornado. --- .gitignore | 3 ++- maint/requirements.txt | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 maint/requirements.txt diff --git a/.gitignore b/.gitignore index f127f38b56..ba7761660e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ _auto2to3* .tox/ .vagrant /.coverage -/htmlcov/ \ No newline at end of file +/htmlcov/ +/env/ diff --git a/maint/requirements.txt b/maint/requirements.txt new file mode 100644 index 0000000000..30e0d32e14 --- /dev/null +++ b/maint/requirements.txt @@ -0,0 +1,21 @@ +# Frozen pip requirements for tools used in the development of tornado + +# Tornado's optional dependencies +MySQL-python==1.2.3 +Twisted==11.1.0 +pycurl==7.19.0 + +# Other useful tools +Sphinx==1.1.2 +coverage==3.5.1 +pyflakes==0.5.0 +tox==1.3 +virtualenv==1.7 + +# Indirect dependencies +Jinja2==2.6 +Pygments==1.4 +docutils==0.8.1 +py==1.4.6 +wsgiref==0.1.2 +zope.interface==3.8.0 From cc671cb55cd788f21cbd6c4fa0fd2abe2119f2c5 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 14:56:43 -0800 Subject: [PATCH 38/53] Allow handlers to override the selection of "ws" or "wss" in the draft76 handshake, to work with SSL proxies that do not insert an X-Scheme header. Closes #437. --- tornado/websocket.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 8fcca20123..7471c4cde1 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -149,6 +149,18 @@ def allow_draft76(self): """ return False + def get_websocket_scheme(self): + """Return the url scheme used for this request, either "ws" or "wss". + + This is normally decided by HTTPServer, but applications + may wish to override this if they are using an SSL proxy + that does not provide the X-Scheme header as understood + by HTTPServer. + + Note that this is only used by the draft76 protocol. + """ + return "wss" if self.request.protocol == "https" else "ws" + def async_callback(self, callback, *args, **kwargs): """Wrap callbacks with this if they are used on asynchronous requests. @@ -228,7 +240,7 @@ def accept_connection(self): logging.debug("Malformed WebSocket request received") self._abort() return - scheme = "wss" if self.request.protocol == "https" else "ws" + scheme = self.handler.get_websocket_scheme() # Write the initial headers before attempting to read the challenge. # This is necessary when using proxies (such as HAProxy), which # need to see the Upgrade headers before passing through the From eee9953dcbadfe1b162ab8c10d105ff4da4ad0be Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 15:34:11 -0800 Subject: [PATCH 39/53] Websocket subprotocol updates. Failure to select a subprotocol is not fatal, so rename method from validate_subprotocol to select_subprotocol. Subprotocols are a comma-separated list in the RFC version, but a single value in draft76. --- tornado/websocket.py | 51 ++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 69cca86683..ca0096322e 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -112,8 +112,17 @@ def write_message(self, message, binary=False): message = tornado.escape.json_encode(message) self.ws_connection.write_message(message, binary=binary) - def validate_subprotocol(self, subprotocols): - """Invoked when a new WebSocket requests specific subprotocols.""" + def select_subprotocol(self, subprotocols): + """Invoked when a new WebSocket requests specific subprotocols. + + ``subprotocols`` is a list of strings identifying the + subprotocols proposed by the client. This method may be + overridden to return one of those strings to select it, or + ``None`` to not select a subprotocol. Failure to select a + subprotocol does not automatically abort the connection, + although clients may close the connection if none of their + proposed subprotocols was selected. + """ return None def open(self, *args, **kwargs): @@ -246,16 +255,14 @@ def accept_connection(self): return scheme = self.handler.get_websocket_scheme() - subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", None) - if subprotocols: - subprotocol = self.handler.validate_subprotocol(subprotocols) - if not subprotocol: - logging.debug("Subprotocol rejected by handler.") - self._abort() - return - subprotocol = "Sec-WebSocket-Protocol: %s\r\n" % subprotocol - else: - subprotocol = '' + # draft76 only allows a single subprotocol + subprotocol_header = '' + subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None) + if subprotocol: + selected = self.handler.select_subprotocol([subprotocol]) + if selected: + assert selected == subprotocol + subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected # Write the initial headers before attempting to read the challenge. # This is necessary when using proxies (such as HAProxy), which @@ -275,7 +282,7 @@ def accept_connection(self): scheme=scheme, host=self.request.host, uri=self.request.uri, - subprotocol=subprotocol)))) + subprotocol=subprotocol_header)))) self.stream.read_bytes(8, self._handle_challenge) def challenge_response(self, challenge): @@ -435,16 +442,14 @@ def _challenge_response(self): return tornado.escape.native_str(base64.b64encode(sha1.digest())) def _accept_connection(self): - subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", None) + subprotocol_header = '' + subprotocols = self.request.headers.get("Sec-WebSocket-Protocol", '') + subprotocols = [s.strip() for s in subprotocols.split(',')] if subprotocols: - subprotocol = self.handler.validate_subprotocol(subprotocols) - if not subprotocol: - logging.debug("Subprotocol rejected by handler.") - self._abort() - return - subprotocol = "Sec-WebSocket-Protocol: %s\r\n" % subprotocol - else: - subprotocol = '' + selected = self.handler.select_subprotocol(subprotocols) + if selected: + assert selected in subprotocols + subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected self.stream.write(tornado.escape.utf8( "HTTP/1.1 101 Switching Protocols\r\n" @@ -452,7 +457,7 @@ def _accept_connection(self): "Connection: Upgrade\r\n" "Sec-WebSocket-Accept: %s\r\n" "%s" - "\r\n" % (self._challenge_response(), subprotocol))) + "\r\n" % (self._challenge_response(), subprotocol_header))) self.async_callback(self.handler.open)(*self.handler.open_args, **self.handler.open_kwargs) self._receive_frame() From ea0291b256264deeea70a77d1add797bc5e9f121 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 15:58:35 -0800 Subject: [PATCH 40/53] Check if the stream is closed before writing final websocket close packet. Closes #390. --- tornado/websocket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index ca0096322e..b5b742c8cd 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -387,8 +387,9 @@ def close(self): """Closes the WebSocket connection.""" if self.client_terminated and self._waiting: tornado.ioloop.IOLoop.instance().remove_timeout(self._waiting) + self._waiting = None self.stream.close() - else: + elif not self.stream.closed(): self.stream.write("\xff\x00") self._waiting = tornado.ioloop.IOLoop.instance().add_timeout( time.time() + 5, self._abort) @@ -603,6 +604,7 @@ def _handle_message(self, opcode, data): def close(self): """Closes the WebSocket connection.""" + if self.stream.closed(): return self._write_frame(True, 0x8, b("")) self._started_closing_handshake = True self._waiting = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 5, self._abort) From bc28966ef5053da6feebb72293ddf09ce98476f9 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 16:16:36 -0800 Subject: [PATCH 41/53] Remove stray log line --- tornado/websocket.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index b5b742c8cd..c4498cfea6 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -86,7 +86,6 @@ def _execute(self, transforms, *args, **kwargs): # The difference between version 8 and 13 is that in 8 the # client sends a "Sec-Websocket-Origin" header and in 13 it's # simply "Origin". - logging.info('starting websocket') if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8", "13"): self.ws_connection = WebSocketProtocol13(self) self.ws_connection.accept_connection() From 5a18d50f44ba2d9c10d14e7cee809de09b2478d8 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 17:12:22 -0800 Subject: [PATCH 42/53] Disable slow websocket performance tests by default. Add pypy to the mix for when we do run the perf tests. Depending on the benchmark pypy is ~twice as fast as cpython 2.7 --- maint/test/websocket/client.py | 2 +- maint/test/websocket/run.sh | 10 +++++++--- maint/test/websocket/server.py | 2 +- maint/test/websocket/tox.ini | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/maint/test/websocket/client.py b/maint/test/websocket/client.py index f549b218f0..1679887942 100644 --- a/maint/test/websocket/client.py +++ b/maint/test/websocket/client.py @@ -11,7 +11,7 @@ define('cases', type=str, multiple=True, default=["*"]) define('exclude', type=str, multiple=True, - default=[]) + default=["9.*"]) if __name__ == '__main__': parse_command_line() diff --git a/maint/test/websocket/run.sh b/maint/test/websocket/run.sh index d11870152b..9478687d47 100755 --- a/maint/test/websocket/run.sh +++ b/maint/test/websocket/run.sh @@ -4,8 +4,8 @@ # python2 and python3. Output goes in ./reports/servers/index.html. # # The --cases and --exclude arguments can be used to run only part of -# the suite. --exclude="9.*" is useful to skip the relatively slow -# performance tests. +# the suite. The default is --exclude="9.*" to skip the relatively slow +# performance tests; pass --exclude="" to override and include them. set -e @@ -21,13 +21,17 @@ PY27_SERVER_PID=$! .tox/py32/bin/python server.py --port=9003 & PY32_SERVER_PID=$! +.tox/pypy/bin/python server.py --port=9004 & +PYPY_SERVER_PID=$! + sleep 1 -.tox/py27/bin/python ./client.py --servers=Tornado/py25=ws://localhost:9001,Tornado/py27=ws://localhost:9002,Tornado/py32=ws://localhost:9003 "$@" +.tox/py27/bin/python ./client.py --servers=Tornado/py25=ws://localhost:9001,Tornado/py27=ws://localhost:9002,Tornado/py32=ws://localhost:9003,Tornado/pypy=ws://localhost:9004 "$@" || true kill $PY25_SERVER_PID kill $PY27_SERVER_PID kill $PY32_SERVER_PID +kill $PYPY_SERVER_PID wait echo "Tests complete. Output is in ./reports/servers/index.html" \ No newline at end of file diff --git a/maint/test/websocket/server.py b/maint/test/websocket/server.py index 83e2da6191..b44056cd63 100644 --- a/maint/test/websocket/server.py +++ b/maint/test/websocket/server.py @@ -17,5 +17,5 @@ def on_message(self, message): app = Application([ ('/', EchoHandler), ]) - app.listen(options.port, address='localhost') + app.listen(options.port, address='127.0.0.1') IOLoop.instance().start() diff --git a/maint/test/websocket/tox.ini b/maint/test/websocket/tox.ini index 7b374a48be..0709749fab 100644 --- a/maint/test/websocket/tox.ini +++ b/maint/test/websocket/tox.ini @@ -2,7 +2,7 @@ # to install autobahn and deal with 2to3 for the python3 version. # See run.sh for the real test runner. [tox] -envlist = py27, py32, py25 +envlist = py27, py32, py25, pypy setupdir=../../.. [testenv] From 442b49f866901c3ec9cc5405891d77e044b8f9e1 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 17:20:10 -0800 Subject: [PATCH 43/53] Refactor websocket close logic; remove dependency on singleton IOLoop. --- tornado/websocket.py | 57 +++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index c4498cfea6..f240b932c5 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -186,15 +186,10 @@ def _not_supported(self, *args, **kwargs): def on_connection_close(self): if self.ws_connection: - self.ws_connection.client_terminated = True + self.ws_connection.on_connection_close() + self.ws_connection = None self.on_close() - def _set_client_terminated(self, value): - self.ws_connection.client_terminated = value - - client_terminated = property(lambda self: self.ws_connection.client_terminated, - _set_client_terminated) - for method in ["write", "redirect", "set_header", "send_error", "set_cookie", "set_status", "flush", "finish"]: @@ -209,6 +204,7 @@ def __init__(self, handler): self.request = handler.request self.stream = handler.stream self.client_terminated = False + self.server_terminated = False def async_callback(self, callback, *args, **kwargs): """Wrap callbacks with this if they are used on asynchronous requests. @@ -227,10 +223,15 @@ def wrapper(*args, **kwargs): self._abort() return wrapper + def on_connection_close(self): + self._abort() + def _abort(self): """Instantly aborts the WebSocket connection by closing the socket""" self.client_terminated = True - self.stream.close() + self.server_terminated = True + self.stream.close() # forcibly tear down the connection + self.close() # let the subclass cleanup class WebSocketProtocol76(WebSocketProtocol): @@ -384,14 +385,18 @@ def write_message(self, message, binary=False): def close(self): """Closes the WebSocket connection.""" - if self.client_terminated and self._waiting: - tornado.ioloop.IOLoop.instance().remove_timeout(self._waiting) + if not self.server_terminated: + if not self.stream.closed(): + self.stream.write("\xff\x00") + self.server_terminated = True + if self.client_terminated: + if self._waiting is not None: + self.stream.io_loop.remove_timeout(self._waiting) self._waiting = None self.stream.close() - elif not self.stream.closed(): - self.stream.write("\xff\x00") - self._waiting = tornado.ioloop.IOLoop.instance().add_timeout( - time.time() + 5, self._abort) + elif self._waiting is None: + self._waiting = self.stream.io_loop.add_timeout( + time.time() + 5, self._abort) class WebSocketProtocol13(WebSocketProtocol): @@ -408,7 +413,7 @@ def __init__(self, handler): self._frame_length = None self._fragmented_message_buffer = None self._fragmented_message_opcode = None - self._started_closing_handshake = False + self._waiting = None def accept_connection(self): try: @@ -589,9 +594,7 @@ def _handle_message(self, opcode, data): elif opcode == 0x8: # Close self.client_terminated = True - if not self._started_closing_handshake: - self._write_frame(True, 0x8, b("")) - self.stream.close() + self.close() elif opcode == 0x9: # Ping self._write_frame(True, 0xA, data) @@ -603,7 +606,17 @@ def _handle_message(self, opcode, data): def close(self): """Closes the WebSocket connection.""" - if self.stream.closed(): return - self._write_frame(True, 0x8, b("")) - self._started_closing_handshake = True - self._waiting = tornado.ioloop.IOLoop.instance().add_timeout(time.time() + 5, self._abort) + if not self.server_terminated: + if not self.stream.closed(): + self._write_frame(True, 0x8, b("")) + self.server_terminated = True + if self.client_terminated: + if self._waiting is not None: + self.stream.io_loop.remove_timeout(self._waiting) + self._waiting = None + self.stream.close() + elif self._waiting is None: + # Give the client a few seconds to complete a clean shutdown, + # otherwise just close the connection. + self._waiting = self.stream.io_loop.add_timeout( + time.time() + 5, self._abort) From 1b38b58da87f97e388c73fcc2c5c271669382e94 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sat, 21 Jan 2012 17:39:37 -0800 Subject: [PATCH 44/53] Update websocket chat demo to work when not addressed as localhost. Turn on draft76 support. --- demos/websocket/chatdemo.py | 4 ++++ demos/websocket/static/chat.js | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/demos/websocket/chatdemo.py b/demos/websocket/chatdemo.py index 21648ebe11..60fb956e88 100755 --- a/demos/websocket/chatdemo.py +++ b/demos/websocket/chatdemo.py @@ -57,6 +57,10 @@ class ChatSocketHandler(tornado.websocket.WebSocketHandler): cache = [] cache_size = 200 + def allow_draft76(self): + # for iOS 5.0 Safari + return True + def open(self): ChatSocketHandler.waiters.add(self) diff --git a/demos/websocket/static/chat.js b/demos/websocket/static/chat.js index 818bc031aa..9d8bcc50d3 100644 --- a/demos/websocket/static/chat.js +++ b/demos/websocket/static/chat.js @@ -50,10 +50,11 @@ var updater = { socket: null, start: function() { + var url = "ws://" + location.host + "/chatsocket"; if ("WebSocket" in window) { - updater.socket = new WebSocket("ws://localhost:8888/chatsocket"); + updater.socket = new WebSocket(url); } else { - updater.socket = new MozWebSocket("ws://localhost:8888/chatsocket"); + updater.socket = new MozWebSocket(url); } updater.socket.onmessage = function(event) { updater.showMessage(JSON.parse(event.data)); From bcb30e56fd3939482daa59b166d21835d77d4f23 Mon Sep 17 00:00:00 2001 From: dave mankoff Date: Sun, 22 Jan 2012 17:15:10 -0500 Subject: [PATCH 45/53] fix curl basic auth --- tornado/curl_httpclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index 8dfcb4a834..7d8faf36e2 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -383,10 +383,10 @@ def ioctl(cmd): else: curl.setopt(pycurl.INFILESIZE, len(request.body)) - if request.auth_username and request.auth_password: - userpwd = "%s:%s" % (request.auth_username, request.auth_password) + if request.auth_username is not None: + userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) - curl.setopt(pycurl.USERPWD, userpwd) + curl.setopt(pycurl.USERPWD, userpwd.encode('ascii')) logging.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: From 3bb80eb83fc230361e8d29ba75fb4d6fb8b45987 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 22 Jan 2012 14:19:19 -0800 Subject: [PATCH 46/53] Always set Etag in StaticFileHandler so it won't break if the default Etag implementation in RequestHandler changes. --- tornado/web.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index 07d576c078..ea57e925e6 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -1518,18 +1518,16 @@ def get(self, path, include_body=True): self.set_status(304) return - if not include_body: - self.set_header("Content-Length", os.path.getsize(abspath)) - with open(abspath, "rb") as file: - hasher = hashlib.sha1() - hasher.update(file.read()) - self.set_header("Etag", '"%s"' % hasher.hexdigest()) - return - file = open(abspath, "rb") - try: - self.write(file.read()) - finally: - file.close() + with open(abspath, "rb") as file: + data = file.read() + hasher = hashlib.sha1() + hasher.update(data) + self.set_header("Etag", '"%s"' % hasher.hexdigest()) + if include_body: + self.write(data) + else: + assert self.request.method == "HEAD" + self.set_header("Content-Length", len(data)) def set_extra_headers(self, path): """For subclass to add extra headers to the response""" From 98b088638ecbe0ca69b5ffdc2476dba5a071af44 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 22 Jan 2012 15:26:35 -0800 Subject: [PATCH 47/53] Make StaticFileHandler.parse_url_path an instance method, and make it responsible for os.path.sep conversion. Assorted doc updates for StaticFileHandler. --- tornado/web.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tornado/web.py b/tornado/web.py index 0d50730d16..c31eb674b2 100644 --- a/tornado/web.py +++ b/tornado/web.py @@ -898,13 +898,11 @@ def static_url(self, path, include_host=None): returned content. The signature is based on the content of the file. - If this handler has a "include_host" attribute, we include the - full host for every static URL, including the "http://". Set - this attribute for handlers whose output needs non-relative static - path names. However, in case the "include_host" argument to this - method is given a value other than None it will override the - attribute value when determening whether to generate a relative - or absolute URL. + By default this method returns URLs relative to the current + host, but if ``include_host`` is true the URL returned will be + absolute. If this handler has an ``include_host`` attribute, + that value will be used as the default for all `static_url` + calls that do not pass ``include_host`` as a keyword argument. """ self.require_setting("static_path", "static_url") static_handler_class = self.settings.get( @@ -1475,8 +1473,6 @@ def head(self, path): self.get(path, include_body=False) def get(self, path, include_body=True): - if os.path.sep != "/": - path = path.replace("/", os.path.sep) path = self.parse_url_path(path) abspath = os.path.abspath(os.path.join(self.root, path)) # os.path.abspath strips a trailing / @@ -1571,11 +1567,16 @@ def make_static_url(cls, settings, path): @classmethod def get_version(cls, settings, path): - """Generate the version string to be appended as a query string - to the static URL - allowing aggressive caching. + """Generate the version string to be used in static URLs. + + This method may be overridden in subclasses (but note that it + is a class method rather than a static method). The default + implementation uses a hash of the file's contents. - ``settings`` is the `Application.settings` dictionary and ```path`` + ``settings`` is the `Application.settings` dictionary and ``path`` is the relative location of the requested asset on the filesystem. + The returned value should be a string, or ``None`` if no version + could be determined. """ abs_path = os.path.join(settings["static_path"], path) with cls._lock: @@ -1593,8 +1594,15 @@ def get_version(cls, settings, path): return hsh[:5] return None - @classmethod - def parse_url_path(cls, url_path): + def parse_url_path(self, url_path): + """Converts a static URL path into a filesystem path. + + ``url_path`` is the path component of the URL with + ``static_url_prefix`` removed. The return value should be + filesystem path relative to ``static_path``. + """ + if os.path.sep != "/": + url_path = url_path.replace("/", os.path.sep) return url_path From 9c3afde89e01d07c56702552eb7752da853bd03a Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 22 Jan 2012 17:04:51 -0800 Subject: [PATCH 48/53] Release note updates --- website/sphinx/releases/next.rst | 62 +++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index 907be793ef..fa940c895a 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -25,9 +25,24 @@ Backwards-incompatible changes of the websocket protocol by default, although this version can be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. +``tornado.httpclient`` +~~~~~~~~~~~~~~~~~~~~~~ -``IOLoop`` and ``IOStream`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* `SimpleAsyncHTTPClient` no longer hangs on ``HEAD`` requests, + responses with no content, or empty ``POST``/``PUT`` response bodies. +* `SimpleAsyncHTTPClient` now supports 303 and 307 redirect codes. +* `tornado.curl_httpclient` now accepts non-integer timeouts. + +``tornado.httpserver`` +~~~~~~~~~~~~~~~~~~~~~~ + +* `HTTPServer` with ``xheaders=True`` will no longer accept + ``X-Real-IP`` headers that don't look like valid IP addresses. +* `HTTPServer` now treats the ``Connection`` request header as + case-insensitive. + +``tornado.ioloop`` and ``tornado.iostream`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * `IOStream.write` now works correctly when given an empty string. * `IOStream.read_until` (and ``read_until_regex``) now perform better @@ -57,44 +72,49 @@ Backwards-incompatible changes status codes other than 301 and 302. * New method `RequestHandler.on_finish` may be overridden for post-request processing (as a counterpart to `RequestHandler.prepare`) +* `StaticFileHandler` now outputs ``Content-Length`` and ``Etag`` headers + on ``HEAD`` requests. +* `StaticFileHandler` now has overridable ``get_version`` and + ``parse_url_path`` methods for use in subclasses. +* `RequestHandler.static_url` now takes an ``include_host`` parameter + (in addition to the old support for the `RequestHandler.include_host` + attribute). ``tornado.websocket`` ~~~~~~~~~~~~~~~~~~~~~ * Updated to support the latest version of the protocol, as finalized in RFC 6455. +* Many bugs were fixed in all supported protocol versions. * `tornado.websocket` no longer supports the older "draft 76" version of the websocket protocol by default, although this version can be enabled by overriding `tornado.websocket.WebSocketHandler.allow_draft76`. * `WebSocketHandler.write_message` now accepts a ``binary`` argument to send binary messages. +* Subprotocols (i.e. the ``Sec-WebSocket-Protocol`` header) are now supported; + see the `WebSocketHandler.select_subprotocol` method for details. +* `WebSocketHandler.get_websocket_scheme` can be used to select the + appropriate url scheme (``ws://`` or ``wss://``) in cases where + `HTTPRequest.protocol` is not set correctly. Other modules ~~~~~~~~~~~~~ -* `SimpleAsyncHTTPClient` no longer hangs on ``HEAD`` requests, - responses with no content, or empty ``POST``/``PUT`` response bodies. -* `SimpleAsyncHTTPClient` now supports 303 and 307 redirect codes. +* `tornado.auth.TwitterMixin.authenticate_redirect` now takes a + ``callback_uri`` parameter. +* `tornado.auth.TwitterMixin.twitter_request` now accepts both URLs and + partial paths (complete URLs are useful for the search API which follows + different patterns). +* Exception handling in `tornado.gen` has been improved. It is now possible + to catch exceptions thrown by a ``Task``. +* `tornado.netutil.bind_sockets` now works when ``getaddrinfo`` returns + duplicate addresses. * `tornado.platform.twisted` compatibility has been significantly improved. Twisted version 11.1.0 is now supported in addition to 11.0.0. +* `tornado.process.fork_processes` correctly reseeds the `random` module + even when `os.urandom` is not implemented. * `tornado.testing.main` supports a new flag ``--exception_on_interrupt``, which can be set to false to make ``Ctrl-C`` kill the process more reliably (at the expense of stack traces when it does so). -* `tornado.process.fork_processes` correctly reseeds the `random` module - even when `os.urandom` is not implemented. -* `HTTPServer` with ``xheaders=True`` will no longer accept - ``X-Real-IP`` headers that don't look like valid IP addresses. -* `HTTPServer` now treats the ``Connection`` request header as - case-insensitive. -* Exception handling in `tornado.gen` has been improved. It is now possible - to catch exceptions thrown by a ``Task``. -* `tornado.netutil.bind_sockets` now works when ``getaddrinfo`` returns - duplicate addresses. * `tornado.version_info` is now a four-tuple so official releases can be distinguished from development branches. -* `tornado.curl_httpclient` now accepts non-integer timeouts. -* `tornado.auth.TwitterMixin.authenticate_redirect` now takes a - ``callback_uri`` parameter. -* `tornado.auth.TwitterMixin.twitter_request` now accepts both URLs and - partial paths (complete URLs are useful for the search API which follows - different patterns). From e04957d535b81f1fbc31ff605cc9fca47a62b3b7 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 22 Jan 2012 17:52:43 -0800 Subject: [PATCH 49/53] Doc updates --- tornado/websocket.py | 12 +++++++++--- website/sphinx/web.rst | 1 + website/sphinx/websocket.rst | 26 +++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/tornado/websocket.py b/tornado/websocket.py index 273dbc183c..8aa7777160 100644 --- a/tornado/websocket.py +++ b/tornado/websocket.py @@ -153,8 +153,13 @@ def select_subprotocol(self, subprotocols): """ return None - def open(self, *args, **kwargs): - """Invoked when a new WebSocket is opened.""" + def open(self): + """Invoked when a new WebSocket is opened. + + The arguments to `open` are extracted from the `tornado.web.URLSpec` + regular expression, just like the arguments to + `tornado.web.RequestHandler.get`. + """ pass def on_message(self, message): @@ -206,7 +211,8 @@ def async_callback(self, callback, *args, **kwargs): """Wrap callbacks with this if they are used on asynchronous requests. Catches exceptions properly and closes this WebSocket if an exception - is uncaught. + is uncaught. (Note that this is usually unnecessary thanks to + `tornado.stack_context`) """ return self.ws_connection.async_callback(callback, *args, **kwargs) diff --git a/website/sphinx/web.rst b/website/sphinx/web.rst index 266ababab0..55d69a2a60 100644 --- a/website/sphinx/web.rst +++ b/website/sphinx/web.rst @@ -12,6 +12,7 @@ .. automethod:: RequestHandler.initialize .. automethod:: RequestHandler.prepare + .. automethod:: RequestHandler.on_finish Implement any of the following methods to handle the corresponding HTTP method. diff --git a/website/sphinx/websocket.rst b/website/sphinx/websocket.rst index 7da05071f3..be7bbc542a 100644 --- a/website/sphinx/websocket.rst +++ b/website/sphinx/websocket.rst @@ -4,4 +4,28 @@ .. automodule:: tornado.websocket .. autoclass:: WebSocketHandler - :members: + + Event handlers + -------------- + + .. automethod:: WebSocketHandler.open + .. automethod:: WebSocketHandler.on_message + .. automethod:: WebSocketHandler.on_close + .. automethod:: WebSocketHandler.select_subprotocol + + Output + ------ + + .. automethod:: WebSocketHandler.write_message + .. automethod:: WebSocketHandler.close + + Configuration + ------------- + + .. automethod:: WebSocketHandler.allow_draft76 + .. automethod:: WebSocketHandler.get_websocket_scheme + + Other + ----- + + .. automethod:: WebSocketHandler.async_callback From 463baf434793ca48cc892bcdcb7d3d8c6e8d887a Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Sun, 22 Jan 2012 18:15:01 -0800 Subject: [PATCH 50/53] Add a timeout to test_sslv2_fail so it passes on cygwin. --- tornado/test/httpserver_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tornado/test/httpserver_test.py b/tornado/test/httpserver_test.py index 9a5c201ee5..1a53a34fd6 100644 --- a/tornado/test/httpserver_test.py +++ b/tornado/test/httpserver_test.py @@ -113,7 +113,12 @@ def test_sslv2_fail(self): # we've got all the other ssl version tests here. # Clients should have SSLv2 disabled by default. try: - response = self.fetch('/') + # The server simply closes the connection when it gets + # an SSLv2 ClientHello packet. + # request_timeout is needed here because on some platforms + # (cygwin, but not native windows python), the close is not + # detected promptly. + response = self.fetch('/', request_timeout=1) except ssl.SSLError: # In some python/ssl builds the PROTOCOL_SSLv2 constant # exists but SSLv2 support is still compiled out, which From df23c039d5cf020333cabaf2190495ea7ab0936b Mon Sep 17 00:00:00 2001 From: dave mankoff Date: Mon, 23 Jan 2012 08:33:51 -0500 Subject: [PATCH 51/53] change encode('ascii') to utf8() --- tornado/curl_httpclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tornado/curl_httpclient.py b/tornado/curl_httpclient.py index 7d8faf36e2..a338cb8d66 100644 --- a/tornado/curl_httpclient.py +++ b/tornado/curl_httpclient.py @@ -386,7 +386,7 @@ def ioctl(cmd): if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) - curl.setopt(pycurl.USERPWD, userpwd.encode('ascii')) + curl.setopt(pycurl.USERPWD, utf8(userpwd)) logging.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: From 7fd178840b45a5d949f1dafd2346fbc89b35eec7 Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 23 Jan 2012 09:45:29 -0800 Subject: [PATCH 52/53] Fix gen.engine docs on decorator order. Add some more test cases that use gen.engine and web.asynchronous together. --- tornado/gen.py | 10 ++++++---- tornado/test/gen_test.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/tornado/gen.py b/tornado/gen.py index 74c3bf9d18..51be537601 100644 --- a/tornado/gen.py +++ b/tornado/gen.py @@ -82,10 +82,12 @@ def engine(func): Any generator that yields objects from this module must be wrapped in this decorator. The decorator only works on functions that are already asynchronous. For `~tornado.web.RequestHandler` - ``get``/``post``/etc methods, this means that both the `tornado.gen.engine` - and `tornado.web.asynchronous` decorators must be used (in either order). - In most other cases, it means that it doesn't make sense to use - ``gen.engine`` on functions that don't already take a callback argument. + ``get``/``post``/etc methods, this means that both the + `tornado.web.asynchronous` and `tornado.gen.engine` decorators + must be used (for proper exception handling, ``asynchronous`` + should come before ``gen.engine``). In most other cases, it means + that it doesn't make sense to use ``gen.engine`` on functions that + don't already take a callback argument. """ @functools.wraps(func) def wrapper(*args, **kwargs): diff --git a/tornado/test/gen_test.py b/tornado/test/gen_test.py index 15c30ab6d7..935b409482 100644 --- a/tornado/test/gen_test.py +++ b/tornado/test/gen_test.py @@ -71,6 +71,7 @@ def f(): self.run_gen(f) def test_exception_in_task_phase2(self): + # This is the case that requires the use of stack_context in gen.engine def fail_task(callback): self.io_loop.add_callback(lambda: 1/0) @@ -273,11 +274,36 @@ def get(self): response.rethrow() self.finish(b("got response: ") + response.body) +class GenExceptionHandler(RequestHandler): + @asynchronous + @gen.engine + def get(self): + # This test depends on the order of the two decorators. + io_loop = self.request.connection.stream.io_loop + yield gen.Task(io_loop.add_callback) + raise Exception("oops") + +class GenYieldExceptionHandler(RequestHandler): + @asynchronous + @gen.engine + def get(self): + io_loop = self.request.connection.stream.io_loop + # Test the interaction of the two stack_contexts. + def fail_task(callback): + io_loop.add_callback(lambda: 1/0) + try: + yield gen.Task(fail_task) + raise Exception("did not get expected exception") + except ZeroDivisionError: + self.finish('ok') + class GenWebTest(AsyncHTTPTestCase, LogTrapTestCase): def get_app(self): return Application([ ('/sequence', GenSequenceHandler), ('/task', GenTaskHandler), + ('/exception', GenExceptionHandler), + ('/yield_exception', GenYieldExceptionHandler), ]) def test_sequence_handler(self): @@ -287,3 +313,12 @@ def test_sequence_handler(self): def test_task_handler(self): response = self.fetch('/task?url=%s' % url_escape(self.get_url('/sequence'))) self.assertEqual(response.body, b("got response: 123")) + + def test_exception_handler(self): + # Make sure we get an error and not a timeout + response = self.fetch('/exception') + self.assertEqual(500, response.code) + + def test_yield_exception_handler(self): + response = self.fetch('/yield_exception') + self.assertEqual(response.body, b('ok')) From 067e465b9267d8065647020a3e764d7c6b18179e Mon Sep 17 00:00:00 2001 From: Ben Darnell Date: Mon, 23 Jan 2012 10:20:35 -0800 Subject: [PATCH 53/53] Document curl_httpclient empty password fix. Accept None as auth_password in simple_httpclient for consistency with curl_httpclient. --- tornado/simple_httpclient.py | 2 +- website/sphinx/releases/next.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tornado/simple_httpclient.py b/tornado/simple_httpclient.py index ad5b7e0987..376d410c56 100644 --- a/tornado/simple_httpclient.py +++ b/tornado/simple_httpclient.py @@ -257,7 +257,7 @@ def _on_connect(self, parsed): username, password = parsed.username, parsed.password elif self.request.auth_username is not None: username = self.request.auth_username - password = self.request.auth_password + password = self.request.auth_password or '' if username is not None: auth = utf8(username) + b(":") + utf8(password) self.request.headers["Authorization"] = (b("Basic ") + diff --git a/website/sphinx/releases/next.rst b/website/sphinx/releases/next.rst index fa940c895a..aec080bf98 100644 --- a/website/sphinx/releases/next.rst +++ b/website/sphinx/releases/next.rst @@ -32,6 +32,8 @@ Backwards-incompatible changes responses with no content, or empty ``POST``/``PUT`` response bodies. * `SimpleAsyncHTTPClient` now supports 303 and 307 redirect codes. * `tornado.curl_httpclient` now accepts non-integer timeouts. +* `tornado.curl_httpclient` now supports basic authentication with an + empty password. ``tornado.httpserver`` ~~~~~~~~~~~~~~~~~~~~~~