Merge branch 'master' into append-handlers

tornado/auth.py

@@ -95,7 +95,7 @@ def get_authenticated_user(self, callback, http_client=None):
         args["openid.mode"] = u"check_authentication"
         url = self._OPENID_ENDPOINT
         if http_client is None:
-            http_client = httpclient.AsyncHTTPClient()
+            http_client = self.get_auth_http_client()
         http_client.fetch(url, self.async_callback(
             self._on_authentication_verified, callback),
             method="POST", body=urllib.urlencode(args))
@@ -208,6 +208,14 @@ def get_ax_arg(uri):
             user["claimed_id"] = claimed_id
         callback(user)
 
+    def get_auth_http_client(self):
+        """Returns the AsyncHTTPClient instance to be used for auth requests.
+
+        May be overridden by subclasses to use an http client other than
+        the default.
+        """
+        return httpclient.AsyncHTTPClient()
+
 
 class OAuthMixin(object):
     """Abstract implementation of OAuth.
@@ -232,7 +240,7 @@ def authorize_redirect(self, callback_uri=None, extra_params=None,
         if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False):
             raise Exception("This service does not support oauth_callback")
         if http_client is None:
-            http_client = httpclient.AsyncHTTPClient()
+            http_client = self.get_auth_http_client()
         if getattr(self, "_OAUTH_VERSION", "1.0a") == "1.0a":
             http_client.fetch(
                 self._oauth_request_token_url(callback_uri=callback_uri,
@@ -277,7 +285,7 @@ def get_authenticated_user(self, callback, http_client=None):
         if oauth_verifier:
             token["verifier"] = oauth_verifier
         if http_client is None:
-            http_client = httpclient.AsyncHTTPClient()
+            http_client = self.get_auth_http_client()
         http_client.fetch(self._oauth_access_token_url(token),
                           self.async_callback(self._on_access_token, callback))
 
@@ -394,6 +402,14 @@ def _oauth_request_parameters(self, url, access_token, parameters={},
         base_args["oauth_signature"] = signature
         return base_args
 
+    def get_auth_http_client(self):
+        """Returns the AsyncHTTPClient instance to be used for auth requests.
+
+        May be overridden by subclasses to use an http client other than
+        the default.
+        """
+        return httpclient.AsyncHTTPClient()
+
 
 class OAuth2Mixin(object):
     """Abstract implementation of OAuth v 2."""
@@ -471,14 +487,15 @@ def _on_auth(self, user):
     _OAUTH_AUTHORIZE_URL = "http://api.twitter.com/oauth/authorize"
     _OAUTH_AUTHENTICATE_URL = "http://api.twitter.com/oauth/authenticate"
     _OAUTH_NO_CALLBACKS = False
+    _TWITTER_BASE_URL = "http://api.twitter.com/1"
 
     def authenticate_redirect(self, callback_uri=None):
         """Just like authorize_redirect(), but auto-redirects if authorized.
 
         This is generally the right interface to use if you are using
         Twitter for single-sign on.
         """
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         http.fetch(self._oauth_request_token_url(callback_uri=callback_uri), self.async_callback(
             self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None))
 
@@ -525,7 +542,7 @@ def _on_post(self, new_entry):
             # usual pattern: http://search.twitter.com/search.json
             url = path
         else:
-            url = "http://api.twitter.com/1" + path + ".json"
+            url = self._TWITTER_BASE_URL + path + ".json"
         # Add the OAuth resource request signature if we have credentials
         if access_token:
             all_args = {}
@@ -538,7 +555,7 @@ def _on_post(self, new_entry):
         if args:
             url += "?" + urllib.urlencode(args)
         callback = self.async_callback(self._on_twitter_request, callback)
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib.urlencode(post_args),
                        callback=callback)
@@ -563,7 +580,7 @@ def _oauth_consumer_token(self):
     def _oauth_get_user(self, access_token, callback):
         callback = self.async_callback(self._parse_user_response, callback)
         self.twitter_request(
-            "/users/show/" + access_token["screen_name"],
+            "/users/show/" + escape.native_str(access_token[b("screen_name")]),
             access_token=access_token, callback=callback)
 
     def _parse_user_response(self, callback, user):
@@ -660,7 +677,7 @@ def _on_post(self, new_entry):
         if args:
             url += "?" + urllib.urlencode(args)
         callback = self.async_callback(self._on_friendfeed_request, callback)
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib.urlencode(post_args),
                        callback=callback)
@@ -751,7 +768,7 @@ def get_authenticated_user(self, callback):
                 break
         token = self.get_argument("openid." + oauth_ns + ".request_token", "")
         if token:
-            http = httpclient.AsyncHTTPClient()
+            http = self.get_auth_http_client()
             token = dict(key=token, secret="")
             http.fetch(self._oauth_access_token_url(token),
                        self.async_callback(self._on_access_token, callback))
@@ -907,7 +924,7 @@ def _on_stream(self, stream):
         args["sig"] = self._signature(args)
         url = "http://api.facebook.com/restserver.php?" + \
             urllib.urlencode(args)
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         http.fetch(url, callback=self.async_callback(
             self._parse_response, callback))
 
@@ -953,6 +970,14 @@ def _signature(self, args):
             body = body.encode("utf-8")
         return hashlib.md5(body).hexdigest()
 
+    def get_auth_http_client(self):
+        """Returns the AsyncHTTPClient instance to be used for auth requests.
+
+        May be overridden by subclasses to use an http client other than
+        the default.
+        """
+        return httpclient.AsyncHTTPClient()
+
 
 class FacebookGraphMixin(OAuth2Mixin):
     """Facebook authentication using the new Graph API and OAuth2."""
@@ -987,7 +1012,7 @@ def _on_login(self, user):
                 self.finish()
 
         """
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         args = {
           "redirect_uri": redirect_uri,
           "code": code,
@@ -1081,7 +1106,7 @@ def _on_post(self, new_entry):
         if all_args:
             url += "?" + urllib.urlencode(all_args)
         callback = self.async_callback(self._on_facebook_request, callback)
-        http = httpclient.AsyncHTTPClient()
+        http = self.get_auth_http_client()
         if post_args is not None:
             http.fetch(url, method="POST", body=urllib.urlencode(post_args),
                        callback=callback)
@@ -1096,6 +1121,14 @@ def _on_facebook_request(self, callback, response):
             return
         callback(escape.json_decode(response.body))
 
+    def get_auth_http_client(self):
+        """Returns the AsyncHTTPClient instance to be used for auth requests.
+
+        May be overridden by subclasses to use an http client other than
+        the default.
+        """
+        return httpclient.AsyncHTTPClient()
+
 
 def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
     """Calculates the HMAC-SHA1 OAuth signature for the given request.

tornado/gen.py

@@ -374,6 +374,7 @@ def run(self):
                             "finished without waiting for callbacks %r" %
                             self.pending_callbacks)
                     self.deactivate_stack_context()
+                    self.deactivate_stack_context = None
                     return
                 except Exception:
                     self.finished = True

tornado/httpclient.py

@@ -38,9 +38,9 @@
 import weakref
 
 from tornado.escape import utf8
-from tornado import httputil
+from tornado import httputil, stack_context
 from tornado.ioloop import IOLoop
-from tornado.util import import_object, bytes_type, Configurable
+from tornado.util import import_object, Configurable
 
 
 class HTTPClient(object):
@@ -232,8 +232,13 @@ def __init__(self, url, method="GET", headers=None, body=None,
            `~HTTPResponse.body` and `~HTTPResponse.buffer` will be empty in
            the final response.
         :arg callable header_callback: If set, `header_callback` will
-           be run with each header line as it is received, and
-           `~HTTPResponse.headers` will be empty in the final response.
+           be run with each header line as it is received (including the
+           first line, e.g. ``HTTP/1.0 200 OK\r\n``, and a final line
+           containing only ``\r\n``.  All lines include the trailing newline
+           characters).  `~HTTPResponse.headers` will be empty in the final
+           response.  This is most useful in conjunction with
+           `streaming_callback`, because it's the only way to get access to
+           header data while the request is in progress.
         :arg callable prepare_curl_callback: If set, will be called with
            a `pycurl.Curl` object to allow the application to make additional
            `setopt` calls.
@@ -281,9 +286,9 @@ def __init__(self, url, method="GET", headers=None, body=None,
         self.user_agent = user_agent
         self.use_gzip = use_gzip
         self.network_interface = network_interface
-        self.streaming_callback = streaming_callback
-        self.header_callback = header_callback
-        self.prepare_curl_callback = prepare_curl_callback
+        self.streaming_callback = stack_context.wrap(streaming_callback)
+        self.header_callback = stack_context.wrap(header_callback)
+        self.prepare_curl_callback = stack_context.wrap(prepare_curl_callback)
         self.allow_nonstandard_methods = allow_nonstandard_methods
         self.validate_cert = validate_cert
         self.ca_certs = ca_certs

tornado/ioloop.py

@@ -406,6 +406,7 @@ def initialize(self, impl, time_func=None):
         self._timeouts = []
         self._running = False
         self._stopped = False
+        self._closing = False
         self._thread_ident = None
         self._blocking_signal_threshold = None
 
@@ -417,6 +418,8 @@ def initialize(self, impl, time_func=None):
                          self.READ)
 
     def close(self, all_fds=False):
+        with self._callback_lock:
+            self._closing = True
         self.remove_handler(self._waker.fileno())
         if all_fds:
             for fd in self._handlers.keys()[:]:
@@ -608,6 +611,8 @@ def remove_timeout(self, timeout):
 
     def add_callback(self, callback):
         with self._callback_lock:
+            if self._closing:
+                raise RuntimeError("IOLoop is closing")
             list_empty = not self._callbacks
             self._callbacks.append(stack_context.wrap(callback))
         if list_empty and thread.get_ident() != self._thread_ident:

tornado/iostream.py

@@ -162,8 +162,14 @@ def read_until_close(self, callback, streaming_callback=None):
         a ``streaming_callback`` is not used.
         """
         self._set_read_callback(callback)
+        self._streaming_callback = stack_context.wrap(streaming_callback)
         if self.closed():
-            self._run_callback(callback, self._consume(self._read_buffer_size))
+            if self._streaming_callback is not None:
+                self._run_callback(self._streaming_callback,
+                                   self._consume(self._read_buffer_size))
+            self._run_callback(self._read_callback,
+                               self._consume(self._read_buffer_size))
+            self._streaming_callback = None
             self._read_callback = None
             return
         self._read_until_close = True

tornado/simple_httpclient.py

@@ -365,8 +365,11 @@ def _on_headers(self, data):
             content_length = None
 
         if self.request.header_callback is not None:
+            # re-attach the newline we split on earlier
+            self.request.header_callback(first_line + _)
             for k, v in self.headers.get_all():
                 self.request.header_callback("%s: %s\r\n" % (k, v))
+            self.request.header_callback('\r\n')
 
         if self.request.method == "HEAD":
             # HEAD requests never have content, even though they may have

tornado/stack_context.py

@@ -160,6 +160,7 @@ def __exit__(self, type, value, traceback):
                 return self.exception_handler(type, value, traceback)
         finally:
             _state.contexts = self.old_contexts
+            self.old_contexts = None
 
 
 class NullContext(object):

tornado/test/auth_test.py

@@ -5,7 +5,7 @@
 
 
 from __future__ import absolute_import, division, with_statement
-from tornado.auth import OpenIdMixin, OAuthMixin, OAuth2Mixin
+from tornado.auth import OpenIdMixin, OAuthMixin, OAuth2Mixin, TwitterMixin
 from tornado.escape import json_decode
 from tornado.testing import AsyncHTTPTestCase
 from tornado.util import b
@@ -101,6 +101,37 @@ def get(self):
         self.authorize_redirect()
 
 
+class TwitterClientLoginHandler(RequestHandler, TwitterMixin):
+    def initialize(self, test):
+        self._OAUTH_REQUEST_TOKEN_URL = test.get_url('/oauth1/server/request_token')
+        self._OAUTH_ACCESS_TOKEN_URL = test.get_url('/twitter/server/access_token')
+        self._OAUTH_AUTHORIZE_URL = test.get_url('/oauth1/server/authorize')
+        self._TWITTER_BASE_URL = test.get_url('/twitter/api')
+
+    @asynchronous
+    def get(self):
+        if self.get_argument("oauth_token", None):
+            self.get_authenticated_user(self.on_user)
+            return
+        self.authorize_redirect()
+
+    def on_user(self, user):
+        if user is None:
+            raise Exception("user is None")
+        self.finish(user)
+
+    def get_auth_http_client(self):
+        return self.settings['http_client']
+
+
+class TwitterServerAccessTokenHandler(RequestHandler):
+    def get(self):
+        self.write('oauth_token=hjkl&oauth_token_secret=vbnm&screen_name=foo')
+
+class TwitterServerShowUserHandler(RequestHandler):
+    def get(self, screen_name):
+        self.write(dict(screen_name=screen_name, name=screen_name.capitalize()))
+
 class AuthTest(AsyncHTTPTestCase):
     def get_app(self):
         return Application(
@@ -119,12 +150,19 @@ def get_app(self):
                  dict(version='1.0a')),
                 ('/oauth2/client/login', OAuth2ClientLoginHandler, dict(test=self)),
 
+                ('/twitter/client/login', TwitterClientLoginHandler, dict(test=self)),
+
                 # simulated servers
                 ('/openid/server/authenticate', OpenIdServerAuthenticateHandler),
                 ('/oauth1/server/request_token', OAuth1ServerRequestTokenHandler),
                 ('/oauth1/server/access_token', OAuth1ServerAccessTokenHandler),
+
+                ('/twitter/server/access_token', TwitterServerAccessTokenHandler),
+                (r'/twitter/api/users/show/(.*)\.json', TwitterServerShowUserHandler),
                 ],
-            http_client=self.http_client)
+            http_client=self.http_client,
+            twitter_consumer_key='test_twitter_consumer_key',
+            twitter_consumer_secret='test_twitter_consumer_secret')
 
     def test_openid_redirect(self):
         response = self.fetch('/openid/client/login', follow_redirects=False)
@@ -198,3 +236,28 @@ def test_oauth2_redirect(self):
         response = self.fetch('/oauth2/client/login', follow_redirects=False)
         self.assertEqual(response.code, 302)
         self.assertTrue('/oauth2/server/authorize?' in response.headers['Location'])
+
+    def test_twitter_redirect(self):
+        # Same as test_oauth10a_redirect
+        response = self.fetch('/twitter/client/login', follow_redirects=False)
+        self.assertEqual(response.code, 302)
+        self.assertTrue(response.headers['Location'].endswith(
+            '/oauth1/server/authorize?oauth_token=zxcv'))
+        # the cookie is base64('zxcv')|base64('1234')
+        self.assertTrue(
+            '_oauth_request_token="enhjdg==|MTIzNA=="' in response.headers['Set-Cookie'],
+            response.headers['Set-Cookie'])
+
+    def test_twitter_get_user(self):
+        response = self.fetch(
+            '/twitter/client/login?oauth_token=zxcv',
+            headers={'Cookie': '_oauth_request_token=enhjdg==|MTIzNA=='})
+        response.rethrow()
+        parsed = json_decode(response.body)
+        self.assertEqual(parsed,
+                         {u'access_token': {u'key': u'hjkl',
+                                            u'screen_name': u'foo',
+                                            u'secret': u'vbnm'},
+                          u'name': u'Foo',
+                          u'screen_name': u'foo',
+                          u'username': u'foo'})