Merge pull request #1033 from ccl0326/master

make graceful shutdown thread-safe

gunicorn/workers/gtornado.py

@@ -33,27 +33,40 @@ def clear(self):
     def handle_exit(self, sig, frame):
         if self.alive:
             super(TornadoWorker, self).handle_exit(sig, frame)
-            self.stop()
 
     def handle_request(self):
         self.nr += 1
         if self.alive and self.nr >= self.max_requests:
-            self.alive = False
             self.log.info("Autorestarting worker after current request.")
-            self.stop()
+            self.alive = False
 
     def watchdog(self):
         if self.alive:
             self.notify()
 
         if self.ppid != os.getppid():
             self.log.info("Parent changed, shutting down: %s", self)
-            self.stop()
+            self.alive = False
+
+    def heartbeat(self):
+        if not self.alive:
+            if self.server_alive:
+                if hasattr(self, 'server'):
+                    try:
+                        self.server.stop()
+                    except Exception:
+                        pass
+                self.server_alive = False
+            else:
+                if not self.ioloop._callbacks and len(self.ioloop._timeouts) <= 1:
+                    self.ioloop.stop()
 
     def run(self):
         self.ioloop = IOLoop.instance()
         self.alive = True
+        self.server_alive = False
         PeriodicCallback(self.watchdog, 1000, io_loop=self.ioloop).start()
+        PeriodicCallback(self.heartbeat, 1000, io_loop=self.ioloop).start()
 
         # Assume the app is a WSGI callable if its not an
         # instance of tornado.web.Application or is an
@@ -95,6 +108,7 @@ def on_close(instance, server_conn):
             server = server_class(app, io_loop=self.ioloop)
 
         self.server = server
+        self.server_alive = True
 
         for s in self.sockets:
             s.setblocking(0)
@@ -107,15 +121,3 @@ def on_close(instance, server_conn):
         server.start(num_processes=1)
 
         self.ioloop.start()
-
-    def stop(self):
-        if hasattr(self, 'server'):
-            try:
-                self.server.stop()
-            except Exception:
-                pass
-        PeriodicCallback(self.stop_ioloop, 1000, io_loop=self.ioloop).start()
-
-    def stop_ioloop(self):
-        if not self.ioloop._callbacks and len(self.ioloop._timeouts) <= 1:
-            self.ioloop.stop()