timeout-start-after cfg for workers with long startup times

Proposed as a solution for issue benoitc#1658

docs/source/settings.rst

@@ -1318,6 +1318,23 @@ you're sure of the repercussions for sync workers. For the non sync
 workers it just means that the worker process is still communicating and
 is not tied to the length of time required to handle a single request.
 
+.. _timeout-start-after:
+
+timeout_start_after
+~~~~~~~~~~~~~~~~~~~
+
+* ``--timeout-start-after INT``
+* ``0``
+
+Wait this many seconds before enforcing timeout on a worker.
+
+Some workers may take a long time to initialize, even though they are
+expected to be highly available once they're ready. Such a worker should
+have a small timeout setting, but Gunicorn needs a way to "forgive" the
+long delay during initialization. When non-zero (zero is the default),
+Gunicorn waits this many seconds after a worker is created before
+enforcing timeout.
+
 .. _graceful-timeout:
 
 graceful_timeout

gunicorn/arbiter.py

@@ -100,6 +100,7 @@ def setup(self, app):
         self.address = self.cfg.address
         self.num_workers = self.cfg.workers
         self.timeout = self.cfg.timeout
+        self.timeout_start_after = self.cfg.timeout_start_after
         self.proc_name = self.cfg.proc_name
 
         self.log.debug('Current configuration:\n{0}'.format(
@@ -492,7 +493,9 @@ def murder_workers(self):
         workers = list(self.WORKERS.items())
         for (pid, worker) in workers:
             try:
-                if time.time() - worker.tmp.last_update() <= self.timeout:
+                now = time.time()
+                grace = now - worker.created < self.timeout_start_after
+                if grace or now - worker.tmp.last_update() <= self.timeout:
                     continue
             except (OSError, ValueError):
                 continue

gunicorn/config.py

@@ -745,6 +745,26 @@ class Timeout(Setting):
         """
 
 
+class TimeoutStartAfter(Setting):
+    name = "timeout_start_after"
+    section = "Worker Processes"
+    cli = ["--timeout-start-after"]
+    meta = "INT"
+    validator = validate_pos_int
+    type = int
+    default = 0
+    desc = """\
+        Wait this many seconds before enforcing timeout on a worker.
+
+        Some workers may take a long time to initialize, even though they are
+        expected to be highly available once they're ready. Such a worker should
+        have a small timeout setting, but Gunicorn needs a way to "forgive" the
+        long delay during initialization. When non-zero (zero is the default),
+        Gunicorn waits this many seconds after a worker is created before
+        enforcing timeout.
+        """
+
+
 class GracefulTimeout(Setting):
     name = "graceful_timeout"
     section = "Worker Processes"

gunicorn/workers/base.py

@@ -49,6 +49,7 @@ def __init__(self, age, ppid, sockets, app, timeout, cfg, log):
         self.booted = False
         self.aborted = False
         self.reloader = None
+        self.created = time.time()
 
         self.nr = 0
 

tests/test_arbiter.py

@@ -4,6 +4,7 @@
 # See the NOTICE for more information.
 
 import os
+import signal
 import unittest.mock as mock
 
 import gunicorn.app.base
@@ -148,6 +149,52 @@ def test_arbiter_reap_workers(mock_os_waitpid):
     arbiter.cfg.child_exit.assert_called_with(arbiter, mock_worker)
 
 
+@mock.patch('os.kill')
+@mock.patch('time.time')
+def test_arbiter_murder_workers(mock_time, mock_kill):
+    mock_time.side_effect = [1000.0]
+
+    mock_worker = mock.Mock()
+    mock_last_update = mock.Mock()
+    mock_last_update.side_effect = [998.0]
+    mock_worker.tmp.last_update = mock_last_update
+    mock_worker.aborted = False
+    mock_worker.created = 1.0
+
+    arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
+    arbiter.log = mock.Mock()
+    arbiter.timeout = 1
+    arbiter.timeout_start_after = 0
+    arbiter.WORKERS = {42: mock_worker}
+
+    arbiter.murder_workers()
+
+    mock_worker.tmp.last_update.assert_called_with()
+    mock_kill.assert_called_with(42, signal.SIGABRT)
+
+
+@mock.patch('os.kill')
+@mock.patch('time.time')
+def test_arbiter_respects_timeout_start_after(mock_time, mock_kill):
+    mock_time.side_effect = [1000.0]
+
+    mock_worker = mock.Mock()
+    mock_last_update = mock.Mock()
+    mock_worker.tmp.last_update = mock_last_update
+    mock_last_update.side_effect = [991.0]
+    mock_worker.created = 991.0
+
+    arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
+    arbiter.timeout = 1
+    arbiter.timeout_start_after = 10
+    arbiter.WORKERS = {42: mock_worker}
+
+    arbiter.murder_workers()
+
+    mock_worker.tmp.last_update.assert_not_called()
+    mock_kill.assert_not_called()
+
+
 class PreloadedAppWithEnvSettings(DummyApplication):
     """
     Simple application that makes use of the 'preload' feature to