diff --git a/docs/source/settings.rst b/docs/source/settings.rst index 16d8961aee..c24594b6d3 100644 --- a/docs/source/settings.rst +++ b/docs/source/settings.rst @@ -1318,6 +1318,23 @@ you're sure of the repercussions for sync workers. For the non sync workers it just means that the worker process is still communicating and is not tied to the length of time required to handle a single request. +.. _timeout-start-after: + +timeout_start_after +~~~~~~~~~~~~~~~~~~~ + +* ``--timeout-start-after INT`` +* ``0`` + +Wait this many seconds before enforcing timeout on a worker. + +Some workers may take a long time to initialize, even though they are +expected to be highly available once they're ready. Such a worker should +have a small timeout setting, but Gunicorn needs a way to "forgive" the +long delay during initialization. When non-zero (zero is the default), +Gunicorn waits this many seconds after a worker is created before +enforcing timeout. + .. _graceful-timeout: graceful_timeout diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index bca671d17b..3a667062e0 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -100,6 +100,7 @@ def setup(self, app): self.address = self.cfg.address self.num_workers = self.cfg.workers self.timeout = self.cfg.timeout + self.timeout_start_after = self.cfg.timeout_start_after self.proc_name = self.cfg.proc_name self.log.debug('Current configuration:\n{0}'.format( @@ -492,7 +493,9 @@ def murder_workers(self): workers = list(self.WORKERS.items()) for (pid, worker) in workers: try: - if time.time() - worker.tmp.last_update() <= self.timeout: + now = time.time() + grace = now - worker.created < self.timeout_start_after + if grace or now - worker.tmp.last_update() <= self.timeout: continue except (OSError, ValueError): continue diff --git a/gunicorn/config.py b/gunicorn/config.py index e8e0f926a5..fd7bd8decb 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -745,6 +745,26 @@ class Timeout(Setting): """ +class TimeoutStartAfter(Setting): + name = "timeout_start_after" + section = "Worker Processes" + cli = ["--timeout-start-after"] + meta = "INT" + validator = validate_pos_int + type = int + default = 0 + desc = """\ + Wait this many seconds before enforcing timeout on a worker. + + Some workers may take a long time to initialize, even though they are + expected to be highly available once they're ready. Such a worker should + have a small timeout setting, but Gunicorn needs a way to "forgive" the + long delay during initialization. When non-zero (zero is the default), + Gunicorn waits this many seconds after a worker is created before + enforcing timeout. + """ + + class GracefulTimeout(Setting): name = "graceful_timeout" section = "Worker Processes" diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index f95994bc78..bdcd9b77e3 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -49,6 +49,7 @@ def __init__(self, age, ppid, sockets, app, timeout, cfg, log): self.booted = False self.aborted = False self.reloader = None + self.created = time.time() self.nr = 0 diff --git a/tests/test_arbiter.py b/tests/test_arbiter.py index dc059edb7b..60fea31431 100644 --- a/tests/test_arbiter.py +++ b/tests/test_arbiter.py @@ -4,6 +4,7 @@ # See the NOTICE for more information. import os +import signal import unittest.mock as mock import gunicorn.app.base @@ -148,6 +149,52 @@ def test_arbiter_reap_workers(mock_os_waitpid): arbiter.cfg.child_exit.assert_called_with(arbiter, mock_worker) +@mock.patch('os.kill') +@mock.patch('time.time') +def test_arbiter_murder_workers(mock_time, mock_kill): + mock_time.side_effect = [1000.0] + + mock_worker = mock.Mock() + mock_last_update = mock.Mock() + mock_last_update.side_effect = [998.0] + mock_worker.tmp.last_update = mock_last_update + mock_worker.aborted = False + mock_worker.created = 1.0 + + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.log = mock.Mock() + arbiter.timeout = 1 + arbiter.timeout_start_after = 0 + arbiter.WORKERS = {42: mock_worker} + + arbiter.murder_workers() + + mock_worker.tmp.last_update.assert_called_with() + mock_kill.assert_called_with(42, signal.SIGABRT) + + +@mock.patch('os.kill') +@mock.patch('time.time') +def test_arbiter_respects_timeout_start_after(mock_time, mock_kill): + mock_time.side_effect = [1000.0] + + mock_worker = mock.Mock() + mock_last_update = mock.Mock() + mock_worker.tmp.last_update = mock_last_update + mock_last_update.side_effect = [991.0] + mock_worker.created = 991.0 + + arbiter = gunicorn.arbiter.Arbiter(DummyApplication()) + arbiter.timeout = 1 + arbiter.timeout_start_after = 10 + arbiter.WORKERS = {42: mock_worker} + + arbiter.murder_workers() + + mock_worker.tmp.last_update.assert_not_called() + mock_kill.assert_not_called() + + class PreloadedAppWithEnvSettings(DummyApplication): """ Simple application that makes use of the 'preload' feature to