timeout-start-after cfg for workers with long startup times

Proposed as a solution for issue benoitc#1658

docs/source/settings.rst

@@ -1312,6 +1312,23 @@ you're sure of the repercussions for sync workers. For the non sync
 workers it just means that the worker process is still communicating and
 is not tied to the length of time required to handle a single request.
 
+.. _timeout-start-after:
+
+timeout_start_after
+~~~~~~~~~~~~~~~~~~~
+
+* ``--timeout-start-after INT``
+* ``0``
+
+Wait this many seconds before enforcing timeout on a worker.
+
+Some workers may take a long time to initialize, even though they are
+expected to be highly available once they're ready. Such a worker should
+have a small timeout setting, but Gunicorn needs a way to "forgive" the
+long delay during initialization. When non-zero (zero is the default),
+Gunicorn waits this many seconds after a worker is created before
+enforcing timeout.
+
 .. _graceful-timeout:
 
 graceful_timeout

gunicorn/arbiter.py

@@ -100,6 +100,7 @@ def setup(self, app):
         self.address = self.cfg.address
         self.num_workers = self.cfg.workers
         self.timeout = self.cfg.timeout
+        self.timeout_start_after = self.cfg.timeout_start_after
         self.proc_name = self.cfg.proc_name
 
         self.log.debug('Current configuration:\n{0}'.format(
@@ -494,7 +495,9 @@ def murder_workers(self):
         workers = list(self.WORKERS.items())
         for (pid, worker) in workers:
             try:
-                if time.time() - worker.tmp.last_update() <= self.timeout:
+                now = time.time()
+                grace = now - worker.created < self.timeout_start_after
+                if grace or now - worker.tmp.last_update() <= self.timeout:
                     continue
             except (OSError, ValueError):
                 continue

gunicorn/config.py

@@ -745,6 +745,26 @@ class Timeout(Setting):
         """
 
 
+class TimeoutStartAfter(Setting):
+    name = "timeout_start_after"
+    section = "Worker Processes"
+    cli = ["--timeout-start-after"]
+    meta = "INT"
+    validator = validate_pos_int
+    type = int
+    default = 0
+    desc = """\
+        Wait this many seconds before enforcing timeout on a worker.
+
+        Some workers may take a long time to initialize, even though they are
+        expected to be highly available once they're ready. Such a worker should
+        have a small timeout setting, but Gunicorn needs a way to "forgive" the
+        long delay during initialization. When non-zero (zero is the default),
+        Gunicorn waits this many seconds after a worker is created before
+        enforcing timeout.
+        """
+
+
 class GracefulTimeout(Setting):
     name = "graceful_timeout"
     section = "Worker Processes"

gunicorn/workers/base.py

@@ -49,6 +49,7 @@ def __init__(self, age, ppid, sockets, app, timeout, cfg, log):
         self.booted = False
         self.aborted = False
         self.reloader = None
+        self.created = time.time()
 
         self.nr = 0
 

tests/test_arbiter.py

@@ -4,6 +4,7 @@
 # See the NOTICE for more information.
 
 import os
+import signal
 
 try:
     import unittest.mock as mock
@@ -152,6 +153,52 @@ def test_arbiter_reap_workers(mock_os_waitpid):
     arbiter.cfg.child_exit.assert_called_with(arbiter, mock_worker)
 
 
+@mock.patch('os.kill')
+@mock.patch('time.time')
+def test_arbiter_murder_workers(mock_time, mock_kill):
+    mock_time.side_effect = [1000.0]
+
+    mock_worker = mock.Mock()
+    mock_last_update = mock.Mock()
+    mock_last_update.side_effect = [998.0]
+    mock_worker.tmp.last_update = mock_last_update
+    mock_worker.aborted = False
+    mock_worker.created = 1.0
+
+    arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
+    arbiter.log = mock.Mock()
+    arbiter.timeout = 1
+    arbiter.timeout_start_after = 0
+    arbiter.WORKERS = {42: mock_worker}
+
+    arbiter.murder_workers()
+
+    mock_worker.tmp.last_update.assert_called_with()
+    mock_kill.assert_called_with(42, signal.SIGABRT)
+
+
+@mock.patch('os.kill')
+@mock.patch('time.time')
+def test_arbiter_respects_timeout_start_after(mock_time, mock_kill):
+    mock_time.side_effect = [1000.0]
+
+    mock_worker = mock.Mock()
+    mock_last_update = mock.Mock()
+    mock_worker.tmp.last_update = mock_last_update
+    mock_last_update.side_effect = [991.0]
+    mock_worker.created = 991.0
+
+    arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
+    arbiter.timeout = 1
+    arbiter.timeout_start_after = 10
+    arbiter.WORKERS = {42: mock_worker}
+
+    arbiter.murder_workers()
+
+    mock_worker.tmp.last_update.assert_not_called()
+    mock_kill.assert_not_called()
+
+
 class PreloadedAppWithEnvSettings(DummyApplication):
     """
     Simple application that makes use of the 'preload' feature to

venv/bin/activate

@@ -0,0 +1,76 @@
+# This file must be used with "source bin/activate" *from bash*
+# you cannot run it directly
+
+deactivate () {
+    # reset old environment variables
+    if [ -n "$_OLD_VIRTUAL_PATH" ] ; then
+        PATH="$_OLD_VIRTUAL_PATH"
+        export PATH
+        unset _OLD_VIRTUAL_PATH
+    fi
+    if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then
+        PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
+        export PYTHONHOME
+        unset _OLD_VIRTUAL_PYTHONHOME
+    fi
+
+    # This should detect bash and zsh, which have a hash command that must
+    # be called to get it to forget past commands.  Without forgetting
+    # past commands the $PATH changes we made may not be respected
+    if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
+        hash -r
+    fi
+
+    if [ -n "$_OLD_VIRTUAL_PS1" ] ; then
+        PS1="$_OLD_VIRTUAL_PS1"
+        export PS1
+        unset _OLD_VIRTUAL_PS1
+    fi
+
+    unset VIRTUAL_ENV
+    if [ ! "$1" = "nondestructive" ] ; then
+    # Self destruct!
+        unset -f deactivate
+    fi
+}
+
+# unset irrelevant variables
+deactivate nondestructive
+
+VIRTUAL_ENV="/Users/richardwinslow/dev/forked/gunicorn/venv"
+export VIRTUAL_ENV
+
+_OLD_VIRTUAL_PATH="$PATH"
+PATH="$VIRTUAL_ENV/bin:$PATH"
+export PATH
+
+# unset PYTHONHOME if set
+# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
+# could use `if (set -u; : $PYTHONHOME) ;` in bash
+if [ -n "$PYTHONHOME" ] ; then
+    _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
+    unset PYTHONHOME
+fi
+
+if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then
+    _OLD_VIRTUAL_PS1="$PS1"
+    if [ "x(venv) " != x ] ; then
+	PS1="(venv) $PS1"
+    else
+    if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then
+        # special case for Aspen magic directories
+        # see http://www.zetadev.com/software/aspen/
+        PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1"
+    else
+        PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1"
+    fi
+    fi
+    export PS1
+fi
+
+# This should detect bash and zsh, which have a hash command that must
+# be called to get it to forget past commands.  Without forgetting
+# past commands the $PATH changes we made may not be respected
+if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then
+    hash -r
+fi

venv/bin/activate.csh

@@ -0,0 +1,37 @@
+# This file must be used with "source bin/activate.csh" *from csh*.
+# You cannot run it directly.
+# Created by Davide Di Blasi <davidedb@gmail.com>.
+# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
+
+alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate'
+
+# Unset irrelevant variables.
+deactivate nondestructive
+
+setenv VIRTUAL_ENV "/Users/richardwinslow/dev/forked/gunicorn/venv"
+
+set _OLD_VIRTUAL_PATH="$PATH"
+setenv PATH "$VIRTUAL_ENV/bin:$PATH"
+
+
+set _OLD_VIRTUAL_PROMPT="$prompt"
+
+if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
+    if ("venv" != "") then
+        set env_name = "venv"
+    else
+        if (`basename "VIRTUAL_ENV"` == "__") then
+            # special case for Aspen magic directories
+            # see http://www.zetadev.com/software/aspen/
+            set env_name = `basename \`dirname "$VIRTUAL_ENV"\``
+        else
+            set env_name = `basename "$VIRTUAL_ENV"`
+        endif
+    endif
+    set prompt = "[$env_name] $prompt"
+    unset env_name
+endif
+
+alias pydoc python -m pydoc
+
+rehash

venv/bin/activate.fish

@@ -0,0 +1,75 @@
+# This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org)
+# you cannot run it directly
+
+function deactivate  -d "Exit virtualenv and return to normal shell environment"
+    # reset old environment variables
+    if test -n "$_OLD_VIRTUAL_PATH"
+        set -gx PATH $_OLD_VIRTUAL_PATH
+        set -e _OLD_VIRTUAL_PATH
+    end
+    if test -n "$_OLD_VIRTUAL_PYTHONHOME"
+        set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
+        set -e _OLD_VIRTUAL_PYTHONHOME
+    end
+
+    if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
+        functions -e fish_prompt
+        set -e _OLD_FISH_PROMPT_OVERRIDE
+        functions -c _old_fish_prompt fish_prompt
+        functions -e _old_fish_prompt
+    end
+
+    set -e VIRTUAL_ENV
+    if test "$argv[1]" != "nondestructive"
+        # Self destruct!
+        functions -e deactivate
+    end
+end
+
+# unset irrelevant variables
+deactivate nondestructive
+
+set -gx VIRTUAL_ENV "/Users/richardwinslow/dev/forked/gunicorn/venv"
+
+set -gx _OLD_VIRTUAL_PATH $PATH
+set -gx PATH "$VIRTUAL_ENV/bin" $PATH
+
+# unset PYTHONHOME if set
+if set -q PYTHONHOME
+    set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
+    set -e PYTHONHOME
+end
+
+if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
+    # fish uses a function instead of an env var to generate the prompt.
+
+    # save the current fish_prompt function as the function _old_fish_prompt
+    functions -c fish_prompt _old_fish_prompt
+
+    # with the original prompt function renamed, we can override with our own.
+    function fish_prompt
+        # Save the return status of the last command
+        set -l old_status $status
+
+        # Prompt override?
+        if test -n "(venv) "            
+            printf "%s%s" "(venv) " (set_color normal)
+        else
+            # ...Otherwise, prepend env
+            set -l _checkbase (basename "$VIRTUAL_ENV")
+            if test $_checkbase = "__"
+                # special case for Aspen magic directories
+                # see http://www.zetadev.com/software/aspen/
+                printf "%s[%s]%s " (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal)
+            else
+                printf "%s(%s)%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal)
+            end
+        end
+
+        # Restore the return status of the previous command.
+        echo "exit $old_status" | .
+        _old_fish_prompt
+    end
+
+    set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
+end

venv/pyvenv.cfg

@@ -0,0 +1,3 @@
+home = /Users/richardwinslow/.venv/zrpy3/bin
+include-system-site-packages = false
+version = 3.6.1