allowing the configuration of the flapping options - global flag + pe…

…r watcher

circus/config.py

@@ -20,6 +20,7 @@ def watcher_defaults():
         'uid': None,
         'gid': None,
         'send_hup': False,
+        'check_flapping': True,
         'flapping_attempts': 2,
         'flapping_window': 1,
         'retry_in': 7,
@@ -110,8 +111,10 @@ def get_config(config_file):
     config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
                                      'tcp://127.0.0.1:5556')
     config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str)
+    config['check_flapping'] = dget('circus', 'check_flapping', True, bool)
 
     stream_backend = dget('circus', 'stream_backend', 'thread')
+
     if stream_backend == 'gevent':
         try:
             import gevent           # NOQA
@@ -162,6 +165,9 @@ def get_config(config_file):
                 elif opt == 'send_hup':
                     watcher['send_hup'] = dget(section, 'send_hup', False,
                             bool)
+                elif opt == 'check_flapping':
+                    watcher['check_flapping'] = dget(section, 'check_flapping',
+                                                     True, bool)
                 elif opt == 'flapping_attempts':
                     watcher['flapping_attempts'] = dget(section,
                                                         "flapping_attempts", 2,

circus/flapping.py

@@ -113,6 +113,11 @@ def check(self, watcher_name):
         else:
             conf = self.update_conf(watcher_name)
 
+        # if the watcher is not activated, we skip it
+        if not conf['check_flapping']:
+            # nothing to do here
+            return
+
         tries = self.tries.get(watcher_name, 0)
 
         if len(timeline) == conf['flapping_attempts']:

circus/watcher.py

@@ -3,7 +3,7 @@
 import signal
 import time
 
-from psutil import STATUS_ZOMBIE, STATUS_DEAD
+from psutil import STATUS_ZOMBIE, STATUS_DEAD, NoSuchProcess
 from zmq.utils.jsonapi import jsonmod as json
 
 from circus.process import Process
@@ -89,6 +89,9 @@ class Watcher(object):
       process. Can be *thread* or *gevent*. When set to *gevent* you need
       to have *gevent* and *gevent_zmq* installed. (default: thread)
 
+    - **check_flapping** -- if set to False, while the global
+      **check_flapping** option is True, will skip the flapping check for this
+      watcher. (default: True)
     - **flapping_attempts** -- number of times a process can restart before we
       start to detect the flapping (default: 2)
     - **flapping_window** -- the time window in seconds to test for flapping.
@@ -110,7 +113,8 @@ def __init__(self, name, cmd, args=None, numprocesses=1, warmup_delay=0.,
                  max_retry=5,
                  graceful_timeout=30., prereload_fn=None,
                  rlimits=None, executable=None, stdout_stream=None,
-                 stderr_stream=None, stream_backend='thread', priority=0):
+                 stderr_stream=None, stream_backend='thread', priority=0,
+                 check_flapping=True):
         self.name = name
         self.res_name = name.lower().replace(" ", "_")
         self.numprocesses = int(numprocesses)
@@ -131,13 +135,14 @@ def __init__(self, name, cmd, args=None, numprocesses=1, warmup_delay=0.,
         self.stdout_stream = stdout_stream
         self.stderr_stream = stderr_stream
         self.stdout_redirector = self.stderr_redirector = None
+        self.check_flapping = check_flapping
 
         self.optnames = ("numprocesses", "warmup_delay", "working_dir",
                          "uid", "gid", "send_hup", "shell", "env",
                          "cmd", "flapping_attempts", "flapping_window",
                          "retry_in", "args",
                          "max_retry", "graceful_timeout", "executable",
-                         "priority")
+                         "priority", "check_flapping")
 
         if not working_dir:
             # working dir hasn't been set
@@ -200,7 +205,8 @@ def load_from_config(cls, config):
                    stdout_stream=config.get('stdout_stream'),
                    stderr_stream=config.get('stderr_stream'),
                    stream_backend=config.get('stream_backend', 'thread'),
-                   priority=int(config.get('priority', 0)))
+                   priority=int(config.get('priority', 0)),
+                   check_flapping=bool(config.get('check_flapping', True)))
 
     @util.debuglog
     def initialize(self, evpub_socket):
@@ -378,7 +384,12 @@ def kill_process(self, process, sig=signal.SIGTERM):
                                "process_pid": process.pid,
                                "time": time.time()})
         logger.debug("%s: kill process %s", self.name, process.pid)
-        process.send_signal(sig)
+        try:
+            process.send_signal(sig)
+        except NoSuchProcess:
+            # already dead !
+            return
+
         process.stop()
 
     @util.debuglog

docs/source/configuration.rst

@@ -35,6 +35,10 @@ circus (single section)
         The ZMQ PUB/SUB socket receiving publications of stats.
         If not configured, this feature is deactivated.
         (default: *tcp://127.0.0.1:5557*)
+    **check_flapping**
+        If True, Circus will detect "flapping" processes and stop
+        the worker after some failed attempts to start it.
+        (default: True)
     **check_delay**
         The polling interval in seconds for the ZMQ socket. (default: 5)
     **include**
@@ -120,11 +124,15 @@ watcher:NAME (as many sections as you want)
         if True, a process reload will be done by sending the SIGHUP signal.
         Defaults to False.
 
+    **check_flapping**
+      If set to False, while the global **check_flapping** option is True, will skip
+      the flapping check for this wacther. Defaults to True.
+
     **flapping_attempts**
         Number of times a process can restart before we start to
         detect the flapping. Defaults to 2.
 
-    **within**
+    **flapping_window**
         The time window in seconds to test for flapping. If the
         process restarts more than **flapping_attempts**
         times, we consider it a flapping process. Defaults to 1.

docs/source/coverage.rst

@@ -7,8 +7,8 @@ Code coverage
 
     Name                           Stmts   Miss  Cover   Missing
     ------------------------------------------------------------
-    circus/__init__                   30     19    37%   1-13, 80-92, 98
-    circus/arbiter                   180     35    81%   65-71, 75-90, 148-152, 187-188, 193, 204-207, 219, 223-228, 247, 263, 288-289, 293, 303-304
+    circus/__init__                   30     19    37%   1-13, 83-95, 101
+    circus/arbiter                   180     31    83%   65-71, 75-90, 148-152, 187-188, 193, 204-207, 219, 223-228, 247, 263, 293
     circus/client                     52     11    79%   34-35, 39-40, 45-49, 60-61, 73
     circus/commands/addwatcher        24     14    42%   1-66, 73, 78
     circus/commands/base              72     55    24%   1-11, 19, 26, 35-79, 82, 86-97, 103-106
@@ -21,6 +21,7 @@ Code coverage
     circus/commands/options           20     18    10%   1-100, 104-110
     circus/commands/quit               7      6    14%   1-36
     circus/commands/reload            17     15    12%   1-68, 70-71
+    circus/commands/restart           15     13    13%   1-56, 58-59
     circus/commands/rmwatcher         12     10    17%   1-54
     circus/commands/sendsignal        47     33    30%   1-109, 114, 118, 124, 127, 130, 138-147
     circus/commands/set               34     22    35%   1-59, 70, 75
@@ -29,23 +30,23 @@ Code coverage
     circus/commands/status            23     20    13%   1-65, 70-80
     circus/commands/stop              12      8    33%   1-50
     circus/commands/util              54     42    22%   1-38, 43, 47, 52, 55-56, 59-60, 64
-    circus/config                    124    112    10%   11, 37-46, 50-68, 72-99, 103-205
+    circus/config                    127     60    53%   44-47, 59, 62-65, 76-100, 119-131, 147, 149, 152, 155, 158, 160, 165-194
     circus/controller                116     15    87%   75, 85-86, 94-96, 104, 116-119, 122, 145, 151, 156-157
-    circus/flapping                  111     20    82%   50-60, 103-107, 131, 137-144
-    circus/process                   119     40    66%   3-9, 92, 97, 100-120, 133, 147, 185-186, 190, 196, 202, 208-211, 216-221, 234-235, 239, 254
+    circus/flapping                  113     21    81%   50-60, 103-107, 118, 135, 141-148
+    circus/process                   119     39    67%   3-9, 92, 97, 100-120, 133, 147, 185-186, 190, 196, 202, 208-211, 216-221, 234-235, 239
     circus/py3compat                  47     44     6%   1-38, 43-67
     circus/sighandler                 36     16    56%   34-44, 47, 50, 53, 56, 59
     circus/stats/__init__             35     23    34%   33-73, 77
     circus/stats/collector            95     77    19%   11-17, 20, 23-40, 43-65, 68, 73-75, 78, 84-90, 93-139, 142-145
     circus/stats/publisher            43     34    21%   12-22, 25-44, 47-50
     circus/stats/streamer            117     92    21%   21-39, 42, 45-47, 51-53, 57-69, 72-75, 78-82, 85-110, 113-135, 138-142
-    circus/stream/__init__            35     11    69%   16, 21-22, 25-26, 29, 34, 37-38, 41, 68
-    circus/stream/base                61     10    84%   22, 39, 55-56, 61-62, 71-74
+    circus/stream/__init__            35      7    80%   16, 29, 34, 37-38, 41, 68
+    circus/stream/base                64     11    83%   22, 39, 51, 58-59, 64-65, 74-77
     circus/stream/sthread             19      0   100%   
     circus/util                      205    119    42%   1-55, 59-62, 68-70, 76, 90-97, 103-111, 116-117, 133-134, 144-145, 149-150, 154-157, 161-162, 168, 173, 182, 191, 204, 212, 224, 232, 234, 238-244, 250-255, 260-313
-    circus/watcher                   297     72    76%   141, 207, 233, 237, 258, 262, 265-266, 271, 298, 314, 334, 342-343, 346-347, 355, 365, 383-385, 395-397, 403-408, 414-415, 425-426, 443, 462, 471, 480-483, 490, 493, 496-498, 520, 536, 538-539, 541-542, 544-545, 547, 549-550, 554-568
+    circus/watcher                   305     69    77%   166, 176, 223, 249, 253, 274, 278, 281-282, 287, 314, 330, 358-359, 362-363, 371, 399-401, 411-413, 419-424, 430-431, 441-442, 478, 498-501, 508, 511, 514-516, 538, 554, 556-557, 559-560, 562-563, 565, 567-568, 572-586
     circus/web/__init__                0      0   100%   
     ------------------------------------------------------------
-    TOTAL                           2221   1143    49%   
+    TOTAL                           2252   1094    51%   
 
 

examples/example3.ini

@@ -0,0 +1,19 @@
+[circus]
+check_delay = 5
+endpoint = tcp://127.0.0.1:5555
+pubsub_endpoint = tcp://127.0.0.1:5556
+
+
+[watcher:dying]
+cmd = bash test.sh
+warmup_delay = 0
+numprocesses = 1
+check_flapping = False
+
+[watcher:dying2]
+cmd = bash test.sh
+warmup_delay = 0
+numprocesses = 1
+check_flapping = True
+retry_in = 1
+max_retry = 2

examples/test.sh

@@ -0,0 +1 @@
+;sleep 1