Merge pull request #6639 from vespa-engine/arnej/delay-crash-loops-more

more restart penalty
vespa-engine · Aug 21, 2018 · d914bbe · d914bbe
2 parents 7f3bd67 + bd87e1b
commit d914bbe
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 7 deletions.
diff --git a/configd/src/apps/sentinel/service.cpp b/configd/src/apps/sentinel/service.cpp
@@ -167,10 +167,13 @@ Service::start()
     // make sure the service does not restart in a tight loop:
     time_t now = time(0);
     int diff = now - _last_start;
-    if (diff < 10) {
+    if (diff < MAX_RESTART_PENALTY) {
         incrementRestartPenalty();
-        now += _restartPenalty; // will delay start this much
     }
+    if (diff > 10 * MAX_RESTART_PENALTY) {
+        resetRestartPenalty();
+    }
+    now += _restartPenalty; // will delay start this much
     _last_start = now;
 
 // make a pipe, close the good ends of it, mark it close-on-exec
@@ -230,7 +233,7 @@ Service::start()
             kill(getpid(), SIGTERM);
         }
         if (_restartPenalty > 0) {
-            LOG(debug, "%s: Applying %u sec restart penalty", name().c_str(),
+            LOG(info, "%s: Applying %u sec restart penalty", name().c_str(),
                 _restartPenalty);
             sleep(_restartPenalty);
         }
@@ -423,9 +426,9 @@ Service::setAutomatic(bool autoStatus)
 void
 Service::incrementRestartPenalty()
 {
-    if (_restartPenalty < MAX_RESTART_PENALTY) {
-        _restartPenalty++;
-    } else {
+    _restartPenalty += 1;
+    _restartPenalty *= 2;
+    if (_restartPenalty > MAX_RESTART_PENALTY) {
         _restartPenalty = MAX_RESTART_PENALTY;
     }
 }

diff --git a/configd/src/apps/sentinel/service.h b/configd/src/apps/sentinel/service.h
@@ -26,7 +26,7 @@ class Service
     SentinelConfig::Service *_config;
     bool _isAutomatic;
 
-    static const unsigned int MAX_RESTART_PENALTY = 60;
+    static const int MAX_RESTART_PENALTY = 1800;
     unsigned int _restartPenalty;
     time_t _last_start;