Merge pull request #658 from jolynch/jolynch_just_use_timers_311

Just use timestamps for autostart (3.11)
Netflix · Feb 1, 2018 · c9aa44f · c9aa44f
2 parents 8d0a73d + 8b359eb
commit c9aa44f
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 9 deletions.
diff --git a/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java b/priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java
@@ -83,6 +83,7 @@ public void start(boolean join_ring) throws IOException {
         logger.info("Start cmd: {}", startCass.command());
         logger.info("Start env: {}", startCass.environment());
 
+        instanceState.markLastStartTime();
         instanceState.setShouldCassandraBeAlive(true);
         Process starter = startCass.start();
 

diff --git a/priam/src/main/java/com/netflix/priam/health/InstanceState.java b/priam/src/main/java/com/netflix/priam/health/InstanceState.java
@@ -20,15 +20,13 @@
 import com.google.inject.Singleton;
 import com.netflix.priam.backup.BackupMetadata;
 import com.netflix.priam.backup.Status;
-import com.netflix.priam.utils.DateUtil;
 import com.netflix.priam.utils.GsonJsonSerializer;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.time.LocalDateTime;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 
 /**
  * Contains the state of the health of processed managed by Priam, and
@@ -53,6 +51,7 @@ public enum NODE_STATE {
     //Cassandra process status
     private final AtomicBoolean isCassandraProcessAlive = new AtomicBoolean(false);
     private final AtomicBoolean shouldCassandraBeAlive = new AtomicBoolean(false);
+    private final AtomicLong lastStartTime = new AtomicLong(Long.MAX_VALUE);
     private final AtomicBoolean isGossipActive = new AtomicBoolean(false);
     private final AtomicBoolean isThriftActive = new AtomicBoolean(false);
     private final AtomicBoolean isNativeTransportActive = new AtomicBoolean(false);
@@ -129,6 +128,14 @@ public void setShouldCassandraBeAlive(boolean shouldCassandraBeAlive) {
         this.shouldCassandraBeAlive.set(shouldCassandraBeAlive);
     }
 
+    public void markLastStartTime() {
+        this.lastStartTime.set(System.currentTimeMillis());
+    }
+
+    public long getLastStartTime() {
+        return this.lastStartTime.get();
+    }
+
     /* Boostrap */
     public boolean isBootstrapping() {
         return isBootstrapping.get();

diff --git a/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java b/priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java
@@ -16,7 +16,6 @@
  */
 package com.netflix.priam.utils;
 
-import com.google.common.util.concurrent.RateLimiter;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.netflix.priam.ICassandraProcess;
@@ -47,14 +46,12 @@ public class CassandraMonitor extends Task {
     private static final AtomicBoolean isCassandraStarted = new AtomicBoolean(false);
     private InstanceState instanceState;
     private ICassandraProcess cassProcess;
-    private RateLimiter startRateLimiter;
 
     @Inject
     protected CassandraMonitor(IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess) {
         super(config);
         this.instanceState = instanceState;
         this.cassProcess = cassProcess;
-        startRateLimiter = RateLimiter.create(1.0);
     }
 
     @Override
@@ -109,8 +106,10 @@ public void execute() throws Exception {
             int rate = config.getRemediateDeadCassandraRate();
             if (rate >= 0 && !config.doesCassandraStartManually()) {
                 if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) {
-                    if (rate == 0 || startRateLimiter.tryAcquire(rate)) {
+                    long msNow = System.currentTimeMillis();
+                    if (rate == 0 || ((instanceState.getLastStartTime() + rate * 1000) < msNow)) {
                         cassProcess.start(true);
+                        instanceState.markLastStartTime();
                     }
                 }
             }

diff --git a/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java b/priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java
@@ -109,10 +109,11 @@ NodeProbe instance(IConfiguration config) {
     public void testAutoRemediationRateLimit() throws Exception {
         final InputStream mockOutput = new ByteArrayInputStream("".getBytes());
         instanceState.setShouldCassandraBeAlive(true);
+        instanceState.markLastStartTime();
         new Expectations() {{
             // 6 calls to execute should = 12 calls to getInputStream();
             mockProcess.getInputStream(); result=mockOutput; times=12;
-            cassProcess.start(true); minTimes=2; maxTimes=4;
+            cassProcess.start(true); times=2;
         }};
         // Mock out the ps call
         final Runtime r = Runtime.getRuntime();
@@ -123,14 +124,14 @@ public void testAutoRemediationRateLimit() throws Exception {
             }
         };
         // Sleep ahead to ensure we have permits in the rate limiter
+        monitor.execute();
         Thread.sleep(1500);
         monitor.execute();
         monitor.execute();
         Thread.sleep(1500);
         monitor.execute();
         monitor.execute();
         monitor.execute();
-        monitor.execute();
 
         new Verifications() {};
     }