Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Just use timestamps for autostart (3.11) #658

Merged
merged 1 commit into from
Feb 1, 2018
Merged

Just use timestamps for autostart (3.11) #658

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions priam/src/main/java/com/netflix/priam/defaultimpl/CassandraProcessManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public void start(boolean join_ring) throws IOException {
logger.info("Start cmd: {}", startCass.command());
logger.info("Start env: {}", startCass.environment());

instanceState.markLastStartTime();
instanceState.setShouldCassandraBeAlive(true);
Process starter = startCass.start();

Expand Down
13 changes: 10 additions & 3 deletions priam/src/main/java/com/netflix/priam/health/InstanceState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,13 @@
import com.google.inject.Singleton;
import com.netflix.priam.backup.BackupMetadata;
import com.netflix.priam.backup.Status;
import com.netflix.priam.utils.DateUtil;
import com.netflix.priam.utils.GsonJsonSerializer;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.time.LocalDateTime;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

/**
* Contains the state of the health of processed managed by Priam, and
Expand All @@ -53,6 +51,7 @@ public enum NODE_STATE {
//Cassandra process status
private final AtomicBoolean isCassandraProcessAlive = new AtomicBoolean(false);
private final AtomicBoolean shouldCassandraBeAlive = new AtomicBoolean(false);
private final AtomicLong lastStartTime = new AtomicLong(Long.MAX_VALUE);
private final AtomicBoolean isGossipActive = new AtomicBoolean(false);
private final AtomicBoolean isThriftActive = new AtomicBoolean(false);
private final AtomicBoolean isNativeTransportActive = new AtomicBoolean(false);
Expand Down Expand Up @@ -129,6 +128,14 @@ public void setShouldCassandraBeAlive(boolean shouldCassandraBeAlive) {
this.shouldCassandraBeAlive.set(shouldCassandraBeAlive);
}

public void markLastStartTime() {
this.lastStartTime.set(System.currentTimeMillis());
}

public long getLastStartTime() {
return this.lastStartTime.get();
}

/* Boostrap */
public boolean isBootstrapping() {
return isBootstrapping.get();
Expand Down
7 changes: 3 additions & 4 deletions priam/src/main/java/com/netflix/priam/utils/CassandraMonitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/
package com.netflix.priam.utils;

import com.google.common.util.concurrent.RateLimiter;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.priam.ICassandraProcess;
Expand Down Expand Up @@ -47,14 +46,12 @@ public class CassandraMonitor extends Task {
private static final AtomicBoolean isCassandraStarted = new AtomicBoolean(false);
private InstanceState instanceState;
private ICassandraProcess cassProcess;
private RateLimiter startRateLimiter;

@Inject
protected CassandraMonitor(IConfiguration config, InstanceState instanceState, ICassandraProcess cassProcess) {
super(config);
this.instanceState = instanceState;
this.cassProcess = cassProcess;
startRateLimiter = RateLimiter.create(1.0);
}

@Override
Expand Down Expand Up @@ -109,8 +106,10 @@ public void execute() throws Exception {
int rate = config.getRemediateDeadCassandraRate();
if (rate >= 0 && !config.doesCassandraStartManually()) {
if (instanceState.shouldCassandraBeAlive() && !instanceState.isCassandraProcessAlive()) {
if (rate == 0 || startRateLimiter.tryAcquire(rate)) {
long msNow = System.currentTimeMillis();
if (rate == 0 || ((instanceState.getLastStartTime() + rate * 1000) < msNow)) {
cassProcess.start(true);
instanceState.markLastStartTime();
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions priam/src/test/java/com/netflix/priam/utils/TestCassandraMonitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,11 @@ NodeProbe instance(IConfiguration config) {
public void testAutoRemediationRateLimit() throws Exception {
final InputStream mockOutput = new ByteArrayInputStream("".getBytes());
instanceState.setShouldCassandraBeAlive(true);
instanceState.markLastStartTime();
new Expectations() {{
// 6 calls to execute should = 12 calls to getInputStream();
mockProcess.getInputStream(); result=mockOutput; times=12;
cassProcess.start(true); minTimes=2; maxTimes=4;
cassProcess.start(true); times=2;
}};
// Mock out the ps call
final Runtime r = Runtime.getRuntime();
Expand All @@ -123,14 +124,14 @@ public void testAutoRemediationRateLimit() throws Exception {
}
};
// Sleep ahead to ensure we have permits in the rate limiter
monitor.execute();
Thread.sleep(1500);
monitor.execute();
monitor.execute();
Thread.sleep(1500);
monitor.execute();
monitor.execute();
monitor.execute();
monitor.execute();

new Verifications() {};
}
Expand Down