-
Notifications
You must be signed in to change notification settings - Fork 210
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1056 from allegro/auto-switching-to-read-only-mod…
…e-1052-2 Auto switching to read only mode #1052
- Loading branch information
Showing
9 changed files
with
295 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
...ava/pl/allegro/tech/hermes/management/domain/health/CouldNotResolveHostNameException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
package pl.allegro.tech.hermes.management.domain.health; | ||
|
||
class CouldNotResolveHostNameException extends RuntimeException { | ||
CouldNotResolveHostNameException(Throwable cause) { | ||
super(cause); | ||
} | ||
} |
47 changes: 47 additions & 0 deletions
47
...t/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckScheduler.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
package pl.allegro.tech.hermes.management.domain.health; | ||
|
||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import com.google.common.util.concurrent.ThreadFactoryBuilder; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import org.springframework.beans.factory.annotation.Value; | ||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; | ||
import org.springframework.stereotype.Component; | ||
import pl.allegro.tech.hermes.infrastructure.zookeeper.ZookeeperPaths; | ||
import pl.allegro.tech.hermes.management.domain.mode.ModeService; | ||
import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClientManager; | ||
|
||
import javax.annotation.PostConstruct; | ||
import java.util.concurrent.Executors; | ||
import java.util.concurrent.ScheduledExecutorService; | ||
import java.util.concurrent.TimeUnit; | ||
|
||
@Component | ||
@ConditionalOnProperty(name = "management.health.enabled", havingValue = "true") | ||
public class HealthCheckScheduler { | ||
|
||
private static final Logger logger = LoggerFactory.getLogger(HealthCheckScheduler.class); | ||
|
||
private final HealthCheckTask healthCheckTask; | ||
private final Long period; | ||
private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor( | ||
new ThreadFactoryBuilder().setNameFormat("storage-health-check-scheduler-%d").build() | ||
); | ||
|
||
public HealthCheckScheduler(ZookeeperClientManager zookeeperClientManager, | ||
ZookeeperPaths zookeeperPaths, | ||
NodeDataProvider nodeDataProvider, | ||
ObjectMapper objectMapper, | ||
ModeService modeService, | ||
@Value("${management.health.periodSeconds}") Long periodSeconds) { | ||
String healthCheckPath = zookeeperPaths.nodeHealthPathForManagementHost(nodeDataProvider.getHostname(), nodeDataProvider.getServerPort()); | ||
this.period = periodSeconds; | ||
this.healthCheckTask = new HealthCheckTask(zookeeperClientManager.getClients(), healthCheckPath, objectMapper, modeService); | ||
} | ||
|
||
@PostConstruct | ||
public void scheduleHealthCheck() { | ||
logger.info("Starting the storage health check scheduler"); | ||
executorService.scheduleAtFixedRate(healthCheckTask, 0, period, TimeUnit.SECONDS); | ||
} | ||
} |
65 changes: 65 additions & 0 deletions
65
...gement/src/main/java/pl/allegro/tech/hermes/management/domain/health/HealthCheckTask.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
package pl.allegro.tech.hermes.management.domain.health; | ||
|
||
import com.fasterxml.jackson.databind.ObjectMapper; | ||
import org.slf4j.Logger; | ||
import org.slf4j.LoggerFactory; | ||
import pl.allegro.tech.hermes.management.domain.mode.ModeService; | ||
import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClient; | ||
|
||
import java.time.LocalDateTime; | ||
import java.time.format.DateTimeFormatter; | ||
import java.util.Collection; | ||
import java.util.List; | ||
import java.util.stream.Collectors; | ||
|
||
class HealthCheckTask implements Runnable { | ||
|
||
private static final Logger logger = LoggerFactory.getLogger(HealthCheckTask.class); | ||
|
||
private final Collection<ZookeeperClient> zookeeperClients; | ||
private final String healthCheckPath; | ||
private final ObjectMapper objectMapper; | ||
private final ModeService modeService; | ||
|
||
HealthCheckTask(Collection<ZookeeperClient> zookeeperClients, String healthCheckPath, ObjectMapper objectMapper, ModeService modeService) { | ||
this.zookeeperClients = zookeeperClients; | ||
this.healthCheckPath = healthCheckPath; | ||
this.objectMapper = objectMapper; | ||
this.modeService = modeService; | ||
} | ||
|
||
@Override | ||
public void run() { | ||
final List<HealthCheckResult> healthCheckResults = zookeeperClients.stream() | ||
.map(this::doHealthCheck) | ||
.collect(Collectors.toList()); | ||
updateMode(healthCheckResults); | ||
} | ||
|
||
private HealthCheckResult doHealthCheck(ZookeeperClient zookeeperClient) { | ||
final String timestamp = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); | ||
try { | ||
zookeeperClient.ensurePathExists(healthCheckPath); | ||
zookeeperClient.getCuratorFramework() | ||
.setData() | ||
.forPath(healthCheckPath, objectMapper.writeValueAsBytes(timestamp)); | ||
logger.info("Storage healthy for datacenter {}", zookeeperClient.getDatacenterName()); | ||
return HealthCheckResult.HEALTHY; | ||
} catch (Exception e) { | ||
logger.error("Storage health check failed for datacenter {}", zookeeperClient.getDatacenterName(), e); | ||
return HealthCheckResult.UNHEALTHY; | ||
} | ||
} | ||
|
||
private void updateMode(List<HealthCheckResult> healthCheckResults) { | ||
if (healthCheckResults.contains(HealthCheckResult.UNHEALTHY)) { | ||
modeService.setMode(ModeService.ManagementMode.READ_ONLY); | ||
} else { | ||
modeService.setMode(ModeService.ManagementMode.READ_WRITE); | ||
} | ||
} | ||
|
||
private enum HealthCheckResult { | ||
HEALTHY, UNHEALTHY | ||
} | ||
} |
29 changes: 29 additions & 0 deletions
29
...ement/src/main/java/pl/allegro/tech/hermes/management/domain/health/NodeDataProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
package pl.allegro.tech.hermes.management.domain.health; | ||
|
||
import org.springframework.beans.factory.annotation.Value; | ||
import org.springframework.stereotype.Component; | ||
|
||
import java.net.InetAddress; | ||
import java.net.UnknownHostException; | ||
|
||
@Component | ||
class NodeDataProvider { | ||
|
||
private final String serverPort; | ||
|
||
NodeDataProvider(@Value("${server.port}") String serverPort) { | ||
this.serverPort = serverPort; | ||
} | ||
|
||
String getHostname() { | ||
try { | ||
return InetAddress.getLocalHost().getHostName(); | ||
} catch (UnknownHostException e) { | ||
throw new CouldNotResolveHostNameException(e); | ||
} | ||
} | ||
|
||
String getServerPort() { | ||
return serverPort; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
109 changes: 109 additions & 0 deletions
109
...rc/test/groovy/pl/allegro/tech/hermes/management/domain/health/HealthCheckTaskTest.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
package pl.allegro.tech.hermes.management.domain.health | ||
|
||
import com.fasterxml.jackson.databind.ObjectMapper | ||
import pl.allegro.tech.hermes.management.config.storage.StorageClustersProperties | ||
import pl.allegro.tech.hermes.management.config.storage.StorageProperties | ||
import pl.allegro.tech.hermes.management.domain.mode.ModeService | ||
import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClient | ||
import pl.allegro.tech.hermes.management.infrastructure.zookeeper.ZookeeperClientManager | ||
import pl.allegro.tech.hermes.management.utils.MultiZookeeperIntegrationTest | ||
|
||
class HealthCheckTaskTest extends MultiZookeeperIntegrationTest { | ||
|
||
def healthCheckPath = '/hermes/storage-health/hostname_8080' | ||
def modeService = new ModeService() | ||
ZookeeperClientManager manager | ||
HealthCheckTask healthCheckTask | ||
|
||
def setup() { | ||
manager = buildZookeeperClientManager() | ||
manager.start() | ||
assertZookeeperClientsConnected(manager.clients) | ||
manager.clients.each { client -> setupZookeeperPath(client, healthCheckPath) } | ||
healthCheckTask = new HealthCheckTask(manager.clients, healthCheckPath, new ObjectMapper(), modeService) | ||
} | ||
|
||
def cleanup() { | ||
manager.stop() | ||
} | ||
|
||
def "should not change mode in case of successful health check"() { | ||
given: | ||
assert !modeService.readOnlyEnabled | ||
|
||
when: | ||
healthCheckTask.run() | ||
|
||
then: | ||
!modeService.readOnlyEnabled | ||
} | ||
|
||
def "should change mode to READ_ONLY in case of failed health check"() { | ||
given: | ||
assert !modeService.readOnlyEnabled | ||
|
||
and: | ||
zookeeper1.stop() | ||
|
||
when: | ||
healthCheckTask.run() | ||
|
||
then: | ||
modeService.readOnlyEnabled | ||
} | ||
|
||
def "should change mode to READ_ONLY in case of failed health check and set READ_WRITE back again in case of successful next connection"() { | ||
given: | ||
assert !modeService.readOnlyEnabled | ||
|
||
and: | ||
zookeeper1.stop() | ||
|
||
when: | ||
healthCheckTask.run() | ||
|
||
then: | ||
modeService.readOnlyEnabled | ||
|
||
and: | ||
zookeeper1.restart() | ||
|
||
and: | ||
healthCheckTask.run() | ||
|
||
and: | ||
!modeService.readOnlyEnabled | ||
} | ||
|
||
static buildZookeeperClientManager(String dc = "dc1") { | ||
def properties = new StorageClustersProperties(clusters: [ | ||
new StorageProperties(connectionString: "localhost:$DC_1_ZOOKEEPER_PORT", datacenter: DC_1_NAME), | ||
new StorageProperties(connectionString: "localhost:$DC_2_ZOOKEEPER_PORT", datacenter: DC_2_NAME) | ||
]) | ||
new ZookeeperClientManager(properties, new TestDatacenterNameProvider(dc)) | ||
} | ||
|
||
static findClientByDc(List<ZookeeperClient> clients, String dcName) { | ||
clients.find { it.datacenterName == dcName } | ||
} | ||
|
||
static setupZookeeperPath(ZookeeperClient zookeeperClient, String path) { | ||
def healthCheckPathExists = zookeeperClient.curatorFramework | ||
.checkExists() | ||
.forPath(path) != null | ||
if (!healthCheckPathExists) { | ||
zookeeperClient.curatorFramework | ||
.create() | ||
.creatingParentContainersIfNeeded() | ||
.forPath(path) | ||
} | ||
} | ||
|
||
static assertZookeeperClientsConnected(List<ZookeeperClient> clients) { | ||
def dc1Client = findClientByDc(clients, DC_1_NAME) | ||
assert assertClientConnected(dc1Client) | ||
|
||
def dc2Client = findClientByDc(clients, DC_2_NAME) | ||
assert assertClientConnected(dc2Client) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters