Skip to content

Commit

Permalink
Add collection of percent repaired metrics for incremental repair sch…
Browse files Browse the repository at this point in the history
…edules
  • Loading branch information
adejanovski committed May 20, 2021
1 parent 5f51fd5 commit aeef47d
Show file tree
Hide file tree
Showing 39 changed files with 875 additions and 149 deletions.
22 changes: 22 additions & 0 deletions .github/workflows/ci.yaml
Expand Up @@ -33,6 +33,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -101,6 +103,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -202,6 +206,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -274,6 +280,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -346,6 +354,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -449,6 +459,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -538,6 +550,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -615,6 +629,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -694,6 +710,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -764,6 +782,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down Expand Up @@ -827,6 +847,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
with:
ref: master

- name: Setup Maven Cache
uses: actions/cache@v2
Expand Down
Expand Up @@ -276,15 +276,14 @@ public void run(ReaperApplicationConfiguration config, Environment environment)
environment.lifecycle().scheduledExecutorService("ReaperApplication-scheduler").threads(3).build(),
context.metricRegistry);

if (context.storage instanceof IDistributedStorage) {
// SIDECAR mode must be distributed. ALL|EACH|LOCAL are lazy: we wait until we see multiple reaper instances
context.isDistributed
.compareAndSet(false, DatacenterAvailability.SIDECAR == context.config.getDatacenterAvailability());

// Allowing multiple Reaper instances require concurrent database polls for repair and metrics statuses
scheduleRepairManager(scheduler);
scheduleHeartbeat(scheduler);
}
// SIDECAR mode must be distributed. ALL|EACH|LOCAL are lazy: we wait until we see multiple reaper instances
context.isDistributed
.compareAndSet(false, DatacenterAvailability.SIDECAR == context.config.getDatacenterAvailability());

// Allowing multiple Reaper instances require concurrent database polls for repair and metrics statuses
scheduleRepairManager(scheduler);
scheduleHeartbeat(scheduler);


context.repairManager.resumeRunningRepairRuns();
schedulePurge(scheduler);
Expand Down
Expand Up @@ -174,6 +174,9 @@ public final class ReaperApplicationConfiguration extends Configuration {
@DefaultValue("true")
private Boolean enableConcurrentMigrations;

@JsonProperty
private Integer percentRepairedCheckIntervalMinutes;

private HttpClientConfiguration httpClient = new HttpClientConfiguration();

@JsonProperty
Expand Down Expand Up @@ -454,13 +457,13 @@ public void setPurgeRecordsAfterInDays(Integer purgeRecordsAfterInDays) {
this.purgeRecordsAfterInDays = purgeRecordsAfterInDays;
}

public Integer getNumberOfRunsToKeepPerUnit() {
return numberOfRunsToKeepPerUnit == null ? 50 : numberOfRunsToKeepPerUnit;
public Integer getPercentRepairedCheckIntervalMinutes() {
return percentRepairedCheckIntervalMinutes == null ? 30 : percentRepairedCheckIntervalMinutes;
}

@JsonProperty("numberOfRunsToKeepPerUnit")
public void setNumberOfRunsToKeepPerUnit(Integer numberOfRunsToKeepPerUnit) {
this.numberOfRunsToKeepPerUnit = numberOfRunsToKeepPerUnit;
@JsonProperty("percentRepairedCheckIntervalMinutes")
public void setpercentRepairedCheckIntervalMinutes(Integer percentRepairedCheckIntervalMinutes) {
this.percentRepairedCheckIntervalMinutes = percentRepairedCheckIntervalMinutes;
}

public Boolean isInSidecarMode() {
Expand Down Expand Up @@ -499,6 +502,15 @@ public void setEnableConcurrentMigrations(boolean enableConcurrentMigrations) {
this.enableConcurrentMigrations = enableConcurrentMigrations;
}

public Integer getNumberOfRunsToKeepPerUnit() {
return numberOfRunsToKeepPerUnit == null ? 50 : numberOfRunsToKeepPerUnit;
}

@JsonProperty("numberOfRunsToKeepPerUnit")
public void setNumberOfRunsToKeepPerUnit(Integer numberOfRunsToKeepPerUnit) {
this.numberOfRunsToKeepPerUnit = numberOfRunsToKeepPerUnit;
}

public HttpClientConfiguration getHttpClientConfiguration() {
return httpClient;
}
Expand Down
Expand Up @@ -71,6 +71,10 @@ public String getMetricFullId() {
return metricDomain + ":type=" + metricType + ",scope=" + metricScope + ",name=" + metricName;
}

public String toString() {
return metricDomain + ":type=" + metricType + ",scope=" + metricScope + ",name=" + metricName + ",value=" + value;
}

public String getHost() {
return host;
}
Expand Down
@@ -0,0 +1,115 @@
/*
* Copyright 2021-2021 DataStax, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.cassandrareaper.core;

import java.util.UUID;

import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;

@JsonDeserialize(builder = PercentRepairedMetric.Builder.class)
public final class PercentRepairedMetric {
private String cluster;
private String node;
private UUID repairScheduleId ;
private String keyspaceName;
private String tableName;
private int percentRepaired;

private PercentRepairedMetric(Builder builder) {
this.cluster = builder.cluster;
this.node = builder.node;
this.repairScheduleId = builder.repairScheduleId ;
this.keyspaceName = builder.keyspaceName;
this.tableName = builder.tableName;
this.percentRepaired = builder.percentRepaired;
}

public String getCluster() {
return cluster;
}

public String getNode() {
return node;
}

public UUID getRepairScheduleId() {
return repairScheduleId;
}

public String getKeyspaceName() {
return keyspaceName;
}

public String getTableName() {
return tableName;
}

public int getPercentRepaired() {
return percentRepaired;
}

public static Builder builder() {
return new Builder();
}

@JsonPOJOBuilder(buildMethodName = "build", withPrefix = "with")
public static final class Builder {
private String cluster;
private String node;
private UUID repairScheduleId ;
private String keyspaceName;
private String tableName;
private int percentRepaired;

private Builder() {}

public Builder withCluster(String cluster) {
this.cluster = cluster;
return this;
}

public Builder withNode(String node) {
this.node = node;
return this;
}

public Builder withKeyspaceName(String keyspaceName) {
this.keyspaceName = keyspaceName;
return this;
}

public Builder withTableName(String tableName) {
this.tableName = tableName;
return this;
}

public Builder withRepairScheduleId(UUID repairScheduleId) {
this.repairScheduleId = repairScheduleId;
return this;
}

public Builder withPercentRepaired(int percentRepaired) {
this.percentRepaired = percentRepaired;
return this;
}

public PercentRepairedMetric build() {
return new PercentRepairedMetric(this);
}
}
}
Expand Up @@ -20,6 +20,7 @@
import io.cassandrareaper.AppContext;
import io.cassandrareaper.ReaperException;
import io.cassandrareaper.core.Cluster;
import io.cassandrareaper.core.PercentRepairedMetric;
import io.cassandrareaper.core.RepairSchedule;
import io.cassandrareaper.core.RepairUnit;
import io.cassandrareaper.resources.view.RepairRunStatus;
Expand Down Expand Up @@ -192,7 +193,7 @@ public Response addRepairSchedule(

if (!parallelism.equals(RepairParallelism.PARALLEL) && incremental) {
return Response.status(Response.Status.BAD_REQUEST)
.entity("Can't mix sequential repair and incremental repairs")
.entity("Incremental repairs only supports PARALLEL parallelism mode.")
.build();
}

Expand Down Expand Up @@ -554,4 +555,13 @@ private DateTime getNextActivationTime(Optional<String> scheduleTriggerTime)
}
return nextActivation;
}

@GET
@Path("/{clusterName}/{id}/percent_repaired")
public List<PercentRepairedMetric> getPercentRepairedMetricsForSchedule(
@PathParam("clusterName") String clusterName,
@PathParam("id") UUID repairScheduleId) throws IllegalArgumentException {
long since = DateTime.now().minusHours(1).getMillis();
return context.storage.getPercentRepairedMetrics(clusterName, repairScheduleId, since);
}
}

0 comments on commit aeef47d

Please sign in to comment.