Fix CircleCI tests since we have 2 DCs there.

The new optimized code path for grabbing metrics will avoid requesting from a remote DC with datacenterAvailability != ALL. Since there's a single Reaper running in CircleCI, we won't get the metrics from the remote DC which will make the tests fail consistently. Switching to LOCAL will fix that issue. Prevent JMX access on nodes 3 and 4 in CircleCI for tests outside of incremental and snapshots (require all nodes to be accessible).
thelastpickle · Apr 6, 2018 · 051be2a · 051be2a
1 parent d7689fa
commit 051be2a
Show file tree

Hide file tree

Showing 19 changed files with 199 additions and 89 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -53,8 +53,6 @@ jobs:
               ccm populate --vnodes -n 2:2
               sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node1/conf/cassandra-env.sh
               sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node2/conf/cassandra-env.sh
-              sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node3/conf/cassandra-env.sh
-              sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node4/conf/cassandra-env.sh
               sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="192m"/' /home/circleci/.ccm/test/node1/conf/cassandra-env.sh
               sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="192m"/' /home/circleci/.ccm/test/node2/conf/cassandra-env.sh
               sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="192m"/' /home/circleci/.ccm/test/node3/conf/cassandra-env.sh
@@ -92,12 +90,19 @@ jobs:
               ccm checklogerror
               MAVEN_OPTS="-Xmx1g" mvn clean install
               # TODO: parallelise the following
-              mvn surefire:test -Dtest=ReaperIT
-              mvn surefire:test -Dtest=ReaperH2IT
+              mvn surefire:test -Dtest=ReaperNoIncrementalIT
+              mvn surefire:test -Dtest=ReaperH2NoIncrementalIT
               #mvn surefire:test -Dtest=ReaperPostgresIT # TODO set up postgres
-              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraIT
-              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraIT -Dgrim.reaper.min=2 -Dgrim.reaper.max=2
-              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraIT -Dgrim.reaper.min=2 -Dgrim.reaper.max=4
+              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraNoIncrementalIT
+              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraNoIncrementalIT -Dgrim.reaper.min=2 -Dgrim.reaper.max=2
+              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraNoIncrementalIT -Dgrim.reaper.min=2 -Dgrim.reaper.max=4
+              sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node3/conf/cassandra-env.sh
+              sed -i 's/etc\/cassandra\/jmxremote.password/home\/circleci\/.local\/jmxremote.password/' /home/circleci/.ccm/test/node4/conf/cassandra-env.sh
+              ccm node3 stop
+              ccm node3 start
+              ccm node4 stop
+              ccm node4 start
+              mvn surefire:test -DsurefireArgLine="-Xmx1g" -Dtest=ReaperCassandraIncrementalIT -Dgrim.reaper.min=2 -Dgrim.reaper.max=4
 
       - store_test_results:
           path: src/server/target/surefire-reports

diff --git a/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java b/src/server/src/main/java/io/cassandrareaper/ReaperApplication.java
@@ -14,6 +14,7 @@
 
 package io.cassandrareaper;
 
+import io.cassandrareaper.ReaperApplicationConfiguration.DatacenterAvailability;
 import io.cassandrareaper.ReaperApplicationConfiguration.JmxCredentials;
 import io.cassandrareaper.jmx.JmxConnectionFactory;
 import io.cassandrareaper.jmx.JmxConnectionsInitializer;
@@ -253,6 +254,11 @@ public void run(ReaperApplicationConfiguration config, Environment environment)
     initializeJmxSeedsForAllClusters();
     LOG.info("resuming pending repair runs");
 
+    Preconditions.checkState(
+        context.storage instanceof IDistributedStorage
+            || DatacenterAvailability.EACH != context.config.getDatacenterAvailability(),
+        "Cassandra backend storage is the only one allowing EACH datacenter availability modes.");
+
     if (context.storage instanceof IDistributedStorage) {
       // Allowing multiple Reaper instances to work concurrently requires
       // us to poll the database for running repairs regularly

diff --git a/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java b/src/server/src/main/java/io/cassandrareaper/service/SegmentRunner.java
@@ -589,17 +589,20 @@ Callable<Pair<String, Optional<NodeMetrics>>> getNodeMetrics(String node, String
     return () -> {
       LOG.debug("getMetricsForHost {} / {} / {}", node, localDc, nodeDc);
 
-      if (DatacenterAvailability.ALL != context.config.getDatacenterAvailability() && !nodeDc.equals(localDc)) {
+      if (DatacenterAvailability.ALL != context.config.getDatacenterAvailability()
+          && !nodeDc.equals(localDc)
+          && context.storage instanceof IDistributedStorage) {
         // If DatacenterAvailability is not ALL, we should assume jmx on remote dc is not reachable.
         return Pair.of(node, getRemoteNodeMetrics(node, nodeDc));
       } else {
         try {
           JmxProxy nodeProxy =
-                    context.jmxConnectionFactory.connect(
-                            Node.builder().withClusterName(clusterName).withHostname(node).build(),
-                            context.config.getJmxConnectionTimeoutInSeconds());
+              context.jmxConnectionFactory.connect(
+                  Node.builder().withClusterName(clusterName).withHostname(node).build(),
+                  context.config.getJmxConnectionTimeoutInSeconds());
 
-          NodeMetrics metrics = NodeMetrics.builder()
+          NodeMetrics metrics =
+              NodeMetrics.builder()
                   .withNode(node)
                   .withDatacenter(nodeDc)
                   .withCluster(nodeProxy.getClusterName())
@@ -610,7 +613,10 @@ Callable<Pair<String, Optional<NodeMetrics>>> getNodeMetrics(String node, String
 
           return Pair.of(node, Optional.of(metrics));
         } catch (RuntimeException | ReaperException e) {
-          LOG.debug("failed to query metrics for host {}, trying to get metrics from storage...", node, e);
+          LOG.debug(
+              "failed to query metrics for host {}, trying to get metrics from storage...",
+              node,
+              e);
           return Pair.of(node, getRemoteNodeMetrics(node, nodeDc));
         }
       }

diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperAuthIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperAuthIT.java
diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIT.java
@@ -39,17 +39,18 @@
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature"
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    plugin = {"pretty"}
     )
-public final class ReaperCassandraIT {
+public class ReaperCassandraIT {
 
   private static final Logger LOG = LoggerFactory.getLogger(ReaperCassandraIT.class);
   private static final List<ReaperTestJettyRunner> RUNNER_INSTANCES = new CopyOnWriteArrayList<>();
   private static final String CASS_CONFIG_FILE = "cassandra-reaper-cassandra-at.yaml";
   private static final Random RAND = new Random(System.nanoTime());
   private static Thread GRIM_REAPER;
 
-  private ReaperCassandraIT() {}
+  protected ReaperCassandraIT() {}
 
   @BeforeClass
   public static void setUp() throws Exception {

diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIncrementalIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraIncrementalIT.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.cassandrareaper.acceptance;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    tags = {"@incremental"}
+    )
+public final class ReaperCassandraIncrementalIT extends ReaperCassandraIT {}
diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraNoIncrementalIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperCassandraNoIncrementalIT.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.cassandrareaper.acceptance;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    tags = {"~@incremental"}
+    )
+public final class ReaperCassandraNoIncrementalIT extends ReaperCassandraIT {}
diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperH2IT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperH2IT.java
@@ -29,15 +29,17 @@
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature"
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    plugin = {"pretty"}
     )
-public final class ReaperH2IT {
+public class ReaperH2IT {
 
   private static final Logger LOG = LoggerFactory.getLogger(ReaperH2IT.class);
   private static ReaperJettyTestSupport runnerInstance;
   private static final String H2_CONFIG_FILE = "cassandra-reaper-h2-at.yaml";
 
-  private ReaperH2IT() {}
+
+  protected ReaperH2IT() {}
 
   @BeforeClass
   public static void setUp() throws Exception {

diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperH2NoIncrementalIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperH2NoIncrementalIT.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.cassandrareaper.acceptance;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    tags = {"~@incremental"}
+    )
+public final class ReaperH2NoIncrementalIT extends ReaperH2IT {}
diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperIT.java
@@ -27,15 +27,16 @@
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature"
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    plugin = {"pretty"}
     )
-public final class ReaperIT {
+public class ReaperIT {
 
   private static final Logger LOG = LoggerFactory.getLogger(ReaperIT.class);
   private static ReaperJettyTestSupport runnerInstance;
   private static final String MEMORY_CONFIG_FILE = "cassandra-reaper-at.yaml";
 
-  private ReaperIT() {}
+  protected ReaperIT() {}
 
   @BeforeClass
   public static void setUp() throws Exception {

diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperNoIncrementalIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperNoIncrementalIT.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.cassandrareaper.acceptance;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    tags = {"~@incremental"}
+    )
+public final class ReaperNoIncrementalIT extends ReaperIT {}
diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperPostgresIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperPostgresIT.java
@@ -27,15 +27,17 @@
 
 @RunWith(Cucumber.class)
 @CucumberOptions(
-    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature"
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    plugin = {"pretty"}
     )
-public final class ReaperPostgresIT {
+public class ReaperPostgresIT {
 
   private static final Logger LOG = LoggerFactory.getLogger(ReaperPostgresIT.class);
   private static ReaperJettyTestSupport runnerInstance;
   private static final String POSTGRES_CONFIG_FILE = "cassandra-reaper-postgres-at.yaml";
 
-  private ReaperPostgresIT() {}
+
+  protected ReaperPostgresIT() {}
 
   @BeforeClass
   public static void setUp() throws Exception {

diff --git a/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperPostgresNoIncrementalIT.java b/src/server/src/test/java/io/cassandrareaper/acceptance/ReaperPostgresNoIncrementalIT.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.cassandrareaper.acceptance;
+
+import cucumber.api.CucumberOptions;
+import cucumber.api.junit.Cucumber;
+import org.junit.runner.RunWith;
+
+@RunWith(Cucumber.class)
+@CucumberOptions(
+    features = "classpath:io.cassandrareaper.acceptance/integration_reaper_functionality.feature",
+    tags = {"~@incremental"}
+    )
+public final class ReaperPostgresNoIncrementalIT extends ReaperPostgresIT {}
diff --git a/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java b/src/server/src/test/java/io/cassandrareaper/service/SegmentRunnerTest.java
@@ -26,6 +26,8 @@
 import io.cassandrareaper.jmx.JmxConnectionFactory;
 import io.cassandrareaper.jmx.JmxProxy;
 import io.cassandrareaper.jmx.RepairStatusHandler;
+import io.cassandrareaper.storage.CassandraStorage;
+import io.cassandrareaper.storage.IDistributedStorage;
 import io.cassandrareaper.storage.IStorage;
 import io.cassandrareaper.storage.MemoryStorage;
 
@@ -551,7 +553,9 @@ public void getTablesToRepairRemoveAllFromListFailingTest() throws ReaperExcepti
   @Test
   public void getNodeMetricsInLocalDCAvailabilityForRemoteDCNodeTest() throws Exception {
     final AppContext context = new AppContext();
-    context.storage = new MemoryStorage();
+    context.storage = Mockito.mock(CassandraStorage.class);
+    when(((IDistributedStorage) context.storage).getNodeMetrics(any(), any()))
+        .thenReturn(Optional.absent());
     JmxConnectionFactory jmxConnectionFactory = mock(JmxConnectionFactory.class);
     when(jmxConnectionFactory.connect(any(), anyInt())).thenReturn(mock(JmxProxy.class));
     context.jmxConnectionFactory = jmxConnectionFactory;
@@ -569,7 +573,7 @@ public void getNodeMetricsInLocalDCAvailabilityForRemoteDCNodeTest() throws Exce
   @Test
   public void getNodeMetricsInLocalDCAvailabilityForLocalDCNodeTest() throws Exception {
     final AppContext context = new AppContext();
-    context.storage = new MemoryStorage();
+    context.storage = Mockito.mock(CassandraStorage.class);
 
     JmxProxy proxy = mock(JmxProxy.class);
     when(proxy.getClusterName()).thenReturn("test");