Mark more tests as flaky

This adds the necessary annotation to repeat flaky tests if necessary for tests described in these issues: * JanusGraph#1457 * JanusGraph#1459 * JanusGraph#1462 * JanusGraph#1464 * JanusGraph#1465 * JanusGraph#1497 * JanusGraph#2272 * JanusGraph#3142 * JanusGraph#3356 * JanusGraph#3392 * JanusGraph#3393 * JanusGraph#3651 * JanusGraph#3931 * JanusGraph#3959 Signed-off-by: Florian Hockmann <fh@florian-hockmann.de>
FlorianHockmann · Sep 6, 2023 · c7f1414 · c7f1414
1 parent e2b0bed
commit c7f1414
Show file tree

Hide file tree

Showing 10 changed files with 104 additions and 33 deletions.
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java
@@ -2188,7 +2188,7 @@ public void testIndexReplay() throws Exception {
         assertEquals(4, recoveryStats[1]); //all 4 index transaction had provoked errors in the indexing backend
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
         final Object[] settings = new Object[]{option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0),
                 option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50),

diff --git a/...ckend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java b/...ckend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphOperationCountingTest.java
@@ -20,6 +20,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -127,7 +128,7 @@ private void verifyLockingOverwrite(long num) {
         }
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void testIdCounts() {
         makeVertexIndexedUniqueKey("uid", Integer.class);
         mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);

diff --git a/...eleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java b/...eleyje/src/test/java/org/janusgraph/graphdb/berkeleyje/BerkeleyOperationCountingTest.java
@@ -14,7 +14,6 @@
 
 package org.janusgraph.graphdb.berkeleyje;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.BerkeleyStorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphOperationCountingTest;
@@ -27,12 +26,6 @@ public WriteConfiguration getBaseConfiguration() {
         return BerkeleyStorageSetup.getBerkeleyJEGraphConfiguration();
     }
 
-    @Override
-    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
-    public void testIdCounts() {
-        super.testIdCounts();
-    }
-
     @AfterEach
     public void resetCounts() {
         resetMetrics(); // Metrics is a singleton, so subsequents test runs have wrong counts if we don't clean up.

diff --git a/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java b/janusgraph-cql/src/test/java/org/janusgraph/core/cql/CQLConfiguredGraphFactoryTest.java
@@ -109,7 +109,7 @@ public void createConfigurationShouldSupportMultiHosts() throws Exception {
         }
     }
 
-    @Test
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
         final MapConfiguration graphConfig = getGraphConfig();
         final String graphName = graphConfig.getString(GRAPH_NAME.toStringWithoutRoot());
@@ -137,5 +137,11 @@ public void dropGraphShouldRemoveGraphKeyspace() throws Exception {
     public void updateConfigurationShouldRemoveGraphFromCache() throws Exception {
         super.updateConfigurationShouldRemoveGraphFromCache();
     }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void dropShouldCleanUpTraversalSourceAndBindings() throws Exception {
+        super.dropShouldCleanUpTraversalSourceAndBindings();
+    }
 }
 
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphCacheTest.java
@@ -14,13 +14,19 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.StorageSetup;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.graphdb.JanusGraphTest;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import java.util.concurrent.ExecutionException;
+
 @Testcontainers
 public class CQLGraphCacheTest extends JanusGraphTest {
 
@@ -31,4 +37,47 @@ public class CQLGraphCacheTest extends JanusGraphTest {
     public WriteConfiguration getConfiguration() {
         return StorageSetup.addPermanentCache(cqlContainer.getConfiguration(getClass().getSimpleName()));
     }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testEdgeTTLTiming() throws Exception {
+        super.testEdgeTTLTiming();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    @Test
+    public void testEdgeTTLWithTransactions() throws Exception {
+        super.testEdgeTTLWithTransactions();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 3)
+    @Override
+    public void testReindexingForEdgeIndex() throws ExecutionException, InterruptedException {
+        super.testReindexingForEdgeIndex();
+    }
 }
diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.ParameterizedRepeatedIfExceptionsTest;
 import io.github.artsok.RepeatedIfExceptionsTest;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -227,18 +228,28 @@ public void testHasTTL() {
         assertTrue(features.hasCellTTL());
     }
 
-    @ParameterizedTest
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @ValueSource(booleans = {true, false})
-    public void simpleLogTest(boolean useStringId) {
-        for (int i = 0; i < 3; i++) {
-            try {
-                super.simpleLogTest(useStringId);
-                // If the test passes, break out of the loop.
-                break;
-            } catch (Exception ex) {
-                log.info("Attempt #{} fails", i, ex);
-            }
-        }
+    public void simpleLogTest(boolean useStringId) throws InterruptedException {
+        super.simpleLogTest(useStringId);
+    }
+
+    @ParameterizedRepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @ValueSource(booleans = {true, false})
+    public void simpleLogTestWithFailure(boolean useStringId) throws InterruptedException {
+        super.simpleLogTestWithFailure(useStringId);
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLWithCompositeIndex() throws Exception {
+        super.testVertexTTLWithCompositeIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testVertexTTLImplicitKey() throws Exception {
+        super.testVertexTTLImplicitKey();
     }
 
     @RepeatedIfExceptionsTest(repeats = 3)

diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLOLAPTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.graphdb.cql;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.WriteConfiguration;
 import org.janusgraph.olap.OLAPTest;
@@ -29,4 +30,10 @@ public class CQLOLAPTest extends OLAPTest {
     public WriteConfiguration getConfiguration() {
         return cqlContainer.getConfiguration(getClass().getSimpleName()).getConfiguration();
     }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testShortestDistance() throws Exception {
+        super.testShortestDistance();
+    }
 }
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/BerkeleyElasticsearchTest.java
@@ -55,9 +55,15 @@ public void testGraphOfTheGodsFactoryCreate() {
         gotg.close();
     }
 
-    @RepeatedIfExceptionsTest(repeats = 3)
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
     @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
+    public void testDisableAndDiscardManuallyAndDropEnabledIndex() throws Exception {
+        super.testDisableAndDiscardManuallyAndDropEnabledIndex();
+    }
+
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testDiscardAndDropRegisteredIndex() throws ExecutionException, InterruptedException {
+        super.testDiscardAndDropRegisteredIndex();
     }
 }
diff --git a/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java b/janusgraph-es/src/test/java/org/janusgraph/diskstorage/es/CQLElasticsearchTest.java
@@ -14,15 +14,12 @@
 
 package org.janusgraph.diskstorage.es;
 
-import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Disabled;
 import org.testcontainers.junit.jupiter.Container;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
-import java.util.concurrent.ExecutionException;
-
 @Testcontainers
 public class CQLElasticsearchTest extends ElasticsearchJanusGraphIndexTest {
 
@@ -37,10 +34,4 @@ public ModifiableConfiguration getStorageConfiguration() {
     @Override
     @Disabled("CQL seems to not clear storage correctly")
     public void testClearStorage() {}
-
-    @RepeatedIfExceptionsTest(repeats = 3)
-    @Override
-    public void testIndexUpdatesWithoutReindex() throws InterruptedException, ExecutionException {
-        super.testIndexUpdatesWithoutReindex();
-    }
 }
diff --git a/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java b/janusgraph-solr/src/test/java/org/janusgraph/diskstorage/solr/CQLSolrTest.java
@@ -14,6 +14,7 @@
 
 package org.janusgraph.diskstorage.solr;
 
+import io.github.artsok.RepeatedIfExceptionsTest;
 import org.janusgraph.JanusGraphCassandraContainer;
 import org.janusgraph.diskstorage.configuration.ModifiableConfiguration;
 import org.junit.jupiter.api.Test;
@@ -38,6 +39,12 @@ public boolean supportsWildcardQuery() {
         return false;
     }
 
+    @RepeatedIfExceptionsTest(repeats = 4, minSuccess = 2)
+    @Override
+    public void testSetIndexing() {
+        super.testSetIndexing();
+    }
+
     @Test
     @Override
     public void testDiscardAndDropRegisteredIndex() {