Fix flaky unit test indexWithFailedInitializationIsNotQueryableAfterP…

…artialRebuild

patch by Andres de la Peña; reviewed by Robert Stupp for CASSANDRA-13963

CHANGES.txt

@@ -1,4 +1,5 @@
 4.0
+ * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963)
  * Introduce leaf-only iterator (CASSANDRA-9988)
  * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997)
  * Allow only one concurrent call to StatusLogger (CASSANDRA-12182)

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -276,6 +276,19 @@ public boolean isIndexQueryable(Index index)
         return queryableIndexes.contains(index.getIndexMetadata().name);
     }
 
+    /**
+     * Checks if the specified index has any running build task.
+     *
+     * @param indexName the index name
+     * @return {@code true} if the index is building, {@code false} otherwise
+     */
+    @VisibleForTesting
+    public synchronized boolean isIndexBuilding(String indexName)
+    {
+        AtomicInteger counter = inProgressBuilds.get(indexName);
+        return counter != null && counter.get() > 0;
+    }
+
     public synchronized void removeIndex(String indexName)
     {
         Index index = unregisterIndex(indexName);

test/unit/org/apache/cassandra/cql3/CQLTester.java

@@ -46,6 +46,7 @@
 
 import org.apache.cassandra.SchemaLoader;
 import org.apache.cassandra.concurrent.ScheduledExecutors;
+import org.apache.cassandra.index.SecondaryIndexManager;
 import org.apache.cassandra.schema.*;
 import org.apache.cassandra.config.DatabaseDescriptor;
 import org.apache.cassandra.cql3.functions.FunctionName;
@@ -736,6 +737,38 @@ protected boolean waitForIndex(String keyspace, String table, String index) thro
         return indexCreated;
     }
 
+    /**
+     * Index creation is asynchronous, this method waits until the specified index hasn't any building task running.
+     * <p>
+     * This method differs from {@link #waitForIndex(String, String, String)} in that it doesn't require the index to be
+     * fully nor successfully built, so it can be used to wait for failing index builds.
+     *
+     * @param keyspace the index keyspace name
+     * @param indexName the index name
+     * @return {@code true} if the index build tasks have finished in 5 seconds, {@code false} otherwise
+     */
+    protected boolean waitForIndexBuilds(String keyspace, String indexName) throws InterruptedException
+    {
+        long start = System.currentTimeMillis();
+        SecondaryIndexManager indexManager = getCurrentColumnFamilyStore(keyspace).indexManager;
+
+        while (true)
+        {
+            if (!indexManager.isIndexBuilding(indexName))
+            {
+                return true;
+            }
+            else if (System.currentTimeMillis() - start > 5000)
+            {
+                return false;
+            }
+            else
+            {
+                Thread.sleep(10);
+            }
+        }
+    }
+
     protected void createIndexMayThrow(String query) throws Throwable
     {
         String fullQuery = formatQuery(query);

test/unit/org/apache/cassandra/index/SecondaryIndexManagerTest.java

@@ -440,7 +440,7 @@ public void initializingIndexNotQueryableAfterPartialRebuild() throws Throwable
     }
 
     @Test
-    public void indexWithfailedInitializationIsQueryableAfterFullRebuild() throws Throwable
+    public void indexWithFailedInitializationIsQueryableAfterFullRebuild() throws Throwable
     {
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
 
@@ -458,11 +458,12 @@ public void indexWithfailedInitializationIsQueryableAfterFullRebuild() throws Th
     }
 
     @Test
-    public void indexWithfailedInitializationIsNotQueryableAfterPartialRebuild() throws Throwable
+    public void indexWithFailedInitializationIsNotQueryableAfterPartialRebuild() throws Throwable
     {
         TestingIndex.shouldFailCreate = true;
         createTable("CREATE TABLE %s (a int, b int, c int, PRIMARY KEY (a, b))");
         String indexName = createIndex(String.format("CREATE CUSTOM INDEX ON %%s(c) USING '%s'", TestingIndex.class.getName()));
+        assertTrue(waitForIndexBuilds(KEYSPACE, indexName));
         TestingIndex.shouldFailCreate = false;
 
         // the index shouldn't be queryable after the failed initialization
@@ -472,6 +473,7 @@ public void indexWithfailedInitializationIsNotQueryableAfterPartialRebuild() thr
 
         // a successful partial build doesn't set the index as queryable
         cfs.indexManager.handleNotification(new SSTableAddedNotification(cfs.getLiveSSTables(), null), this);
+        assertTrue(waitForIndexBuilds(KEYSPACE, indexName));
         assertFalse(cfs.indexManager.isIndexQueryable(index));
     }