Test utility for failing native indexes in various scenarios

And updates tests that were previously targetting Lucene-specifics
in order to get them to fail.

community/community-it/cypher-it/src/test/scala/org/neo4j/cypher/IndexOpAcceptanceTest.scala

@@ -19,14 +19,15 @@
  */
 package org.neo4j.cypher
 
-import java.io.{File, FileOutputStream}
+import java.io.File
 import java.util.concurrent.TimeUnit
 
 import org.neo4j.cypher.ExecutionEngineHelper.createEngine
 import org.neo4j.cypher.internal.javacompat.GraphDatabaseCypherService
 import org.neo4j.graphdb.GraphDatabaseService
 import org.neo4j.kernel.api.exceptions.schema.{DropIndexFailureException, NoSuchIndexException}
-import org.neo4j.kernel.api.impl.schema.{LuceneIndexProviderFactory, NativeLuceneFusionIndexProviderFactory20}
+import org.neo4j.kernel.impl.index.schema.FailingGenericNativeIndexProviderFactory
+import org.neo4j.kernel.impl.index.schema.FailingGenericNativeIndexProviderFactory.FailureType.POPULATION
 import org.neo4j.test.TestGraphDatabaseFactory
 import org.neo4j.test.rule.TestDirectory
 
@@ -102,28 +103,25 @@ class IndexOpAcceptanceTest extends ExecutionEngineFunSuite with QueryStatistics
     testDirectory.prepareDirectory(getClass, "createDbWithFailedIndex")
     val storeDir = testDirectory.databaseDir()
     graph.shutdown()
-    graph = new GraphDatabaseCypherService(new TestGraphDatabaseFactory().newEmbeddedDatabase(storeDir))
+    val dbFactory = new TestGraphDatabaseFactory()
+    // Build a properly failing index provider which is a wrapper around the default provider, but which throws exception
+    // in its populator when trying to add updates to it
+    val providerFactory = new FailingGenericNativeIndexProviderFactory(POPULATION)
+    dbFactory.removeKernelExtensions(TestGraphDatabaseFactory.INDEX_PROVIDERS_FILTER)
+    dbFactory.addKernelExtension(providerFactory)
+    graph = new GraphDatabaseCypherService(dbFactory.newEmbeddedDatabase(storeDir))
     eengine = createEngine(graph)
-    execute("CREATE INDEX ON :Person(name)")
     execute("create (:Person {name:42})")
+    execute("CREATE INDEX ON :Person(name)")
     val tx = graph.getGraphDatabaseService.beginTx()
     try {
       graph.schema().awaitIndexesOnline(3, TimeUnit.SECONDS)
       tx.success()
+    } catch {
+      case e:IllegalStateException => assert(e.getMessage.contains("FAILED"), "Was expecting FAILED state")
     } finally {
       tx.close()
     }
-
-    val indexDirectory = NativeLuceneFusionIndexProviderFactory20.subProviderDirectoryStructure( testDirectory.databaseDir() )
-        .forProvider( LuceneIndexProviderFactory.PROVIDER_DESCRIPTOR ).directoryForIndex( 1 )
-    graph.shutdown()
-
-    val stream = new FileOutputStream( new File( indexDirectory, "failure-message" ) )
-    stream.write(65)
-    stream.close()
-
-    graph = new GraphDatabaseCypherService(new TestGraphDatabaseFactory().newEmbeddedDatabase(storeDir))
-    eengine = createEngine(graph)
     graph.getGraphDatabaseService
   }
 }

community/community-it/index-it/src/test/java/org/neo4j/index/IndexFailureOnStartupTest.java

@@ -24,22 +24,17 @@
 import org.junit.Test;
 
 import java.io.File;
-import java.io.IOException;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Stream;
 
 import org.neo4j.graphdb.Label;
 import org.neo4j.graphdb.Node;
 import org.neo4j.graphdb.Transaction;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.graphdb.schema.IndexDefinition;
 import org.neo4j.graphdb.schema.Schema;
-import org.neo4j.io.fs.DefaultFileSystemAbstraction;
-import org.neo4j.io.fs.FileSystemAbstraction;
-import org.neo4j.io.layout.DatabaseLayout;
-import org.neo4j.kernel.api.index.IndexDirectoryStructure;
 import org.neo4j.test.rule.DatabaseRule;
 import org.neo4j.test.rule.EmbeddedDatabaseRule;
+import org.neo4j.test.rule.RandomRule;
 import org.neo4j.values.storable.CoordinateReferenceSystem;
 import org.neo4j.values.storable.Values;
 
@@ -51,12 +46,14 @@
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
 import static org.neo4j.graphdb.schema.Schema.IndexState.ONLINE;
-import static org.neo4j.kernel.api.impl.schema.LuceneIndexProviderFactory.PROVIDER_DESCRIPTOR;
-import static org.neo4j.kernel.api.impl.schema.NativeLuceneFusionIndexProviderFactory20.subProviderDirectoryStructure;
+import static org.neo4j.index.SabotageNativeIndex.nativeIndexDirectoryStructure;
 
 public class IndexFailureOnStartupTest
 {
     private static final Label PERSON = Label.label( "Person" );
+
+    @Rule
+    public final RandomRule random = new RandomRule();
     @Rule
     public final DatabaseRule db = new EmbeddedDatabaseRule().startLazily();
 
@@ -72,7 +69,7 @@ public void failedIndexShouldRepairAutomatically() throws Exception
         awaitIndexesOnline( 5, SECONDS );
         createNamed( PERSON, "Johan" );
         // when - we restart the database in a state where the index is not operational
-        db.restartDatabase( new DeleteIndexFile( "_0.cfs" ) );
+        db.restartDatabase( new SabotageNativeIndex( random.random() ) );
         // then - the database should still be operational
         createNamed( PERSON, "Lars" );
         awaitIndexesOnline( 5, SECONDS );
@@ -91,7 +88,7 @@ public void shouldNotBeAbleToViolateConstraintWhenBackingIndexFailsToOpen() thro
         }
         createNamed( PERSON, "Lars" );
         // when - we restart the database in a state where the index is not operational
-        db.restartDatabase( new DeleteIndexFile( "_0.cfs" ) );
+        db.restartDatabase( new SabotageNativeIndex( random.random() ) );
         // then - we must not be able to violate the constraint
         createNamed( PERSON, "Johan" );
         Throwable failure = null;
@@ -134,26 +131,23 @@ public void shouldArchiveFailedIndex() throws Exception
         assertThat( archiveFile(), nullValue() );
 
         // when
-        db.restartDatabase( new DeleteIndexFile( "segments_" ) );
+        db.restartDatabase( new SabotageNativeIndex( random.random() ) );
 
         // then
         indexStateShouldBe( equalTo( ONLINE ) );
         assertThat( archiveFile(), notNullValue() );
     }
 
-    private File archiveFile() throws IOException
+    private File archiveFile()
     {
-        try ( FileSystemAbstraction fs = new DefaultFileSystemAbstraction() )
+        File indexDir = nativeIndexDirectoryStructure( db.databaseLayout() ).rootDirectory();
+        File[] files = indexDir.listFiles( pathname -> pathname.isFile() && pathname.getName().startsWith( "archive-" ) );
+        if ( files == null || files.length == 0 )
         {
-            File indexDir = indexRootDirectory( db.databaseLayout().databaseDirectory() );
-            File[] files = indexDir.listFiles( pathname -> pathname.isFile() && pathname.getName().startsWith( "archive-" ) );
-            if ( files == null || files.length == 0 )
-            {
-                return null;
-            }
-            assertEquals( 1, files.length );
-            return files[0];
+            return null;
         }
+        assertEquals( 1, files.length );
+        return files[0];
     }
 
     private void awaitIndexesOnline( int timeout, TimeUnit unit )
@@ -195,37 +189,4 @@ private void createNamed( Label label, String name )
             tx.success();
         }
     }
-
-    private static class DeleteIndexFile implements DatabaseRule.RestartAction
-    {
-        private final String prefix;
-
-        DeleteIndexFile( String prefix )
-        {
-            this.prefix = prefix;
-        }
-
-        @Override
-        public void run( FileSystemAbstraction fs, DatabaseLayout databaseLayout )
-        {
-            File indexRootDirectory = new File( soleIndexDir( databaseLayout.databaseDirectory() ), "1" /*the partition*/ );
-            File[] files = fs.listFiles( indexRootDirectory, ( dir, name ) -> name.startsWith( prefix ) );
-            Stream.of( files ).forEach( fs::deleteFile );
-        }
-    }
-
-    private static File indexRootDirectory( File base )
-    {
-        return providerDirectoryStructure( base ).rootDirectory();
-    }
-
-    private static File soleIndexDir( File base )
-    {
-        return providerDirectoryStructure( base ).directoryForIndex( 1 );
-    }
-
-    private static IndexDirectoryStructure providerDirectoryStructure( File base )
-    {
-        return subProviderDirectoryStructure( base ).forProvider( PROVIDER_DESCRIPTOR );
-    }
 }

community/community-it/index-it/src/test/java/org/neo4j/index/SabotageNativeIndex.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2002-2018 "Neo4j,"
+ * Neo4j Sweden AB [http://neo4j.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.index;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.fs.OpenMode;
+import org.neo4j.io.fs.StoreChannel;
+import org.neo4j.io.layout.DatabaseLayout;
+import org.neo4j.kernel.api.index.IndexDirectoryStructure;
+import org.neo4j.kernel.impl.index.schema.GenericNativeIndexProvider;
+import org.neo4j.test.rule.DatabaseRule;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.neo4j.io.ByteUnit.mebiBytes;
+
+public class SabotageNativeIndex implements DatabaseRule.RestartAction
+{
+    private final Random random;
+
+    public SabotageNativeIndex( Random random )
+    {
+        this.random = random;
+    }
+
+    @Override
+    public void run( FileSystemAbstraction fs, DatabaseLayout databaseLayout ) throws IOException
+    {
+        int files = scrambleIndexFiles( fs, nativeIndexDirectoryStructure( databaseLayout ).rootDirectory(), 0 );
+        assertThat( files, greaterThanOrEqualTo( 1 ) );
+    }
+
+    private int scrambleIndexFiles( FileSystemAbstraction fs, File fileOrDir, int count ) throws IOException
+    {
+        if ( fs.isDirectory( fileOrDir ) )
+        {
+            File[] children = fs.listFiles( fileOrDir );
+            if ( children != null )
+            {
+                for ( File child : children )
+                {
+                    return scrambleIndexFiles( fs, child, count );
+                }
+            }
+        }
+        else
+        {
+            // Completely scramble file, assuming small files
+            System.out.println( "scrambling " + fileOrDir );
+            try ( StoreChannel channel = fs.open( fileOrDir, OpenMode.READ_WRITE ) )
+            {
+                if ( channel.size() > mebiBytes( 10 ) )
+                {
+                    throw new IllegalArgumentException( "Was expecting small files here" );
+                }
+                ByteBuffer buffer = ByteBuffer.allocate( (int) channel.size() );
+                channel.readAll( buffer );
+                buffer.flip();
+                for ( int i = 0; i < buffer.limit(); i++ )
+                {
+                    byte existing = buffer.get( i );
+                    byte scrambled = existing;
+                    while ( scrambled == existing )
+                    {
+                        scrambled = (byte) random.nextInt();
+                    }
+                    buffer.put( i, scrambled );
+                }
+                channel.position( 0 );
+                channel.writeAll( buffer );
+            }
+            count++;
+        }
+        return count;
+    }
+
+    public static IndexDirectoryStructure nativeIndexDirectoryStructure( DatabaseLayout databaseLayout )
+    {
+        return IndexDirectoryStructure.directoriesByProvider( databaseLayout.databaseDirectory() ).forProvider(
+                GenericNativeIndexProvider.DESCRIPTOR );
+    }
+}

community/community-it/it-test-support/src/main/java/org/neo4j/test/TestGraphDatabaseFactory.java

@@ -23,6 +23,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.function.Function;
+import java.util.function.Predicate;
 import javax.annotation.Nonnull;
 
 import org.neo4j.graphdb.GraphDatabaseService;
@@ -44,6 +45,7 @@
 import org.neo4j.kernel.configuration.Settings;
 import org.neo4j.kernel.extension.KernelExtensionFactory;
 import org.neo4j.kernel.impl.factory.DatabaseInfo;
+import org.neo4j.kernel.impl.index.schema.AbstractIndexProviderFactory;
 import org.neo4j.kernel.internal.locker.StoreLocker;
 import org.neo4j.kernel.monitoring.Monitors;
 import org.neo4j.logging.LogProvider;
@@ -64,6 +66,8 @@
  */
 public class TestGraphDatabaseFactory extends GraphDatabaseFactory
 {
+    public static final Predicate<KernelExtensionFactory<?>> INDEX_PROVIDERS_FILTER = extension -> extension instanceof AbstractIndexProviderFactory;
+
     public TestGraphDatabaseFactory()
     {
         this( NullLogProvider.getInstance() );
@@ -199,6 +203,12 @@ public TestGraphDatabaseFactory setKernelExtensions( Iterable<KernelExtensionFac
         return this;
     }
 
+    public TestGraphDatabaseFactory removeKernelExtensions( Predicate<KernelExtensionFactory<?>> filter )
+    {
+        getCurrentState().removeKernelExtensions( filter );
+        return this;
+    }
+
     @Override
     public TestGraphDatabaseFactory addURLAccessRule( String protocol, URLAccessRule rule )
     {

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/AbstractIndexProviderFactory.java

@@ -31,7 +31,6 @@
 import org.neo4j.kernel.extension.KernelExtensionFactory;
 import org.neo4j.kernel.impl.factory.OperationalMode;
 import org.neo4j.kernel.impl.spi.KernelContext;
-import org.neo4j.kernel.lifecycle.Lifecycle;
 import org.neo4j.kernel.monitoring.Monitors;
 import org.neo4j.logging.Log;
 import org.neo4j.logging.internal.LogService;
@@ -44,7 +43,7 @@ protected AbstractIndexProviderFactory( String key )
     }
 
     @Override
-    public Lifecycle newInstance( KernelContext context, Dependencies dependencies )
+    public IndexProvider newInstance( KernelContext context, Dependencies dependencies )
     {
         PageCache pageCache = dependencies.pageCache();
         File databaseDir = context.directory();