Integration tests for schema and scan store indexes

Add javadocs, integration test for basic usage scenarious of schema and label scan indexes,
label scan store index usages.
Integration tests currently checks possibility to create and write into multi partitioned index,
and aware about huge limitations on reading data back.
Commit do not include any multi threaded integration tests.

community/kernel/src/test/java/org/neo4j/graphdb/LabelScanStoreIT.java

@@ -19,23 +19,29 @@
  */
 package org.neo4j.graphdb;
 
-import java.util.Set;
-
 import org.junit.Rule;
 import org.junit.Test;
+
+import java.util.Set;
+
 import org.neo4j.kernel.GraphDatabaseAPI;
 import org.neo4j.test.DatabaseRule;
-import org.neo4j.test.ImpermanentDatabaseRule;
+import org.neo4j.test.EmbeddedDatabaseRule;
 
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 import static org.neo4j.graphdb.Label.label;
 import static org.neo4j.helpers.collection.IteratorUtil.asSet;
 import static org.neo4j.helpers.collection.IteratorUtil.emptySetOf;
 import static org.neo4j.helpers.collection.IteratorUtil.single;
 
 public class LabelScanStoreIT
 {
+
+    @Rule
+    public final DatabaseRule dbRule = new EmbeddedDatabaseRule();
+
     @Test
     public void shouldGetNodesWithCreatedLabel() throws Exception
     {
@@ -139,30 +145,30 @@ public void shouldHandleLargeAmountsOfNodesAddedAndRemovedInSameTx() throws Exce
 
         // When
         Node node;
-        try( Transaction tx = db.beginTx() )
+        try ( Transaction tx = db.beginTx() )
         {
             node = db.createNode();
 
             // I create a lot of labels, enough to push the store to use two dynamic records
-            for(int l=0;l<labelsToAdd;l++)
+            for ( int l = 0; l < labelsToAdd; l++ )
             {
-                node.addLabel( label("Label-" + l) );
+                node.addLabel( label( "Label-" + l ) );
             }
 
             // and I delete some of them, enough to bring the number of dynamic records needed down to 1
-            for(int l=0;l<labelsToRemove;l++)
+            for ( int l = 0; l < labelsToRemove; l++ )
             {
-                node.removeLabel( label("Label-" + l) );
+                node.removeLabel( label( "Label-" + l ) );
             }
 
             tx.success();
         }
 
         // Then
-        try( Transaction ignore = db.beginTx() )
+        try ( Transaction ignore = db.beginTx() )
         {
             // All the labels remaining should be in the label scan store
-            for(int l=labelsToAdd-1;l>=labelsToRemove;l--)
+            for ( int l = labelsToAdd - 1; l >= labelsToRemove; l-- )
             {
                 Label label = label( "Label-" + l );
                 assertThat( "Should have founnd node when looking for label " + label,
@@ -222,9 +228,7 @@ private void addLabels( Node node, Label... labels )
         }
     }
 
-    public final @Rule DatabaseRule dbRule = new ImpermanentDatabaseRule();
-
-    private static enum Labels implements Label
+    private enum Labels implements Label
     {
         First,
         Second,

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java

@@ -59,6 +59,14 @@ public AbstractLuceneIndex( PartitionedIndexStorage indexStorage )
         this.indexStorage = indexStorage;
     }
 
+    /**
+     * Creates new index.
+     * As part of creation process index will allocate all required folders, index failure storage
+     * and will create its first partition.
+     * <p></p>
+     * <b>Index creation do not automatically open it. To be able to use index please open it first.</b>
+     * @throws IOException
+     */
     public void create() throws IOException
     {
         ensureNotOpen();
@@ -67,6 +75,10 @@ public void create() throws IOException
         createNewPartitionFolder();
     }
 
+    /**
+     * Open index with all allocated partitions.
+     * @throws IOException
+     */
     public void open() throws IOException
     {
         Map<File,Directory> indexDirectories = indexStorage.openIndexDirectories();
@@ -82,6 +94,11 @@ boolean isOpen()
         return open;
     }
 
+    /**
+     * Check lucene index existence within all allocated partitions.
+     * @return true if index exist in all partitions, false when index is empty or does not exist
+     * @throws IOException
+     */
     public boolean exists() throws IOException
     {
         List<File> folders = indexStorage.listFolders();
@@ -99,6 +116,12 @@ public boolean exists() throws IOException
         return true;
     }
 
+    /**
+     * Verify state of the index.
+     * If index is already open and in use method assume that index is valid since lucene already operating with it,
+     * otherwise necessary checks perform.
+     * @return true if lucene confirm that index is in valid clean state or index is already open.
+     */
     public boolean isValid()
     {
         if ( open )
@@ -145,12 +168,23 @@ public void drop() throws IOException
 
     public void flush() throws IOException
     {
-        List<IndexPartition> partitions = getPartitions();
-        for ( IndexPartition partition : partitions )
+        if (!open)
         {
-            partition.getIndexWriter().commit();
+            return;
+        }
+        commitCloseLock.lock();
+        try
+        {
+            List<IndexPartition> partitions = getPartitions();
+            for ( IndexPartition partition : partitions )
+            {
+                partition.getIndexWriter().commit();
+            }
+        }
+        finally
+        {
+            commitCloseLock.unlock();
         }
-
     }
 
     @Override

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanIndex.java

@@ -53,9 +53,7 @@ public LabelScanReader getLabelScanReader()
             List<IndexPartition> partitions = getPartitions();
             if ( hasSinglePartition( partitions ) )
             {
-                IndexPartition partition = getFirstPartition( partitions );
-                PartitionSearcher searcher = partition.acquireSearcher();
-                return new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy );
+                return createSimpleReader( partitions );
             }
             throw new UnsupportedOperationException();
         }
@@ -88,4 +86,11 @@ public AllEntriesLabelScanReader allNodeLabelRanges()
     {
         return storageStrategy.newNodeLabelReader( allDocumentsReader() );
     }
+
+    private LabelScanReader createSimpleReader( List<IndexPartition> partitions ) throws IOException
+    {
+        IndexPartition partition = getFirstPartition( partitions );
+        PartitionSearcher searcher = partition.acquireSearcher();
+        return new SimpleLuceneLabelScanStoreReader( searcher, storageStrategy );
+    }
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneLabelScanStore.java

@@ -206,6 +206,7 @@ public LabelScanWriter newWriter()
     {
         // Only a single writer is allowed at any point in time. For that this lock is used and passed
         // onto the writer to release in its close()
+        // TODO:
         lock.lock();
         return luceneIndex.getLabelScanWriter(lock);
     }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/LuceneSchemaIndex.java

@@ -169,19 +169,19 @@ private SimpleIndexReader createSimpleReader( List<IndexPartition> partitions )
         return new SimpleIndexReader( singlePartition.acquireSearcher(), config, samplingConfig, taskCoordinator );
     }
 
-    private PartitionedIndexReader createPartitionedReader( List<IndexPartition> partitions ) throws IOException
-    {
-        List<PartitionSearcher> searchers = acquireSearchers( partitions );
-        return new PartitionedIndexReader( searchers );
-    }
-
     private UniquenessVerifier createSimpleUniquenessVerifier( List<IndexPartition> partitions ) throws IOException
     {
         IndexPartition singlePartition = getFirstPartition( partitions );
         PartitionSearcher partitionSearcher = singlePartition.acquireSearcher();
         return new SimpleUniquenessVerifier( partitionSearcher );
     }
 
+    private PartitionedIndexReader createPartitionedReader( List<IndexPartition> partitions ) throws IOException
+    {
+        List<PartitionSearcher> searchers = acquireSearchers( partitions );
+        return new PartitionedIndexReader( searchers );
+    }
+
     private UniquenessVerifier createPartitionedUniquenessVerifier( List<IndexPartition> partitions ) throws IOException
     {
         List<PartitionSearcher> searchers = acquireSearchers( partitions );

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/PartitionedIndexWriter.java

@@ -29,15 +29,13 @@
 import java.util.Optional;
 
 import org.neo4j.kernel.api.impl.index.partition.IndexPartition;
-import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles;
 
 public class PartitionedIndexWriter implements LuceneIndexWriter
 {
     private LuceneSchemaIndex index;
 
-    private static final Integer MAXIMUM_PARTITION_SIZE =
-            FeatureToggles.getInteger( PartitionedIndexWriter.class, "partitionSize",
-                    Integer.MAX_VALUE - (Integer.MAX_VALUE / 10) );
+    private final Integer MAXIMUM_PARTITION_SIZE = Integer.getInteger( "luceneSchemaIndex.maxPartitionSize",
+            Integer.MAX_VALUE - (Integer.MAX_VALUE / 10) );
 
     public PartitionedIndexWriter( LuceneSchemaIndex index ) throws IOException
     {
@@ -84,7 +82,7 @@ private synchronized IndexWriter getIndexWriter() throws IOException
     {
         List<IndexPartition> indexPartitions = index.getPartitions();
         Optional<IndexPartition> writablePartition = indexPartitions.stream()
-                .filter( PartitionedIndexWriter::writablePartition )
+                .filter( this::writablePartition )
                 .findFirst();
         if ( writablePartition.isPresent() )
         {
@@ -97,7 +95,7 @@ private synchronized IndexWriter getIndexWriter() throws IOException
         }
     }
 
-    private static boolean writablePartition( IndexPartition partition )
+    private boolean writablePartition( IndexPartition partition )
     {
         return partition.getIndexWriter().numDocs() < MAXIMUM_PARTITION_SIZE;
     }

community/lucene-index/src/test/java/org/neo4j/graphdb/LuceneLabelScanStoreIT.java

@@ -19,9 +19,125 @@
  */
 package org.neo4j.graphdb;
 
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
+
+import org.neo4j.helpers.collection.Iterables;
+import org.neo4j.io.fs.FileUtils;
+import org.neo4j.kernel.NeoStoreDataSource;
+import org.neo4j.kernel.api.labelscan.LabelScanStore;
+import org.neo4j.kernel.api.labelscan.LabelScanWriter;
+import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
+import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
+import org.neo4j.kernel.impl.transaction.state.DataSourceManager;
+import org.neo4j.storageengine.api.schema.LabelScanReader;
+
+import static org.junit.Assert.assertEquals;
+
 public class LuceneLabelScanStoreIT extends LabelScanStoreIT
 {
-    // Just extending the IT from kernel which pulls in the same tests, but with the important difference
-    // that the LuceneLasbelScanStore is on the class path, and will therefore be selected instead of
-    // an in-memory store.
+
+    @Test
+    public void scanStoreStartWithoutExistentIndex() throws IOException
+    {
+        NeoStoreDataSource dataSource = getDataSource();
+        LabelScanStore labelScanStore = dataSource.getLabelScanStore();
+        labelScanStore.shutdown();
+
+        File labelScanStoreDirectory = getLabelScanStoreDirectory( dataSource );
+        FileUtils.deleteRecursively( labelScanStoreDirectory  );
+
+        labelScanStore.init();
+        labelScanStore.start();
+
+        checkLabelScanStoreAccessible( labelScanStore );
+    }
+
+    @Test
+    public void scanStoreRecreateCorruptedIndexOnStartup() throws IOException
+    {
+        NeoStoreDataSource dataSource = getDataSource();
+        LabelScanStore labelScanStore = dataSource.getLabelScanStore();
+
+        Node node = createTestNode();
+        List<Long> labels = readNodeLabels( labelScanStore, node );
+        assertEquals( "Label scan store see 1 label for node", 1, labels.size() );
+        labelScanStore.force();
+        labelScanStore.shutdown();
+
+        corruptIndex( dataSource );
+
+        labelScanStore.init();
+        labelScanStore.start();
+
+        List<Long> rebuildLabels = readNodeLabels( labelScanStore, node );
+        assertEquals( "Store should rebuild corrupted index", labels, rebuildLabels );
+    }
+
+    private List<Long> readNodeLabels( LabelScanStore labelScanStore, Node node )
+    {
+        try ( LabelScanReader reader = labelScanStore.newReader() )
+        {
+            return Iterables.toList( reader.labelsForNode( node.getId() ) );
+        }
+    }
+
+    private Node createTestNode()
+    {
+        Node node = null;
+        try (Transaction transaction = dbRule.beginTx())
+        {
+            node = dbRule.createNode( Label.label( "testLabel" ));
+            transaction.success();
+        }
+        return node;
+    }
+
+    private void corruptIndex( NeoStoreDataSource dataSource ) throws IOException
+    {
+        File labelScanStoreDirectory = getLabelScanStoreDirectory( dataSource );
+        Files.walkFileTree( labelScanStoreDirectory.toPath(), new SimpleFileVisitor<Path>()
+        {
+            @Override
+            public FileVisitResult visitFile( Path file, BasicFileAttributes attrs ) throws IOException
+            {
+                Files.write( file, ArrayUtils.add(Files.readAllBytes( file ), (byte) 7 ));
+                return FileVisitResult.CONTINUE;
+            }
+        } );
+    }
+
+    private File getLabelScanStoreDirectory( NeoStoreDataSource dataSource )
+    {
+        return LabelScanStoreProvider.getStoreDirectory( dataSource.getStoreDir() );
+    }
+
+    private NeoStoreDataSource getDataSource()
+    {
+        DependencyResolver dependencyResolver = dbRule.getDependencyResolver();
+        DataSourceManager dataSourceManager = dependencyResolver.resolveDependency( DataSourceManager.class );
+        return dataSourceManager.getDataSource();
+    }
+
+    private void checkLabelScanStoreAccessible( LabelScanStore labelScanStore ) throws IOException
+    {
+        try ( LabelScanWriter labelScanWriter = labelScanStore.newWriter() )
+        {
+            labelScanWriter.write( NodeLabelUpdate.labelChanges( 1, new long[]{}, new long[]{1} ) );
+        }
+        try ( LabelScanReader labelScanReader = labelScanStore.newReader() )
+        {
+            assertEquals( 1, labelScanReader.labelsForNode( 1 ).next().intValue() );
+        }
+    }
+
 }