Label scan store moved to sharded lucene infrastructure

Label scan store is capable of tracking labels for more than Integer.MAX_VALUE
nodes. It uses lucene documents with bitmaps encoded in int or long fields.
However even with this efficient format it's capacity is not infinite.

This commit makes label scan store use sharded lucene index underneath. Though
only reads/writes from single partition are supported. Basic functionality of a
sharded lucene index now lives in AbstractLuceneIndex class. Implementation
for schema index - LuceneSchemaIndex and for label scan store -
LuceneLabelScanStoreIndex. These implementations mainly differ in readers and
writers that they expose.

Interfaces and implementations are not finalized. There are still some
todo items and ignored tests present.

community/io/src/main/java/org/neo4j/io/IOUtils.java

@@ -78,19 +78,22 @@ public static <T extends AutoCloseable> void closeAll( T... closeables ) throws
         Exception closeException = null;
         for ( T closeable : closeables )
         {
-            try
+            if ( closeable != null )
             {
-                closeable.close();
-            }
-            catch ( Exception e )
-            {
-                if ( closeException == null )
+                try
                 {
-                    closeException = e;
+                    closeable.close();
                 }
-                else
+                catch ( Exception e )
                 {
-                    closeException.addSuppressed( e );
+                    if ( closeException == null )
+                    {
+                        closeException = e;
+                    }
+                    else
+                    {
+                        closeException.addSuppressed( e );
+                    }
                 }
             }
         }
@@ -99,4 +102,22 @@ public static <T extends AutoCloseable> void closeAll( T... closeables ) throws
             throw new IOException( "Exception closing multiple resources", closeException );
         }
     }
+
+    /**
+     * Closes given array of {@link AutoCloseable closeables} ignoring all exceptions.
+     *
+     * @param closeables the closeables to close
+     * @param <T> the type of closeable
+     */
+    @SafeVarargs
+    public static <T extends AutoCloseable> void closeAllSilently( T... closeables )
+    {
+        try
+        {
+            closeAll( closeables );
+        }
+        catch ( IOException ignored )
+        {
+        }
+    }
 }

community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexAccessor.java

@@ -135,11 +135,13 @@ public long maxCount()
                     return 0;
                 }
 
-                @Override public void close() throws IOException
+                @Override
+                public void close() throws IOException
                 {
                 }
 
-                @Override public Iterator<Long> iterator()
+                @Override
+                public Iterator<Long> iterator()
                 {
                     return emptyIterator();
                 }

community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexReader.java

@@ -139,7 +139,7 @@ public int countIndexedNodes( long nodeId, Object propertyValue )
         @Override
         public IndexSampler createSampler()
         {
-            return null;
+            return IndexSampler.EMPTY;
         }
 
         @Override

community/kernel/src/main/java/org/neo4j/storageengine/api/schema/IndexSampler.java

@@ -24,6 +24,11 @@
 
 public interface IndexSampler
 {
+    IndexSampler EMPTY = result -> {
+        result.write( 0, 0 );
+        return 0;
+    };
+
     /**
      * Sample this index (on the current thread)
      *

community/kernel/src/main/java/org/neo4j/storageengine/api/schema/LabelScanReader.java

@@ -43,4 +43,8 @@ public interface LabelScanReader extends Resource
     Iterator<Long> labelsForNode( long nodeId );
 
     AllEntriesLabelScanReader allNodeLabelRanges();
+
+    Iterator getAllDocsIterator(); // todo: should not be here...
+
+    long getMaxDoc(); // todo: should not be here...
 }

community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexRestartIT.java

@@ -54,14 +54,14 @@
 
 public class IndexRestartIT
 {
+    @Rule
+    public final EphemeralFileSystemRule fs = new EphemeralFileSystemRule();
 
     private GraphDatabaseService db;
-    @Rule public EphemeralFileSystemRule fs = new EphemeralFileSystemRule();
     private TestGraphDatabaseFactory factory;
     private final ControlledPopulationSchemaIndexProvider provider = new ControlledPopulationSchemaIndexProvider();
     private final Label myLabel = label( "MyLabel" );
 
-
     @Before
     public void before() throws Exception
     {

community/kernel/src/test/java/org/neo4j/kernel/impl/api/scan/InMemoryLabelScanStore.java

@@ -19,6 +19,8 @@
  */
 package org.neo4j.kernel.impl.api.scan;
 
+import org.apache.lucene.document.Document;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -102,6 +104,12 @@ public void close()
             {   // Nothing to close
             }
 
+            @Override
+            public long getMaxDoc()
+            {
+                return 0;
+            }
+
             @Override
             public Iterator<Long> labelsForNode( long nodeId )
             {
@@ -121,6 +129,12 @@ public AllEntriesLabelScanReader allNodeLabelRanges()
             {
                 return newAllEntriesReader();
             }
+
+            @Override
+            public Iterator<Document> getAllDocsIterator()
+            {
+                return null;
+            }
         };
     }
 

community/lucene-index/src/main/java/org/neo4j/index/lucene/LuceneLabelScanStoreBuilder.java

@@ -23,10 +23,10 @@
 import java.io.IOException;
 
 import org.neo4j.io.fs.FileSystemAbstraction;
-import org.neo4j.kernel.api.impl.index.LuceneIndex;
-import org.neo4j.kernel.api.impl.index.LuceneIndexBuilder;
+import org.neo4j.kernel.api.impl.index.LuceneLabelScanIndex;
 import org.neo4j.kernel.api.impl.index.LuceneLabelScanStore;
-import org.neo4j.kernel.api.impl.index.NodeRangeDocumentLabelScanStorageStrategy;
+import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
+import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
 import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.FullStoreChangeStream;
@@ -64,14 +64,12 @@ public LabelScanStore build()
         if ( null == labelScanStore )
         {
             // TODO: Replace with kernel extension based lookup
-            LuceneIndex index = LuceneIndexBuilder.create()
-                    .withIndexRootFolder( LabelScanStoreProvider.getStoreDirectory( storeDir ) )
-                    .withFileSystem( fileSystem )
-                    .withIndexIdentifier( LuceneLabelScanStore.INDEX_IDENTIFIER )
-                    .build();
-
-            labelScanStore = new LuceneLabelScanStore(new NodeRangeDocumentLabelScanStorageStrategy(), index,
-                    fullStoreStream, LuceneLabelScanStore.loggerMonitor( logProvider ) );
+            PartitionedIndexStorage indexStorage = new PartitionedIndexStorage( DirectoryFactory.PERSISTENT, fileSystem,
+                    LabelScanStoreProvider.getStoreDirectory( storeDir ),
+                    LuceneLabelScanStore.INDEX_IDENTIFIER );
+            LuceneLabelScanIndex index = new LuceneLabelScanIndex( indexStorage );
+            labelScanStore = new LuceneLabelScanStore( index, fullStoreStream,
+                    LuceneLabelScanStore.loggerMonitor( logProvider ) );
 
             try
             {