Better lucene index writer config for index population

This commit makes IndexWriterConfig configurable when LuceneSchemaIndex is
created. Then during population an improved version of the config is used. It
disables flushing by number of documents and enables flushing by RAM buffer
size. Value of this size is also increased from default 16M to 50M so flushes
of newly added documents happen less frequently.

Each IndexWriterConfig is attached to a single IndexWriter and can't be
reused to create other writers. That is why Supplier<IndexWriterConfig>
is used everywhere until actual writer creation.

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndex.java

@@ -21,6 +21,7 @@
 
 import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.Directory;
 
 import java.io.Closeable;
@@ -33,6 +34,7 @@
 import java.util.Map;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
 
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.helpers.ArrayUtil;
@@ -58,12 +60,14 @@ public abstract class AbstractLuceneIndex implements Closeable
     protected final ReentrantLock partitionsLock = new ReentrantLock();
 
     protected final PartitionedIndexStorage indexStorage;
+    private final Supplier<IndexWriterConfig> writerConfigSupplier;
     private List<IndexPartition> partitions = new CopyOnWriteArrayList<>();
     private volatile boolean open;
 
-    public AbstractLuceneIndex( PartitionedIndexStorage indexStorage )
+    public AbstractLuceneIndex( PartitionedIndexStorage indexStorage, Supplier<IndexWriterConfig> writerConfigSupplier )
     {
         this.indexStorage = indexStorage;
+        this.writerConfigSupplier = writerConfigSupplier;
     }
 
     /**
@@ -93,7 +97,8 @@ public void open() throws IOException
         Map<File,Directory> indexDirectories = indexStorage.openIndexDirectories();
         for ( Map.Entry<File,Directory> indexDirectory : indexDirectories.entrySet() )
         {
-            partitions.add( new IndexPartition( indexDirectory.getKey(), indexDirectory.getValue() ) );
+            partitions.add( new IndexPartition( indexDirectory.getKey(), indexDirectory.getValue(),
+                    writerConfigSupplier.get() ) );
         }
         open = true;
     }
@@ -363,8 +368,8 @@ public IndexPartition addNewPartition() throws IOException
         try
         {
             File partitionFolder = createNewPartitionFolder();
-            IndexPartition indexPartition = new IndexPartition( partitionFolder,
-                    indexStorage.openDirectory( partitionFolder ) );
+            Directory directory = indexStorage.openDirectory( partitionFolder );
+            IndexPartition indexPartition = new IndexPartition( partitionFolder, directory, writerConfigSupplier.get() );
             partitions.add( indexPartition );
             return indexPartition;
         }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/IndexWriterConfigs.java

@@ -40,12 +40,15 @@ public final class IndexWriterConfigs
     private static final double MERGE_POLICY_MIN_MERGE_MB =
             FeatureToggles.getDouble( IndexWriterConfigs.class, "min.merge", 0.1 );
 
+    private static final int POPULATION_RAM_BUFFER_SIZE_MB =
+            FeatureToggles.getInteger( IndexWriterConfigs.class, "population.ram.buffer.size", 50 );
+
     private IndexWriterConfigs()
     {
         throw new AssertionError( "Not for instantiation!" );
     }
 
-    public static IndexWriterConfig standardConfig()
+    public static IndexWriterConfig standard()
     {
         IndexWriterConfig writerConfig = new IndexWriterConfig( LuceneDataSource.KEYWORD_ANALYZER );
 
@@ -61,4 +64,12 @@ public static IndexWriterConfig standardConfig()
 
         return writerConfig;
     }
+
+    public static IndexWriterConfig population()
+    {
+        IndexWriterConfig writerConfig = standard();
+        writerConfig.setMaxBufferedDocs( IndexWriterConfig.DISABLE_AUTO_FLUSH );
+        writerConfig.setRAMBufferSizeMB( POPULATION_RAM_BUFFER_SIZE_MB );
+        return writerConfig;
+    }
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/index/partition/IndexPartition.java

@@ -20,6 +20,7 @@
 package org.neo4j.kernel.api.impl.index.partition;
 
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
 import org.apache.lucene.store.Directory;
@@ -30,7 +31,6 @@
 
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.io.IOUtils;
-import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.backup.LuceneIndexSnapshotFileIterator;
 
 /**
@@ -45,11 +45,12 @@ public class IndexPartition implements Closeable
     private final SearcherManager searcherManager;
     private final File indexFolder;
 
-    public IndexPartition( File partitionFolder, Directory directory ) throws IOException
+    public IndexPartition( File partitionFolder, Directory directory, IndexWriterConfig writerConfig )
+            throws IOException
     {
         this.indexFolder = partitionFolder;
         this.directory = directory;
-        this.indexWriter = new IndexWriter( directory, IndexWriterConfigs.standardConfig() );
+        this.indexWriter = new IndexWriter( directory, writerConfig );
         this.searcherManager = new SearcherManager( indexWriter, true, new SearcherFactory() );
     }
 

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/labelscan/LuceneLabelScanIndex.java

@@ -24,6 +24,7 @@
 import java.util.List;
 
 import org.neo4j.kernel.api.impl.index.AbstractLuceneIndex;
+import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.partition.IndexPartition;
 import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
@@ -51,7 +52,7 @@ public class LuceneLabelScanIndex extends AbstractLuceneIndex
 
     public LuceneLabelScanIndex( BitmapDocumentFormat format, PartitionedIndexStorage indexStorage )
     {
-        super( indexStorage );
+        super( indexStorage, IndexWriterConfigs::standard );
         this.format = format;
         this.storageStrategy = new NodeRangeDocumentLabelScanStorageStrategy( format );
     }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndex.java

@@ -21,12 +21,14 @@
 
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.Directory;
 
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
 
 import org.neo4j.helpers.TaskCoordinator;
 import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
@@ -63,9 +65,9 @@ public class LuceneSchemaIndex extends AbstractLuceneIndex
     private final TaskCoordinator taskCoordinator = new TaskCoordinator( 10, TimeUnit.MILLISECONDS );
 
     public LuceneSchemaIndex( PartitionedIndexStorage indexStorage, IndexConfiguration config,
-            IndexSamplingConfig samplingConfig )
+            IndexSamplingConfig samplingConfig, Supplier<IndexWriterConfig> writerConfigSupplier )
     {
-        super( indexStorage );
+        super( indexStorage, writerConfigSupplier );
         this.config = config;
         this.samplingConfig = samplingConfig;
     }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexBuilder.java

@@ -19,9 +19,14 @@
  */
 package org.neo4j.kernel.api.impl.schema;
 
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+
 import java.util.Map;
+import java.util.function.Supplier;
 
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
+import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.builder.AbstractLuceneIndexBuilder;
 import org.neo4j.kernel.api.index.IndexConfiguration;
 import org.neo4j.kernel.configuration.Config;
@@ -41,13 +46,15 @@ public class LuceneSchemaIndexBuilder extends AbstractLuceneIndexBuilder<LuceneS
 {
     private IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
     private IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
+    private Supplier<IndexWriterConfig> writerConfigSupplier = IndexWriterConfigs::standard;
 
     private LuceneSchemaIndexBuilder()
     {
     }
 
     /**
      * Create new lucene schema index builder.
+     *
      * @return new LuceneSchemaIndexBuilder
      */
     public static LuceneSchemaIndexBuilder create()
@@ -57,6 +64,7 @@ public static LuceneSchemaIndexBuilder create()
 
     /**
      * Specify lucene schema index sampling config
+     *
      * @param samplingConfig sampling config
      * @return index builder
      */
@@ -68,6 +76,7 @@ public LuceneSchemaIndexBuilder withSamplingConfig( IndexSamplingConfig sampling
 
     /**
      * Specify lucene schema index sampling buffer size
+     *
      * @param size sampling buffer size
      * @return index builder
      */
@@ -81,6 +90,7 @@ public LuceneSchemaIndexBuilder withSamplingBufferSize( int size )
 
     /**
      * Specify lucene schema index config
+     *
      * @param indexConfig index config
      * @return index builder
      */
@@ -90,8 +100,21 @@ public LuceneSchemaIndexBuilder withIndexConfig( IndexConfiguration indexConfig
         return this;
     }
 
+    /**
+     * Specify {@link Supplier} of lucene {@link IndexWriterConfig} to create {@link IndexWriter}s.
+     *
+     * @param writerConfigSupplier the supplier of writer configs
+     * @return index builder
+     */
+    public LuceneSchemaIndexBuilder withWriterConfig( Supplier<IndexWriterConfig> writerConfigSupplier )
+    {
+        this.writerConfigSupplier = writerConfigSupplier;
+        return this;
+    }
+
     /**
      * Transform builder to build unique index
+     *
      * @return index builder
      */
     public LuceneSchemaIndexBuilder uniqueIndex()
@@ -102,11 +125,12 @@ public LuceneSchemaIndexBuilder uniqueIndex()
 
     /**
      * Build lucene schema index with specified configuration
+     *
      * @return lucene schema index
      */
     public LuceneSchemaIndex build()
     {
-        return new LuceneSchemaIndex( storageBuilder.build(), indexConfig, samplingConfig );
+        return new LuceneSchemaIndex( storageBuilder.build(), indexConfig, samplingConfig, writerConfigSupplier );
     }
 
 }

community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexProvider.java

@@ -30,6 +30,7 @@
 
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
 import org.neo4j.kernel.api.impl.index.storage.IndexStorageFactory;
 import org.neo4j.kernel.api.impl.index.storage.PartitionedIndexStorage;
@@ -76,6 +77,7 @@ public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor,
                                         .withIndexConfig( config )
                                         .withSamplingConfig( samplingConfig )
                                         .withIndexStorage( getIndexStorage( indexId ) )
+                                        .withWriterConfig( IndexWriterConfigs::population )
                                         .build();
         if ( config.isUnique() )
         {

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/AbstractLuceneIndexIntegrationTest.java

@@ -181,7 +181,7 @@ private static class TestLuceneIndex extends AbstractLuceneIndex
     {
         TestLuceneIndex( PartitionedIndexStorage indexStorage )
         {
-            super( indexStorage );
+            super( indexStorage, IndexWriterConfigs::standard );
         }
     }
 

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/backup/LuceneIndexSnapshotFileIteratorTest.java

@@ -59,7 +59,7 @@ public void initializeLuceneResources() throws IOException
     {
         indexDir = testDir.directory();
         dir = new RAMDirectory();
-        writer = new IndexWriter( dir, IndexWriterConfigs.standardConfig() );
+        writer = new IndexWriter( dir, IndexWriterConfigs.standard() );
     }
 
     @After

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/storage/PartitionedIndexStorageTest.java

@@ -186,7 +186,7 @@ private Directory createRandomLuceneDir( File rootFolder ) throws IOException
         File folder = createRandomFolder( rootFolder );
         DirectoryFactory directoryFactory = getOrCreateDirFactory( fs );
         Directory directory = directoryFactory.open( folder );
-        try ( IndexWriter writer = new IndexWriter( directory, IndexWriterConfigs.standardConfig() ) )
+        try ( IndexWriter writer = new IndexWriter( directory, IndexWriterConfigs.standard() ) )
         {
             writer.addDocument( randomDocument() );
             writer.commit();

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/index/verification/SimpleUniquenessVerifierTest.java

@@ -75,7 +75,7 @@ public void initLuceneResources() throws Exception
     {
         dirFactory = new DirectoryFactory.InMemoryDirectoryFactory();
         Directory dir = dirFactory.open( testDir.directory( "test" ) );
-        writer = new IndexWriter( dir, IndexWriterConfigs.standardConfig() );
+        writer = new IndexWriter( dir, IndexWriterConfigs.standard() );
         searcherManager = new SearcherManager( writer, true, new SearcherFactory() );
     }
 

community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/labelscan/LuceneLabelScanStoreWriterTest.java

@@ -51,10 +51,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.locks.Lock;
 import java.util.stream.Stream;
 
 import org.neo4j.kernel.api.impl.index.IndexReaderStub;
+import org.neo4j.kernel.api.impl.index.IndexWriterConfigs;
 import org.neo4j.kernel.api.impl.index.partition.IndexPartition;
 import org.neo4j.kernel.api.impl.index.partition.PartitionSearcher;
 import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
@@ -344,7 +344,7 @@ private static class StubIndexPartition extends IndexPartition
 
         StubIndexPartition( File folder, Directory directory ) throws IOException
         {
-            super( folder, directory );
+            super( folder, directory, IndexWriterConfigs.standard() );
             this.directory = directory;
         }