Combined unique/non-unique native index populator into one

so that it matches index accessor design and more easily
can provide reader for the added deferred constraint checking

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexPopulator.java

@@ -42,9 +42,16 @@
 import org.neo4j.kernel.api.index.PropertyAccessor;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
 import org.neo4j.kernel.api.schema.index.IndexDescriptor;
+import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler;
+import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler;
+import org.neo4j.kernel.impl.api.index.sampling.UniqueIndexSampler;
+import org.neo4j.storageengine.api.schema.IndexReader;
+import org.neo4j.storageengine.api.schema.IndexSample;
 
 import static org.neo4j.index.internal.gbptree.GBPTree.NO_HEADER_WRITER;
 import static org.neo4j.kernel.api.schema.index.IndexDescriptor.Type.GENERAL;
+import static org.neo4j.kernel.api.schema.index.IndexDescriptor.Type.UNIQUE;
 
 /**
  * {@link IndexPopulator} backed by a {@link GBPTree}.
@@ -61,18 +68,36 @@ abstract class NativeSchemaIndexPopulator<KEY extends NativeSchemaKey, VALUE ext
 
     private final KEY treeKey;
     private final VALUE treeValue;
+    private final UniqueIndexSampler uniqueSampler;
+    private final NonUniqueIndexSampler nonUniqueSampler;
+    final IndexSamplingConfig samplingConfig;
+
     private WorkSync<IndexUpdateApply<KEY,VALUE>,IndexUpdateWork<KEY,VALUE>> additionsWorkSync;
     private WorkSync<IndexUpdateApply<KEY,VALUE>,IndexUpdateWork<KEY,VALUE>> updatesWorkSync;
 
     private byte[] failureBytes;
     private boolean dropped;
 
     NativeSchemaIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, Layout<KEY,VALUE> layout,
-            SchemaIndexProvider.Monitor monitor, IndexDescriptor descriptor, long indexId )
+            SchemaIndexProvider.Monitor monitor, IndexDescriptor descriptor, long indexId, IndexSamplingConfig samplingConfig )
     {
         super( pageCache, fs, storeFile, layout, monitor, descriptor, indexId );
         this.treeKey = layout.newKey();
         this.treeValue = layout.newValue();
+        this.samplingConfig = samplingConfig;
+        switch ( descriptor.type() )
+        {
+        case GENERAL:
+            uniqueSampler = null;
+            nonUniqueSampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() );
+            break;
+        case UNIQUE:
+            uniqueSampler = new UniqueIndexSampler();
+            nonUniqueSampler = null;
+            break;
+        default:
+            throw new IllegalArgumentException( "Unexpected index type " + descriptor.type() );
+        }
     }
 
     public void clear() throws IOException
@@ -89,11 +114,11 @@ public synchronized void create() throws IOException
         // true:  tree uniqueness is (value,entityId)
         // false: tree uniqueness is (value) <-- i.e. more strict
         boolean compareIds = descriptor.type() == GENERAL;
-        additionsWorkSync = new WorkSync<>( new IndexUpdateApply( tree, treeKey, treeValue, new ConflictDetectingValueMerger<>( compareIds ) ) );
+        additionsWorkSync = new WorkSync<>( new IndexUpdateApply<>( tree, treeKey, treeValue, new ConflictDetectingValueMerger<>( compareIds ) ) );
 
         // for updates we have to have uniqueness on (value,entityId) to allow for intermediary violating updates.
         // there are added conflict checks after updates have been applied.
-        updatesWorkSync = new WorkSync<>( new IndexUpdateApply( tree, treeKey, treeValue, new ConflictDetectingValueMerger<>( true ) ) );
+        updatesWorkSync = new WorkSync<>( new IndexUpdateApply<>( tree, treeKey, treeValue, new ConflictDetectingValueMerger<>( true ) ) );
     }
 
     @Override
@@ -125,7 +150,7 @@ public void verifyDeferredConstraints( PropertyAccessor propertyAccessor )
     @Override
     public IndexUpdater newPopulatingUpdater( PropertyAccessor accessor )
     {
-        return new IndexUpdater()
+        IndexUpdater updater = new IndexUpdater()
         {
             private boolean closed;
             private final Collection<IndexEntryUpdate<?>> updates = new ArrayList<>();
@@ -152,8 +177,18 @@ private void assertOpen()
                 }
             }
         };
+
+        if ( descriptor.type() == UNIQUE )
+        {
+            // The index population detects conflicts on the fly, however for updates coming in we're in a position
+            // where we cannot detect conflicts while applying, but instead afterwards.
+            updater = new DeferredConflictCheckingIndexUpdater( updater, this::newReader, descriptor );
+        }
+        return updater;
     }
 
+    abstract IndexReader newReader();
+
     @Override
     public synchronized void close( boolean populationCompletedSuccessfully ) throws IOException
     {
@@ -187,7 +222,7 @@ private void applyWithWorkSync( WorkSync<IndexUpdateApply<KEY,VALUE>,IndexUpdate
     {
         try
         {
-            workSync.apply( new IndexUpdateWork( updates ) );
+            workSync.apply( new IndexUpdateWork<>( updates ) );
         }
         catch ( ExecutionException e )
         {
@@ -286,17 +321,47 @@ static class IndexUpdateWork<KEY extends NativeSchemaKey, VALUE extends NativeSc
         }
 
         @Override
-        public IndexUpdateWork combine( IndexUpdateWork work )
+        public IndexUpdateWork<KEY,VALUE> combine( IndexUpdateWork work )
         {
             ArrayList<IndexEntryUpdate<?>> combined = new ArrayList<>( updates );
             combined.addAll( work.updates );
-            return new IndexUpdateWork( combined );
+            return new IndexUpdateWork<>( combined );
         }
 
         @Override
-        public void apply( IndexUpdateApply indexUpdateApply ) throws Exception
+        public void apply( IndexUpdateApply<KEY,VALUE> indexUpdateApply ) throws Exception
         {
             indexUpdateApply.process( updates );
         }
     }
+
+    @Override
+    public void includeSample( IndexEntryUpdate<?> update )
+    {
+        switch ( descriptor.type() )
+        {
+        case GENERAL:
+            nonUniqueSampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) update.values() ) );
+            break;
+        case UNIQUE:
+            uniqueSampler.increment( 1 );
+            break;
+        default:
+            throw new IllegalArgumentException( "Unexpected index type " + descriptor.type() );
+        }
+    }
+
+    @Override
+    public IndexSample sampleResult()
+    {
+        switch ( descriptor.type() )
+        {
+        case GENERAL:
+            return nonUniqueSampler.result();
+        case UNIQUE:
+            return uniqueSampler.result();
+        default:
+            throw new IllegalArgumentException( "Unexpected index type " + descriptor.type() );
+        }
+    }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexProvider.java

@@ -19,15 +19,6 @@
  */
 package org.neo4j.kernel.impl.index.schema;
 
-import org.neo4j.kernel.api.index.IndexDirectoryStructure.Factory;
-import org.neo4j.kernel.api.schema.index.IndexDescriptor;
-import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
-import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
-
-import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_FAILED;
-import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_ONLINE;
-import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_POPULATING;
-
 import java.io.File;
 import java.io.IOException;
 
@@ -38,8 +29,12 @@
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.api.index.IndexAccessor;
+import org.neo4j.kernel.api.index.IndexDirectoryStructure.Factory;
 import org.neo4j.kernel.api.index.IndexPopulator;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
+import org.neo4j.kernel.api.schema.index.IndexDescriptor;
+import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
 
 /**
  * Base class for native indexes on top of {@link GBPTree}.
@@ -75,19 +70,13 @@ public IndexPopulator getPopulator( long indexId, IndexDescriptor descriptor, In
         }
 
         File storeFile = nativeIndexFileFromIndexId( indexId );
-        switch ( descriptor.type() )
-        {
-        case GENERAL:
-            return new NativeNonUniqueSchemaIndexPopulator<>( pageCache, fs, storeFile, layoutNonUnique(), samplingConfig,
-                    monitor, descriptor, indexId );
-        case UNIQUE:
-            return new NativeUniqueSchemaIndexPopulator<>( pageCache, fs, storeFile, layoutUnique(), samplingConfig, monitor, descriptor,
-                    indexId );
-        default:
-            throw new UnsupportedOperationException( "Can not create index populator of type " + descriptor.type() );
-        }
+        Layout<KEY,VALUE> layout = layout( descriptor );
+        return newIndexPopulator( storeFile, layout, descriptor, indexId, samplingConfig );
     }
 
+    protected abstract IndexPopulator newIndexPopulator( File storeFile, Layout<KEY, VALUE> layout, IndexDescriptor descriptor, long indexId,
+            IndexSamplingConfig samplingConfig );
+
     @Override
     public IndexAccessor getOnlineAccessor(
             long indexId, IndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) throws IOException

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeNonUniqueSchemaIndexPopulator.java → community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexPopulator.java

@@ -24,38 +24,22 @@
 import org.neo4j.index.internal.gbptree.Layout;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
-import org.neo4j.kernel.api.index.IndexEntryUpdate;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
 import org.neo4j.kernel.api.schema.index.IndexDescriptor;
-import org.neo4j.kernel.impl.api.index.sampling.DefaultNonUniqueIndexSampler;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
-import org.neo4j.kernel.impl.api.index.sampling.NonUniqueIndexSampler;
-import org.neo4j.storageengine.api.schema.IndexSample;
+import org.neo4j.storageengine.api.schema.IndexReader;
 
-/**
- * {@link NativeSchemaIndexPopulator} which can accept duplicate values (for different entity ids).
- */
-class NativeNonUniqueSchemaIndexPopulator<KEY extends NativeSchemaKey, VALUE extends NativeSchemaValue>
-        extends NativeSchemaIndexPopulator<KEY,VALUE>
+class NumberSchemaIndexPopulator extends NativeSchemaIndexPopulator<NumberSchemaKey,NativeSchemaValue>
 {
-    private NonUniqueIndexSampler sampler;
-
-    NativeNonUniqueSchemaIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, Layout<KEY,VALUE> layout,
-            IndexSamplingConfig samplingConfig, SchemaIndexProvider.Monitor monitor, IndexDescriptor descriptor, long indexId )
-    {
-        super( pageCache, fs, storeFile, layout, monitor, descriptor, indexId );
-        this.sampler = new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() );
-    }
-
-    @Override
-    public void includeSample( IndexEntryUpdate<?> update )
+    NumberSchemaIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, Layout<NumberSchemaKey,NativeSchemaValue> layout,
+            SchemaIndexProvider.Monitor monitor, IndexDescriptor descriptor, long indexId, IndexSamplingConfig samplingConfig )
     {
-        sampler.include( SamplingUtil.encodedStringValuesForSampling( (Object[]) update.values() ) );
+        super( pageCache, fs, storeFile, layout, monitor, descriptor, indexId, samplingConfig );
     }
 
     @Override
-    public IndexSample sampleResult()
+    IndexReader newReader()
     {
-        return sampler.result();
+        return new NumberSchemaIndexReader<>( tree, layout, samplingConfig, descriptor );
     }
 }

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexProvider.java

@@ -32,6 +32,7 @@
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.api.index.IndexAccessor;
 import org.neo4j.kernel.api.index.IndexDirectoryStructure;
+import org.neo4j.kernel.api.index.IndexPopulator;
 import org.neo4j.kernel.api.schema.index.IndexDescriptor;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
 import org.neo4j.values.storable.ValueGroup;
@@ -64,6 +65,13 @@ protected NumberLayoutNonUnique layoutNonUnique()
         return new NumberLayoutNonUnique();
     }
 
+    @Override
+    protected IndexPopulator newIndexPopulator( File storeFile, Layout<NumberSchemaKey,NativeSchemaValue> layout, IndexDescriptor descriptor, long indexId,
+            IndexSamplingConfig samplingConfig )
+    {
+        return new NumberSchemaIndexPopulator( pageCache, fs, storeFile, layout, monitor, descriptor, indexId, samplingConfig );
+    }
+
     @Override
     protected IndexAccessor newIndexAccessor( File storeFile, Layout<NumberSchemaKey,NativeSchemaValue> layout, IndexDescriptor descriptor,
             long indexId, IndexSamplingConfig samplingConfig ) throws IOException

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexPopulator.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import java.io.File;
+
+import org.neo4j.index.internal.gbptree.Layout;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.api.index.SchemaIndexProvider;
+import org.neo4j.kernel.api.schema.index.IndexDescriptor;
+import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.storageengine.api.schema.IndexReader;
+
+public class StringSchemaIndexPopulator extends NativeSchemaIndexPopulator<StringSchemaKey,NativeSchemaValue>
+{
+    StringSchemaIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, Layout<StringSchemaKey,NativeSchemaValue> layout,
+            SchemaIndexProvider.Monitor monitor, IndexDescriptor descriptor, long indexId, IndexSamplingConfig samplingConfig )
+    {
+        super( pageCache, fs, storeFile, layout, monitor, descriptor, indexId, samplingConfig );
+    }
+
+    @Override
+    IndexReader newReader()
+    {
+        return new StringSchemaIndexReader( tree, layout, samplingConfig, descriptor );
+    }
+}