Merge pull request #11152 from tinwelint/3.4-nsni-fusion

Native string index in fusion index

community/kernel-api/src/test/java/org/neo4j/internal/kernel/api/NodeValueIndexCursorTestBase.java

@@ -195,7 +195,7 @@ public void shouldPerformExactLookupInCompositeIndex() throws Exception
               PrimitiveLongSet uniqueIds = Primitive.longSet() )
         {
             // when
-            IndexValueCapability valueCapability = index.valueCapability( ValueGroup.TEXT );
+            IndexValueCapability valueCapability = index.valueCapability( ValueGroup.TEXT, ValueGroup.TEXT );
             read.nodeIndexSeek( index, node, IndexOrder.NONE, IndexQuery.exact( firstName, "Joe" ),
                     IndexQuery.exact( surname, "Dalton" ) );
 

community/kernel/src/main/java/org/neo4j/kernel/impl/api/state/ConstraintIndexCreator.java

@@ -244,7 +244,7 @@ private void awaitConstrainIndexPopulation( UniquenessConstraintDescriptor const
             }
             else
             {
-                throw new UniquePropertyValueValidationException( constraint, VERIFICATION, cause );
+                throw new UniquePropertyValueValidationException( constraint, VERIFICATION, e );
             }
         }
     }

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIndexProgressor.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.neo4j.cursor.RawCursor;
+import org.neo4j.index.internal.gbptree.Hit;
+import org.neo4j.internal.kernel.api.IndexQuery;
+import org.neo4j.values.storable.Value;
+
+class FilteringNativeHitIndexProgressor<KEY extends NativeSchemaKey, VALUE extends NativeSchemaValue> extends NativeHitIndexProgressor<KEY,VALUE>
+{
+    private final IndexQuery[] filter;
+
+    FilteringNativeHitIndexProgressor( RawCursor<Hit<KEY,VALUE>,IOException> seeker, NodeValueClient client,
+            Collection<RawCursor<Hit<KEY,VALUE>,IOException>> toRemoveFromOnClose, IndexQuery[] filter )
+    {
+        super( seeker, client, toRemoveFromOnClose );
+        this.filter = filter;
+    }
+
+    @Override
+    boolean acceptValue( Value[] values )
+    {
+        for ( int i = 0; i < values.length; i++ )
+        {
+            if ( !filter[i].acceptsValue( values[i] ) )
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIterator.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.index.schema;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import org.neo4j.cursor.RawCursor;
+import org.neo4j.index.internal.gbptree.Hit;
+import org.neo4j.internal.kernel.api.IndexQuery;
+import org.neo4j.values.storable.Value;
+
+class FilteringNativeHitIterator<KEY extends NativeSchemaKey, VALUE extends NativeSchemaValue> extends NativeHitIterator<KEY,VALUE>
+{
+    private final IndexQuery[] filters;
+
+    FilteringNativeHitIterator( RawCursor<Hit<KEY,VALUE>,IOException> seeker,
+            Collection<RawCursor<Hit<KEY,VALUE>,IOException>> toRemoveFromWhenExhausted, IndexQuery[] filters )
+    {
+        super( seeker, toRemoveFromWhenExhausted );
+        this.filters = filters;
+    }
+
+    @Override
+    boolean acceptValue( Value value )
+    {
+        return filters[0].acceptsValue( value );
+    }
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIndexProgressor.java

@@ -52,7 +52,7 @@ public boolean next()
             {
                 KEY key = seeker.get().key();
                 Value[] values = extractValues( key );
-                if ( client.acceptNode( key.getEntityId(), values ) )
+                if ( acceptValue( values ) && client.acceptNode( key.getEntityId(), values ) )
                 {
                     return true;
                 }
@@ -65,6 +65,11 @@ public boolean next()
         }
     }
 
+    boolean acceptValue( Value[] values )
+    {
+        return true;
+    }
+
     Value[] extractValues( KEY key )
     {
         return client.needsValues() ? new Value[]{ key.asValue()} : null;

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberHitIterator.java → community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIterator.java

@@ -28,22 +28,23 @@
 import org.neo4j.collection.primitive.PrimitiveLongResourceIterator;
 import org.neo4j.cursor.RawCursor;
 import org.neo4j.index.internal.gbptree.Hit;
+import org.neo4j.values.storable.Value;
 
 /**
  * Wraps number key/value results in a {@link PrimitiveLongIterator}.
  *
  * @param <KEY> type of {@link NumberSchemaKey}.
  * @param <VALUE> type of {@link NativeSchemaValue}.
  */
-public class NumberHitIterator<KEY extends NativeSchemaKey, VALUE extends NativeSchemaValue>
+public class NativeHitIterator<KEY extends NativeSchemaKey, VALUE extends NativeSchemaValue>
         extends PrimitiveLongCollections.PrimitiveLongBaseIterator
         implements PrimitiveLongResourceIterator
 {
     private final RawCursor<Hit<KEY,VALUE>,IOException> seeker;
     private final Collection<RawCursor<Hit<KEY,VALUE>,IOException>> toRemoveFromWhenExhausted;
     private boolean closed;
 
-    NumberHitIterator( RawCursor<Hit<KEY,VALUE>,IOException> seeker,
+    NativeHitIterator( RawCursor<Hit<KEY,VALUE>,IOException> seeker,
             Collection<RawCursor<Hit<KEY,VALUE>,IOException>> toRemoveFromWhenExhausted )
     {
         this.seeker = seeker;
@@ -55,9 +56,13 @@ protected boolean fetchNext()
     {
         try
         {
-            if ( seeker.next() )
+            while ( seeker.next() )
             {
-                return next( seeker.get().key().getEntityId() );
+                KEY key = seeker.get().key();
+                if ( acceptValue( key.asValue() ) )
+                {
+                    return next( key.getEntityId() );
+                }
             }
             return false;
         }
@@ -67,6 +72,11 @@ protected boolean fetchNext()
         }
     }
 
+    boolean acceptValue( Value value )
+    {
+        return true;
+    }
+
     private void ensureCursorClosed() throws IOException
     {
         if ( !closed )

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexReader.java

@@ -70,8 +70,8 @@ public void close()
     @Override
     public IndexSampler createSampler()
     {
-        // For an unique index there's an optimization, knowing that all values in it are unique, to simply count
-        // the number of indexes values and create a sample for that count. The GBPTree doesn't have an O(1)
+        // For a unique index there's an optimization, knowing that all values in it are unique, to simply count
+        // the number of indexed values and create a sample for that count. The GBPTree doesn't have an O(1)
         // count mechanism, it will have to manually count the indexed values in it to get it.
         // For that reason this implementation opts for keeping complexity down by just using the existing
         // non-unique sampler which scans the index and counts (potentially duplicates, of which there will
@@ -113,7 +113,7 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates )
         KEY treeKeyFrom = layout.newKey();
         KEY treeKeyTo = layout.newKey();
 
-        initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates );
+        boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates );
         if ( isBackwardsSeek( treeKeyFrom, treeKeyTo ) )
         {
             return PrimitiveLongResourceCollections.emptyIterator();
@@ -123,14 +123,20 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates )
         {
             RawCursor<Hit<KEY,VALUE>,IOException> seeker = tree.seek( treeKeyFrom, treeKeyTo );
             openSeekers.add( seeker );
-            return new NumberHitIterator<>( seeker, openSeekers );
+            return getHitIterator( seeker, needFilter, predicates );
         }
         catch ( IOException e )
         {
             throw new UncheckedIOException( e );
         }
     }
 
+    private PrimitiveLongResourceIterator getHitIterator( RawCursor<Hit<KEY,VALUE>,IOException> seeker, boolean needFilter, IndexQuery[] predicates )
+    {
+        return needFilter ? new FilteringNativeHitIterator<KEY,VALUE>( seeker, openSeekers, predicates )
+                          : new NativeHitIterator<KEY,VALUE>( seeker, openSeekers );
+    }
+
     @Override
     public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder, IndexQuery... predicates )
     {
@@ -139,18 +145,21 @@ public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder
         KEY treeKeyFrom = layout.newKey();
         KEY treeKeyTo = layout.newKey();
 
-        initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates );
-        startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates );
+        boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates );
+        startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates, needFilter );
     }
 
     @Override
     public abstract boolean hasFullValuePrecision( IndexQuery... predicates );
 
     abstract void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates );
 
-    abstract void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates );
+    /**
+     * @return true if query results from seek will need to be filtered through the predicates, else false
+     */
+    abstract boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates );
 
-    void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query )
+    void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query, boolean needFilter )
     {
         if ( isBackwardsSeek( treeKeyFrom, treeKeyTo ) )
         {
@@ -160,7 +169,7 @@ void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY t
         try
         {
             RawCursor<Hit<KEY,VALUE>,IOException> seeker = makeIndexSeeker( treeKeyFrom, treeKeyTo );
-            IndexProgressor hitProgressor = new NativeHitIndexProgressor<>( seeker, client, openSeekers );
+            IndexProgressor hitProgressor = getIndexProgressor( seeker, client, needFilter, query );
             client.initialize( descriptor, hitProgressor, query );
         }
         catch ( IOException e )
@@ -176,6 +185,13 @@ RawCursor<Hit<KEY,VALUE>,IOException> makeIndexSeeker( KEY treeKeyFrom, KEY tree
         return seeker;
     }
 
+    private IndexProgressor getIndexProgressor( RawCursor<Hit<KEY,VALUE>,IOException> seeker, IndexProgressor.NodeValueClient client, boolean needFilter,
+            IndexQuery[] query )
+    {
+        return needFilter ? new FilteringNativeHitIndexProgressor<KEY,VALUE>( seeker, client, openSeekers, query )
+                          : new NativeHitIndexProgressor<KEY,VALUE>( seeker, client, openSeekers );
+    }
+
     private boolean isBackwardsSeek( KEY treeKeyFrom, KEY treeKeyTo )
     {
         return layout.compare( treeKeyFrom, treeKeyTo ) > 0;

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexReader.java

@@ -49,7 +49,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates )
     }
 
     @Override
-    void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates )
+    boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates )
     {
         IndexQuery predicate = predicates[0];
         switch ( predicate.type() )
@@ -71,6 +71,7 @@ void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predi
         default:
             throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." );
         }
+        return false;
     }
 
     private void initToForRange( NumberRangePredicate rangePredicate, KEY treeKeyTo )

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java

@@ -73,7 +73,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates )
     }
 
     @Override
-    void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates )
+    boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates )
     {
         throw new UnsupportedOperationException( "Cannot initialize 1D range in multidimensional spatial index reader" );
     }
@@ -111,7 +111,7 @@ private void startSeekForExists( IndexProgressor.NodeValueClient client, IndexQu
         KEY treeKeyTo = layout.newKey();
         treeKeyFrom.initAsLowest();
         treeKeyTo.initAsHighest();
-        startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates );
+        startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates, false );
     }
 
     private void startSeekForExact( IndexProgressor.NodeValueClient client, Value value, IndexQuery... predicates )
@@ -120,7 +120,7 @@ private void startSeekForExact( IndexProgressor.NodeValueClient client, Value va
         KEY treeKeyTo = layout.newKey();
         treeKeyFrom.from( Long.MIN_VALUE, value );
         treeKeyTo.from( Long.MAX_VALUE, value );
-        startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates );
+        startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates, false );
     }
 
     private void startSeekForRange( IndexProgressor.NodeValueClient client, GeometryRangePredicate rangePredicate, IndexQuery[] query )
@@ -158,7 +158,7 @@ private void startSeekForRange( IndexProgressor.NodeValueClient client, Geometry
     }
 
     @Override
-    void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query )
+    void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query, boolean needFilter )
     {
         if ( layout.compare( treeKeyFrom, treeKeyTo ) > 0 )
         {

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringIndexProvider.java

@@ -46,9 +46,9 @@ public class StringIndexProvider extends NativeIndexProvider<StringSchemaKey,Nat
     static final IndexCapability CAPABILITY = new StringIndexCapability();
     private static final Descriptor STRING_PROVIDER_DESCRIPTOR = new Descriptor( KEY, "1.0" );
 
-    StringIndexProvider( PageCache pageCache, FileSystemAbstraction fs,
-                         IndexDirectoryStructure.Factory directoryStructure, Monitor monitor, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector,
-                         boolean readOnly )
+    public StringIndexProvider( PageCache pageCache, FileSystemAbstraction fs,
+            IndexDirectoryStructure.Factory directoryStructure, Monitor monitor, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector,
+            boolean readOnly )
     {
         super( STRING_PROVIDER_DESCRIPTOR, 0, directoryStructure, pageCache, fs, monitor, recoveryCleanupWorkCollector, readOnly );
     }

community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexReader.java

@@ -30,6 +30,8 @@
 import org.neo4j.values.storable.Value;
 import org.neo4j.values.storable.ValueGroup;
 
+import static org.neo4j.internal.kernel.api.IndexQuery.StringPrefixPredicate;
+
 class StringSchemaIndexReader extends NativeSchemaIndexReader<StringSchemaKey,NativeSchemaValue>
 {
     StringSchemaIndexReader( GBPTree<StringSchemaKey,NativeSchemaValue> tree, Layout<StringSchemaKey,NativeSchemaValue> layout,
@@ -50,25 +52,35 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates )
     }
 
     @Override
-    void initializeRangeForQuery( StringSchemaKey treeKeyFrom, StringSchemaKey treeKeyTo, IndexQuery[] predicates )
+    boolean initializeRangeForQuery( StringSchemaKey treeKeyFrom, StringSchemaKey treeKeyTo, IndexQuery[] predicates )
     {
         IndexQuery predicate = predicates[0];
         switch ( predicate.type() )
         {
         case exists:
             treeKeyFrom.initAsLowest();
             treeKeyTo.initAsHighest();
-            break;
+            return false;
         case exact:
             ExactPredicate exactPredicate = (ExactPredicate) predicate;
             treeKeyFrom.from( Long.MIN_VALUE, exactPredicate.value() );
             treeKeyTo.from( Long.MAX_VALUE, exactPredicate.value() );
-            break;
+            return false;
         case rangeString:
             StringRangePredicate rangePredicate = (StringRangePredicate)predicate;
             initFromForRange( rangePredicate, treeKeyFrom );
             initToForRange( rangePredicate, treeKeyTo );
-            break;
+            return false;
+        case stringPrefix:
+            StringPrefixPredicate prefixPredicate = (StringPrefixPredicate) predicate;
+            treeKeyFrom.initAsPrefixLow( prefixPredicate.prefix() );
+            treeKeyTo.initAsPrefixHigh( prefixPredicate.prefix() );
+            return false;
+        case stringSuffix:
+        case stringContains:
+            treeKeyFrom.initAsLowest();
+            treeKeyTo.initAsHighest();
+            return true;
         default:
             throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." );
         }