From 5ff481b943e36ef49c4c4cc62238f6ca36fa798f Mon Sep 17 00:00:00 2001 From: Anton Persson Date: Thu, 1 Mar 2018 10:42:31 +0100 Subject: [PATCH] Implement contains and suffix seek for native string index This is done by filtering with IndexQuery just outside of SeekCursor (GBPTree) on deserialized value. --- .../FilteringNativeHitIndexProgressor.java | 53 +++++++++++++++++++ .../schema/FilteringNumberHitIterator.java | 46 ++++++++++++++++ .../schema/NativeHitIndexProgressor.java | 7 ++- .../index/schema/NativeSchemaIndexReader.java | 42 ++++++++++++--- .../impl/index/schema/NumberHitIterator.java | 13 ++++- .../index/schema/NumberSchemaIndexReader.java | 3 +- .../schema/SpatialSchemaIndexReader.java | 8 +-- .../index/schema/StringSchemaIndexReader.java | 19 +++---- .../index/schema/TemporalIndexPartReader.java | 3 +- .../schema/fusion/FusionIndexReader.java | 5 +- 10 files changed, 173 insertions(+), 26 deletions(-) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIndexProgressor.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNumberHitIterator.java diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIndexProgressor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIndexProgressor.java new file mode 100644 index 0000000000000..1ce8af9ee8469 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNativeHitIndexProgressor.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.IOException; +import java.util.Collection; + +import org.neo4j.cursor.RawCursor; +import org.neo4j.index.internal.gbptree.Hit; +import org.neo4j.internal.kernel.api.IndexQuery; +import org.neo4j.values.storable.Value; + +class FilteringNativeHitIndexProgressor extends NativeHitIndexProgressor +{ + private final IndexQuery[] filter; + + FilteringNativeHitIndexProgressor( RawCursor,IOException> seeker, NodeValueClient client, + Collection,IOException>> toRemoveFromOnClose, IndexQuery[] filter ) + { + super( seeker, client, toRemoveFromOnClose ); + this.filter = filter; + } + + @Override + boolean acceptValue( Value[] values ) + { + for ( int i = 0; i < values.length; i++ ) + { + if ( !filter[i].acceptsValue( values[i] ) ) + { + return false; + } + } + return true; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNumberHitIterator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNumberHitIterator.java new file mode 100644 index 0000000000000..bd749e106c705 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/FilteringNumberHitIterator.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2002-2018 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.index.schema; + +import java.io.IOException; +import java.util.Collection; + +import org.neo4j.cursor.RawCursor; +import org.neo4j.index.internal.gbptree.Hit; +import org.neo4j.internal.kernel.api.IndexQuery; +import org.neo4j.values.storable.Value; + +class FilteringNumberHitIterator extends NumberHitIterator +{ + private final IndexQuery[] filters; + + FilteringNumberHitIterator( RawCursor,IOException> seeker, + Collection,IOException>> toRemoveFromWhenExhausted, IndexQuery[] filters ) + { + super( seeker, toRemoveFromWhenExhausted ); + this.filters = filters; + } + + @Override + boolean acceptValue( Value value ) + { + return filters[0].acceptsValue( value ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIndexProgressor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIndexProgressor.java index 778a58c5e6021..eb6276776492c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIndexProgressor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeHitIndexProgressor.java @@ -52,7 +52,7 @@ public boolean next() { KEY key = seeker.get().key(); Value[] values = extractValues( key ); - if ( client.acceptNode( key.getEntityId(), values ) ) + if ( acceptValue( values ) && client.acceptNode( key.getEntityId(), values ) ) { return true; } @@ -65,6 +65,11 @@ public boolean next() } } + boolean acceptValue( Value[] values ) + { + return true; + } + Value[] extractValues( KEY key ) { return client.needsValues() ? new Value[]{ key.asValue()} : null; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexReader.java index e988f852c8eed..c15c404c40208 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeSchemaIndexReader.java @@ -113,7 +113,7 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates ) KEY treeKeyFrom = layout.newKey(); KEY treeKeyTo = layout.newKey(); - initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); + boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); if ( isBackwardsSeek( treeKeyFrom, treeKeyTo ) ) { return PrimitiveLongResourceCollections.emptyIterator(); @@ -123,7 +123,7 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates ) { RawCursor,IOException> seeker = tree.seek( treeKeyFrom, treeKeyTo ); openSeekers.add( seeker ); - return new NumberHitIterator<>( seeker, openSeekers ); + return getNumberHitIterator( seeker, needFilter, predicates ); } catch ( IOException e ) { @@ -131,6 +131,18 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates ) } } + private PrimitiveLongResourceIterator getNumberHitIterator( RawCursor,IOException> seeker, boolean needFilter, IndexQuery[] predicates ) + { + if ( needFilter ) + { + return new FilteringNumberHitIterator<>( seeker, openSeekers, predicates ); + } + else + { + return new NumberHitIterator<>( seeker, openSeekers ); + } + } + @Override public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder, IndexQuery... predicates ) { @@ -139,8 +151,8 @@ public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder KEY treeKeyFrom = layout.newKey(); KEY treeKeyTo = layout.newKey(); - initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); - startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates ); + boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); + startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates, needFilter ); } @Override @@ -148,9 +160,12 @@ public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder abstract void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ); - abstract void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ); + /** + * @return true if query results from seek will need to be filtered through the predicates, else false + */ + abstract boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ); - void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query ) + void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query, boolean needFilter ) { if ( isBackwardsSeek( treeKeyFrom, treeKeyTo ) ) { @@ -160,7 +175,7 @@ void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY t try { RawCursor,IOException> seeker = makeIndexSeeker( treeKeyFrom, treeKeyTo ); - IndexProgressor hitProgressor = new NativeHitIndexProgressor<>( seeker, client, openSeekers ); + IndexProgressor hitProgressor = getIndexProgressor( seeker, client, needFilter, query ); client.initialize( descriptor, hitProgressor, query ); } catch ( IOException e ) @@ -176,6 +191,19 @@ RawCursor,IOException> makeIndexSeeker( KEY treeKeyFrom, KEY tree return seeker; } + private IndexProgressor getIndexProgressor( RawCursor,IOException> seeker, IndexProgressor.NodeValueClient client, boolean needFilter, + IndexQuery[] query ) + { + if ( needFilter ) + { + return new FilteringNativeHitIndexProgressor<>( seeker, client, openSeekers, query ); + } + else + { + return new NativeHitIndexProgressor<>( seeker, client, openSeekers ); + } + } + private boolean isBackwardsSeek( KEY treeKeyFrom, KEY treeKeyTo ) { return layout.compare( treeKeyFrom, treeKeyTo ) > 0; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberHitIterator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberHitIterator.java index 050bafc46a3f5..2edf92abcbbbe 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberHitIterator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberHitIterator.java @@ -28,6 +28,7 @@ import org.neo4j.collection.primitive.PrimitiveLongResourceIterator; import org.neo4j.cursor.RawCursor; import org.neo4j.index.internal.gbptree.Hit; +import org.neo4j.values.storable.Value; /** * Wraps number key/value results in a {@link PrimitiveLongIterator}. @@ -55,9 +56,12 @@ protected boolean fetchNext() { try { - if ( seeker.next() ) + while ( seeker.next() ) { - return next( seeker.get().key().getEntityId() ); + if ( acceptValue( seeker.get().key().asValue() ) ) + { + return next( seeker.get().key().getEntityId() ); + } } return false; } @@ -67,6 +71,11 @@ protected boolean fetchNext() } } + boolean acceptValue( Value value ) + { + return true; + } + private void ensureCursorClosed() throws IOException { if ( !closed ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexReader.java index 9c48a3b984700..73ffd921fcd46 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberSchemaIndexReader.java @@ -49,7 +49,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) @@ -71,6 +71,7 @@ void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predi default: throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." ); } + return false; } private void initToForRange( NumberRangePredicate rangePredicate, KEY treeKeyTo ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java index 8d89a865e36f6..7cfef25d1ba8f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java @@ -73,7 +73,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) { throw new UnsupportedOperationException( "Cannot initialize 1D range in multidimensional spatial index reader" ); } @@ -111,7 +111,7 @@ private void startSeekForExists( IndexProgressor.NodeValueClient client, IndexQu KEY treeKeyTo = layout.newKey(); treeKeyFrom.initAsLowest(); treeKeyTo.initAsHighest(); - startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates ); + startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates, false ); } private void startSeekForExact( IndexProgressor.NodeValueClient client, Value value, IndexQuery... predicates ) @@ -120,7 +120,7 @@ private void startSeekForExact( IndexProgressor.NodeValueClient client, Value va KEY treeKeyTo = layout.newKey(); treeKeyFrom.from( Long.MIN_VALUE, value ); treeKeyTo.from( Long.MAX_VALUE, value ); - startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates ); + startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, predicates, false ); } private void startSeekForRange( IndexProgressor.NodeValueClient client, GeometryRangePredicate rangePredicate, IndexQuery[] query ) @@ -158,7 +158,7 @@ private void startSeekForRange( IndexProgressor.NodeValueClient client, Geometry } @Override - void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query ) + void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query, boolean needFilter ) { if ( layout.compare( treeKeyFrom, treeKeyTo ) > 0 ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexReader.java index 3b895851e68b1..7eace4e2577ce 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringSchemaIndexReader.java @@ -52,7 +52,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - void initializeRangeForQuery( StringSchemaKey treeKeyFrom, StringSchemaKey treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( StringSchemaKey treeKeyFrom, StringSchemaKey treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) @@ -60,26 +60,27 @@ void initializeRangeForQuery( StringSchemaKey treeKeyFrom, StringSchemaKey treeK case exists: treeKeyFrom.initAsLowest(); treeKeyTo.initAsHighest(); - break; + return false; case exact: ExactPredicate exactPredicate = (ExactPredicate) predicate; treeKeyFrom.from( Long.MIN_VALUE, exactPredicate.value() ); treeKeyTo.from( Long.MAX_VALUE, exactPredicate.value() ); - break; + return false; case rangeString: StringRangePredicate rangePredicate = (StringRangePredicate)predicate; initFromForRange( rangePredicate, treeKeyFrom ); initToForRange( rangePredicate, treeKeyTo ); - break; + return false; case stringPrefix: StringPrefixPredicate prefixPredicate = (StringPrefixPredicate) predicate; treeKeyFrom.initAsPrefixLow( prefixPredicate.prefix() ); treeKeyTo.initAsPrefixHigh( prefixPredicate.prefix() ); - break; -// case stringSuffix: -// break; -// case stringContains: -// break; + return false; + case stringSuffix: + case stringContains: + treeKeyFrom.initAsLowest(); + treeKeyTo.initAsHighest(); + return true; default: throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java index db4590de5a1f5..c314d019c7d47 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java @@ -49,7 +49,7 @@ protected void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - protected void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) + protected boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) @@ -67,6 +67,7 @@ protected void initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQue default: throw new IllegalArgumentException( "IndexQuery of type " + predicate.type() + " is not supported." ); } + return false; // no filtering } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java index 3fdaa979d9e7e..4a87c1188d5f2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/FusionIndexReader.java @@ -122,7 +122,10 @@ public PrimitiveLongResourceIterator query( IndexQuery... predicates ) throws In return numberReader.query( predicates ); } - if ( predicate instanceof StringRangePredicate ) + if ( predicate instanceof StringRangePredicate || + predicate instanceof StringPrefixPredicate || + predicate instanceof StringSuffixPredicate || + predicate instanceof StringContainsPredicate ) { return stringReader.query( predicate ); }