diff --git a/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/CompositeIndexAccessorCompatibility.java b/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/CompositeIndexAccessorCompatibility.java index bcd91b6eaddf..7e99933baeb5 100644 --- a/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/CompositeIndexAccessorCompatibility.java +++ b/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/CompositeIndexAccessorCompatibility.java @@ -56,6 +56,7 @@ import static org.neo4j.values.storable.ValueGroup.GEOMETRY; import static org.neo4j.values.storable.ValueGroup.GEOMETRY_ARRAY; import static org.neo4j.values.storable.Values.booleanArray; +import static org.neo4j.values.storable.Values.intValue; import static org.neo4j.values.storable.Values.longArray; import static org.neo4j.values.storable.Values.pointArray; import static org.neo4j.values.storable.Values.pointValue; @@ -264,26 +265,12 @@ public void testIndexSeekExactWithRangeByTemporalArray() throws Exception @Test public void testIndexSeekExactWithRangeBySpatial() throws Exception { - testIndexSeekExactWithRange( GEOMETRY, pointValue( WGS84, 100D, 100D ), pointValue( WGS84, 0D, 0D ), + testIndexSeekExactWithRange( GEOMETRY, intValue( 100 ), intValue( 10 ), + pointValue( WGS84, -10D, -10D ), + pointValue( WGS84, -1D, -1D ), + pointValue( WGS84, 0D, 0D ), pointValue( WGS84, 1D, 1D ), - pointValue( WGS84, 10D, 10D ), - pointValue( WGS84, 110D, 110D ), - pointValue( WGS84, 200D, 200D ), - pointValue( WGS84, 300D, 300D ) ); - } - - @Test - public void testIndexSeekExactWithRangeBySpatialArray() throws Exception - { - testIndexSeekExactWithRange( GEOMETRY, - pointArray( new PointValue[] {pointValue( WGS84, 100D, 100D ), pointValue( WGS84, 100D, 101D )} ), - pointArray( new PointValue[] {pointValue( WGS84, 0D, 0D ), pointValue( WGS84, 0D, 1D )} ), - - pointArray( new PointValue[] {pointValue( WGS84, 1D, 1D ), pointValue( WGS84, 1D, 2D )} ), - pointArray( new PointValue[] {pointValue( WGS84, 10D, 10D ), pointValue( WGS84, 10D, 11D )} ), - pointArray( new PointValue[] {pointValue( WGS84, 100D, 100D ), pointValue( WGS84, 100D, 101D )} ), - pointArray( new PointValue[] {pointValue( WGS84, 200D, 200D ), pointValue( WGS84, 200D, 201D )} ), - pointArray( new PointValue[] {pointValue( WGS84, 300D, 300D ), pointValue( WGS84, 300D, 301D )} ) ); + pointValue( WGS84, 10D, 10D ) ); } private void testIndexSeekExactWithRange( ValueGroup valueGroup, Value base1, Value base2, Value obj1, Value obj2, Value obj3, Value obj4, Value obj5 ) @@ -309,7 +296,6 @@ private void testIndexSeekExactWithRange( ValueGroup valueGroup, Value base1, Va assertThat( query( exact( 0, base1 ), range( 1, obj5, false, obj2, true ) ), equalTo( EMPTY_LIST ) ); assertThat( query( exact( 0, base1 ), range( 1, null, false, obj3, false ) ), equalTo( asList( 1L, 2L ) ) ); assertThat( query( exact( 0, base1 ), range( 1, null, true, obj3, true ) ), equalTo( asList( 1L, 2L, 3L ) ) ); - assertThat( query( exact( 0, base1 ), range( 1, valueGroup ) ), equalTo( asList( 1L, 2L, 3L, 4L, 5L ) ) ); assertThat( query( exact( 0, base1 ), range( 1, obj1, false, obj2, true ) ), equalTo( singletonList( 2L ) ) ); assertThat( query( exact( 0, base1 ), range( 1, obj1, false, obj3, false ) ), equalTo( singletonList( 2L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj2, true, obj4, false ) ), equalTo( asList( 7L, 8L ) ) ); @@ -318,9 +304,14 @@ private void testIndexSeekExactWithRange( ValueGroup valueGroup, Value base1, Va assertThat( query( exact( 0, base2 ), range( 1, obj5, false, obj2, true ) ), equalTo( EMPTY_LIST ) ); assertThat( query( exact( 0, base2 ), range( 1, null, false, obj3, false ) ), equalTo( asList( 6L, 7L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, null, true, obj3, true ) ), equalTo( asList( 6L, 7L, 8L ) ) ); - assertThat( query( exact( 0, base2 ), range( 1, valueGroup ) ), equalTo( asList( 6L, 7L, 8L, 9L, 10L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj1, false, obj2, true ) ), equalTo( singletonList( 7L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj1, false, obj3, false ) ), equalTo( singletonList( 7L ) ) ); + + if ( valueGroup != GEOMETRY && valueGroup != GEOMETRY_ARRAY ) + { + assertThat( query( exact( 0, base1 ), range( 1, obj1.valueGroup() ) ), equalTo( asList( 1L, 2L, 3L, 4L, 5L ) ) ); + assertThat( query( exact( 0, base2 ), range( 1, obj1.valueGroup() ) ), equalTo( asList( 6L, 7L, 8L, 9L, 10L ) ) ); + } } private void testIndexSeekExactWithRangeByBooleanType( ValueGroup valueGroup, Value base1, Value base2, Value obj1, Value obj2 ) throws Exception @@ -337,7 +328,7 @@ private void testIndexSeekExactWithRangeByBooleanType( ValueGroup valueGroup, Va assertThat( query( exact( 0, base1 ), range( 1, obj1, false, obj2, false ) ), equalTo( EMPTY_LIST ) ); assertThat( query( exact( 0, base1 ), range( 1, null, true, obj2, true ) ), equalTo( asList( 1L, 2L ) ) ); assertThat( query( exact( 0, base1 ), range( 1, obj1, true, null, true ) ), equalTo( asList( 1L, 2L ) ) ); - assertThat( query( exact( 0, base1 ), range( 1, valueGroup ) ), equalTo( asList( 1L, 2L ) ) ); + assertThat( query( exact( 0, base1 ), range( 1, obj1.valueGroup() ) ), equalTo( asList( 1L, 2L ) ) ); assertThat( query( exact( 0, base1 ), range( 1, obj2, true, obj1, true ) ), equalTo( EMPTY_LIST ) ); assertThat( query( exact( 0, base2 ), range( 1, obj1, true, obj2, true ) ), equalTo( asList( 3L, 4L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj1, false, obj2, true ) ), equalTo( singletonList( 4L ) ) ); @@ -345,7 +336,7 @@ private void testIndexSeekExactWithRangeByBooleanType( ValueGroup valueGroup, Va assertThat( query( exact( 0, base2 ), range( 1, obj1, false, obj2, false ) ), equalTo( EMPTY_LIST ) ); assertThat( query( exact( 0, base2 ), range( 1, null, true, obj2, true ) ), equalTo( asList( 3L, 4L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj1, true, null, true ) ), equalTo( asList( 3L, 4L ) ) ); - assertThat( query( exact( 0, base2 ), range( 1, valueGroup ) ), equalTo( asList( 3L, 4L ) ) ); + assertThat( query( exact( 0, base2 ), range( 1, obj1.valueGroup() ) ), equalTo( asList( 3L, 4L ) ) ); assertThat( query( exact( 0, base2 ), range( 1, obj2, true, obj1, true ) ), equalTo( EMPTY_LIST ) ); } @@ -576,10 +567,10 @@ public void testIndexSeekRangeWithExistsBySpatial() throws Exception { testIndexSeekRangeWithExists( GEOMETRY, pointValue( Cartesian, 0D, 0D ), - pointValue( Cartesian, 100D, 100D ), - pointValue( Cartesian, 200D, 200D ), - pointValue( Cartesian, 300D, 300D ), - pointValue( Cartesian, 400D, 400D ) ); + pointValue( Cartesian, 1D, 1D ), + pointValue( Cartesian, 2D, 2D ), + pointValue( Cartesian, 3D, 3D ), + pointValue( Cartesian, 4D, 4D ) ); } @Test @@ -587,10 +578,10 @@ public void testIndexSeekRangeWithExistsBySpatialArray() throws Exception { testIndexSeekRangeWithExists( ValueGroup.GEOMETRY_ARRAY, pointArray( new PointValue[] {pointValue( Cartesian, 0D, 0D ), pointValue( Cartesian, 0D, 1D )} ), - pointArray( new PointValue[] {pointValue( Cartesian, 100D, 100D ), pointValue( Cartesian, 100D, 100D )} ), - pointArray( new PointValue[] {pointValue( Cartesian, 200D, 200D ), pointValue( Cartesian, 200D, 200D )} ), - pointArray( new PointValue[] {pointValue( Cartesian, 300D, 300D ), pointValue( Cartesian, 300D, 300D )} ), - pointArray( new PointValue[] {pointValue( Cartesian, 400D, 400D ), pointValue( Cartesian, 400D, 400D )} ) ); + pointArray( new PointValue[] {pointValue( Cartesian, 10D, 1D ), pointValue( Cartesian, 10D, 2D )} ), + pointArray( new PointValue[] {pointValue( Cartesian, 20D, 2D ), pointValue( Cartesian, 20D, 3D )} ), + pointArray( new PointValue[] {pointValue( Cartesian, 30D, 3D ), pointValue( Cartesian, 30D, 4D )} ), + pointArray( new PointValue[] {pointValue( Cartesian, 40D, 4D ), pointValue( Cartesian, 40D, 5D )} ) ); } private void testIndexSeekRangeWithExists( ValueGroup valueGroup, Object obj1, Object obj2, Object obj3, Object obj4, Object obj5 ) throws Exception @@ -618,7 +609,7 @@ private void testIndexSeekRangeWithExists( ValueGroup valueGroup, Value obj1, Va { // This cannot be done for spatial values because each bound in a spatial query needs a coordinate reference system, // and those are provided by Value instances, e.g. PointValue - assertThat( query( range( 0, valueGroup ), exists( 1 ) ), equalTo( asList( 1L, 2L, 3L, 4L, 5L ) ) ); + assertThat( query( range( 0, obj1.valueGroup() ), exists( 1 ) ), equalTo( asList( 1L, 2L, 3L, 4L, 5L ) ) ); } assertThat( query( range( 0, obj1, false, obj2, true ), exists( 1 ) ), equalTo( singletonList( 2L ) ) ); assertThat( query( range( 0, obj1, false, obj3, false ), exists( 1 ) ), equalTo( singletonList( 2L ) ) ); diff --git a/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java b/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java index bea44ec7961f..5e518a659e0d 100644 --- a/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java +++ b/community/community-it/kernel-it/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java @@ -19,14 +19,16 @@ */ package org.neo4j.kernel.api.index; -import org.eclipse.collections.api.iterator.LongIterator; import org.junit.After; import org.junit.Before; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import org.neo4j.collection.PrimitiveLongResourceIterator; import org.neo4j.internal.kernel.api.IndexQuery; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.configuration.Config; @@ -34,10 +36,13 @@ import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.storageengine.api.schema.IndexDescriptor; import org.neo4j.storageengine.api.schema.IndexReader; +import org.neo4j.values.storable.Value; public abstract class IndexAccessorCompatibility extends IndexProviderCompatibilityTestSuite.Compatibility { protected IndexAccessor accessor; + // This map is for spatial values, so that the #query method can lookup the values for the results and filter properly + private Map committedValues = new HashMap<>(); public IndexAccessorCompatibility( IndexProviderCompatibilityTestSuite testSuite, IndexDescriptor descriptor ) { @@ -63,28 +68,48 @@ public void after() protected List query( IndexQuery... predicates ) throws Exception { - return metaGet( reader -> reader.query( predicates ) ); - } - - private List metaGet( ReaderInteraction interaction ) throws Exception - { - try ( IndexReader reader = accessor.newReader() ) + try ( IndexReader reader = accessor.newReader(); + PrimitiveLongResourceIterator unfilteredResults = reader.query( predicates )) { List list = new LinkedList<>(); - for ( LongIterator iterator = interaction.results( reader ); iterator.hasNext(); ) + while ( unfilteredResults.hasNext() ) { - list.add( iterator.next() ); + long entityId = unfilteredResults.next(); + if ( passesFilter( entityId, predicates ) ) + { + list.add( entityId ); + } } Collections.sort( list ); return list; } } - interface ReaderInteraction + /** + * Run the Value[] from a particular entityId through the list of IndexQuery[] predicates to see if they all accept the value. + */ + private boolean passesFilter( long entityId, IndexQuery[] predicates ) { - LongIterator results( IndexReader reader ) throws Exception; + if ( predicates.length == 1 && predicates[0] instanceof IndexQuery.ExistsPredicate ) + { + return true; + } + + Value[] values = committedValues.get( entityId ); + for ( int i = 0; i < values.length; i++ ) + { + if ( !predicates[i].acceptsValue( values[i] ) ) + { + return false; + } + } + return true; } + /** + * Commit these updates to the index. Also store the values, which currently are stored for all types except geometry, + * so therefore it's done explicitly here so that we can filter on them later. + */ void updateAndCommit( List> updates ) throws IndexEntryConflictException { try ( IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE ) ) @@ -92,6 +117,18 @@ void updateAndCommit( List> updates ) throws IndexEntryConfl for ( IndexEntryUpdate update : updates ) { updater.process( update ); + switch ( update.updateMode() ) + { + case ADDED: + case CHANGED: + committedValues.put( update.getEntityId(), update.values() ); + break; + case REMOVED: + committedValues.remove( update.getEntityId() ); + break; + default: + throw new IllegalArgumentException( "Unknown update mode of " + update ); + } } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/CompositeGenericKey.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/CompositeGenericKey.java index ed2ce8e07780..3dd238ddab72 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/CompositeGenericKey.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/CompositeGenericKey.java @@ -22,9 +22,13 @@ import java.util.Arrays; import java.util.StringJoiner; +import org.neo4j.gis.spatial.index.curves.SpaceFillingCurve; +import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache; import org.neo4j.util.Preconditions; +import org.neo4j.values.storable.CoordinateReferenceSystem; +import org.neo4j.values.storable.PointValue; import org.neo4j.values.storable.Value; import org.neo4j.values.storable.ValueGroup; @@ -43,6 +47,17 @@ class CompositeGenericKey extends NativeIndexKey } } + /** + * Special init method for when doing sub-queries for geometry range. This given slot will not be initialized with a {@link PointValue}, + * but a 1D value which is derived from that point, calculated based on intersecting tiles. + * + * @see SpaceFillingCurve#getTilesIntersectingEnvelope(double[], double[], SpaceFillingCurveConfiguration) + */ + void initFromDerivedSpatialValue( int stateSlot, CoordinateReferenceSystem crs, long derivedValue, Inclusion inclusion ) + { + states[stateSlot].writePointDerived( crs, derivedValue, inclusion ); + } + @Override void writeValue( int stateSlot, Value value, Inclusion inclusion ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericKeyState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericKeyState.java index e132e551355f..76e4ae7c6ed3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericKeyState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericKeyState.java @@ -1783,6 +1783,18 @@ else if ( this.crs != crs ) } } + void writePointDerived( CoordinateReferenceSystem crs, long derivedValue, NativeIndexKey.Inclusion inclusion ) + { + if ( isArray ) + { + throw new IllegalStateException( "Not sure we're doing arrays like this, are we?" ); + } + type = Type.GEOMETRY; + this.inclusion = inclusion; + long0 = derivedValue; + updateCurve( crs ); + } + private void updateCurve( CoordinateReferenceSystem crs ) { if ( this.crs == null || this.crs != crs ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java index 412b4412caa4..e133af46b950 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexAccessor.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; +import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration; import org.neo4j.index.internal.gbptree.GBPTree; import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector; import org.neo4j.io.fs.FileSystemAbstraction; @@ -38,14 +39,19 @@ class GenericNativeIndexAccessor extends NativeIndexAccessor { + private final IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings; + private final SpaceFillingCurveConfiguration configuration; private Validator validator; GenericNativeIndexAccessor( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout layout, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor, - IndexSamplingConfig samplingConfig, IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings ) throws IOException + IndexSamplingConfig samplingConfig, IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings, + SpaceFillingCurveConfiguration configuration ) throws IOException { super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, samplingConfig, new SpaceFillingCurveSettingsWriter( spaceFillingCurveSettings ) ); + this.spaceFillingCurveSettings = spaceFillingCurveSettings; + this.configuration = configuration; } @Override @@ -57,7 +63,7 @@ protected void afterTreeInstantiation( GBPTree { + private final IndexSpecificSpaceFillingCurveSettingsCache spatialSettings; + private final SpaceFillingCurveConfiguration configuration; + GenericNativeIndexPopulator( PageCache pageCache, FileSystemAbstraction fs, File storeFile, IndexLayout layout, IndexProvider.Monitor monitor, StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig, - IndexSpecificSpaceFillingCurveSettingsCache spatialSettings ) + IndexSpecificSpaceFillingCurveSettingsCache spatialSettings, SpaceFillingCurveConfiguration configuration ) { super( pageCache, fs, storeFile, layout, monitor, descriptor, samplingConfig, new SpaceFillingCurveSettingsWriter( spatialSettings ) ); + this.spatialSettings = spatialSettings; + this.configuration = configuration; } @Override IndexReader newReader() { - return new GenericNativeIndexReader( tree, layout, samplingConfig, descriptor ); + return new GenericNativeIndexReader( tree, layout, samplingConfig, descriptor, spatialSettings, configuration ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java index 1630d3cdcc70..62687b8ccd09 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/GenericNativeIndexProvider.java @@ -46,6 +46,7 @@ import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettings; import org.neo4j.storageengine.api.schema.StoreIndexDescriptor; import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettingsReader; +import org.neo4j.storageengine.api.schema.StoreIndexDescriptor; import org.neo4j.values.storable.CoordinateReferenceSystem; import org.neo4j.values.storable.ValueCategory; @@ -173,7 +174,7 @@ protected IndexPopulator newIndexPopulator( File storeFile, IndexLayout { + private final IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings; + private final SpaceFillingCurveConfiguration configuration; + GenericNativeIndexReader( GBPTree tree, IndexLayout layout, - IndexSamplingConfig samplingConfig, IndexDescriptor descriptor ) + IndexSamplingConfig samplingConfig, IndexDescriptor descriptor, IndexSpecificSpaceFillingCurveSettingsCache spaceFillingCurveSettings, + SpaceFillingCurveConfiguration configuration ) { super( tree, layout, samplingConfig, descriptor ); + this.spaceFillingCurveSettings = spaceFillingCurveSettings; + this.configuration = configuration; } @Override @@ -57,12 +71,61 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - boolean initializeRangeForQuery( CompositeGenericKey treeKeyFrom, CompositeGenericKey treeKeyTo, IndexQuery[] predicates ) + public void query( IndexProgressor.NodeValueClient client, IndexOrder indexOrder, boolean needsValues, IndexQuery... query ) + { + validateQuery( indexOrder, query ); + + IndexQuery.GeometryRangePredicate geometryRangePredicate = getGeometryRangePredicateIfAny( query ); + if ( geometryRangePredicate != null ) + { + try + { + // If there's a GeometryRangeQuery among the predicates then this query changes from a straight-forward: build from/to and seek... + // into a query that is split into multiple sub-queries. Predicates both before and after will have to be accompanied each sub-query. + BridgingIndexProgressor multiProgressor = new BridgingIndexProgressor( client, descriptor.schema().getPropertyIds() ); + client.initialize( descriptor, multiProgressor, query, false ); + double[] from = geometryRangePredicate.from() == null ? null : geometryRangePredicate.from().coordinate(); + double[] to = geometryRangePredicate.to() == null ? null : geometryRangePredicate.to().coordinate(); + CoordinateReferenceSystem crs = geometryRangePredicate.crs(); + SpaceFillingCurve curve = spaceFillingCurveSettings.forCrs( crs, false ); + List ranges = curve.getTilesIntersectingEnvelope( from, to, configuration ); + for ( SpaceFillingCurve.LongRange range : ranges ) + { + // Here's a sub-query that we'll have to do for this geometry range. Build this query from all predicates + // and when getting to the geometry range predicate that sparked these sub-query chenanigans, swap in this sub-query in its place. + CompositeGenericKey treeKeyFrom = layout.newKey(); + CompositeGenericKey treeKeyTo = layout.newKey(); + initializeFromToKeys( treeKeyFrom, treeKeyTo ); + boolean needFiltering = initializeRangeForGeometrySubQuery( multiProgressor, treeKeyFrom, treeKeyTo, query, crs, range ); + + // TODO needsValues==true could be problematic, no? + startSeekForInitializedRange( multiProgressor, treeKeyFrom, treeKeyTo, query, needFiltering, needsValues ); + } + } + catch ( IllegalArgumentException e ) + { + // Invalid query ranges will cause this state (eg. min>max) + client.initialize( descriptor, IndexProgressor.EMPTY, query, false ); + } + } + else + { + CompositeGenericKey treeKeyFrom = layout.newKey(); + CompositeGenericKey treeKeyTo = layout.newKey(); + initializeFromToKeys( treeKeyFrom, treeKeyTo ); + + boolean needFilter = initializeRangeForQuery( client, treeKeyFrom, treeKeyTo, query ); + startSeekForInitializedRange( client, treeKeyFrom, treeKeyTo, query, needFilter, needsValues ); + } + } + + private boolean initializeRangeForGeometrySubQuery( IndexProgressor.NodeValueClient client, CompositeGenericKey treeKeyFrom, CompositeGenericKey treeKeyTo, + IndexQuery[] query, CoordinateReferenceSystem crs, SpaceFillingCurve.LongRange range ) { boolean needsFiltering = false; - for ( int i = 0; i < predicates.length; i++ ) + for ( int i = 0; i < query.length; i++ ) { - IndexQuery predicate = predicates[i]; + IndexQuery predicate = query[i]; switch ( predicate.type() ) { case exists: @@ -75,9 +138,18 @@ boolean initializeRangeForQuery( CompositeGenericKey treeKeyFrom, CompositeGener treeKeyTo.initFromValue( i, exactPredicate.value(), NEUTRAL ); break; case range: - RangePredicate rangePredicate = (RangePredicate) predicate; - initFromForRange( i, rangePredicate, treeKeyFrom ); - initToForRange( i, rangePredicate, treeKeyTo ); + if ( isGeometryRangeQuery( predicate ) ) + { + // Use the supplied SpaceFillingCurve range instead + treeKeyFrom.initFromDerivedSpatialValue( i, crs, range.min, fromInclusion( (RangePredicate) predicate ) ); + treeKeyTo.initFromDerivedSpatialValue( i, crs, range.max + 1, toInclusion( (RangePredicate) predicate ) ); + } + else + { + RangePredicate rangePredicate = (RangePredicate) predicate; + initFromForRange( i, rangePredicate, treeKeyFrom ); + initToForRange( i, rangePredicate, treeKeyTo ); + } break; case stringPrefix: StringPrefixPredicate prefixPredicate = (StringPrefixPredicate) predicate; @@ -97,6 +169,13 @@ boolean initializeRangeForQuery( CompositeGenericKey treeKeyFrom, CompositeGener return needsFiltering; } + @Override + boolean initializeRangeForQuery( IndexProgressor.NodeValueClient client, CompositeGenericKey treeKeyFrom, CompositeGenericKey treeKeyTo, + IndexQuery[] query ) + { + return initializeRangeForGeometrySubQuery( client, treeKeyFrom, treeKeyTo, query, null, null ); + } + private static void initFromForRange( int stateSlot, RangePredicate rangePredicate, CompositeGenericKey treeKeyFrom ) { Value fromValue = rangePredicate.fromValue(); @@ -106,7 +185,7 @@ private static void initFromForRange( int stateSlot, RangePredicate rangePred } else { - treeKeyFrom.initFromValue( stateSlot, fromValue, rangePredicate.fromInclusive() ? LOW : HIGH ); + treeKeyFrom.initFromValue( stateSlot, fromValue, fromInclusion( rangePredicate ) ); treeKeyFrom.setCompareId( true ); } } @@ -120,8 +199,36 @@ private static void initToForRange( int stateSlot, RangePredicate rangePredic } else { - treeKeyTo.initFromValue( stateSlot, toValue, rangePredicate.toInclusive() ? HIGH : LOW ); + treeKeyTo.initFromValue( stateSlot, toValue, toInclusion( rangePredicate ) ); treeKeyTo.setCompareId( true ); } } + + private static NativeIndexKey.Inclusion fromInclusion( RangePredicate rangePredicate ) + { + return rangePredicate.fromInclusive() ? LOW : HIGH; + } + + private static NativeIndexKey.Inclusion toInclusion( RangePredicate rangePredicate ) + { + return rangePredicate.toInclusive() ? HIGH : LOW; + } + + private IndexQuery.GeometryRangePredicate getGeometryRangePredicateIfAny( IndexQuery[] predicates ) + { + for ( IndexQuery predicate : predicates ) + { + if ( isGeometryRangeQuery( predicate ) ) + { + return (IndexQuery.GeometryRangePredicate) predicate; + } + } + return null; + } + + private boolean isGeometryRangeQuery( IndexQuery predicate ) + { + return predicate instanceof IndexQuery.GeometryRangePredicate; + } + } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexReader.java index a9ce949b1c8c..bf65df49a84e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NativeIndexReader.java @@ -24,7 +24,6 @@ import java.util.HashSet; import java.util.Set; -import org.neo4j.collection.PrimitiveLongResourceCollections; import org.neo4j.collection.PrimitiveLongResourceIterator; import org.neo4j.cursor.RawCursor; import org.neo4j.index.internal.gbptree.GBPTree; @@ -114,34 +113,10 @@ public long countIndexedNodes( long nodeId, Value... propertyValues ) @Override public PrimitiveLongResourceIterator query( IndexQuery... predicates ) { - KEY treeKeyFrom = layout.newKey(); - KEY treeKeyTo = layout.newKey(); - - treeKeyFrom.initialize( Long.MIN_VALUE ); - treeKeyTo.initialize( Long.MAX_VALUE ); - - boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); - if ( isBackwardsSeek( treeKeyFrom, treeKeyTo ) ) - { - return PrimitiveLongResourceCollections.emptyIterator(); - } - - try - { - RawCursor,IOException> seeker = tree.seek( treeKeyFrom, treeKeyTo ); - openSeekers.add( seeker ); - return getHitIterator( seeker, needFilter, predicates ); - } - catch ( IOException e ) - { - throw new UncheckedIOException( e ); - } - } - - private PrimitiveLongResourceIterator getHitIterator( RawCursor,IOException> seeker, boolean needFilter, IndexQuery[] predicates ) - { - return needFilter ? new FilteringNativeHitIterator<>( seeker, openSeekers, predicates ) - : new NativeHitIterator<>( seeker, openSeekers ); + // This method isn't called from main product code anymore so it can might as well delegate to the "real" method + NodeValueIterator nodeValueIterator = new NodeValueIterator(); + query( nodeValueIterator, IndexOrder.NONE, nodeValueIterator.needsValues(), predicates ); + return nodeValueIterator; } @Override @@ -151,12 +126,16 @@ public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder KEY treeKeyFrom = layout.newKey(); KEY treeKeyTo = layout.newKey(); + initializeFromToKeys( treeKeyFrom, treeKeyTo ); + boolean needFilter = initializeRangeForQuery( cursor, treeKeyFrom, treeKeyTo, predicates ); + startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates, needFilter, needsValues ); + } + + void initializeFromToKeys( KEY treeKeyFrom, KEY treeKeyTo ) + { treeKeyFrom.initialize( Long.MIN_VALUE ); treeKeyTo.initialize( Long.MAX_VALUE ); - - boolean needFilter = initializeRangeForQuery( treeKeyFrom, treeKeyTo, predicates ); - startSeekForInitializedRange( cursor, treeKeyFrom, treeKeyTo, predicates, needFilter, needsValues ); } @Override @@ -167,7 +146,7 @@ public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder /** * @return true if query results from seek will need to be filtered through the predicates, else false */ - abstract boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ); + abstract boolean initializeRangeForQuery( IndexProgressor.NodeValueClient cursor, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ); void startSeekForInitializedRange( IndexProgressor.NodeValueClient client, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] query, boolean needFilter, boolean needsValues ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexReader.java index 4a36d15e3ea3..a149a7787e69 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/NumberIndexReader.java @@ -26,6 +26,7 @@ import org.neo4j.internal.kernel.api.IndexQuery.RangePredicate; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.storageengine.api.schema.IndexDescriptor; +import org.neo4j.storageengine.api.schema.IndexProgressor; import org.neo4j.values.storable.Value; import org.neo4j.values.storable.ValueGroup; import org.neo4j.values.storable.Values; @@ -50,7 +51,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - boolean initializeRangeForQuery( NumberIndexKey treeKeyFrom, NumberIndexKey treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( IndexProgressor.NodeValueClient cursor, NumberIndexKey treeKeyFrom, NumberIndexKey treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexPartReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexPartReader.java index 9d2d46759e6f..4fb0c95085fb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexPartReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialIndexPartReader.java @@ -23,7 +23,6 @@ import java.io.UncheckedIOException; import java.util.List; -import org.neo4j.collection.PrimitiveLongResourceIterator; import org.neo4j.cursor.RawCursor; import org.neo4j.gis.spatial.index.curves.SpaceFillingCurve; import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration; @@ -38,7 +37,6 @@ import org.neo4j.storageengine.api.schema.IndexDescriptor; import org.neo4j.storageengine.api.schema.IndexProgressor; import org.neo4j.values.storable.Value; - import org.neo4j.values.storable.ValueGroup; public class SpatialIndexPartReader extends NativeIndexReader @@ -67,19 +65,11 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - boolean initializeRangeForQuery( SpatialIndexKey treeKeyFrom, SpatialIndexKey treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( IndexProgressor.NodeValueClient cursor, SpatialIndexKey treeKeyFrom, SpatialIndexKey treeKeyTo, IndexQuery[] predicates ) { throw new UnsupportedOperationException( "Cannot initialize 1D range in multidimensional spatial index reader" ); } - @Override - public PrimitiveLongResourceIterator query( IndexQuery... predicates ) - { - NodeValueIterator nodeValueIterator = new NodeValueIterator(); - query( nodeValueIterator, IndexOrder.NONE, nodeValueIterator.needsValues(), predicates ); - return nodeValueIterator; - } - @Override public void query( IndexProgressor.NodeValueClient cursor, IndexOrder indexOrder, boolean needsValues, IndexQuery... predicates ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringIndexReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringIndexReader.java index 9bfc17f4186a..baa1a40f7a9a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringIndexReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/StringIndexReader.java @@ -26,6 +26,7 @@ import org.neo4j.internal.kernel.api.IndexQuery.RangePredicate; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.storageengine.api.schema.IndexDescriptor; +import org.neo4j.storageengine.api.schema.IndexProgressor; import org.neo4j.values.storable.Value; import org.neo4j.values.storable.ValueGroup; import org.neo4j.values.storable.Values; @@ -52,7 +53,7 @@ void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - boolean initializeRangeForQuery( StringIndexKey treeKeyFrom, StringIndexKey treeKeyTo, IndexQuery[] predicates ) + boolean initializeRangeForQuery( IndexProgressor.NodeValueClient cursor, StringIndexKey treeKeyFrom, StringIndexKey treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java index a6f9dfc36462..c6832cab10f4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/TemporalIndexPartReader.java @@ -24,6 +24,7 @@ import org.neo4j.internal.kernel.api.IndexQuery; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.storageengine.api.schema.IndexDescriptor; +import org.neo4j.storageengine.api.schema.IndexProgressor; import org.neo4j.values.storable.Value; import org.neo4j.values.storable.ValueGroup; import org.neo4j.values.storable.Values; @@ -50,7 +51,7 @@ protected void validateQuery( IndexOrder indexOrder, IndexQuery[] predicates ) } @Override - protected boolean initializeRangeForQuery( KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) + protected boolean initializeRangeForQuery( IndexProgressor.NodeValueClient cursor, KEY treeKeyFrom, KEY treeKeyTo, IndexQuery[] predicates ) { IndexQuery predicate = predicates[0]; switch ( predicate.type() ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/config/IndexSpecificSpaceFillingCurveSettingsCache.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/config/IndexSpecificSpaceFillingCurveSettingsCache.java index 5afd4877fa45..847a6d3604a9 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/config/IndexSpecificSpaceFillingCurveSettingsCache.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/config/IndexSpecificSpaceFillingCurveSettingsCache.java @@ -57,7 +57,11 @@ public IndexSpecificSpaceFillingCurveSettingsCache( public SpaceFillingCurve forCrs( int crsTableId, int crsCodePoint, boolean assignToIndexIfNotYetAssigned ) { CoordinateReferenceSystem crs = CoordinateReferenceSystem.get( crsTableId, crsCodePoint ); + return forCrs( crs, assignToIndexIfNotYetAssigned ); + } + public SpaceFillingCurve forCrs( CoordinateReferenceSystem crs, boolean assignToIndexIfNotYetAssigned ) + { // Index-specific SpaceFillingCurveSettings specificSetting = specificIndexConfigCache.get( crs ); if ( specificSetting != null )