Skip to content

Commit

Permalink
Change when checking for existing spatial indexes
Browse files Browse the repository at this point in the history 
Moving the check to getInitialState ensures that we have the
IndexDescriptor available on creation.

Also renamed SpatialKnownIndex to SpatialCRSSSchemaIndex
  • Loading branch information
@OliviaYtterbrink @fickludd
OliviaYtterbrink authored and fickludd committed Mar 1, 2018
1 parent 9a3c928 commit 0f1a200
Show file tree
Hide file tree
Showing 11 changed files with 150 additions and 155 deletions.
57 changes: 30 additions & 27 deletions .../impl/index/schema/SpatialKnownIndex.java → ...l/index/schema/SpatialCRSSchemaIndex.java
View file
Expand Up @@ -65,18 +65,17 @@
import static org.neo4j.kernel.impl.index.schema.NativeSchemaIndexPopulator.BYTE_POPULATING;

/**
* An instance of this class represents a dynamically created sub-index specific to a particular coordinate reference system.
* A dynamically created sub-index specific to a particular coordinate reference system.
* This allows the fusion index design to be extended to an unknown number of sub-indexes, one for each CRS.
*/
public class SpatialKnownIndex
public class SpatialCRSSchemaIndex
{
private UniqueIndexSampler uniqueSampler;
private final File indexFile;
private final PageCache pageCache;
private final IndexDescriptor descriptor;
private final CoordinateReferenceSystem crs;
private final long indexId;
private final FileSystemAbstraction fs;
private final SchemaIndexProvider.Monitor monitor;
private final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector;
private final SpaceFillingCurve curve;

Expand All @@ -92,18 +91,19 @@ public class SpatialKnownIndex
private WorkSync<IndexUpdateApply<SpatialSchemaKey,NativeSchemaValue>,IndexUpdateWork<SpatialSchemaKey,NativeSchemaValue>> workSync;
private DefaultNonUniqueIndexSampler generalSampler;

/**
* Create a representation of a spatial index for a specific coordinate reference system.
* This constructor should be used for first time creation.
*/
public SpatialKnownIndex( IndexDirectoryStructure directoryStructure, CoordinateReferenceSystem crs, long indexId, PageCache pageCache,
FileSystemAbstraction fs, SchemaIndexProvider.Monitor monitor, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector )
public SpatialCRSSchemaIndex( IndexDescriptor descriptor,
IndexDirectoryStructure directoryStructure,
CoordinateReferenceSystem crs,
long indexId,
PageCache pageCache,
FileSystemAbstraction fs,
SchemaIndexProvider.Monitor monitor,
RecoveryCleanupWorkCollector recoveryCleanupWorkCollector )
{
this.descriptor = descriptor;
this.crs = crs;
this.indexId = indexId;
this.pageCache = pageCache;
this.fs = fs;
this.monitor = monitor;
this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector;

// Depends on crs
Expand All @@ -125,25 +125,31 @@ else if ( crs.getDimension() == 3 )
throw new IllegalArgumentException( "Cannot create spatial index with other than 2D or 3D coordinate reference system: " + crs );
}
state = State.NONE;

layout = layout( descriptor );
treeKey = layout.newKey();
treeValue = layout.newValue();
schemaIndex = new NativeSchemaIndex<>( pageCache, fs, indexFile, layout, monitor, descriptor, indexId );

}

/**
* Makes sure that the index is initialized
*/
public void init( IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
public void init( IndexSamplingConfig samplingConfig )
{
if ( state == State.NONE )
{
initialize( descriptor, samplingConfig );
initialize( samplingConfig );
}
}

/**
* Makes sure that the index is ready to populate
*/
public void startPopulation( IndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) throws IOException
public void startPopulation( IndexSamplingConfig samplingConfig ) throws IOException
{
init( descriptor, samplingConfig );
init( samplingConfig );
if ( state == State.INIT )
{
// First add to sub-index, make sure to create
Expand All @@ -158,9 +164,9 @@ public void startPopulation( IndexDescriptor descriptor, IndexSamplingConfig sam
/**
* Makes sure that the index is online
*/
public void takeOnline( IndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) throws IOException
public void takeOnline( IndexSamplingConfig samplingConfig ) throws IOException
{
init( descriptor, samplingConfig );
init( samplingConfig );
if ( !indexExists() )
{
throw new IOException( "Index file does not exist." );
Expand All @@ -175,14 +181,14 @@ public void takeOnline( IndexDescriptor descriptor, IndexSamplingConfig sampling
}
}

public IndexUpdater updaterWithCreate( IndexDescriptor descriptor, IndexSamplingConfig samplingConfig, boolean populating ) throws IOException
public IndexUpdater updaterWithCreate( IndexSamplingConfig samplingConfig, boolean populating ) throws IOException
{
if ( populating )
{
if ( state == State.NONE )
{
// sub-index didn't exist, create in populating mode
initialize( descriptor, samplingConfig );
initialize( samplingConfig );
create();
}
return newPopulatingUpdater();
Expand All @@ -192,7 +198,7 @@ public IndexUpdater updaterWithCreate( IndexDescriptor descriptor, IndexSampling
if ( state == State.NONE )
{
// sub-index didn't exist, create and make it online
initialize( descriptor, samplingConfig );
initialize( samplingConfig );
create();
finishPopulation( true );
online();
Expand Down Expand Up @@ -434,13 +440,9 @@ private synchronized void create() throws IOException
state = State.POPULATING;
}

private void initialize( IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
private void initialize( IndexSamplingConfig samplingConfig )
{
assert state == State.NONE;
layout = layout( descriptor );
treeKey = layout.newKey();
treeValue = layout.newValue();
schemaIndex = new NativeSchemaIndex<>( pageCache, fs, indexFile, layout, monitor, descriptor, indexId );
if ( isUnique( descriptor ) )
{
uniqueSampler = new UniqueIndexSampler();
Expand Down Expand Up @@ -552,7 +554,8 @@ private enum State

public interface Factory
{
SpatialKnownIndex selectAndCreate( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId,
SpatialCRSSchemaIndex selectAndCreate( IndexDescriptor descriptor,
Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId,
CoordinateReferenceSystem crs );
}

Expand Down
4 changes: 1 addition & 3 deletions ...ity/kernel/src/main/java/org/neo4j/kernel/impl/index/schema/SpatialSchemaIndexReader.java
View file
Expand Up @@ -34,12 +34,10 @@
import org.neo4j.internal.kernel.api.IndexQuery;
import org.neo4j.internal.kernel.api.IndexQuery.ExactPredicate;
import org.neo4j.internal.kernel.api.IndexQuery.GeometryRangePredicate;
import org.neo4j.kernel.api.exceptions.index.IndexNotApplicableKernelException;
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.fusion.BridgingIndexProgressor;
import org.neo4j.storageengine.api.schema.IndexProgressor;
import org.neo4j.values.storable.PointValue;
import org.neo4j.values.storable.Value;

import static java.lang.String.format;
Expand Down Expand Up @@ -128,7 +126,7 @@ private void startSeekForRange( IndexProgressor.NodeValueClient client, Geometry
BridgingIndexProgressor multiProgressor = new BridgingIndexProgressor( client, descriptor.schema().getPropertyIds() );
client.initialize( descriptor, multiProgressor, query );
SpaceFillingCurve curve = spatial.getSpaceFillingCurve();
Envelope completeEnvelope = SpatialKnownIndex.envelopeFromCRS( spatial.crs );
Envelope completeEnvelope = SpatialCRSSchemaIndex.envelopeFromCRS( spatial.crs );
double[] from = rangePredicate.from() == null ? completeEnvelope.getMin() : rangePredicate.from().coordinate();
double[] to = rangePredicate.to() == null ? completeEnvelope.getMax() : rangePredicate.to().coordinate();
Envelope envelope = new Envelope( from, to );
Expand Down
28 changes: 14 additions & 14 deletions ...l/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/SpatialFusionIndexAccessor.java
View file
Expand Up @@ -38,7 +38,7 @@
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.SpatialKnownIndex;
import org.neo4j.kernel.impl.index.schema.SpatialCRSSchemaIndex;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.values.storable.CoordinateReferenceSystem;

Expand All @@ -47,30 +47,30 @@

class SpatialFusionIndexAccessor implements IndexAccessor
{
private final Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap;
private final Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap;
private final long indexId;
private final IndexDescriptor descriptor;
private final IndexSamplingConfig samplingConfig;
private final SpatialKnownIndex.Factory indexFactory;
private final SpatialCRSSchemaIndex.Factory indexFactory;

SpatialFusionIndexAccessor( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId, IndexDescriptor descriptor,
IndexSamplingConfig samplingConfig, SpatialKnownIndex.Factory indexFactory ) throws IOException
SpatialFusionIndexAccessor( Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId, IndexDescriptor descriptor,
IndexSamplingConfig samplingConfig, SpatialCRSSchemaIndex.Factory indexFactory ) throws IOException
{
this.indexMap = indexMap;
this.indexId = indexId;
this.descriptor = descriptor;
this.samplingConfig = samplingConfig;
this.indexFactory = indexFactory;
for ( SpatialKnownIndex index : indexMap.values() )
for ( SpatialCRSSchemaIndex index : indexMap.values() )
{
index.takeOnline(descriptor, samplingConfig);
index.takeOnline( samplingConfig );
}
}

@Override
public void drop() throws IOException
{
forAll( SpatialKnownIndex::drop, indexMap.values() );
forAll( SpatialCRSSchemaIndex::drop, indexMap.values() );
indexMap.clear();
}

Expand All @@ -83,7 +83,7 @@ public IndexUpdater newUpdater( IndexUpdateMode mode )
@Override
public void force( IOLimiter ioLimiter ) throws IOException
{
forAll( spatialKnownIndex -> spatialKnownIndex.force( ioLimiter ), indexMap.values() );
forAll( spatialCRSSchemaIndex -> spatialCRSSchemaIndex.force( ioLimiter ), indexMap.values() );
}

@Override
Expand All @@ -95,14 +95,14 @@ public void refresh()
@Override
public void close() throws IOException
{
forAll( SpatialKnownIndex::close, indexMap.values() );
forAll( SpatialCRSSchemaIndex::close, indexMap.values() );
}

@Override
public IndexReader newReader()
{
Map<CoordinateReferenceSystem,IndexReader> indexReaders = new HashMap<>();
for ( Map.Entry<CoordinateReferenceSystem,SpatialKnownIndex> index : indexMap.entrySet() )
for ( Map.Entry<CoordinateReferenceSystem,SpatialCRSSchemaIndex> index : indexMap.entrySet() )
{
// TODO should this be populated here, or delegate to SpatialFusionIndexReader?
indexReaders.put( index.getKey(), index.getValue().newReader( samplingConfig, descriptor ) );
Expand All @@ -114,7 +114,7 @@ public IndexReader newReader()
public BoundedIterable<Long> newAllEntriesReader()
{
ArrayList<BoundedIterable<Long>> allEntriesReader = new ArrayList<>();
for ( SpatialKnownIndex index : indexMap.values() )
for ( SpatialCRSSchemaIndex index : indexMap.values() )
{
allEntriesReader.add( index.newAllEntriesReader() );
}
Expand Down Expand Up @@ -151,7 +151,7 @@ public Iterator<Long> iterator()
public ResourceIterator<File> snapshotFiles() throws IOException
{
List<ResourceIterator<File>> snapshotFiles = new ArrayList<>();
for ( SpatialKnownIndex index : indexMap.values() )
for ( SpatialCRSSchemaIndex index : indexMap.values() )
{
snapshotFiles.add( index.snapshotFiles() );
}
Expand All @@ -168,6 +168,6 @@ public void verifyDeferredConstraints( PropertyAccessor propertyAccessor )
@Override
public boolean isDirty()
{
return indexMap.values().stream().anyMatch( SpatialKnownIndex::wasDirtyOnStartup );
return indexMap.values().stream().anyMatch( SpatialCRSSchemaIndex::wasDirtyOnStartup );
}
}
22 changes: 11 additions & 11 deletions .../src/main/java/org/neo4j/kernel/impl/index/schema/fusion/SpatialFusionIndexPopulator.java
View file
Expand Up @@ -32,7 +32,7 @@
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.SpatialKnownIndex;
import org.neo4j.kernel.impl.index.schema.SpatialCRSSchemaIndex;
import org.neo4j.storageengine.api.schema.IndexSample;
import org.neo4j.values.storable.CoordinateReferenceSystem;
import org.neo4j.values.storable.PointValue;
Expand All @@ -46,11 +46,11 @@ class SpatialFusionIndexPopulator implements IndexPopulator
private final long indexId;
private final IndexDescriptor descriptor;
private final IndexSamplingConfig samplingConfig;
private final SpatialKnownIndex.Factory indexFactory;
private final Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap;
private final SpatialCRSSchemaIndex.Factory indexFactory;
private final Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap;

SpatialFusionIndexPopulator( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId, IndexDescriptor descriptor,
IndexSamplingConfig samplingConfig, SpatialKnownIndex.Factory indexFactory )
SpatialFusionIndexPopulator( Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId, IndexDescriptor descriptor,
IndexSamplingConfig samplingConfig, SpatialCRSSchemaIndex.Factory indexFactory )
{
this.indexMap = indexMap;
this.indexId = indexId;
Expand All @@ -73,7 +73,7 @@ public void create() throws IOException
@Override
public void drop() throws IOException
{
forAll( SpatialKnownIndex::drop, indexMap.values() );
forAll( SpatialCRSSchemaIndex::drop, indexMap.values() );
indexMap.clear();
}

Expand All @@ -87,8 +87,8 @@ public void add( Collection<? extends IndexEntryUpdate<?>> updates ) throws Inde
}
for ( CoordinateReferenceSystem crs : batchMap.keySet() )
{
SpatialKnownIndex index = indexFactory.selectAndCreate( indexMap, indexId, crs );
index.startPopulation( descriptor, samplingConfig );
SpatialCRSSchemaIndex index = indexFactory.selectAndCreate( descriptor, indexMap, indexId, crs );
index.startPopulation( samplingConfig );
index.add( batchMap.get( crs ) );
}
}
Expand Down Expand Up @@ -131,14 +131,14 @@ public void includeSample( IndexEntryUpdate<?> update )
Value[] values = update.values();
assert values.length == 1;
CoordinateReferenceSystem crs = ((PointValue) values[0]).getCoordinateReferenceSystem();
SpatialKnownIndex index = indexFactory.selectAndCreate( indexMap, indexId, crs );
index.init( descriptor, samplingConfig );
SpatialCRSSchemaIndex index = indexFactory.selectAndCreate( descriptor, indexMap, indexId, crs );
index.init( samplingConfig );
index.includeSample( update );
}

@Override
public IndexSample sampleResult()
{
return combineSamples( indexMap.values().stream().map( SpatialKnownIndex::sampleResult ).toArray( IndexSample[]::new ) );
return combineSamples( indexMap.values().stream().map( SpatialCRSSchemaIndex::sampleResult ).toArray( IndexSample[]::new ) );
}
}
20 changes: 10 additions & 10 deletions ...el/src/main/java/org/neo4j/kernel/impl/index/schema/fusion/SpatialFusionIndexUpdater.java
View file
Expand Up @@ -28,7 +28,7 @@
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.kernel.api.schema.index.IndexDescriptor;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.SpatialKnownIndex;
import org.neo4j.kernel.impl.index.schema.SpatialCRSSchemaIndex;
import org.neo4j.values.storable.CoordinateReferenceSystem;
import org.neo4j.values.storable.PointValue;
import org.neo4j.values.storable.Value;
Expand All @@ -37,27 +37,27 @@

class SpatialFusionIndexUpdater implements IndexUpdater
{
private final Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap;
private final Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap;
private final Map<CoordinateReferenceSystem,IndexUpdater> currentUpdaters = new HashMap<>();
private final long indexId;
private final SpatialKnownIndex.Factory indexFactory;
private final SpatialCRSSchemaIndex.Factory indexFactory;
private final IndexDescriptor descriptor;
private final IndexSamplingConfig samplingConfig;
private final boolean populating;

static SpatialFusionIndexUpdater updaterForAccessor( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId,
SpatialKnownIndex.Factory indexFactory, IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
static SpatialFusionIndexUpdater updaterForAccessor( Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId,
SpatialCRSSchemaIndex.Factory indexFactory, IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
{
return new SpatialFusionIndexUpdater( indexMap, indexId, indexFactory, descriptor, samplingConfig, false );
}

static SpatialFusionIndexUpdater updaterForPopulator( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId,
SpatialKnownIndex.Factory indexFactory, IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
static SpatialFusionIndexUpdater updaterForPopulator( Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId,
SpatialCRSSchemaIndex.Factory indexFactory, IndexDescriptor descriptor, IndexSamplingConfig samplingConfig )
{
return new SpatialFusionIndexUpdater( indexMap, indexId, indexFactory, descriptor, samplingConfig, true );
}

private SpatialFusionIndexUpdater( Map<CoordinateReferenceSystem,SpatialKnownIndex> indexMap, long indexId, SpatialKnownIndex.Factory indexFactory,
private SpatialFusionIndexUpdater( Map<CoordinateReferenceSystem,SpatialCRSSchemaIndex> indexMap, long indexId, SpatialCRSSchemaIndex.Factory indexFactory,
IndexDescriptor descriptor, IndexSamplingConfig samplingConfig, boolean populating )
{
this.indexMap = indexMap;
Expand Down Expand Up @@ -112,8 +112,8 @@ private IndexUpdater selectUpdater( Value... values ) throws IOException
{
return updater;
}
SpatialKnownIndex index = indexFactory.selectAndCreate( indexMap, indexId, crs );
IndexUpdater indexUpdater = index.updaterWithCreate( descriptor, samplingConfig, populating );
SpatialCRSSchemaIndex index = indexFactory.selectAndCreate( descriptor, indexMap, indexId, crs );
IndexUpdater indexUpdater = index.updaterWithCreate( samplingConfig, populating );
return remember( crs, indexUpdater );
}

Expand Down

0 comments on commit 0f1a200

Please sign in to comment.