Skip to content

Commit

Permalink
Refactored spatial index config settings to improve clarity
Browse files Browse the repository at this point in the history
  • Loading branch information
craigtaverner authored and burqen committed Jun 28, 2018
1 parent c5a7772 commit fe464ac
Show file tree
Hide file tree
Showing 12 changed files with 317 additions and 260 deletions.
Expand Up @@ -31,20 +31,22 @@
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.index.IndexDirectoryStructure;
import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettings;
import org.neo4j.kernel.impl.index.schema.config.ConfiguredSpaceFillingCurveSettingsCache;
import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettingsFactory;
import org.neo4j.values.storable.CoordinateReferenceSystem;

class SpatialIndexFiles
{
private static final Pattern CRS_DIR_PATTERN = Pattern.compile( "(\\d+)-(\\d+)" );
private final FileSystemAbstraction fs;
private final SpaceFillingCurveSettingsFactory settingsFactory;
private final ConfiguredSpaceFillingCurveSettingsCache configuredSettings;
private final File indexDirectory;

SpatialIndexFiles( IndexDirectoryStructure directoryStructure, long indexId, FileSystemAbstraction fs, SpaceFillingCurveSettingsFactory settingsFactory )
SpatialIndexFiles( IndexDirectoryStructure directoryStructure, long indexId, FileSystemAbstraction fs,
ConfiguredSpaceFillingCurveSettingsCache settingsCache )
{
this.fs = fs;
this.settingsFactory = settingsFactory;
this.configuredSettings = settingsCache;
indexDirectory = directoryStructure.directoryForIndex( indexId );
}

Expand All @@ -65,7 +67,7 @@ <T> void loadExistingIndexes( SpatialIndexCache<T> indexCache ) throws IOExcepti

SpatialFile forCrs( CoordinateReferenceSystem crs )
{
return new SpatialFile( crs, settingsFactory, indexDirectory );
return new SpatialFile( crs, configuredSettings, indexDirectory );
}

private void addExistingFiles( List<SpatialFile> existing )
Expand All @@ -91,13 +93,13 @@ private void addExistingFiles( List<SpatialFile> existing )
static class SpatialFile
{
final File indexFile;
final SpaceFillingCurveSettingsFactory settings;
final ConfiguredSpaceFillingCurveSettingsCache configuredSettings;
private final CoordinateReferenceSystem crs;

SpatialFile( CoordinateReferenceSystem crs, SpaceFillingCurveSettingsFactory settingsFactory, File indexDirectory )
SpatialFile( CoordinateReferenceSystem crs, ConfiguredSpaceFillingCurveSettingsCache configuredSettings, File indexDirectory )
{
this.crs = crs;
this.settings = settingsFactory;
this.configuredSettings = configuredSettings;
String s = crs.getTable().getTableId() + "-" + Integer.toString( crs.getCode() );
this.indexFile = new File( indexDirectory, s );
}
Expand All @@ -107,7 +109,7 @@ static class SpatialFile
*/
SpatialFileLayout getLayoutForNewIndex()
{
return new SpatialFileLayout( this, settings.settingsFor( crs ) );
return new SpatialFileLayout( this, configuredSettings.forCRS( crs ) );
}

/**
Expand All @@ -116,7 +118,7 @@ SpatialFileLayout getLayoutForNewIndex()
SpatialFileLayout getLayoutForExistingIndex( PageCache pageCache ) throws IOException
{
SpaceFillingCurveSettings settings =
SpaceFillingCurveSettings.fromGBPTree( indexFile, pageCache, NativeIndexHeaderReader::readFailureMessage );
SpaceFillingCurveSettingsFactory.fromGBPTree( indexFile, pageCache, NativeIndexHeaderReader::readFailureMessage );
return new SpatialFileLayout( this, settings );
}
}
Expand Down
Expand Up @@ -39,7 +39,7 @@
import org.neo4j.kernel.api.schema.index.StoreIndexDescriptor;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettingsFactory;
import org.neo4j.kernel.impl.index.schema.config.ConfiguredSpaceFillingCurveSettingsCache;
import org.neo4j.kernel.impl.index.schema.config.SpatialIndexSettings;
import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
import org.neo4j.values.storable.ValueCategory;
Expand All @@ -56,7 +56,7 @@ public class SpatialIndexProvider extends IndexProvider
private final RecoveryCleanupWorkCollector recoveryCleanupWorkCollector;
private final boolean readOnly;
private final SpaceFillingCurveConfiguration configuration;
private final SpaceFillingCurveSettingsFactory settingsFactory;
private final ConfiguredSpaceFillingCurveSettingsCache configuredSettings;

public SpatialIndexProvider( PageCache pageCache, FileSystemAbstraction fs,
IndexDirectoryStructure.Factory directoryStructure, Monitor monitor,
Expand All @@ -69,12 +69,12 @@ public SpatialIndexProvider( PageCache pageCache, FileSystemAbstraction fs,
this.recoveryCleanupWorkCollector = recoveryCleanupWorkCollector;
this.readOnly = readOnly;
this.configuration = getConfiguredSpaceFillingCurveConfiguration( config );
this.settingsFactory = getConfiguredSpaceFillingCurveSettings( config );
this.configuredSettings = getConfiguredSpaceFillingCurveSettings( config );
}

private SpaceFillingCurveSettingsFactory getConfiguredSpaceFillingCurveSettings( Config config )
private ConfiguredSpaceFillingCurveSettingsCache getConfiguredSpaceFillingCurveSettings( Config config )
{
return new SpaceFillingCurveSettingsFactory( config );
return new ConfiguredSpaceFillingCurveSettingsCache( config );
}

private static SpaceFillingCurveConfiguration getConfiguredSpaceFillingCurveConfiguration( Config config )
Expand All @@ -100,21 +100,21 @@ public IndexPopulator getPopulator( StoreIndexDescriptor descriptor, IndexSampli
{
throw new UnsupportedOperationException( "Can't create populator for read only index" );
}
SpatialIndexFiles files = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, settingsFactory );
SpatialIndexFiles files = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, configuredSettings );
return new SpatialIndexPopulator( descriptor, samplingConfig, files, pageCache, fs, monitor, configuration );
}

@Override
public IndexAccessor getOnlineAccessor( StoreIndexDescriptor descriptor, IndexSamplingConfig samplingConfig ) throws IOException
{
SpatialIndexFiles files = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, settingsFactory );
SpatialIndexFiles files = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, configuredSettings );
return new SpatialIndexAccessor( descriptor, samplingConfig, pageCache, fs, recoveryCleanupWorkCollector, monitor, files, configuration );
}

@Override
public String getPopulationFailure( StoreIndexDescriptor descriptor ) throws IllegalStateException
{
SpatialIndexFiles spatialIndexFiles = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, settingsFactory );
SpatialIndexFiles spatialIndexFiles = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, configuredSettings );

try
{
Expand All @@ -137,7 +137,7 @@ public String getPopulationFailure( StoreIndexDescriptor descriptor ) throws Ill
@Override
public InternalIndexState getInitialState( StoreIndexDescriptor descriptor )
{
SpatialIndexFiles spatialIndexFiles = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, settingsFactory );
SpatialIndexFiles spatialIndexFiles = new SpatialIndexFiles( directoryStructure(), descriptor.getId(), fs, configuredSettings );

final Iterable<SpatialIndexFiles.SpatialFile> existing = spatialIndexFiles.existing();
InternalIndexState state = InternalIndexState.ONLINE;
Expand Down
@@ -0,0 +1,110 @@
/*
* Copyright (c) 2002-2018 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.impl.index.schema.config;

import java.util.HashMap;
import java.util.Map;

import org.neo4j.configuration.ConfigValue;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.index.schema.config.SpaceFillingCurveSettingsFactory.EnvelopeSettings;
import org.neo4j.values.storable.CoordinateReferenceSystem;

/**
* These settings affect the creation of the 2D (or 3D) to 1D mapper.
* Changing these will change the values of the 1D mapping, and require re-indexing, so
* once data has been indexed, do not change these without recreating the index.
*/
public class ConfiguredSpaceFillingCurveSettingsCache
{
private int maxBits;
private HashMap<CoordinateReferenceSystem,SpaceFillingCurveSettings> settings = new HashMap<>();
private static final String SPATIAL_SETTING_PREFIX = "unsupported.dbms.db.spatial.crs.";

public ConfiguredSpaceFillingCurveSettingsCache( Config config )
{
this.maxBits = config.get( SpatialIndexSettings.space_filling_curve_max_bits );
HashMap<CoordinateReferenceSystem,EnvelopeSettings> env = new HashMap<>();
for ( Map.Entry<String,ConfigValue> entry : config.getConfigValues().entrySet() )
{
String key = entry.getKey();
String value = entry.getValue().toString();
if ( key.startsWith( SPATIAL_SETTING_PREFIX ) )
{
String[] fields = key.replace( SPATIAL_SETTING_PREFIX, "" ).split( "\\." );
if ( fields.length != 3 )
{
throw new IllegalArgumentException(
"Invalid spatial config settings, expected three fields after '" + SPATIAL_SETTING_PREFIX + "': " + key );
}
else
{
CoordinateReferenceSystem crs = CoordinateReferenceSystem.byName( fields[0] );
EnvelopeSettings envelopeSettings = env.computeIfAbsent( crs, EnvelopeSettings::new );
int index = "xyz".indexOf( fields[1].toLowerCase() );
if ( index < 0 )
{
throw new IllegalArgumentException( "Invalid spatial coordinate key (should be one of 'x', 'y' or 'z'): " + fields[1] );
}
if ( index >= crs.getDimension() )
{
throw new IllegalArgumentException( "Invalid spatial coordinate key for " + crs.getDimension() + "D: " + fields[1] );
}
switch ( fields[2].toLowerCase() )
{
case "min":
envelopeSettings.min[index] = Double.parseDouble( value );
break;
case "max":
envelopeSettings.max[index] = Double.parseDouble( value );
break;
default:
throw new IllegalArgumentException( "Invalid spatial coordinate range key (should be one of 'max' or 'min'): " + fields[2] );
}
}
}
}
for ( Map.Entry<CoordinateReferenceSystem,EnvelopeSettings> entry : env.entrySet() )
{
CoordinateReferenceSystem crs = entry.getKey();
settings.put( crs, SpaceFillingCurveSettingsFactory.fromConfig( this.maxBits, entry.getValue() ) );
}
}

/**
* The space filling curve is configured up front to cover a specific region of 2D (or 3D) space,
* and the mapping tree is configured up front to have a specific maximum depth. These settings
* are stored in an instance of SpaceFillingCurveSettings and are determined by the Coordinate
* Reference System, and any neo4j.conf settings to override the CRS defaults.
*
* @return The default settings for the specified coordinate reference system
*/
public SpaceFillingCurveSettings forCRS( CoordinateReferenceSystem crs )
{
if ( settings.containsKey( crs ) )
{
return settings.get( crs );
}
else
{
return SpaceFillingCurveSettingsFactory.fromConfig( maxBits, new EnvelopeSettings( crs ) );
}
}
}

0 comments on commit fe464ac

Please sign in to comment.