Skip to content

Commit

Permalink
A first insertion of space filling curve settings into layout/key
Browse files Browse the repository at this point in the history
  • Loading branch information
tinwelint committed Sep 5, 2018
1 parent b558d75 commit 920f500
Show file tree
Hide file tree
Showing 10 changed files with 165 additions and 34 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.StringJoiner;

import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
import org.neo4j.util.Preconditions;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueGroup;
Expand All @@ -33,12 +34,12 @@ class CompositeGenericKey extends NativeIndexKey<CompositeGenericKey>
{
private GenericKeyState[] states;

CompositeGenericKey( int slots )
CompositeGenericKey( int slots, IndexSpecificSpaceFillingCurveSettingsCache spatialSettings )
{
states = new GenericKeyState[slots];
for ( int i = 0; i < states.length; i++ )
{
states[i] = new GenericKeyState();
states[i] = new GenericKeyState( spatialSettings );
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@
import java.time.ZonedDateTime;
import java.util.Arrays;

import org.neo4j.gis.spatial.index.curves.SpaceFillingCurve;
import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.kernel.impl.index.schema.GenericLayout.Type;
import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
import org.neo4j.kernel.impl.store.TemporalValueWriterAdapter;
import org.neo4j.string.UTF8;
import org.neo4j.values.storable.BooleanValue;
import org.neo4j.values.storable.CoordinateReferenceSystem;
import org.neo4j.values.storable.DateTimeValue;
import org.neo4j.values.storable.DateValue;
import org.neo4j.values.storable.DurationValue;
Expand Down Expand Up @@ -73,6 +76,9 @@ public class GenericKeyState extends TemporalValueWriterAdapter<RuntimeException

// TODO copy-pasted from individual keys
// TODO also put this in Type enum
private static final int SIZE_GEOMETRY = Long.BYTES + /* rawValueBits */
Integer.BYTES + /* coordinate reference system tableId */
Integer.BYTES; /* coordinate reference system codePointId */
public static final int SIZE_ZONED_DATE_TIME = Long.BYTES + /* epochSecond */
Integer.BYTES + /* nanoOfSecond */
Integer.BYTES; /* timeZone */
Expand Down Expand Up @@ -108,7 +114,9 @@ public class GenericKeyState extends TemporalValueWriterAdapter<RuntimeException
private boolean isHighestArray;
private int arrayLength;
private int currentArrayOffset;
private final IndexSpecificSpaceFillingCurveSettingsCache settings;

// spatial long0 (rawValueBits), long1 (coordinate reference system tableId), long2 (coordinate reference system tableId)
// zoned date time: long0 (epochSecondUTC), long1 (nanoOfSecond), long2 (zoneId), long3 (zoneOffsetSeconds)
// local date time: long0 (nanoOfSecond), long1 (epochSecond)
// date: long0 (epochDay)
Expand All @@ -118,7 +126,6 @@ public class GenericKeyState extends TemporalValueWriterAdapter<RuntimeException
// text: long0 (length), long1 (bytesDereferenced), long2 (ignoreLength), long3 (isHighest), byteArray
// boolean: long0
// number: long0 (value), long1 (number type)
// TODO spatial

// for non-array values
private long long0;
Expand All @@ -134,6 +141,15 @@ public class GenericKeyState extends TemporalValueWriterAdapter<RuntimeException
private long[] long3Array;
private byte[][] byteArrayArray;

// for spatial values
private CoordinateReferenceSystem crs;
private SpaceFillingCurve spaceFillingCurve;

GenericKeyState( IndexSpecificSpaceFillingCurveSettingsCache settings )
{
this.settings = settings;
}

/* <initializers> */
void clear()
{
Expand Down Expand Up @@ -498,6 +514,8 @@ private int valueSize()
{
switch ( type )
{
case GEOMETRY:
return SIZE_GEOMETRY;
case ZONED_DATE_TIME:
return SIZE_ZONED_DATE_TIME;
case LOCAL_DATE_TIME:
Expand All @@ -517,6 +535,8 @@ private int valueSize()
return SIZE_BOOLEAN;
case NUMBER:
return numberKeySize( long1 ) + SIZE_NUMBER_TYPE;
case GEOMETRY_ARRAY:
return arrayKeySize( SIZE_GEOMETRY );
case ZONED_DATE_TIME_ARRAY:
return arrayKeySize( SIZE_ZONED_DATE_TIME );
case LOCAL_DATE_TIME_ARRAY:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,39 +24,39 @@
import java.util.Comparator;

import org.neo4j.io.pagecache.PageCursor;
import org.neo4j.kernel.impl.index.schema.config.IndexSpecificSpaceFillingCurveSettingsCache;
import org.neo4j.values.storable.ValueGroup;

import static java.util.Comparator.comparing;

class GenericLayout extends IndexLayout<CompositeGenericKey,NativeIndexValue>
{
static final Comparator<Type> TYPE_COMPARATOR = comparing( t -> t.valueGroup );
private final int numberOfSlots;

// Order doesn't matter since it's each Type's ValueGroup that matters and will be used for comparison
enum Type
{
ZONED_DATE_TIME( ValueGroup.ZONED_DATE_TIME, (byte) 0 ),
LOCAL_DATE_TIME( ValueGroup.LOCAL_DATE_TIME, (byte) 1 ),
DATE( ValueGroup.DATE, (byte) 2 ),
ZONED_TIME( ValueGroup.ZONED_TIME, (byte) 3 ),
LOCAL_TIME( ValueGroup.LOCAL_TIME, (byte) 4 ),
DURATION( ValueGroup.DURATION, (byte) 5 ),
TEXT( ValueGroup.TEXT, (byte) 6 ),
BOOLEAN( ValueGroup.BOOLEAN, (byte) 7 ),
NUMBER( ValueGroup.NUMBER, (byte) 8 ),
// TODO SPATIAL

ZONED_DATE_TIME_ARRAY( ValueGroup.ZONED_DATE_TIME_ARRAY, (byte) 9 ),
LOCAL_DATE_TIME_ARRAY( ValueGroup.LOCAL_DATE_TIME_ARRAY, (byte) 10 ),
DATE_ARRAY( ValueGroup.DATE_ARRAY, (byte) 11 ),
ZONED_TIME_ARRAY( ValueGroup.ZONED_TIME_ARRAY, (byte) 12 ),
LOCAL_TIME_ARRAY( ValueGroup.LOCAL_TIME_ARRAY, (byte) 13 ),
DURATION_ARRAY( ValueGroup.DURATION_ARRAY, (byte) 14 ),
TEXT_ARRAY( ValueGroup.TEXT_ARRAY, (byte) 15 ),
BOOLEAN_ARRAY( ValueGroup.BOOLEAN_ARRAY, (byte) 16 ),
NUMBER_ARRAY( ValueGroup.NUMBER_ARRAY, (byte) 17 );
// TODO SPATIAL_ARRAY
GEOMETRY( ValueGroup.GEOMETRY, (byte) 0 ),
ZONED_DATE_TIME( ValueGroup.ZONED_DATE_TIME, (byte) 1 ),
LOCAL_DATE_TIME( ValueGroup.LOCAL_DATE_TIME, (byte) 2 ),
DATE( ValueGroup.DATE, (byte) 3 ),
ZONED_TIME( ValueGroup.ZONED_TIME, (byte) 4 ),
LOCAL_TIME( ValueGroup.LOCAL_TIME, (byte) 5 ),
DURATION( ValueGroup.DURATION, (byte) 6 ),
TEXT( ValueGroup.TEXT, (byte) 7 ),
BOOLEAN( ValueGroup.BOOLEAN, (byte) 8 ),
NUMBER( ValueGroup.NUMBER, (byte) 9 ),

GEOMETRY_ARRAY( ValueGroup.GEOMETRY_ARRAY, (byte) 10 ),
ZONED_DATE_TIME_ARRAY( ValueGroup.ZONED_DATE_TIME_ARRAY, (byte) 11 ),
LOCAL_DATE_TIME_ARRAY( ValueGroup.LOCAL_DATE_TIME_ARRAY, (byte) 12 ),
DATE_ARRAY( ValueGroup.DATE_ARRAY, (byte) 13 ),
ZONED_TIME_ARRAY( ValueGroup.ZONED_TIME_ARRAY, (byte) 14 ),
LOCAL_TIME_ARRAY( ValueGroup.LOCAL_TIME_ARRAY, (byte) 15 ),
DURATION_ARRAY( ValueGroup.DURATION_ARRAY, (byte) 16 ),
TEXT_ARRAY( ValueGroup.TEXT_ARRAY, (byte) 17 ),
BOOLEAN_ARRAY( ValueGroup.BOOLEAN_ARRAY, (byte) 18 ),
NUMBER_ARRAY( ValueGroup.NUMBER_ARRAY, (byte) 19 );

private final ValueGroup valueGroup;
final byte typeId;
Expand Down Expand Up @@ -85,16 +85,20 @@ enum Type
}
}

GenericLayout( int numberOfSlots )
private final int numberOfSlots;
private final IndexSpecificSpaceFillingCurveSettingsCache spatialSettings;

GenericLayout( int numberOfSlots, IndexSpecificSpaceFillingCurveSettingsCache spatialSettings )
{
super( "NSIL", 0, 1 );
this.numberOfSlots = numberOfSlots;
this.spatialSettings = spatialSettings;
}

@Override
public CompositeGenericKey newKey()
{
return new CompositeGenericKey( numberOfSlots );
return new CompositeGenericKey( numberOfSlots, spatialSettings );
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.io.File;
import java.io.IOException;

import org.neo4j.gis.spatial.index.curves.SpaceFillingCurveConfiguration;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
import org.neo4j.internal.kernel.api.IndexCapability;
Expand All @@ -33,10 +34,14 @@
import org.neo4j.kernel.api.index.IndexAccessor;
import org.neo4j.kernel.api.index.IndexDirectoryStructure;
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
import org.neo4j.kernel.impl.index.schema.config.ConfiguredSpaceFillingCurveSettingsCache;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
import org.neo4j.values.storable.ValueCategory;

import static org.neo4j.kernel.impl.index.schema.SpatialIndexProvider.getConfiguredSpaceFillingCurveConfiguration;

/**
* Single-value all-in-one native index
*
Expand Down Expand Up @@ -110,17 +115,32 @@ public IndexValueCapability valueCapability( ValueCategory... valueCategories )
}
};

/**
* Cache of all setting for various specific CRS's found in the config at instantiation of this provider.
* The config is read once and all relevant CRS configs cached here.
*/
private final ConfiguredSpaceFillingCurveSettingsCache configuredSettings;

/**
* A space filling curve configuration used when reading spatial index values.
*/
private final SpaceFillingCurveConfiguration configuration;

public GenericNativeIndexProvider( int priority, IndexDirectoryStructure.Factory directoryStructureFactory, PageCache pageCache,
FileSystemAbstraction fs, Monitor monitor, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, boolean readOnly )
FileSystemAbstraction fs, Monitor monitor, RecoveryCleanupWorkCollector recoveryCleanupWorkCollector, boolean readOnly, Config config )
{
super( DESCRIPTOR, priority, directoryStructureFactory, pageCache, fs, monitor, recoveryCleanupWorkCollector, readOnly );

this.configuredSettings = new ConfiguredSpaceFillingCurveSettingsCache( config );
this.configuration = getConfiguredSpaceFillingCurveConfiguration( config );
}

@Override
IndexLayout<CompositeGenericKey,NativeIndexValue> layout( StoreIndexDescriptor descriptor )
{
int numberOfSlots = descriptor.properties().length;
return new GenericLayout( numberOfSlots );
// TODO read header from the tree and build a IndexSpecificSpaceFillingCurveSettingsCache and pass in
return new GenericLayout( numberOfSlots, null );
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public static GenericNativeIndexProvider create( PageCache pageCache, File store

IndexDirectoryStructure.Factory directoryStructure = directoriesByProvider( storeDir );
boolean readOnly = config.get( GraphDatabaseSettings.read_only ) && (OperationalMode.single == mode);
return new GenericNativeIndexProvider( priority, directoryStructure, pageCache, fs, monitor, recoveryCleanupWorkCollector, readOnly );
return new GenericNativeIndexProvider( priority, directoryStructure, pageCache, fs, monitor, recoveryCleanupWorkCollector, readOnly, config );
}

public interface Dependencies extends AbstractIndexProviderFactory.Dependencies
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ private ConfiguredSpaceFillingCurveSettingsCache getConfiguredSpaceFillingCurveS
return new ConfiguredSpaceFillingCurveSettingsCache( config );
}

private static SpaceFillingCurveConfiguration getConfiguredSpaceFillingCurveConfiguration( Config config )
static SpaceFillingCurveConfiguration getConfiguredSpaceFillingCurveConfiguration( Config config )
{
int extraLevels = config.get( SpatialIndexSettings.space_filling_curve_extra_levels );
double topThreshold = config.get( SpatialIndexSettings.space_filling_curve_top_threshold );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,9 @@ public SpaceFillingCurveSettings forCRS( CoordinateReferenceSystem crs )
return SpaceFillingCurveSettingsFactory.fromConfig( maxBits, new EnvelopeSettings( crs ) );
}
}

public int getMaxBits()
{
return maxBits;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package org.neo4j.kernel.impl.index.schema.config;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;

import org.neo4j.gis.spatial.index.curves.SpaceFillingCurve;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.values.storable.CoordinateReferenceSystem;

/**
* A combination of {@link ConfiguredSpaceFillingCurveSettingsCache}, which contains all settings from {@link Config},
* but also settings for a specific index. Settings for a specific index can change over time as new {@link CoordinateReferenceSystem}
* are used in this index.
*/
public class IndexSpecificSpaceFillingCurveSettingsCache
{
private final ConfiguredSpaceFillingCurveSettingsCache globalConfigCache;
private final ConcurrentMap<CoordinateReferenceSystem,SpaceFillingCurveSettings> indexConfigCache = new ConcurrentHashMap<>();

public IndexSpecificSpaceFillingCurveSettingsCache(
ConfiguredSpaceFillingCurveSettingsCache globalConfigCache,
Map<CoordinateReferenceSystem,SpaceFillingCurveSettings> indexConfigCache )
{
this.globalConfigCache = globalConfigCache;
this.indexConfigCache.putAll( indexConfigCache );
}

/**
* Gets {@link SpaceFillingCurve} for a particular coordinate reference system's crsTableId and code point.
*
* @param crsTableId table id of the {@link CoordinateReferenceSystem}.
* @param crsCodePoint code of the {@link CoordinateReferenceSystem}.
* @param assignToIndexIfNotYetAssigned whether or not to make a snapshot of this setting index-specific if this is the
* first time it's accessed for this index. It will then show up in {@link #visitIndexSpecificSettings(SettingVisitor)}.
* @return the {@link SpaceFillingCurve} for the given coordinate reference system.
*/
public SpaceFillingCurve forCrs( int crsTableId, int crsCodePoint, boolean assignToIndexIfNotYetAssigned )
{
CoordinateReferenceSystem crs = CoordinateReferenceSystem.get( crsTableId, crsCodePoint );

// Index-specific
SpaceFillingCurveSettings specificSetting = indexConfigCache.get( crs );
if ( specificSetting != null )
{
return specificSetting.curve();
}

// Global config
SpaceFillingCurveSettings configuredSetting = fromConfig( crs );
if ( assignToIndexIfNotYetAssigned )
{
indexConfigCache.put( crs, configuredSetting );
}
return configuredSetting.curve();
}

/**
* Mostly for checkpoints to serialize index-specific settings into the index header.
*/
public void visitIndexSpecificSettings( SettingVisitor visitor )
{
indexConfigCache.forEach( (crs, settings) -> visitor.visit( crs, settings ) );
}

private SpaceFillingCurveSettings fromConfig( CoordinateReferenceSystem crs )
{
SpaceFillingCurveSettings global = globalConfigCache.forCRS( crs );
if ( global != null )
{
return global;
}

// Fall back to creating one (TODO cache this?)
return SpaceFillingCurveSettingsFactory.fromConfig( globalConfigCache.getMaxBits(), new EnvelopeSettings( crs ) );
}

public interface SettingVisitor
{
void visit( CoordinateReferenceSystem crs, SpaceFillingCurveSettings settings );
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@

class GenericKeyStateCompareTest
{

@Test
void compareGenericKeyState()
{
Expand Down Expand Up @@ -121,7 +120,7 @@ void compareGenericKeyState()
List<GenericKeyState> states = new ArrayList<>();
for ( Value value : allValues )
{
GenericKeyState state = new GenericKeyState();
GenericKeyState state = new GenericKeyState( null ); // TODO <-- also do something with this one when spatial is supported
state.writeValue( value, NativeIndexKey.Inclusion.NEUTRAL );
states.add( state );
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

/**
* These settings define how to optimize the 2D (or 3D) to 1D mapping of the space filling curve.
* They will affect the number of 1D ranges produced as well as the number of false positives expcted from the 1D index.
* They will affect the number of 1D ranges produced as well as the number of false positives expected from the 1D index.
* The ideal performance depends on the behaviour of the underlying 1D index, whether it costs more to have more 1D searches,
* or have more false positives for post filtering.
*/
Expand Down

0 comments on commit 920f500

Please sign in to comment.