Skip to content

Commit

Permalink
Uses efficient schema matching on constraints checking
Browse files Browse the repository at this point in the history 
IndexMap caches constraints in addition to indexes. This allows for
more efficient schema matching based on the change, instead of
going through all constraints and filter.

The reading was further cleaned up by making a property key list
read in one place and pass in array instead of doing that read
arbitrarily in multiple places. This makes the code clearer
in what it does and prevents duplicate work.
  • Loading branch information
@tinwelint
tinwelint committed Apr 8, 2019
1 parent 786e070 commit 3f3b57b
Show file tree
Hide file tree
Showing 20 changed files with 469 additions and 316 deletions.
5 changes: 3 additions & 2 deletions ...y/community-it/index-it/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java
View file
Expand Up @@ -85,6 +85,7 @@
import org.neo4j.storageengine.api.StorageReader; import org.neo4j.storageengine.api.StorageReader;
import org.neo4j.storageengine.api.schema.IndexDescriptorFactory; import org.neo4j.storageengine.api.schema.IndexDescriptorFactory;
import org.neo4j.storageengine.api.schema.IndexReader; import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.SchemaRule;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor; import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
import org.neo4j.test.rule.EmbeddedDatabaseRule; import org.neo4j.test.rule.EmbeddedDatabaseRule;
import org.neo4j.values.storable.Values; import org.neo4j.values.storable.Values;
Expand Down Expand Up @@ -386,9 +387,9 @@ private StoreIndexDescriptor[] createIndexRules( Map<String,Integer> labelNameId
.toArray( StoreIndexDescriptor[]::new ); .toArray( StoreIndexDescriptor[]::new );
} }


private List<StoreIndexDescriptor> getIndexRules( NeoStores neoStores ) private List<SchemaRule> getIndexRules( NeoStores neoStores )
{ {
return Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).indexesGetAll() ); return Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).loadAllSchemaRules() );
} }


private Map<String, Integer> getLabelIdsByName( String... names ) private Map<String, Integer> getLabelIdsByName( String... names )
Expand Down
2 changes: 1 addition & 1 deletion community/community-it/kernel-it/src/test/java/org/neo4j/kernel/RecoveryIT.java
View file
Expand Up @@ -72,7 +72,6 @@
import org.neo4j.kernel.api.index.IndexPopulator; import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider; import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater; import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory; import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode; import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
Expand Down Expand Up @@ -104,6 +103,7 @@
import org.neo4j.logging.NullLog; import org.neo4j.logging.NullLog;
import org.neo4j.logging.NullLogProvider; import org.neo4j.logging.NullLogProvider;
import org.neo4j.scheduler.ThreadPoolJobScheduler; import org.neo4j.scheduler.ThreadPoolJobScheduler;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.StorageEngine; import org.neo4j.storageengine.api.StorageEngine;
import org.neo4j.storageengine.api.schema.IndexReader; import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor; import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
Expand Down
2 changes: 1 addition & 1 deletion ...unity/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java
View file
Expand Up @@ -229,7 +229,7 @@ public KernelTransactionImplementation( Config config, StatementOperationParts s
this.operations = this.operations =
new Operations( new Operations(
allStoreHolder, allStoreHolder,
new IndexTxStateUpdater( storageReader, allStoreHolder, indexingService ), storageReader, new IndexTxStateUpdater( allStoreHolder, indexingService ), storageReader,
this, this,
new KernelToken( storageReader, this, tokenHolders ), new KernelToken( storageReader, this, tokenHolders ),
cursors, cursors,
Expand Down
152 changes: 107 additions & 45 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexMap.java
View file
Expand Up @@ -36,6 +36,9 @@
import java.util.Set; import java.util.Set;


import org.neo4j.internal.kernel.api.schema.SchemaDescriptor; import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptorSupplier;
import org.neo4j.kernel.api.schema.constraints.IndexBackedConstraintDescriptor;
import org.neo4j.kernel.impl.store.record.ConstraintRule;
import org.neo4j.storageengine.api.EntityType; import org.neo4j.storageengine.api.EntityType;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor; import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;


Expand All @@ -48,35 +51,41 @@
public final class IndexMap implements Cloneable public final class IndexMap implements Cloneable
{ {
private final MutableLongObjectMap<IndexProxy> indexesById; private final MutableLongObjectMap<IndexProxy> indexesById;
private final MutableLongObjectMap<IndexBackedConstraintDescriptor> uniquenessConstraintsById;
private final Map<SchemaDescriptor,IndexProxy> indexesByDescriptor; private final Map<SchemaDescriptor,IndexProxy> indexesByDescriptor;
private final MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor; private final MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor;
private final LabelPropertyMultiSet descriptorsByLabelThenProperty; private final LabelPropertyMultiSet<SchemaDescriptor> descriptorsByLabelThenProperty;
private final LabelPropertyMultiSet descriptorsByReltypeThenProperty; private final LabelPropertyMultiSet<SchemaDescriptor> descriptorsByReltypeThenProperty;
private final LabelPropertyMultiSet<IndexBackedConstraintDescriptor> constraintsByLabelThenProperty;
private final LabelPropertyMultiSet<IndexBackedConstraintDescriptor> constraintsByRelTypeThenProperty;


public IndexMap() public IndexMap()
{ {
this( new LongObjectHashMap<>(), new HashMap<>(), new ObjectLongHashMap<>() ); this( new LongObjectHashMap<>(), new HashMap<>(), new ObjectLongHashMap<>(), new LongObjectHashMap<>() );
}

IndexMap( MutableLongObjectMap<IndexProxy> indexesById )
{
this( indexesById, indexesByDescriptor( indexesById ), indexIdsByDescriptor( indexesById ) );
} }


private IndexMap( private IndexMap(
MutableLongObjectMap<IndexProxy> indexesById, MutableLongObjectMap<IndexProxy> indexesById,
Map<SchemaDescriptor,IndexProxy> indexesByDescriptor, Map<SchemaDescriptor,IndexProxy> indexesByDescriptor,
MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor ) MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor,
MutableLongObjectMap<IndexBackedConstraintDescriptor> uniquenessConstraintsById )
{ {
this.indexesById = indexesById; this.indexesById = indexesById;
this.indexesByDescriptor = indexesByDescriptor; this.indexesByDescriptor = indexesByDescriptor;
this.indexIdsByDescriptor = indexIdsByDescriptor; this.indexIdsByDescriptor = indexIdsByDescriptor;
this.descriptorsByLabelThenProperty = new LabelPropertyMultiSet(); this.uniquenessConstraintsById = uniquenessConstraintsById;
this.descriptorsByReltypeThenProperty = new LabelPropertyMultiSet(); this.descriptorsByLabelThenProperty = new LabelPropertyMultiSet<>();
this.descriptorsByReltypeThenProperty = new LabelPropertyMultiSet<>();
for ( SchemaDescriptor schema : indexesByDescriptor.keySet() ) for ( SchemaDescriptor schema : indexesByDescriptor.keySet() )
{ {
addDescriptorToLookups( schema ); addDescriptorToLookups( schema );
} }
this.constraintsByLabelThenProperty = new LabelPropertyMultiSet<>();
this.constraintsByRelTypeThenProperty = new LabelPropertyMultiSet<>();
for ( IndexBackedConstraintDescriptor constraint : uniquenessConstraintsById.values() )
{
addConstraintToLookups( constraint );
}
} }


public IndexProxy getIndexProxy( long indexId ) public IndexProxy getIndexProxy( long indexId )
Expand Down Expand Up @@ -114,14 +123,7 @@ IndexProxy removeIndexProxy( long indexId )


SchemaDescriptor schema = removedProxy.getDescriptor().schema(); SchemaDescriptor schema = removedProxy.getDescriptor().schema();
indexesByDescriptor.remove( schema ); indexesByDescriptor.remove( schema );
if ( schema.entityType() == EntityType.NODE ) selectIndexesByEntityType( schema.entityType() ).remove( schema );
{
descriptorsByLabelThenProperty.remove( schema );
}
else if ( schema.entityType() == EntityType.RELATIONSHIP )
{
descriptorsByReltypeThenProperty.remove( schema );
}


return removedProxy; return removedProxy;
} }
Expand All @@ -136,9 +138,61 @@ Iterable<IndexProxy> getAllIndexProxies()
return indexesById.values(); return indexesById.values();
} }


void putUniquenessConstraint( ConstraintRule rule )
{
IndexBackedConstraintDescriptor constraintDescriptor = (IndexBackedConstraintDescriptor) rule.getConstraintDescriptor();
uniquenessConstraintsById.put( rule.getId(), constraintDescriptor );
constraintsByLabelThenProperty.add( constraintDescriptor );
}

void removeUniquenessConstraint( long constraintId )
{
IndexBackedConstraintDescriptor constraint = uniquenessConstraintsById.remove( constraintId );
if ( constraint != null )
{
selectConstraintsByEntityType( constraint.schema().entityType() ).remove( constraint );
}
}

private LabelPropertyMultiSet<IndexBackedConstraintDescriptor> selectConstraintsByEntityType( EntityType entityType )
{
switch ( entityType )
{
case NODE:
return constraintsByLabelThenProperty;
case RELATIONSHIP:
return constraintsByRelTypeThenProperty;
default:
throw new IllegalArgumentException( "Unknown entity type " + entityType );
}
}

private LabelPropertyMultiSet<SchemaDescriptor> selectIndexesByEntityType( EntityType entityType )
{
switch ( entityType )
{
case NODE:
return descriptorsByLabelThenProperty;
case RELATIONSHIP:
return descriptorsByReltypeThenProperty;
default:
throw new IllegalArgumentException( "Unknown entity type " + entityType );
}
}

boolean hasRelatedSchema( long[] labels, int propertyKey, EntityType entityType )
{
return selectIndexesByEntityType( entityType ).has( labels, propertyKey ) || selectConstraintsByEntityType( entityType ).has( labels, propertyKey );
}

boolean hasRelatedSchema( int label, EntityType entityType )
{
return selectIndexesByEntityType( entityType ).has( label ) || selectConstraintsByEntityType( entityType ).has( label );
}

/** /**
* Get all indexes that would be affected by changes in the input labels and/or properties. The returned * Get all descriptors that would be affected by changes in the input labels and/or properties. The returned
* indexes are guaranteed to contain all affected indexes, but might also contain unaffected indexes as * descriptors are guaranteed to contain all affected indexes, but might also contain unaffected indexes as
* we cannot provide matching without checking unaffected properties for composite indexes. * we cannot provide matching without checking unaffected properties for composite indexes.
* *
* @param changedEntityTokens set of labels that have changed * @param changedEntityTokens set of labels that have changed
Expand All @@ -150,17 +204,26 @@ Iterable<IndexProxy> getAllIndexProxies()
public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties, public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType ) boolean propertyListIsComplete, EntityType entityType )
{ {
switch ( entityType ) return getRelatedDescriptors( selectIndexesByEntityType( entityType ), changedEntityTokens, unchangedEntityTokens, sortedProperties,
{ propertyListIsComplete );
case NODE: }
return getRelatedDescriptors( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete,
descriptorsByLabelThenProperty ); /**
case RELATIONSHIP: * Get all uniqueness constraints that would be affected by changes in the input labels and/or properties. The returned
return getRelatedDescriptors( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete, * set is guaranteed to contain all affected constraints, but might also contain unaffected constraints as
descriptorsByReltypeThenProperty ); * we cannot provide matching without checking unaffected properties for composite indexes.
default: *
throw new IllegalArgumentException( "The given EntityType cannot be indexed: " + entityType ); * @param changedEntityTokens set of labels that have changed
} * @param unchangedEntityTokens set of labels that are unchanged
* @param sortedProperties sorted list of properties
* @param entityType type of indexes to get
* @return set of SchemaDescriptors describing the potentially affected indexes
*/
public Set<IndexBackedConstraintDescriptor> getRelatedConstraints( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType )
{
return getRelatedDescriptors( selectConstraintsByEntityType( entityType ), changedEntityTokens, unchangedEntityTokens, sortedProperties,
propertyListIsComplete );
} }


/** /**
Expand All @@ -170,15 +233,15 @@ public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long
* @param propertyListIsComplete whether or not the property list is complete. For CREATE/DELETE the list is complete, but may not be for UPDATEs. * @param propertyListIsComplete whether or not the property list is complete. For CREATE/DELETE the list is complete, but may not be for UPDATEs.
* @return set of SchemaDescriptors describing the potentially affected indexes * @return set of SchemaDescriptors describing the potentially affected indexes
*/ */
Set<SchemaDescriptor> getRelatedDescriptors( long[] changedLabels, long[] unchangedLabels, int[] sortedProperties, boolean propertyListIsComplete, private <T extends SchemaDescriptorSupplier> Set<T> getRelatedDescriptors( LabelPropertyMultiSet<T> set, long[] changedLabels, long[] unchangedLabels,
LabelPropertyMultiSet set ) int[] sortedProperties, boolean propertyListIsComplete )
{ {
if ( indexesById.isEmpty() ) if ( set.isEmpty() )
{ {
return Collections.emptySet(); return Collections.emptySet();
} }


Set<SchemaDescriptor> descriptors = new HashSet<>(); Set<T> descriptors = new HashSet<>();
if ( propertyListIsComplete ) if ( propertyListIsComplete )
{ {
set.matchingDescriptorsForCompleteListOfProperties( descriptors, changedLabels, sortedProperties ); set.matchingDescriptorsForCompleteListOfProperties( descriptors, changedLabels, sortedProperties );
Expand Down Expand Up @@ -214,7 +277,8 @@ else if ( changedLabels.length == 0 )
@Override @Override
public IndexMap clone() public IndexMap clone()
{ {
return new IndexMap( LongObjectHashMap.newMap( indexesById ), cloneMap( indexesByDescriptor ), new ObjectLongHashMap<>( indexIdsByDescriptor ) ); return new IndexMap( LongObjectHashMap.newMap( indexesById ), cloneMap( indexesByDescriptor ), new ObjectLongHashMap<>( indexIdsByDescriptor ),
LongObjectHashMap.newMap( uniquenessConstraintsById ) );
} }


public Iterator<SchemaDescriptor> descriptors() public Iterator<SchemaDescriptor> descriptors()
Expand Down Expand Up @@ -243,14 +307,12 @@ private <K, V> Map<K, V> cloneMap( Map<K, V> map )


private void addDescriptorToLookups( SchemaDescriptor schema ) private void addDescriptorToLookups( SchemaDescriptor schema )
{ {
if ( schema.entityType() == EntityType.NODE ) selectIndexesByEntityType( schema.entityType() ).add( schema );
{ }
descriptorsByLabelThenProperty.add( schema );
} private void addConstraintToLookups( IndexBackedConstraintDescriptor constraint )
else if ( schema.entityType() == EntityType.RELATIONSHIP ) {
{ selectConstraintsByEntityType( constraint.schema().entityType() ).add( constraint );
descriptorsByReltypeThenProperty.add( schema );
}
} }


private static Map<SchemaDescriptor, IndexProxy> indexesByDescriptor( LongObjectMap<IndexProxy> indexesById ) private static Map<SchemaDescriptor, IndexProxy> indexesByDescriptor( LongObjectMap<IndexProxy> indexesById )
Expand Down
17 changes: 17 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexMapReference.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.neo4j.function.ThrowingFunction; import org.neo4j.function.ThrowingFunction;
import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException; import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptor; import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.kernel.api.schema.constraints.IndexBackedConstraintDescriptor;
import org.neo4j.storageengine.api.EntityType; import org.neo4j.storageengine.api.EntityType;
import org.neo4j.values.storable.Value; import org.neo4j.values.storable.Value;


Expand Down Expand Up @@ -109,6 +110,22 @@ public Collection<SchemaDescriptor> getRelatedIndexes( long[] changedEntityToken
return indexMap.getRelatedIndexes( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete, entityType ); return indexMap.getRelatedIndexes( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete, entityType );
} }


public Collection<IndexBackedConstraintDescriptor> getRelatedConstraints( long[] changedLabels, long[] unchangedLabels, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType )
{
return indexMap.getRelatedConstraints( changedLabels, unchangedLabels, sortedProperties, propertyListIsComplete, entityType );
}

public boolean hasRelatedSchema( long[] labels, int propertyKey, EntityType entityType )
{
return indexMap.hasRelatedSchema( labels, propertyKey, entityType );
}

public boolean hasRelatedSchema( int label, EntityType entityType )
{
return indexMap.hasRelatedSchema( label, entityType );
}

public IndexUpdaterMap createIndexUpdaterMap( IndexUpdateMode mode ) public IndexUpdaterMap createIndexUpdaterMap( IndexUpdateMode mode )
{ {
return new IndexUpdaterMap( indexMap, mode ); return new IndexUpdaterMap( indexMap, mode );
Expand Down

0 comments on commit 3f3b57b

Please sign in to comment.