Skip to content

Commit

Permalink
Uses efficient schema matching on constraints checking
Browse files Browse the repository at this point in the history 
IndexMap caches constraints in addition to indexes. This allows for
more efficient schema matching based on the change, instead of
going through all constraints and filter.

The reading was further cleaned up by making a property key list
read in one place and pass in array instead of doing that read
arbitrarily in multiple places. This makes the code clearer
in what it does and prevents duplicate work.
  • Loading branch information
@tinwelint
tinwelint committed Apr 8, 2019
1 parent 786e070 commit 3f3b57b
Show file tree
Hide file tree
Showing 20 changed files with 469 additions and 316 deletions.
5 changes: 3 additions & 2 deletions ...y/community-it/index-it/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java
View file
Expand Up @@ -85,6 +85,7 @@
import org.neo4j.storageengine.api.StorageReader;
import org.neo4j.storageengine.api.schema.IndexDescriptorFactory;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.SchemaRule;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
import org.neo4j.test.rule.EmbeddedDatabaseRule;
import org.neo4j.values.storable.Values;
Expand Down Expand Up @@ -386,9 +387,9 @@ private StoreIndexDescriptor[] createIndexRules( Map<String,Integer> labelNameId
.toArray( StoreIndexDescriptor[]::new );
}

private List<StoreIndexDescriptor> getIndexRules( NeoStores neoStores )
private List<SchemaRule> getIndexRules( NeoStores neoStores )
{
return Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).indexesGetAll() );
return Iterators.asList( new SchemaStorage( neoStores.getSchemaStore() ).loadAllSchemaRules() );
}

private Map<String, Integer> getLabelIdsByName( String... names )
Expand Down
2 changes: 1 addition & 1 deletion community/community-it/kernel-it/src/test/java/org/neo4j/kernel/RecoveryIT.java
View file
Expand Up @@ -72,7 +72,6 @@
import org.neo4j.kernel.api.index.IndexPopulator;
import org.neo4j.kernel.api.index.IndexProvider;
import org.neo4j.kernel.api.index.IndexUpdater;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.extension.KernelExtensionFactory;
import org.neo4j.kernel.impl.api.index.IndexUpdateMode;
Expand Down Expand Up @@ -104,6 +103,7 @@
import org.neo4j.logging.NullLog;
import org.neo4j.logging.NullLogProvider;
import org.neo4j.scheduler.ThreadPoolJobScheduler;
import org.neo4j.storageengine.api.NodePropertyAccessor;
import org.neo4j.storageengine.api.StorageEngine;
import org.neo4j.storageengine.api.schema.IndexReader;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;
Expand Down
2 changes: 1 addition & 1 deletion ...unity/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java
View file
Expand Up @@ -229,7 +229,7 @@ public KernelTransactionImplementation( Config config, StatementOperationParts s
this.operations =
new Operations(
allStoreHolder,
new IndexTxStateUpdater( storageReader, allStoreHolder, indexingService ), storageReader,
new IndexTxStateUpdater( allStoreHolder, indexingService ), storageReader,
this,
new KernelToken( storageReader, this, tokenHolders ),
cursors,
Expand Down
152 changes: 107 additions & 45 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexMap.java
View file
Expand Up @@ -36,6 +36,9 @@
import java.util.Set;

import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptorSupplier;
import org.neo4j.kernel.api.schema.constraints.IndexBackedConstraintDescriptor;
import org.neo4j.kernel.impl.store.record.ConstraintRule;
import org.neo4j.storageengine.api.EntityType;
import org.neo4j.storageengine.api.schema.StoreIndexDescriptor;

Expand All @@ -48,35 +51,41 @@
public final class IndexMap implements Cloneable
{
private final MutableLongObjectMap<IndexProxy> indexesById;
private final MutableLongObjectMap<IndexBackedConstraintDescriptor> uniquenessConstraintsById;
private final Map<SchemaDescriptor,IndexProxy> indexesByDescriptor;
private final MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor;
private final LabelPropertyMultiSet descriptorsByLabelThenProperty;
private final LabelPropertyMultiSet descriptorsByReltypeThenProperty;
private final LabelPropertyMultiSet<SchemaDescriptor> descriptorsByLabelThenProperty;
private final LabelPropertyMultiSet<SchemaDescriptor> descriptorsByReltypeThenProperty;
private final LabelPropertyMultiSet<IndexBackedConstraintDescriptor> constraintsByLabelThenProperty;
private final LabelPropertyMultiSet<IndexBackedConstraintDescriptor> constraintsByRelTypeThenProperty;

public IndexMap()
{
this( new LongObjectHashMap<>(), new HashMap<>(), new ObjectLongHashMap<>() );
}

IndexMap( MutableLongObjectMap<IndexProxy> indexesById )
{
this( indexesById, indexesByDescriptor( indexesById ), indexIdsByDescriptor( indexesById ) );
this( new LongObjectHashMap<>(), new HashMap<>(), new ObjectLongHashMap<>(), new LongObjectHashMap<>() );
}

private IndexMap(
MutableLongObjectMap<IndexProxy> indexesById,
Map<SchemaDescriptor,IndexProxy> indexesByDescriptor,
MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor )
MutableObjectLongMap<SchemaDescriptor> indexIdsByDescriptor,
MutableLongObjectMap<IndexBackedConstraintDescriptor> uniquenessConstraintsById )
{
this.indexesById = indexesById;
this.indexesByDescriptor = indexesByDescriptor;
this.indexIdsByDescriptor = indexIdsByDescriptor;
this.descriptorsByLabelThenProperty = new LabelPropertyMultiSet();
this.descriptorsByReltypeThenProperty = new LabelPropertyMultiSet();
this.uniquenessConstraintsById = uniquenessConstraintsById;
this.descriptorsByLabelThenProperty = new LabelPropertyMultiSet<>();
this.descriptorsByReltypeThenProperty = new LabelPropertyMultiSet<>();
for ( SchemaDescriptor schema : indexesByDescriptor.keySet() )
{
addDescriptorToLookups( schema );
}
this.constraintsByLabelThenProperty = new LabelPropertyMultiSet<>();
this.constraintsByRelTypeThenProperty = new LabelPropertyMultiSet<>();
for ( IndexBackedConstraintDescriptor constraint : uniquenessConstraintsById.values() )
{
addConstraintToLookups( constraint );
}
}

public IndexProxy getIndexProxy( long indexId )
Expand Down Expand Up @@ -114,14 +123,7 @@ IndexProxy removeIndexProxy( long indexId )

SchemaDescriptor schema = removedProxy.getDescriptor().schema();
indexesByDescriptor.remove( schema );
if ( schema.entityType() == EntityType.NODE )
{
descriptorsByLabelThenProperty.remove( schema );
}
else if ( schema.entityType() == EntityType.RELATIONSHIP )
{
descriptorsByReltypeThenProperty.remove( schema );
}
selectIndexesByEntityType( schema.entityType() ).remove( schema );

return removedProxy;
}
Expand All @@ -136,9 +138,61 @@ Iterable<IndexProxy> getAllIndexProxies()
return indexesById.values();
}

void putUniquenessConstraint( ConstraintRule rule )
{
IndexBackedConstraintDescriptor constraintDescriptor = (IndexBackedConstraintDescriptor) rule.getConstraintDescriptor();
uniquenessConstraintsById.put( rule.getId(), constraintDescriptor );
constraintsByLabelThenProperty.add( constraintDescriptor );
}

void removeUniquenessConstraint( long constraintId )
{
IndexBackedConstraintDescriptor constraint = uniquenessConstraintsById.remove( constraintId );
if ( constraint != null )
{
selectConstraintsByEntityType( constraint.schema().entityType() ).remove( constraint );
}
}

private LabelPropertyMultiSet<IndexBackedConstraintDescriptor> selectConstraintsByEntityType( EntityType entityType )
{
switch ( entityType )
{
case NODE:
return constraintsByLabelThenProperty;
case RELATIONSHIP:
return constraintsByRelTypeThenProperty;
default:
throw new IllegalArgumentException( "Unknown entity type " + entityType );
}
}

private LabelPropertyMultiSet<SchemaDescriptor> selectIndexesByEntityType( EntityType entityType )
{
switch ( entityType )
{
case NODE:
return descriptorsByLabelThenProperty;
case RELATIONSHIP:
return descriptorsByReltypeThenProperty;
default:
throw new IllegalArgumentException( "Unknown entity type " + entityType );
}
}

boolean hasRelatedSchema( long[] labels, int propertyKey, EntityType entityType )
{
return selectIndexesByEntityType( entityType ).has( labels, propertyKey ) || selectConstraintsByEntityType( entityType ).has( labels, propertyKey );
}

boolean hasRelatedSchema( int label, EntityType entityType )
{
return selectIndexesByEntityType( entityType ).has( label ) || selectConstraintsByEntityType( entityType ).has( label );
}

/**
* Get all indexes that would be affected by changes in the input labels and/or properties. The returned
* indexes are guaranteed to contain all affected indexes, but might also contain unaffected indexes as
* Get all descriptors that would be affected by changes in the input labels and/or properties. The returned
* descriptors are guaranteed to contain all affected indexes, but might also contain unaffected indexes as
* we cannot provide matching without checking unaffected properties for composite indexes.
*
* @param changedEntityTokens set of labels that have changed
Expand All @@ -150,17 +204,26 @@ Iterable<IndexProxy> getAllIndexProxies()
public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType )
{
switch ( entityType )
{
case NODE:
return getRelatedDescriptors( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete,
descriptorsByLabelThenProperty );
case RELATIONSHIP:
return getRelatedDescriptors( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete,
descriptorsByReltypeThenProperty );
default:
throw new IllegalArgumentException( "The given EntityType cannot be indexed: " + entityType );
}
return getRelatedDescriptors( selectIndexesByEntityType( entityType ), changedEntityTokens, unchangedEntityTokens, sortedProperties,
propertyListIsComplete );
}

/**
* Get all uniqueness constraints that would be affected by changes in the input labels and/or properties. The returned
* set is guaranteed to contain all affected constraints, but might also contain unaffected constraints as
* we cannot provide matching without checking unaffected properties for composite indexes.
*
* @param changedEntityTokens set of labels that have changed
* @param unchangedEntityTokens set of labels that are unchanged
* @param sortedProperties sorted list of properties
* @param entityType type of indexes to get
* @return set of SchemaDescriptors describing the potentially affected indexes
*/
public Set<IndexBackedConstraintDescriptor> getRelatedConstraints( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType )
{
return getRelatedDescriptors( selectConstraintsByEntityType( entityType ), changedEntityTokens, unchangedEntityTokens, sortedProperties,
propertyListIsComplete );
}

/**
Expand All @@ -170,15 +233,15 @@ public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long
* @param propertyListIsComplete whether or not the property list is complete. For CREATE/DELETE the list is complete, but may not be for UPDATEs.
* @return set of SchemaDescriptors describing the potentially affected indexes
*/
Set<SchemaDescriptor> getRelatedDescriptors( long[] changedLabels, long[] unchangedLabels, int[] sortedProperties, boolean propertyListIsComplete,
LabelPropertyMultiSet set )
private <T extends SchemaDescriptorSupplier> Set<T> getRelatedDescriptors( LabelPropertyMultiSet<T> set, long[] changedLabels, long[] unchangedLabels,
int[] sortedProperties, boolean propertyListIsComplete )
{
if ( indexesById.isEmpty() )
if ( set.isEmpty() )
{
return Collections.emptySet();
}

Set<SchemaDescriptor> descriptors = new HashSet<>();
Set<T> descriptors = new HashSet<>();
if ( propertyListIsComplete )
{
set.matchingDescriptorsForCompleteListOfProperties( descriptors, changedLabels, sortedProperties );
Expand Down Expand Up @@ -214,7 +277,8 @@ else if ( changedLabels.length == 0 )
@Override
public IndexMap clone()
{
return new IndexMap( LongObjectHashMap.newMap( indexesById ), cloneMap( indexesByDescriptor ), new ObjectLongHashMap<>( indexIdsByDescriptor ) );
return new IndexMap( LongObjectHashMap.newMap( indexesById ), cloneMap( indexesByDescriptor ), new ObjectLongHashMap<>( indexIdsByDescriptor ),
LongObjectHashMap.newMap( uniquenessConstraintsById ) );
}

public Iterator<SchemaDescriptor> descriptors()
Expand Down Expand Up @@ -243,14 +307,12 @@ private <K, V> Map<K, V> cloneMap( Map<K, V> map )

private void addDescriptorToLookups( SchemaDescriptor schema )
{
if ( schema.entityType() == EntityType.NODE )
{
descriptorsByLabelThenProperty.add( schema );
}
else if ( schema.entityType() == EntityType.RELATIONSHIP )
{
descriptorsByReltypeThenProperty.add( schema );
}
selectIndexesByEntityType( schema.entityType() ).add( schema );
}

private void addConstraintToLookups( IndexBackedConstraintDescriptor constraint )
{
selectConstraintsByEntityType( constraint.schema().entityType() ).add( constraint );
}

private static Map<SchemaDescriptor, IndexProxy> indexesByDescriptor( LongObjectMap<IndexProxy> indexesById )
Expand Down
17 changes: 17 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexMapReference.java
View file
Expand Up @@ -25,6 +25,7 @@
import org.neo4j.function.ThrowingFunction;
import org.neo4j.internal.kernel.api.exceptions.schema.IndexNotFoundKernelException;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.kernel.api.schema.constraints.IndexBackedConstraintDescriptor;
import org.neo4j.storageengine.api.EntityType;
import org.neo4j.values.storable.Value;

Expand Down Expand Up @@ -109,6 +110,22 @@ public Collection<SchemaDescriptor> getRelatedIndexes( long[] changedEntityToken
return indexMap.getRelatedIndexes( changedEntityTokens, unchangedEntityTokens, sortedProperties, propertyListIsComplete, entityType );
}

public Collection<IndexBackedConstraintDescriptor> getRelatedConstraints( long[] changedLabels, long[] unchangedLabels, int[] sortedProperties,
boolean propertyListIsComplete, EntityType entityType )
{
return indexMap.getRelatedConstraints( changedLabels, unchangedLabels, sortedProperties, propertyListIsComplete, entityType );
}

public boolean hasRelatedSchema( long[] labels, int propertyKey, EntityType entityType )
{
return indexMap.hasRelatedSchema( labels, propertyKey, entityType );
}

public boolean hasRelatedSchema( int label, EntityType entityType )
{
return indexMap.hasRelatedSchema( label, entityType );
}

public IndexUpdaterMap createIndexUpdaterMap( IndexUpdateMode mode )
{
return new IndexUpdaterMap( indexMap, mode );
Expand Down

0 comments on commit 3f3b57b

Please sign in to comment.