Skip to content

Commit

Permalink
More efficient schema cache/lookup in IndexMap
Browse files Browse the repository at this point in the history 
Introduces a sort of multi-set which combines label and property keys
so that a more narrow view of related indexes can be selected.
For CREATE/DELETE of nodes the selection can be made 100% precise
since all the data is there (the previous lookup method wasn't precise),
but for UPDATE where node data is incomplete, more specifically
the property keys, the selection isn't as precise.
  • Loading branch information
@tinwelint
tinwelint committed Apr 8, 2019
1 parent 65c9611 commit 6f7eee6
Show file tree
Hide file tree
Showing 18 changed files with 582 additions and 224 deletions.
12 changes: 6 additions & 6 deletions ...y/community-it/index-it/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java
View file
Expand Up @@ -159,9 +159,9 @@ public void setUp()
public void applyConcurrentDeletesToPopulatedIndex() throws Throwable public void applyConcurrentDeletesToPopulatedIndex() throws Throwable
{ {
List<EntityUpdates> updates = new ArrayList<>( 2 ); List<EntityUpdates> updates = new ArrayList<>( 2 );
updates.add( EntityUpdates.forEntity( country1.getId() ).withTokens( id( COUNTRY_LABEL ) ) updates.add( EntityUpdates.forEntity( country1.getId(), false ).withTokens( id( COUNTRY_LABEL ) )
.removed( propertyId, Values.of( "Sweden" ) ).build() ); .removed( propertyId, Values.of( "Sweden" ) ).build() );
updates.add( EntityUpdates.forEntity( color2.getId() ).withTokens( id( COLOR_LABEL ) ) updates.add( EntityUpdates.forEntity( color2.getId(), false ).withTokens( id( COLOR_LABEL ) )
.removed( propertyId, Values.of( "green" ) ).build() ); .removed( propertyId, Values.of( "green" ) ).build() );


launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) ); launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) );
Expand Down Expand Up @@ -189,9 +189,9 @@ public void applyConcurrentDeletesToPopulatedIndex() throws Throwable
public void applyConcurrentAddsToPopulatedIndex() throws Throwable public void applyConcurrentAddsToPopulatedIndex() throws Throwable
{ {
List<EntityUpdates> updates = new ArrayList<>( 2 ); List<EntityUpdates> updates = new ArrayList<>( 2 );
updates.add( EntityUpdates.forEntity( otherNodes[0].getId() ).withTokens( id( COUNTRY_LABEL ) ) updates.add( EntityUpdates.forEntity( otherNodes[0].getId(), false ).withTokens( id( COUNTRY_LABEL ) )
.added( propertyId, Values.of( "Denmark" ) ).build() ); .added( propertyId, Values.of( "Denmark" ) ).build() );
updates.add( EntityUpdates.forEntity( otherNodes[1].getId() ).withTokens( id( CAR_LABEL ) ) updates.add( EntityUpdates.forEntity( otherNodes[1].getId(), false ).withTokens( id( CAR_LABEL ) )
.added( propertyId, Values.of( "BMW" ) ).build() ); .added( propertyId, Values.of( "BMW" ) ).build() );


launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) ); launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) );
Expand Down Expand Up @@ -219,9 +219,9 @@ public void applyConcurrentAddsToPopulatedIndex() throws Throwable
public void applyConcurrentChangesToPopulatedIndex() throws Exception public void applyConcurrentChangesToPopulatedIndex() throws Exception
{ {
List<EntityUpdates> updates = new ArrayList<>( 2 ); List<EntityUpdates> updates = new ArrayList<>( 2 );
updates.add( EntityUpdates.forEntity( color2.getId() ).withTokens( id( COLOR_LABEL ) ) updates.add( EntityUpdates.forEntity( color2.getId(), false ).withTokens( id( COLOR_LABEL ) )
.changed( propertyId, Values.of( "green" ), Values.of( "pink" ) ).build() ); .changed( propertyId, Values.of( "green" ), Values.of( "pink" ) ).build() );
updates.add( EntityUpdates.forEntity( car2.getId() ).withTokens( id( CAR_LABEL ) ) updates.add( EntityUpdates.forEntity( car2.getId(), false ).withTokens( id( CAR_LABEL ) )
.changed( propertyId, Values.of( "Ford" ), Values.of( "SAAB" ) ).build() ); .changed( propertyId, Values.of( "Ford" ), Values.of( "SAAB" ) ).build() );


launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) ); launchCustomIndexPopulation( labelsNameIdMap, propertyId, new UpdateGenerator( updates ) );
Expand Down
2 changes: 1 addition & 1 deletion ...st/java/org/neo4j/kernel/impl/transaction/state/storeview/NeoStoreIndexStoreViewTest.java
View file
Expand Up @@ -328,7 +328,7 @@ public void processAllRelationshipProperties() throws Exception


EntityUpdates add( long nodeId, int propertyKeyId, Object value, long[] labels ) EntityUpdates add( long nodeId, int propertyKeyId, Object value, long[] labels )
{ {
return EntityUpdates.forEntity( nodeId ).withTokens( labels ).added( propertyKeyId, Values.of( value ) ).build(); return EntityUpdates.forEntity( nodeId, true ).withTokens( labels ).added( propertyKeyId, Values.of( value ) ).build();
} }


private void createAlistairAndStefanNodes() private void createAlistairAndStefanNodes()
Expand Down
42 changes: 31 additions & 11 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/EntityUpdates.java
View file
Expand Up @@ -21,7 +21,6 @@


import org.eclipse.collections.api.iterator.IntIterator; import org.eclipse.collections.api.iterator.IntIterator;
import org.eclipse.collections.api.map.primitive.MutableIntObjectMap; import org.eclipse.collections.api.map.primitive.MutableIntObjectMap;
import org.eclipse.collections.api.set.primitive.IntSet;
import org.eclipse.collections.api.set.primitive.MutableIntSet; import org.eclipse.collections.api.set.primitive.MutableIntSet;
import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap; import org.eclipse.collections.impl.map.mutable.primitive.IntObjectHashMap;
import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet; import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet;
Expand Down Expand Up @@ -57,8 +56,10 @@ public class EntityUpdates implements PropertyLoader.PropertyLoadSink
// ASSUMPTION: these long arrays are actually sorted sets // ASSUMPTION: these long arrays are actually sorted sets
private long[] entityTokensBefore; private long[] entityTokensBefore;
private long[] entityTokensAfter; private long[] entityTokensAfter;

private final boolean propertyListComplete;
private final MutableIntObjectMap<PropertyValue> knownProperties; private final MutableIntObjectMap<PropertyValue> knownProperties;
private int[] propertyKeyIds;
private int propertyKeyIdsCursor;
private boolean hasLoadedAdditionalProperties; private boolean hasLoadedAdditionalProperties;


public static class Builder public static class Builder
Expand Down Expand Up @@ -115,20 +116,30 @@ public EntityUpdates build()


private void put( int propertyKeyId, PropertyValue propertyValue ) private void put( int propertyKeyId, PropertyValue propertyValue )
{ {
knownProperties.put( propertyKeyId, propertyValue ); PropertyValue existing = knownProperties.put( propertyKeyId, propertyValue );
if ( existing == null )
{
if ( propertyKeyIdsCursor >= propertyKeyIds.length )
{
propertyKeyIds = Arrays.copyOf( propertyKeyIds, propertyKeyIdsCursor * 2 );
}
propertyKeyIds[propertyKeyIdsCursor++] = propertyKeyId;
}
} }


public static Builder forEntity( long entityId ) public static Builder forEntity( long entityId, boolean propertyListIsComplete )
{ {
return new Builder( new EntityUpdates( entityId, EMPTY_LONG_ARRAY, EMPTY_LONG_ARRAY ) ); return new Builder( new EntityUpdates( entityId, EMPTY_LONG_ARRAY, EMPTY_LONG_ARRAY, propertyListIsComplete ) );
} }


private EntityUpdates( long entityId, long[] entityTokensBefore, long[] entityTokensAfter ) private EntityUpdates( long entityId, long[] entityTokensBefore, long[] entityTokensAfter, boolean propertyListComplete )
{ {
this.entityId = entityId; this.entityId = entityId;
this.entityTokensBefore = entityTokensBefore; this.entityTokensBefore = entityTokensBefore;
this.entityTokensAfter = entityTokensAfter; this.entityTokensAfter = entityTokensAfter;
this.propertyListComplete = propertyListComplete;
this.knownProperties = new IntObjectHashMap<>(); this.knownProperties = new IntObjectHashMap<>();
this.propertyKeyIds = new int[8];
} }


public final long getEntityId() public final long getEntityId()
Expand All @@ -146,17 +157,27 @@ long[] entityTokensUnchanged()
return PrimitiveArrays.intersect( entityTokensBefore, entityTokensAfter ); return PrimitiveArrays.intersect( entityTokensBefore, entityTokensAfter );
} }


IntSet propertiesChanged() int[] propertiesChanged()
{ {
assert !hasLoadedAdditionalProperties : "Calling propertiesChanged() is not valid after non-changed " + assert !hasLoadedAdditionalProperties : "Calling propertiesChanged() is not valid after non-changed " +
"properties have already been loaded."; "properties have already been loaded.";
return knownProperties.keySet().toImmutable(); Arrays.sort( propertyKeyIds, 0, propertyKeyIdsCursor );
return propertyKeyIdsCursor == propertyKeyIds.length ? propertyKeyIds : Arrays.copyOf( propertyKeyIds, propertyKeyIdsCursor );
}

/**
* @return whether or not the list provided from {@link #propertiesChanged()} is the complete list of properties on this node.
* If {@code false} then the list may contain some properties, whereas there may be other unloaded properties on the persisted existing node.
*/
boolean isPropertyListComplete()
{
return propertyListComplete;
} }


@Override @Override
public void onProperty( int propertyId, Value value ) public void onProperty( int propertyId, Value value )
{ {
knownProperties.put( propertyId, unchanged( value ) ); put( propertyId, unchanged( value ) );
} }


/** /**
Expand Down Expand Up @@ -242,7 +263,6 @@ else if ( relevantBefore && relevantAfter )
} }
} }
} }

return indexUpdates; return indexUpdates;
} }


Expand All @@ -265,7 +285,7 @@ private void loadProperties( PropertyLoader propertyLoader, MutableIntSet additi
final IntIterator propertiesWithNoValue = additionalPropertiesToLoad.intIterator(); final IntIterator propertiesWithNoValue = additionalPropertiesToLoad.intIterator();
while ( propertiesWithNoValue.hasNext() ) while ( propertiesWithNoValue.hasNext() )
{ {
knownProperties.put( propertiesWithNoValue.next(), noValue ); put( propertiesWithNoValue.next(), noValue );
} }
} }


Expand Down

0 comments on commit 6f7eee6

Please sign in to comment.