Skip to content

Commit

Permalink
Avoids unnecessary re-reading of node data on tx-state index updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@tinwelint
tinwelint committed Apr 8, 2019
1 parent 6f7eee6 commit 9055a87
Show file tree
Hide file tree
Showing 10 changed files with 224 additions and 142 deletions.
8 changes: 4 additions & 4 deletions ...unity/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactionImplementation.java
View file
Expand Up @@ -73,7 +73,7 @@
import org.neo4j.kernel.api.txstate.auxiliary.AuxiliaryTransactionStateHolder;
import org.neo4j.kernel.api.txstate.auxiliary.AuxiliaryTransactionStateManager;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.IndexingProvidersService;
import org.neo4j.kernel.impl.api.index.IndexingService;
import org.neo4j.kernel.impl.api.state.ConstraintIndexCreator;
import org.neo4j.kernel.impl.api.state.TxState;
import org.neo4j.kernel.impl.constraints.ConstraintSemantics;
Expand Down Expand Up @@ -199,7 +199,7 @@ public KernelTransactionImplementation( Config config, StatementOperationParts s
LockTracer lockTracer, PageCursorTracerSupplier cursorTracerSupplier, StorageEngine storageEngine, AccessCapability accessCapability,
AutoIndexing autoIndexing, ExplicitIndexStore explicitIndexStore, VersionContextSupplier versionContextSupplier,
CollectionsFactorySupplier collectionsFactorySupplier, ConstraintSemantics constraintSemantics, SchemaState schemaState,
IndexingProvidersService indexProviders, TokenHolders tokenHolders, Dependencies dataSourceDependencies )
IndexingService indexingService, TokenHolders tokenHolders, Dependencies dataSourceDependencies )
{
this.schemaWriteGuard = schemaWriteGuard;
this.hooks = hooks;
Expand Down Expand Up @@ -229,14 +229,14 @@ public KernelTransactionImplementation( Config config, StatementOperationParts s
this.operations =
new Operations(
allStoreHolder,
new IndexTxStateUpdater( storageReader, allStoreHolder, indexProviders ), storageReader,
new IndexTxStateUpdater( storageReader, allStoreHolder, indexingService ), storageReader,
this,
new KernelToken( storageReader, this, tokenHolders ),
cursors,
autoIndexing,
constraintIndexCreator,
constraintSemantics,
indexProviders,
indexingService,
config );
this.collectionsFactory = collectionsFactorySupplier.create();
}
Expand Down
9 changes: 5 additions & 4 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/KernelTransactions.java
View file
Expand Up @@ -42,6 +42,7 @@
import org.neo4j.kernel.availability.AvailabilityGuard;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.api.index.IndexingProvidersService;
import org.neo4j.kernel.impl.api.index.IndexingService;
import org.neo4j.kernel.impl.api.state.ConstraintIndexCreator;
import org.neo4j.kernel.impl.constraints.ConstraintSemantics;
import org.neo4j.kernel.impl.core.TokenHolders;
Expand Down Expand Up @@ -100,7 +101,7 @@ public class KernelTransactions extends LifecycleAdapter implements Supplier<Ker
private final MonotonicCounter userTransactionIdCounter = MonotonicCounter.newAtomicMonotonicCounter();
private final AutoIndexing autoIndexing;
private final ExplicitIndexStore explicitIndexStore;
private final IndexingProvidersService indexProviders;
private final IndexingService indexingService;
private final TokenHolders tokenHolders;
private final String currentDatabaseName;
private final Dependencies dataSourceDependencies;
Expand Down Expand Up @@ -143,7 +144,7 @@ public KernelTransactions( Config config, StatementLocksFactory statementLocksFa
Procedures procedures, TransactionIdStore transactionIdStore, SystemNanoClock clock, AtomicReference<CpuClock> cpuClockRef,
AtomicReference<HeapAllocation> heapAllocationRef, AccessCapability accessCapability, AutoIndexing autoIndexing,
ExplicitIndexStore explicitIndexStore, VersionContextSupplier versionContextSupplier, CollectionsFactorySupplier collectionsFactorySupplier,
ConstraintSemantics constraintSemantics, SchemaState schemaState, IndexingProvidersService indexProviders, TokenHolders tokenHolders,
ConstraintSemantics constraintSemantics, SchemaState schemaState, IndexingService indexingService, TokenHolders tokenHolders,
String currentDatabaseName, Dependencies dataSourceDependencies )
{
this.config = config;
Expand All @@ -166,7 +167,7 @@ public KernelTransactions( Config config, StatementLocksFactory statementLocksFa
this.accessCapability = accessCapability;
this.autoIndexing = autoIndexing;
this.explicitIndexStore = explicitIndexStore;
this.indexProviders = indexProviders;
this.indexingService = indexingService;
this.tokenHolders = tokenHolders;
this.currentDatabaseName = currentDatabaseName;
this.dataSourceDependencies = dataSourceDependencies;
Expand Down Expand Up @@ -368,7 +369,7 @@ public KernelTransactionImplementation newInstance()
tracers.pageCursorTracerSupplier, storageEngine, accessCapability,
autoIndexing,
explicitIndexStore, versionContextSupplier, collectionsFactorySupplier, constraintSemantics,
schemaState, indexProviders, tokenHolders, dataSourceDependencies );
schemaState, indexingService, tokenHolders, dataSourceDependencies );
this.transactions.add( tx );
return tx;
}
Expand Down
4 changes: 2 additions & 2 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexMapReference.java
View file
Expand Up @@ -19,7 +19,7 @@
*/
package org.neo4j.kernel.impl.api.index;

import java.util.Set;
import java.util.Collection;
import java.util.function.Function;

import org.neo4j.function.ThrowingFunction;
Expand Down Expand Up @@ -103,7 +103,7 @@ public Iterable<IndexProxy> getAllIndexProxies()
return indexMap.getAllIndexProxies();
}

public Set<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties, boolean mode,
public Collection<SchemaDescriptor> getRelatedIndexes( long[] changedEntityTokens, long[] unchangedEntityTokens, int[] sortedProperties, boolean mode,
EntityType entityType )
{
return indexMap.getRelatedIndexes( changedEntityTokens, unchangedEntityTokens, sortedProperties, mode, entityType );
Expand Down
6 changes: 6 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java
View file
Expand Up @@ -45,6 +45,7 @@
import java.util.function.Function;
import java.util.stream.Collectors;

import org.neo4j.collection.PrimitiveLongCollections;
import org.neo4j.function.ThrowingConsumer;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
Expand Down Expand Up @@ -826,6 +827,11 @@ private void logIndexProviderSummary( Map<IndexProviderDescriptor,List<IndexLogR
}
}

public Collection<SchemaDescriptor> getRelatedIndexes( long[] labels, int propertyKeyId, EntityType entityType )
{
return indexMapRef.getRelatedIndexes( PrimitiveLongCollections.EMPTY_LONG_ARRAY, labels, new int[] {propertyKeyId}, false, entityType );
}

private final class IndexPopulationStarter implements Function<IndexMap,IndexMap>
{
private final boolean verifyBeforeFlipping;
Expand Down
162 changes: 86 additions & 76 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/newapi/IndexTxStateUpdater.java
View file
Expand Up @@ -20,21 +20,26 @@
package org.neo4j.kernel.impl.newapi;

import org.apache.commons.lang3.ArrayUtils;
import org.eclipse.collections.api.map.primitive.MutableIntObjectMap;
import org.eclipse.collections.api.set.primitive.IntSet;
import org.eclipse.collections.api.set.primitive.MutableIntSet;
import org.eclipse.collections.impl.factory.primitive.IntObjectMaps;
import org.eclipse.collections.impl.set.mutable.primitive.IntHashSet;

import java.util.Collection;
import java.util.Iterator;

import org.neo4j.internal.kernel.api.NodeCursor;
import org.neo4j.internal.kernel.api.PropertyCursor;
import org.neo4j.kernel.impl.api.index.IndexingProvidersService;
import org.neo4j.internal.kernel.api.schema.SchemaDescriptor;
import org.neo4j.kernel.impl.api.index.IndexingService;
import org.neo4j.storageengine.api.StorageReader;
import org.neo4j.storageengine.api.schema.IndexDescriptor;
import org.neo4j.values.storable.Value;
import org.neo4j.values.storable.ValueTuple;

import static org.neo4j.kernel.api.StatementConstants.NO_SUCH_PROPERTY_KEY;
import static org.neo4j.storageengine.api.EntityType.NODE;
import static org.neo4j.values.storable.Values.NO_VALUE;

/**
Expand All @@ -44,15 +49,15 @@ public class IndexTxStateUpdater
{
private final StorageReader storageReader;
private final Read read;
private final IndexingProvidersService indexProviders;
private final IndexingService indexingService;

// We can use the StorageReader directly instead of the SchemaReadOps, because we know that in transactions
// where this class is needed we will never have index changes.
public IndexTxStateUpdater( StorageReader storageReader, Read read, IndexingProvidersService indexProviders )
public IndexTxStateUpdater( StorageReader storageReader, Read read, IndexingService indexingService )
{
this.storageReader = storageReader;
this.read = read;
this.indexProviders = indexProviders;
this.indexingService = indexingService;
}

// LABEL CHANGES
Expand Down Expand Up @@ -91,17 +96,19 @@ void onLabelChange( int labelId, NodeCursor node, PropertyCursor propertyCursor,
nodePropertyIds.add( propertyCursor.propertyKey() );
}

// Check all indexes of the changed label
MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
while ( indexes.hasNext() )
{
IndexDescriptor index = indexes.next();
int[] indexPropertyIds = index.schema().getPropertyIds();
if ( nodeHasIndexProperties( nodePropertyIds, indexPropertyIds ) )
{
Value[] values = getValueTuple( node, propertyCursor, indexPropertyIds );
Value[] values = getValueTuple( node, propertyCursor, NO_SUCH_PROPERTY_KEY, NO_VALUE, indexPropertyIds, materializedProperties );
switch ( changeType )
{
case ADDED_LABEL:
indexProviders.validateBeforeCommit( index.schema(), values );
indexingService.validateBeforeCommit( index.schema(), values );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(), null, ValueTuple.of( values ) );
break;
case REMOVED_LABEL:
Expand All @@ -121,99 +128,102 @@ private boolean noSchemaChangedInTx()

//PROPERTY CHANGES

void onPropertyAdd( NodeCursor node, PropertyCursor propertyCursor, int propertyKeyId, Value value )
void onPropertyAdd( NodeCursor node, PropertyCursor propertyCursor, long[] labels, int propertyKeyId, Value value )
{
assert noSchemaChangedInTx();
Iterator<? extends IndexDescriptor> indexes =
storageReader.indexesGetRelatedToProperty( propertyKeyId );
NodeSchemaMatcher.onMatchingSchema( indexes, node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
Value[] values = getValueTuple( node, propertyCursor, propertyKeyId, value, index.schema().getPropertyIds() );
indexProviders.validateBeforeCommit( index.schema(), values );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(), null, ValueTuple.of( values ) );
} );
Collection<SchemaDescriptor> indexes = indexingService.getRelatedIndexes( labels, propertyKeyId, NODE );
if ( !indexes.isEmpty() )
{
MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
NodeSchemaMatcher.onMatchingSchema( indexes.iterator(), node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
Value[] values = getValueTuple( node, propertyCursor, propertyKeyId, value, index.schema().getPropertyIds(), materializedProperties );
indexingService.validateBeforeCommit( index.schema(), values );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(), null, ValueTuple.of( values ) );
} );
}
}

void onPropertyRemove( NodeCursor node, PropertyCursor propertyCursor, int propertyKeyId, Value value )
void onPropertyRemove( NodeCursor node, PropertyCursor propertyCursor, long[] labels, int propertyKeyId, Value value )
{
assert noSchemaChangedInTx();
Iterator<? extends IndexDescriptor> indexes =
storageReader.indexesGetRelatedToProperty( propertyKeyId );
NodeSchemaMatcher.onMatchingSchema( indexes, node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
Value[] values = getValueTuple( node, propertyCursor, propertyKeyId, value, index.schema().getPropertyIds() );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(), ValueTuple.of( values ), null );
} );
Collection<SchemaDescriptor> indexes = indexingService.getRelatedIndexes( labels, propertyKeyId, NODE );
if ( !indexes.isEmpty() )
{
MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
NodeSchemaMatcher.onMatchingSchema( indexes.iterator(), node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
Value[] values = getValueTuple( node, propertyCursor, propertyKeyId, value, index.schema().getPropertyIds(), materializedProperties );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(), ValueTuple.of( values ), null );
} );
}
}

void onPropertyChange( NodeCursor node, PropertyCursor propertyCursor, int propertyKeyId,
void onPropertyChange( NodeCursor node, PropertyCursor propertyCursor, long[] labels, int propertyKeyId,
Value beforeValue, Value afterValue )
{
assert noSchemaChangedInTx();
Iterator<? extends IndexDescriptor> indexes = storageReader.indexesGetRelatedToProperty( propertyKeyId );
NodeSchemaMatcher.onMatchingSchema( indexes, node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
int[] indexPropertyIds = index.schema().getPropertyIds();

Value[] valuesBefore = new Value[indexPropertyIds.length];
Value[] valuesAfter = new Value[indexPropertyIds.length];
for ( int i = 0; i < indexPropertyIds.length; i++ )
Collection<SchemaDescriptor> indexes = indexingService.getRelatedIndexes( labels, propertyKeyId, NODE );
if ( !indexes.isEmpty() )
{
MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
NodeSchemaMatcher.onMatchingSchema( indexes.iterator(), node, propertyCursor, propertyKeyId,
( index, propertyKeyIds ) ->
{
int indexPropertyId = indexPropertyIds[i];
if ( indexPropertyId == propertyKeyId )
{
valuesBefore[i] = beforeValue;
valuesAfter[i] = afterValue;
}
else
{
node.properties( propertyCursor );
Value value = NO_VALUE;
while ( propertyCursor.next() )
{
if ( propertyCursor.propertyKey() == indexPropertyId )
{
value = propertyCursor.propertyValue();
}
}
valuesBefore[i] = value;
valuesAfter[i] = value;
}
}
indexProviders.validateBeforeCommit( index.schema(), valuesAfter );
read.txState().indexDoUpdateEntry( index.schema(), node.nodeReference(),
ValueTuple.of( valuesBefore ), ValueTuple.of( valuesAfter ) );
} );
}

private static Value[] getValueTuple( NodeCursor node, PropertyCursor propertyCursor, int[] indexPropertyIds )
{
return getValueTuple( node, propertyCursor, NO_SUCH_PROPERTY_KEY, NO_VALUE, indexPropertyIds );
int[] propertyIds = index.getPropertyIds();
Value[] valuesAfter = getValueTuple( node, propertyCursor, propertyKeyId, afterValue, propertyIds, materializedProperties );

// The valuesBefore tuple is just like valuesAfter, except is has the afterValue instead of the beforeValue
Value[] valuesBefore = valuesAfter.clone();
int k = ArrayUtils.indexOf( propertyIds, propertyKeyId );
valuesBefore[k] = beforeValue;

indexingService.validateBeforeCommit( index, valuesAfter );
read.txState().indexDoUpdateEntry( index, node.nodeReference(),
ValueTuple.of( valuesBefore ), ValueTuple.of( valuesAfter ) );
} );
}
}

private static Value[] getValueTuple( NodeCursor node, PropertyCursor propertyCursor, int changedPropertyKeyId, Value changedValue, int[] indexPropertyIds )
private Value[] getValueTuple( NodeCursor node, PropertyCursor propertyCursor,
int changedPropertyKeyId, Value changedValue, int[] indexPropertyIds,
MutableIntObjectMap<Value> materializedValues )
{
Value[] values = new Value[indexPropertyIds.length];
node.properties( propertyCursor );
while ( propertyCursor.next() )
int missing = 0;

// First get whatever values we already have on the stack, like the value change that provoked this update in the first place
// and already loaded values that we can get from the map of materialized values.
for ( int k = 0; k < indexPropertyIds.length; k++ )
{
int k = ArrayUtils.indexOf( indexPropertyIds, propertyCursor.propertyKey() );
if ( k >= 0 )
values[k] = indexPropertyIds[k] == changedPropertyKeyId ? changedValue : materializedValues.get( indexPropertyIds[k] );
if ( values[k] == null )
{
values[k] = indexPropertyIds[k] == changedPropertyKeyId
? changedValue : propertyCursor.propertyValue();
missing++;
}
}

if ( changedPropertyKeyId != NO_SUCH_PROPERTY_KEY )
// If we couldn't get all values that we wanted we need to load from the node. While we're loading values
// we'll place those values in the map so that other index updates from this change can just used them.
if ( missing > 0 )
{
int k = ArrayUtils.indexOf( indexPropertyIds, changedPropertyKeyId );
if ( k >= 0 )
node.properties( propertyCursor );
while ( missing > 0 && propertyCursor.next() )
{
values[k] = changedValue;
int k = ArrayUtils.indexOf( indexPropertyIds, propertyCursor.propertyKey() );
if ( k >= 0 && values[k] == null )
{
int propertyKeyId = indexPropertyIds[k];
boolean thisIsTheChangedProperty = propertyKeyId == changedPropertyKeyId;
values[k] = thisIsTheChangedProperty ? changedValue : propertyCursor.propertyValue();
if ( !thisIsTheChangedProperty )
{
materializedValues.put( propertyKeyId, values[k] );
}
missing--;
}
}
}

Expand Down

0 comments on commit 9055a87

Please sign in to comment.