Skip to content

Commit

Permalink
Locking for Rel and multitoken indexes
Browse files Browse the repository at this point in the history
  • Loading branch information
@ragadeeshu
ragadeeshu committed Jun 15, 2018
1 parent 7888879 commit 2f6d5cd
Show file tree
Hide file tree
Showing 17 changed files with 164 additions and 114 deletions.
6 changes: 4 additions & 2 deletions ...unity/consistency-check/src/main/java/org/neo4j/consistency/checking/full/IndexCheck.java
View file
Expand Up @@ -19,6 +19,8 @@
*/
package org.neo4j.consistency.checking.full;

import java.util.Arrays;

import org.neo4j.consistency.checking.CheckerEngine;
import org.neo4j.consistency.checking.RecordCheck;
import org.neo4j.consistency.report.ConsistencyReport;
Expand All @@ -38,8 +40,8 @@ public IndexCheck( StoreIndexDescriptor indexRule )
@Override
public void check( IndexEntry record, CheckerEngine<IndexEntry, ConsistencyReport.IndexConsistencyReport> engine, RecordAccess records )
{
int labelId = indexRule.schema().keyId();
int[] entityTokenIds = indexRule.schema().getEntityTokenIds();
engine.comparativeCheck( records.node( record.getId() ),
new NodeInUseWithCorrectLabelsCheck<>( new long[]{labelId}, false ) );
new NodeInUseWithCorrectLabelsCheck<>( Arrays.stream( entityTokenIds ).asLongStream().toArray(), false ) );
}
}
8 changes: 5 additions & 3 deletions ...neo4j/cypher/internal/compiler/v3_5/planner/DbStructureLogicalPlanningConfiguration.scala
View file
Expand Up @@ -24,9 +24,10 @@ import java.util
import org.neo4j.cypher.internal.compiler.v3_5.CypherPlannerConfiguration
import org.opencypher.v9_0.ast.semantics.SemanticTable
import org.neo4j.cypher.internal.planner.v3_5.spi.{GraphStatistics, StatisticsCompletingGraphStatistics}
import org.opencypher.v9_0.util.{LabelId, PropertyKeyId, RelTypeId}
import org.neo4j.helpers.collection.{Pair, Visitable}
import org.neo4j.kernel.impl.util.dbstructure.{DbStructureCollector, DbStructureLookup, DbStructureVisitor}
import org.opencypher.v9_0.ast.semantics.SemanticTable
import org.opencypher.v9_0.util.{LabelId, PropertyKeyId, RelTypeId}

import scala.collection.JavaConverters._
import scala.collection.mutable
Expand Down Expand Up @@ -64,8 +65,9 @@ case class DbStructureLogicalPlanningConfiguration(cypherCompilerConfig: CypherP
}
}

private def indexSet(indices: util.Iterator[Pair[String, Array[String]]]): Set[(String, Seq[String])] =
indices.asScala.map { pair => pair.first() -> pair.other().to[Seq] }.toSet
private def indexSet(indices: util.Iterator[Pair[Array[String], Array[String]]]): Set[(String, Seq[String])] =
//We use a zero index here as to not bleed multi-token descriptors into cypher.
indices.asScala.map { pair => pair.first().apply(0) -> pair.other().to[Seq] }.toSet

private def resolveTokens[T](iterator: util.Iterator[Pair[Integer, String]])(f: Int => T): mutable.Map[String, T] = {
val builder = mutable.Map.newBuilder[String, T]
Expand Down
8 changes: 8 additions & 0 deletions community/kernel-api/src/main/java/org/neo4j/internal/kernel/api/SchemaRead.java
View file
Expand Up @@ -45,6 +45,14 @@ public interface SchemaRead
*/
IndexReference index( int label, int... properties );

/**
* Acquire a reference to the index mapping the given {@code SchemaDescriptor}.
*
* @param schema {@link SchemaDescriptor} for the index
* @return the IndexReference, or {@link IndexReference#NO_INDEX} if such an index does not exist.
*/
IndexReference index( SchemaDescriptor schema );

/**
* Acquire an index reference of the given {@code label} and {@code properties}. This method does not assert
* that the created reference points to a valid online index.
Expand Down
14 changes: 2 additions & 12 deletions community/kernel/src/main/java/org/neo4j/kernel/api/schema/MultiTokenSchemaDescriptor.java
View file
Expand Up @@ -111,23 +111,13 @@ public int[] getEntityTokenIds()
@Override
public int keyId()
{
//TODO MultiTokenSchema support for the new kernel api. This method is only used in the AllStoreHolder
return 1;
return hashCode();
}

@Override
public ResourceType keyType()
{
if ( entityType == EntityType.NODE )
{
return ResourceTypes.LABEL;
}
else if ( entityType == EntityType.RELATIONSHIP )
{
return ResourceTypes.RELATIONSHIP_TYPE;
}
throw new UnsupportedOperationException(
"Keys for non-schema indexes of type " + entityType + " is not supported." );
return ResourceTypes.SCHEMA;
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...y/kernel/src/main/java/org/neo4j/kernel/impl/api/index/TentativeConstraintIndexProxy.java
View file
Expand Up @@ -163,7 +163,7 @@ public void validate() throws UniquePropertyValueValidationException
{
SchemaDescriptor descriptor = getDescriptor().schema();
throw new UniquePropertyValueValidationException(
ConstraintDescriptorFactory.uniqueForLabel( descriptor.keyId(), descriptor.getPropertyIds() ),
ConstraintDescriptorFactory.uniqueForSchema( descriptor ),
ConstraintValidationException.Phase.VERIFICATION,
new HashSet<>( failures )
);
Expand Down
21 changes: 5 additions & 16 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/state/ConstraintIndexCreator.java
View file
Expand Up @@ -57,7 +57,6 @@
import static org.neo4j.internal.kernel.api.exceptions.schema.ConstraintValidationException.Phase.VERIFICATION;
import static org.neo4j.internal.kernel.api.exceptions.schema.SchemaKernelException.OperationContext.CONSTRAINT_CREATION;
import static org.neo4j.internal.kernel.api.security.SecurityContext.AUTH_DISABLED;
import static org.neo4j.kernel.impl.locking.ResourceTypes.LABEL;

public class ConstraintIndexCreator
{
Expand Down Expand Up @@ -128,7 +127,7 @@ public long createUniquenessConstraintIndex( KernelTransactionImplementation tra
// At this point the integrity of the constraint to be created was checked
// while holding the lock and the index rule backing the soon-to-be-created constraint
// has been created. Now it's just the population left, which can take a long time
releaseLabelLock( locks, descriptor.keyId() );
locks.releaseExclusive( descriptor.keyType(), descriptor.keyId() );

awaitConstrainIndexPopulation( constraint, proxy );
log.info( "Constraint %s populated, starting verification.", constraint.ownedIndexDescriptor() );
Expand All @@ -137,7 +136,7 @@ public long createUniquenessConstraintIndex( KernelTransactionImplementation tra
// Acquire LABEL WRITE lock and verify the constraints here in this user transaction
// and if everything checks out then it will be held until after the constraint has been
// created and activated.
acquireLabelLock( transaction, locks, descriptor.keyId() );
locks.acquireExclusive( transaction.lockTracer(), descriptor.keyType(), descriptor.keyId() );
reacquiredLabelLock = true;

indexingService.getIndexProxy( indexId ).verifyDeferredConstraints( propertyAccessor );
Expand Down Expand Up @@ -170,7 +169,7 @@ public long createUniquenessConstraintIndex( KernelTransactionImplementation tra
{
if ( !reacquiredLabelLock )
{
acquireLabelLock( transaction, locks, descriptor.keyId() );
locks.acquireExclusive( transaction.lockTracer(), descriptor.keyType(), descriptor.keyId() );
}

if ( indexStillExists( schemaRead, descriptor, index ) )
Expand All @@ -183,20 +182,10 @@ public long createUniquenessConstraintIndex( KernelTransactionImplementation tra

private boolean indexStillExists( SchemaRead schemaRead, SchemaDescriptor descriptor, IndexReference index )
{
IndexReference existingIndex = schemaRead.index( descriptor.keyId(), descriptor.getPropertyIds() );
IndexReference existingIndex = schemaRead.index( descriptor );
return existingIndex != IndexReference.NO_INDEX && existingIndex.equals( index );
}

private void acquireLabelLock( KernelTransactionImplementation state, Client locks, int labelId )
{
locks.acquireExclusive( state.lockTracer(), LABEL, labelId );
}

private void releaseLabelLock( Client locks, int labelId )
{
locks.releaseExclusive( LABEL, labelId );
}

/**
* You MUST hold a schema write lock before you call this method.
*/
Expand Down Expand Up @@ -237,7 +226,7 @@ private IndexReference getOrCreateUniquenessConstraintIndex( SchemaRead schemaRe
Optional<String> provider )
throws SchemaKernelException, IndexNotFoundKernelException
{
IndexReference descriptor = schemaRead.index( schema.keyId(), schema.getPropertyIds() );
IndexReference descriptor = schemaRead.index( schema );
if ( descriptor != IndexReference.NO_INDEX )
{
if ( descriptor.isUnique() )
Expand Down
6 changes: 6 additions & 0 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/state/TxState.java
View file
Expand Up @@ -537,6 +537,12 @@ public ReadableDiffSets<IndexDescriptor> indexDiffSetsByLabel( int labelId )
return indexChangesDiffSets().filterAdded( SchemaDescriptorPredicates.hasLabel( labelId ) );
}

@Override
public ReadableDiffSets<IndexDescriptor> indexDiffSetsBySchema( SchemaDescriptor schema )
{
return indexChangesDiffSets().filterAdded( indexDescriptor -> indexDescriptor.schema().equals( schema ) );
}

@Override
public ReadableDiffSets<IndexDescriptor> indexChanges()
{
Expand Down
3 changes: 2 additions & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/locking/ResourceTypes.java
View file
Expand Up @@ -41,7 +41,8 @@ public enum ResourceTypes implements ResourceType
INDEX_ENTRY( 4, LockWaitStrategies.INCREMENTAL_BACKOFF ),
EXPLICIT_INDEX( 5, LockWaitStrategies.INCREMENTAL_BACKOFF ),
LABEL( 6, LockWaitStrategies.INCREMENTAL_BACKOFF ),
RELATIONSHIP_TYPE( 7, LockWaitStrategies.INCREMENTAL_BACKOFF );
RELATIONSHIP_TYPE( 7, LockWaitStrategies.INCREMENTAL_BACKOFF ),
SCHEMA( 8, LockWaitStrategies.INCREMENTAL_BACKOFF );

private static final boolean useStrongHashing =
FeatureToggles.flag( ResourceTypes.class, "useStrongHashing", false );
Expand Down
81 changes: 64 additions & 17 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/newapi/AllStoreHolder.java
View file
Expand Up @@ -352,6 +352,44 @@ public IndexReference index( int label, int... properties )
return indexDescriptor != null ? indexDescriptor : IndexReference.NO_INDEX;
}

@Override
public IndexReference index( SchemaDescriptor schema )
{
ktx.assertOpen();

CapableIndexDescriptor indexDescriptor = storageReader.indexGetForSchema( schema );
if ( ktx.hasTxStateWithChanges() )
{
ReadableDiffSets<IndexDescriptor> diffSets = ktx.txState().indexDiffSetsBySchema( schema );
if ( indexDescriptor != null )
{
if ( diffSets.isRemoved( indexDescriptor ) )
{
return IndexReference.NO_INDEX;
}
else
{
return indexDescriptor;
}
}
else
{
Iterator<IndexDescriptor> fromTxState =
filter( SchemaDescriptor.equalTo( schema ), diffSets.getAdded().iterator() );
if ( fromTxState.hasNext() )
{
return fromTxState.next();
}
else
{
return IndexReference.NO_INDEX;
}
}
}

return indexDescriptor != null ? indexDescriptor : IndexReference.NO_INDEX;
}

@Override
public IndexReference indexReferenceUnchecked( int label, int... properties )
{
Expand Down Expand Up @@ -387,7 +425,7 @@ public Iterator<IndexReference> indexesGetAll()

return Iterators.map( indexDescriptor ->
{
sharedOptimisticLock( ResourceTypes.LABEL, indexDescriptor.schema().keyId() );
sharedOptimisticLock( indexDescriptor.schema().keyType(), indexDescriptor.schema().keyId() );
return indexDescriptor;
}, iterator );
}
Expand All @@ -396,23 +434,24 @@ public Iterator<IndexReference> indexesGetAll()
public InternalIndexState indexGetState( IndexReference index ) throws IndexNotFoundKernelException
{
assertValidIndex( index );
sharedOptimisticLock( ResourceTypes.LABEL, index.label() );
IndexDescriptor descriptor = (IndexDescriptor) index;
sharedOptimisticLock( descriptor.schema().keyType(), descriptor.schema().keyId() );
ktx.assertOpen();
return indexGetState( (IndexDescriptor) index );
return indexGetState( descriptor );
}

@Override
public PopulationProgress indexGetPopulationProgress( IndexReference index )
throws IndexNotFoundKernelException
{
sharedOptimisticLock( ResourceTypes.LABEL, index.label() );
ktx.assertOpen();
IndexDescriptor descriptor = (IndexDescriptor) index;
sharedOptimisticLock( descriptor.schema().keyType(), descriptor.schema().keyId() );
ktx.assertOpen();

if ( ktx.hasTxStateWithChanges() )
{
if ( checkIndexState( descriptor,
ktx.txState().indexDiffSetsByLabel( index.label() ) ) )
ktx.txState().indexDiffSetsBySchema( descriptor.schema() ) ) )
{
return PopulationProgress.NONE;
}
Expand All @@ -424,11 +463,12 @@ public PopulationProgress indexGetPopulationProgress( IndexReference index )
@Override
public Long indexGetOwningUniquenessConstraintId( IndexReference index )
{
sharedOptimisticLock( ResourceTypes.LABEL, index.label() );
IndexDescriptor descriptor = (IndexDescriptor) index;
sharedOptimisticLock( descriptor.schema().keyType(), descriptor.schema().keyId() );
ktx.assertOpen();
if ( index instanceof StoreIndexDescriptor )
if ( descriptor instanceof StoreIndexDescriptor )
{
return ((StoreIndexDescriptor) index).getOwningConstraint();
return ((StoreIndexDescriptor) descriptor).getOwningConstraint();
}
else
{
Expand All @@ -439,11 +479,12 @@ public Long indexGetOwningUniquenessConstraintId( IndexReference index )
@Override
public long indexGetCommittedId( IndexReference index ) throws SchemaRuleNotFoundException
{
sharedOptimisticLock( ResourceTypes.LABEL, index.label() );
IndexDescriptor descriptor = (IndexDescriptor) index;
sharedOptimisticLock( descriptor.schema().keyType(), descriptor.schema().keyId() );
ktx.assertOpen();
if ( index instanceof StoreIndexDescriptor )
if ( descriptor instanceof StoreIndexDescriptor )
{
return ((StoreIndexDescriptor) index).getId();
return ((StoreIndexDescriptor) descriptor).getId();
}
else
{
Expand All @@ -454,24 +495,30 @@ public long indexGetCommittedId( IndexReference index ) throws SchemaRuleNotFoun
@Override
public String indexGetFailure( IndexReference index ) throws IndexNotFoundKernelException
{
return storageReader.indexGetFailure( SchemaDescriptorFactory.forLabel( index.label(), index.properties() ) );
assertValidIndex( index );
IndexDescriptor descriptor = (IndexDescriptor) index;
return storageReader.indexGetFailure( descriptor.schema() );
}

@Override
public double indexUniqueValuesSelectivity( IndexReference index ) throws IndexNotFoundKernelException
{
acquireSharedLabelLock( index.label() );
assertValidIndex( index );
SchemaDescriptor schema = ((IndexDescriptor) index).schema();
sharedOptimisticLock( schema.keyType(), schema.keyId() );
ktx.assertOpen();
return storageReader
.indexUniqueValuesPercentage( SchemaDescriptorFactory.forLabel( index.label(), index.properties() ) );
.indexUniqueValuesPercentage( schema );
}

@Override
public long indexSize( IndexReference index ) throws IndexNotFoundKernelException
{
acquireSharedLabelLock( index.label() );
assertValidIndex( index );
SchemaDescriptor schema = ((IndexDescriptor) index).schema();
sharedOptimisticLock( schema.keyType(), schema.keyId() );
ktx.assertOpen();
return storageReader.indexSize( SchemaDescriptorFactory.forLabel( index.label(), index.properties() ) );
return storageReader.indexSize( schema );
}

@Override
Expand Down

0 comments on commit 2f6d5cd

Please sign in to comment.