Skip to content

Commit

Permalink
More efficient store scan data structures
Browse files Browse the repository at this point in the history
  • Loading branch information
@tinwelint
tinwelint committed Jan 5, 2016
1 parent d54db9c commit 2706b78
Show file tree
Hide file tree
Showing 21 changed files with 89 additions and 58 deletions.
2 changes: 1 addition & 1 deletion ...heck-legacy/src/main/java/org/neo4j/legacy/consistency/checking/index/IndexAccessors.java
View file
Expand Up @@ -77,7 +77,7 @@ public IndexAccessors( SchemaIndexProvider provider,
for ( IndexRule indexRule : indexRules )
{
long indexId = indexRule.getId();
IndexConfiguration indexConfig = new IndexConfiguration( indexRule.isConstraintIndex() );
IndexConfiguration indexConfig = IndexConfiguration.of( indexRule );
accessors.put( indexId, provider.getOnlineAccessor( indexId, indexConfig, samplingConfig ) );
}
}
Expand Down
6 changes: 3 additions & 3 deletions ...cy/src/test/java/org/neo4j/legacy/consistency/checking/full/FullCheckIntegrationTest.java
View file
Expand Up @@ -456,7 +456,7 @@ public void shouldNotReportIndexInconsistenciesIfIndexIsFailed() throws Exceptio
{
IndexRule rule = rules.next();
IndexDescriptor descriptor = new IndexDescriptor( rule.getLabel(), rule.getPropertyKey() );
IndexConfiguration indexConfig = new IndexConfiguration( false );
IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator =
storeAccess.indexes().getPopulator( rule.getId(), descriptor, indexConfig, samplingConfig );
Expand Down Expand Up @@ -568,7 +568,7 @@ public void shouldReportNodesThatAreNotIndexed() throws Exception
for ( IndexRule indexRule : loadAllIndexRules( fixture.directStoreAccess().nativeStores().getSchemaStore() ) )
{
IndexAccessor accessor = fixture.directStoreAccess().indexes().getOnlineAccessor(
indexRule.getId(), new IndexConfiguration( indexRule.isConstraintIndex() ), samplingConfig );
indexRule.getId(), IndexConfiguration.of( indexRule ), samplingConfig );
IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE );
updater.remove( asPrimitiveLongSet( indexedNodes ) );
updater.close();
Expand All @@ -587,7 +587,7 @@ public void shouldReportNodesThatAreNotIndexed() throws Exception
public void shouldReportNodesWithDuplicatePropertyValueInUniqueIndex() throws Exception
{
// given
IndexConfiguration indexConfig = new IndexConfiguration( false );
IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
for ( IndexRule indexRule : loadAllIndexRules( fixture.directStoreAccess().nativeStores().getSchemaStore() ) )
{
Expand Down
2 changes: 1 addition & 1 deletion .../consistency-check/src/main/java/org/neo4j/consistency/checking/index/IndexAccessors.java
View file
Expand Up @@ -77,7 +77,7 @@ public IndexAccessors( SchemaIndexProvider provider,
for ( IndexRule indexRule : indexRules )
{
long indexId = indexRule.getId();
IndexConfiguration indexConfig = new IndexConfiguration( indexRule.isConstraintIndex() );
IndexConfiguration indexConfig = IndexConfiguration.of( indexRule );
accessors.put( indexId, provider.getOnlineAccessor( indexId, indexConfig, samplingConfig ) );
}
}
Expand Down
6 changes: 3 additions & 3 deletions ...ncy-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java
View file
Expand Up @@ -491,7 +491,7 @@ public void shouldNotReportIndexInconsistenciesIfIndexIsFailed() throws Exceptio
{
IndexRule rule = rules.next();
IndexDescriptor descriptor = new IndexDescriptor( rule.getLabel(), rule.getPropertyKey() );
IndexConfiguration indexConfig = new IndexConfiguration( false );
IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator =
storeAccess.indexes().getPopulator( rule.getId(), descriptor, indexConfig, samplingConfig );
Expand Down Expand Up @@ -603,7 +603,7 @@ public void shouldReportNodesThatAreNotIndexed() throws Exception
for ( IndexRule indexRule : loadAllIndexRules( fixture.directStoreAccess().nativeStores().getSchemaStore() ) )
{
IndexAccessor accessor = fixture.directStoreAccess().indexes().getOnlineAccessor(
indexRule.getId(), new IndexConfiguration( indexRule.isConstraintIndex() ), samplingConfig );
indexRule.getId(), IndexConfiguration.of( indexRule ), samplingConfig );
IndexUpdater updater = accessor.newUpdater( IndexUpdateMode.ONLINE );
updater.remove( asPrimitiveLongSet( indexedNodes ) );
updater.close();
Expand All @@ -622,7 +622,7 @@ public void shouldReportNodesThatAreNotIndexed() throws Exception
public void shouldReportNodesWithDuplicatePropertyValueInUniqueIndex() throws Exception
{
// given
IndexConfiguration indexConfig = new IndexConfiguration( false );
IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig samplingConfig = new IndexSamplingConfig( new Config() );
for ( IndexRule indexRule : loadAllIndexRules( fixture.directStoreAccess().nativeStores().getSchemaStore() ) )
{
Expand Down
21 changes: 11 additions & 10 deletions community/kernel/src/main/java/org/neo4j/kernel/api/index/IndexConfiguration.java
View file
Expand Up @@ -19,11 +19,16 @@
*/
package org.neo4j.kernel.api.index;

public class IndexConfiguration
import org.neo4j.kernel.impl.store.record.IndexRule;

public enum IndexConfiguration
{
NON_UNIQUE( false ),
UNIQUE( true );

private final boolean unique;

public IndexConfiguration( boolean unique )
private IndexConfiguration( boolean unique )
{
this.unique = unique;
}
Expand All @@ -33,17 +38,13 @@ public boolean isUnique()
return unique;
}

@Override
public boolean equals( Object o )
public static IndexConfiguration of( boolean unique )
{
return this == o ||
!(o == null || getClass() != o.getClass()) &&
unique == ((IndexConfiguration) o).unique;
return unique ? UNIQUE : NON_UNIQUE;
}

@Override
public int hashCode()
public static IndexConfiguration of( IndexRule rule )
{
return (unique ? 1 : 0);
return of( rule.isConstraintIndex() );
}
}
8 changes: 4 additions & 4 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexProxySetup.java
View file
Expand Up @@ -73,7 +73,7 @@ public IndexProxy createPopulatingIndexProxy( final long ruleId,

// TODO: This is here because there is a circular dependency from PopulatingIndexProxy to FlippableIndexProxy
final String indexUserDescription = indexUserDescription( descriptor, providerDescriptor );
final IndexConfiguration config = new IndexConfiguration( constraint );
final IndexConfiguration config = IndexConfiguration.of( constraint );
IndexPopulator populator = populatorFromProvider( providerDescriptor, ruleId, descriptor, config,
samplingConfig );

Expand Down Expand Up @@ -126,7 +126,7 @@ public IndexProxy createRecoveringIndexProxy( IndexDescriptor descriptor,
SchemaIndexProvider.Descriptor providerDescriptor,
boolean constraint )
{
IndexConfiguration configuration = new IndexConfiguration( constraint );
IndexConfiguration configuration = IndexConfiguration.of( constraint );
IndexProxy proxy;
proxy = new RecoveringIndexProxy( descriptor, providerDescriptor, configuration );
proxy = new ContractCheckingIndexProxy( proxy, true );
Expand All @@ -141,7 +141,7 @@ public IndexProxy createOnlineIndexProxy( long ruleId,
// TODO Hook in version verification/migration calls to the SchemaIndexProvider here
try
{
IndexConfiguration config = new IndexConfiguration( unique );
IndexConfiguration config = IndexConfiguration.of( unique );
IndexAccessor onlineAccessor =
onlineAccessorFromProvider( providerDescriptor, ruleId, config, samplingConfig );
IndexProxy proxy;
Expand All @@ -161,7 +161,7 @@ public IndexProxy createFailedIndexProxy( long ruleId,
boolean unique,
IndexPopulationFailure populationFailure )
{
IndexConfiguration config = new IndexConfiguration( unique );
IndexConfiguration config = IndexConfiguration.of( unique );
IndexPopulator indexPopulator =
populatorFromProvider( providerDescriptor, ruleId, descriptor, config, samplingConfig );
String indexUserDescription = indexUserDescription(descriptor, providerDescriptor);
Expand Down
13 changes: 12 additions & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexStoreView.java
View file
Expand Up @@ -19,11 +19,14 @@
*/
package org.neo4j.kernel.impl.api.index;

import java.util.Collection;

import org.neo4j.helpers.collection.Visitor;
import org.neo4j.kernel.api.index.IndexDescriptor;
import org.neo4j.kernel.api.index.NodePropertyUpdate;
import org.neo4j.kernel.api.index.PropertyAccessor;
import org.neo4j.kernel.api.labelscan.NodeLabelUpdate;
import org.neo4j.kernel.impl.store.record.NodeRecord;
import org.neo4j.register.Register.DoubleLongRegister;

/** The indexing services view of the universe. */
Expand All @@ -47,7 +50,15 @@ <FAILURE extends Exception> StoreScan<FAILURE> visitNodes( int[] labelIds, int[]
Visitor<NodePropertyUpdate, FAILURE> propertyUpdateVisitor,
Visitor<NodeLabelUpdate, FAILURE> labelUpdateVisitor );

Iterable<NodePropertyUpdate> nodeAsUpdates( long nodeId );
/**
* Produces {@link NodePropertyUpdate} objects from reading node {@code nodeId}, its labels and properties
* and puts those updates into {@code target}.
*
* @param nodeId id of node to load.
* @param reusableRecord {@link NodeRecord} to load node record into.
* @param target {@link Collection} to add updates into.
*/
void nodeAsUpdates( long nodeId, NodeRecord reusableRecord, Collection<NodePropertyUpdate> target );

DoubleLongRegister indexUpdatesAndSize( IndexDescriptor descriptor, DoubleLongRegister output );

Expand Down
6 changes: 4 additions & 2 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexingService.java
View file
Expand Up @@ -34,7 +34,6 @@
import org.neo4j.collection.primitive.PrimitiveLongSet;
import org.neo4j.collection.primitive.PrimitiveLongVisitor;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.helpers.collection.Iterables;
import org.neo4j.kernel.api.TokenNameLookup;
import org.neo4j.kernel.api.exceptions.index.IndexActivationFailedKernelException;
import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException;
Expand All @@ -55,6 +54,7 @@
import org.neo4j.kernel.impl.store.record.IndexRule;
import org.neo4j.kernel.impl.transaction.state.DirectIndexUpdates;
import org.neo4j.kernel.impl.transaction.state.IndexUpdates;
import org.neo4j.kernel.impl.store.record.NodeRecord;
import org.neo4j.kernel.impl.util.JobScheduler;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;
import org.neo4j.logging.Log;
Expand Down Expand Up @@ -581,10 +581,12 @@ private IndexUpdates readRecoveredUpdatesFromStore()
final List<NodePropertyUpdate> recoveredUpdates = new ArrayList<>();
recoveredNodeIds.visitKeys( new PrimitiveLongVisitor<RuntimeException>()
{
private final NodeRecord nodeRecord = new NodeRecord( -1 );

@Override
public boolean visited( long nodeId )
{
Iterables.addAll( recoveredUpdates, storeView.nodeAsUpdates( nodeId ) );
storeView.nodeAsUpdates( nodeId, nodeRecord, recoveredUpdates );
return false;
}
} );
Expand Down
4 changes: 2 additions & 2 deletions ...g/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/IndexLookup.java
View file
Expand Up @@ -44,7 +44,7 @@ class IndexLookup implements AutoCloseable
private final Map<IndexRule,IndexReader> readerCache;
private final SchemaIndexProvider schemaIndexProvider;
private final PrimitiveIntObjectMap<List<IndexRule>> indexRuleIndex;
private IndexSamplingConfig samplingConfig;
private final IndexSamplingConfig samplingConfig;

public IndexLookup( SchemaStore store, SchemaIndexProvider schemaIndexProvider )
{
Expand Down Expand Up @@ -108,7 +108,7 @@ private IndexReader getIndexReader( IndexRule rule ) throws IOException
IndexReader reader = readerCache.get( rule );
if ( reader == null )
{
IndexConfiguration indexConfig = new IndexConfiguration( rule.isConstraintIndex() );
IndexConfiguration indexConfig = IndexConfiguration.of( rule );
IndexAccessor accessor = schemaIndexProvider.getOnlineAccessor(
rule.getId(), indexConfig, samplingConfig );
indexAccessors.add( accessor );
Expand Down
21 changes: 10 additions & 11 deletions .../kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreIndexStoreView.java
View file
Expand Up @@ -20,6 +20,7 @@
package org.neo4j.kernel.impl.transaction.state;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

Expand Down Expand Up @@ -55,6 +56,7 @@

import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY;
import static org.neo4j.kernel.api.CountsRead.ANY_LABEL;
import static org.neo4j.kernel.api.index.NodePropertyUpdate.add;
import static org.neo4j.kernel.api.labelscan.NodeLabelUpdate.labelChanges;
import static org.neo4j.kernel.impl.store.NodeLabelsField.parseLabelsField;

Expand Down Expand Up @@ -172,8 +174,7 @@ protected Update read( NodeRecord node )
{
if ( propertyKeyId == sought )
{
update.add( NodePropertyUpdate
.add( node.getId(), propertyKeyId, valueOf( property ), labels ) );
update.add( add( node.getId(), propertyKeyId, valueOf( property ), labels ) );
continue properties;
}
}
Expand All @@ -194,33 +195,31 @@ protected void process( Update update ) throws FAILURE
}

@Override
public Iterable<NodePropertyUpdate> nodeAsUpdates( long nodeId )
public void nodeAsUpdates( long nodeId, NodeRecord record, Collection<NodePropertyUpdate> target )
{
NodeRecord node = nodeStore.forceGetRecord( nodeId );
if ( !node.inUse() )
NodeRecord node = nodeStore.loadRecord( nodeId, record );
if ( node == null || !node.inUse() )
{
return Iterables.empty(); // node not in use => no updates
return;
}
long firstPropertyId = node.getNextProp();
if ( firstPropertyId == Record.NO_NEXT_PROPERTY.intValue() )
{
return Iterables.empty(); // no properties => no updates (it's not going to be in any index)
return; // no properties => no updates (it's not going to be in any index)
}
long[] labels = parseLabelsField( node ).get( nodeStore );
if ( labels.length == 0 )
{
return Iterables.empty(); // no labels => no updates (it's not going to be in any index)
return; // no labels => no updates (it's not going to be in any index)
}
ArrayList<NodePropertyUpdate> updates = new ArrayList<>();
for ( PropertyRecord propertyRecord : propertyStore.getPropertyRecordChain( firstPropertyId ) )
{
for ( PropertyBlock property : propertyRecord )
{
Object value = property.getType().getValue( property, propertyStore );
updates.add( NodePropertyUpdate.add( node.getId(), property.getKeyIndexId(), value, labels ) );
target.add( add( node.getId(), property.getKeyIndexId(), value, labels ) );
}
}
return updates;
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java
View file
Expand Up @@ -524,11 +524,10 @@ private void repopulateAllIndexes() throws IOException, IndexEntryConflictExcept
propertyKeyIds[i] = propertyKeyId;

IndexDescriptor descriptor = new IndexDescriptor( labelId, propertyKeyId );
boolean isConstraint = rule.isConstraintIndex();
populators[i] = schemaIndexProviders.apply( rule.getProviderDescriptor() )
.getPopulator( rule.getId(),
descriptor,
new IndexConfiguration( isConstraint ),
IndexConfiguration.of( rule ),
new IndexSamplingConfig( config ) );
populators[i].create();
}
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/test/java/org/neo4j/kernel/api/index/IndexAccessorCompatibility.java
View file
Expand Up @@ -58,7 +58,7 @@ public IndexAccessorCompatibility( IndexProviderCompatibilityTestSuite testSuite
@Before
public void before() throws Exception
{
IndexConfiguration indexConfig = new IndexConfiguration( isUnique );
IndexConfiguration indexConfig = IndexConfiguration.of( isUnique );
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, indexConfig, indexSamplingConfig );
populator.create();
Expand Down
12 changes: 6 additions & 6 deletions ...kernel/src/test/java/org/neo4j/kernel/api/index/NonUniqueIndexPopulatorCompatibility.java
View file
Expand Up @@ -51,7 +51,7 @@ public NonUniqueIndexPopulatorCompatibility( IndexProviderCompatibilityTestSuite
public void shouldProvidePopulatorThatAcceptsDuplicateEntries() throws Exception
{
// when
IndexConfiguration config = new IndexConfiguration( false );
IndexConfiguration config = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig );
populator.create();
Expand All @@ -73,7 +73,7 @@ public void shouldProvidePopulatorThatAcceptsDuplicateEntries() throws Exception
public void shouldStorePopulationFailedForRetrievalFromProviderLater() throws Exception
{
// GIVEN
IndexConfiguration config = new IndexConfiguration( false );
IndexConfiguration config = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig );
String failure = "The contrived failure";
Expand All @@ -90,7 +90,7 @@ public void shouldStorePopulationFailedForRetrievalFromProviderLater() throws Ex
public void shouldReportInitialStateAsFailedIfPopulationFailed() throws Exception
{
// GIVEN
IndexConfiguration config = new IndexConfiguration( false );
IndexConfiguration config = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig );
String failure = "The contrived failure";
Expand All @@ -107,7 +107,7 @@ public void shouldReportInitialStateAsFailedIfPopulationFailed() throws Exceptio
public void shouldBeAbleToDropAClosedIndexPopulator() throws Exception
{
// GIVEN
IndexConfiguration config = new IndexConfiguration( false );
IndexConfiguration config = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig );
populator.close( false );
Expand All @@ -122,7 +122,7 @@ public void shouldBeAbleToDropAClosedIndexPopulator() throws Exception
public void shouldApplyUpdatesIdempotently() throws Exception
{
// GIVEN
IndexConfiguration config = new IndexConfiguration( false );
IndexConfiguration config = IndexConfiguration.NON_UNIQUE;
IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( new Config() );
IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig );
populator.create();
Expand Down Expand Up @@ -151,7 +151,7 @@ public Property getProperty( long nodeId, int propertyKeyId ) throws EntityNotFo


// then
IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, new IndexConfiguration( false ), indexSamplingConfig );
IndexAccessor accessor = indexProvider.getOnlineAccessor( 17, IndexConfiguration.NON_UNIQUE, indexSamplingConfig );
try ( IndexReader reader = accessor.newReader() )
{
PrimitiveLongIterator nodes = reader.seek( propertyValue );
Expand Down

0 comments on commit 2706b78

Please sign in to comment.