diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java index ae54129b995f..d061e6db3402 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulator.java @@ -39,6 +39,8 @@ import org.neo4j.function.Predicates; import org.neo4j.helpers.Exceptions; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException; import org.neo4j.kernel.api.index.IndexConfiguration; @@ -407,10 +409,30 @@ public void stop() delegate.stop(); } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + delegate.complete( indexPopulator, descriptor ); + } + + @Override + public void acceptUpdate( MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + delegate.acceptUpdate( updater, update, currentlyIndexedNodeId ); + } + @Override public PopulationProgress getProgress() { return delegate.getProgress(); } + + @Override + public void configure( List populations ) + { + delegate.configure( populations ); + } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexStoreView.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexStoreView.java index 20a5e974cd14..57c3b34102c2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexStoreView.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/IndexStoreView.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import java.util.function.IntPredicate; import org.neo4j.helpers.collection.Visitor; @@ -68,21 +69,6 @@ StoreScan visitNodes( void incrementIndexUpdates( IndexDescriptor descriptor, long updatesDelta ); - /** - * Check if provided node update is applicable in a context of current store view. - * - * @param updater - * @param update update to check - * @param currentlyIndexedNodeId id of currently indexed node - */ - void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, - long currentlyIndexedNodeId ); - - void complete( IndexPopulator populator, IndexDescriptor descriptor ) - throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException; - - boolean isFullScan(); - StoreScan EMPTY_SCAN = new StoreScan() { @Override @@ -95,11 +81,31 @@ public void stop() { } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + + } + @Override public PopulationProgress getProgress() { return PopulationProgress.DONE; } + + @Override + public void configure( List list ) + { + + } }; IndexStoreView EMPTY = new IndexStoreView() @@ -146,25 +152,5 @@ public void incrementIndexUpdates( IndexDescriptor descriptor, long updatesDelta { } - @Override - public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, - long currentlyIndexedNodeId ) - { - } - - @Override - public void complete( IndexPopulator populator, IndexDescriptor descriptor ) - { - - } - - @Override - public boolean isFullScan() - { - return true; - } }; - - - } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java index 8da249659d54..63eb634c7925 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulator.java @@ -93,6 +93,7 @@ public class MultipleIndexPopulator implements IndexPopulator private final IndexStoreView storeView; private final LogProvider logProvider; protected final Log log; + private StoreScan storeScan; public MultipleIndexPopulator( IndexStoreView storeView, LogProvider logProvider ) { @@ -155,13 +156,9 @@ public StoreScan indexAllNodes() int[] propertyKeyIds = propertyKeyIds(); IntPredicate propertyKeyIdFilter = (propertyKeyId) -> contains( propertyKeyIds, propertyKeyId ); - StoreScan storeScan = - storeView.visitNodes( labelIds, propertyKeyIdFilter, new NodePopulationVisitor(), null ); - - populations.forEach( population -> population.populator.configureSampling( storeView.isFullScan() ) ); - + storeScan = storeView.visitNodes( labelIds, propertyKeyIdFilter, new NodePopulationVisitor(), null ); + storeScan.configure(populations); return storeScan; - } /** @@ -337,7 +334,7 @@ private void populateFromQueueIfAvailable( long currentlyIndexedNodeId ) { // no need to check for null as nobody else is emptying this queue NodePropertyUpdate update = queue.poll(); - storeView.acceptUpdate( updater, update, currentlyIndexedNodeId ); + storeScan.acceptUpdate( updater, update, currentlyIndexedNodeId ); } while ( !queue.isEmpty() ); } @@ -435,9 +432,9 @@ public void close() } } - protected class IndexPopulation + public class IndexPopulation { - final IndexPopulator populator; + public final IndexPopulator populator; final IndexDescriptor descriptor; final IndexConfiguration config; final SchemaIndexProvider.Descriptor providerDescriptor; @@ -506,7 +503,7 @@ private void flip() throws FlipFailedKernelException { flipper.flip( () -> { populateFromQueueIfAvailable( Long.MAX_VALUE ); - storeView.complete(populator, descriptor); + storeScan.complete(populator, descriptor); IndexSample sample = populator.sampleResult(); storeView.replaceIndexCounts( descriptor, sample.uniqueValues(), sample.sampleSize(), sample.indexSize() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/StoreScan.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/StoreScan.java index 045cc6bd73e1..139f45d1d850 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/StoreScan.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/index/StoreScan.java @@ -19,6 +19,15 @@ */ package org.neo4j.kernel.impl.api.index; +import java.io.IOException; +import java.util.List; + +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexDescriptor; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.storageengine.api.schema.PopulationProgress; public interface StoreScan @@ -27,5 +36,13 @@ public interface StoreScan void stop(); + void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException; + + void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ); + PopulationProgress getProgress(); + + void configure( List populations ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/AdaptableIndexStoreView.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/AdaptableIndexStoreView.java index 584d79d27fd7..559b1bfd9d3c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/AdaptableIndexStoreView.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/AdaptableIndexStoreView.java @@ -20,35 +20,18 @@ package org.neo4j.kernel.impl.transaction.state.storeview; import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import java.io.IOException; -import java.util.Arrays; import java.util.function.IntPredicate; +import java.util.stream.IntStream; -import org.neo4j.collection.primitive.Primitive; -import org.neo4j.collection.primitive.PrimitiveLongIterator; -import org.neo4j.collection.primitive.PrimitiveLongObjectMap; -import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.helpers.collection.Visitor; -import org.neo4j.kernel.api.exceptions.EntityNotFoundException; -import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; -import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; -import org.neo4j.kernel.api.index.IndexDescriptor; -import org.neo4j.kernel.api.index.IndexPopulator; -import org.neo4j.kernel.api.index.IndexUpdater; -import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; -import org.neo4j.kernel.api.properties.Property; -import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.NodePropertyUpdates; import org.neo4j.kernel.impl.api.index.StoreScan; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.counts.CountsTracker; -import org.neo4j.kernel.impl.store.record.NodeRecord; -import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.register.Registers; import org.neo4j.unsafe.impl.internal.dragons.FeatureToggles; @@ -63,11 +46,6 @@ public class AdaptableIndexStoreView extends NeoStoreIndexStoreView private final LabelScanStore labelScanStore; private final CountsTracker counts; - //TODO: we need to remember what kind of scan we do currently to handle updates in proper way - private boolean usingLabelScan = true; - - PrimitiveLongObjectMap> propertyLabelNodes = Primitive.longObjectMap(); - public AdaptableIndexStoreView( LabelScanStore labelScanStore, LockService locks, NeoStores neoStores ) { @@ -83,131 +61,12 @@ public StoreScan visitNodes( int[] labelIds { if ( ArrayUtils.isEmpty( labelIds ) || isEmptyLabelScanStore() || isNumberOfLabeledNodesExceedThreshold( labelIds ) ) { - usingLabelScan = false; return super.visitNodes( labelIds, propertyKeyIdFilter, propertyUpdatesVisitor, labelUpdateVisitor ); } - usingLabelScan = true; - return new LabelScanViewNodeStoreScan<>( nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, + return new LabelScanViewNodeStoreScan<>( this, nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, propertyUpdatesVisitor, labelIds, propertyKeyIdFilter ); } - /** - * Checks if provided node property update is applicable for current store view. - * In case if we use label store view - updates are always applicable, otherwise fallback to default behaviour of - * neo store view. - * - * @param updater updater that should process updates in required - * @param update node property update - * @param currentlyIndexedNodeId currently indexed node id - */ - @Override - public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, - long currentlyIndexedNodeId ) - { - if ( usingLabelScan ) - { - long[] labels; - switch ( update.getUpdateMode() ) - { - case ADDED: - labels = update.getLabelsAfter(); - break; - case CHANGED: - //TODO: should it be like that???? - labels = Arrays.stream( update.getLabelsBefore() ) - .filter( item -> Arrays.binarySearch( update.getLabelsAfter(), item ) >= 0 ) - .toArray(); - break; - case REMOVED: - labels = update.getLabelsBefore(); - break; - default: - throw new UnsupportedOperationException( "unsupported update mode" ); - } - int property = update.getPropertyKeyId(); - long nodeId = update.getNodeId(); - for ( long label : labels ) - { - // TODO: put if absent ? - PrimitiveLongObjectMap labelsNodes = propertyLabelNodes.get( property ); - if ( labelsNodes == null) - { - labelsNodes = Primitive.longObjectMap(); - propertyLabelNodes.put( property, labelsNodes ); - } - PrimitiveLongSet nodes = labelsNodes.get( label ); - if (nodes == null) - { - nodes = Primitive.longSet(); - labelsNodes.put( label, nodes ); - } - nodes.add( nodeId ); - } - } - else - { - super.acceptUpdate( updater, update, currentlyIndexedNodeId ); - } - } - - @Override - public Property getProperty( long nodeId, int propertyKeyId ) throws EntityNotFoundException - { - return super.getProperty( nodeId, propertyKeyId ); - } - - @Override - public boolean isFullScan() - { - return !usingLabelScan; - } - - // @Override - public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) - throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException - { - if (usingLabelScan) - { - PrimitiveLongObjectMap labelNodes = propertyLabelNodes.get( descriptor.getPropertyKeyId() ); - if ( labelNodes != null ) - { - int labelId = descriptor.getLabelId(); - PrimitiveLongSet nodes = labelNodes.get( labelId ); - if ( nodes != null ) - { - long[] labels = new long[labelId]; - PrimitiveLongIterator nodeIterator = nodes.iterator(); - NodeRecord nodeRecord = new NodeRecord( -1 ); - try ( IndexUpdater updater = indexPopulator.newPopulatingUpdater( this ) ) - { - while ( nodeIterator.hasNext() ) - { - long nodeId = nodeIterator.next(); - NodeRecord record = nodeStore.getRecord( nodeId, nodeRecord, RecordLoad.FORCE ); - - int propertyKeyId = descriptor.getPropertyKeyId(); - - updater.process( - NodePropertyUpdate.remove( nodeId, propertyKeyId, StringUtils.EMPTY, labels ) ); - if ( record.inUse() ) - { - Property property = getProperty( nodeId, propertyKeyId ); - if ( property.isDefined() ) - { - Object propertyValue = property.value(); - updater.process( - NodePropertyUpdate.change( nodeId, propertyKeyId, StringUtils.EMPTY, - labels, propertyValue, labels ) ); - } - } - - } - } - } - } - } - } - private boolean isEmptyLabelScanStore() { return labelScanStore.allNodeLabelRanges().maxCount() == 0; @@ -225,7 +84,7 @@ private long getVisitAllNodesThreshold() private long getNumberOfLabeledNodes( int[] labelIds ) { - return Arrays.stream( labelIds ) + return IntStream.of( labelIds ) .mapToLong( labelId -> counts.nodeCount( labelId, Registers.newDoubleLongRegister() ).readSecond() ) .reduce( Math::addExact ) .orElse( 0L ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScan.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScan.java index 50243ce69e9b..e01831da98ff 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScan.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScan.java @@ -19,17 +19,36 @@ */ package org.neo4j.kernel.impl.transaction.state.storeview; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; import java.util.function.IntPredicate; +import org.neo4j.collection.primitive.Primitive; import org.neo4j.collection.primitive.PrimitiveLongIterator; +import org.neo4j.collection.primitive.PrimitiveLongObjectMap; import org.neo4j.collection.primitive.PrimitiveLongResourceIterator; +import org.neo4j.collection.primitive.PrimitiveLongSet; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexDescriptor; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.IndexUpdater; +import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; +import org.neo4j.kernel.api.properties.Property; +import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.NodePropertyUpdates; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; +import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.storageengine.api.schema.LabelScanReader; /** @@ -39,15 +58,20 @@ */ public class LabelScanViewNodeStoreScan extends StoreViewNodeStoreScan { + private NeoStoreIndexStoreView storeView; private final LabelScanStore labelScanStore; - public LabelScanViewNodeStoreScan( NodeStore nodeStore, LockService locks, PropertyStore propertyStore, + PrimitiveLongObjectMap> propertyLabelNodes = Primitive.longObjectMap(); + + public LabelScanViewNodeStoreScan( NeoStoreIndexStoreView storeView, NodeStore nodeStore, LockService locks, + PropertyStore propertyStore, LabelScanStore labelScanStore, Visitor labelUpdateVisitor, Visitor propertyUpdatesVisitor, int[] labelIds, IntPredicate propertyKeyIdFilter ) { super( nodeStore, locks, propertyStore, labelUpdateVisitor, propertyUpdatesVisitor, labelIds, propertyKeyIdFilter ); + this.storeView = storeView; this.labelScanStore = labelScanStore; } @@ -57,6 +81,107 @@ public PrimitiveLongResourceIterator getNodeIdIterator() return new LabelScanViewIdIterator( labelScanStore, labelIds ); } + @Override + public void configure( List populations ) + { + populations.forEach( population -> population.populator.configureSampling( false ) ); + } + + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + + PrimitiveLongObjectMap labelNodes = propertyLabelNodes.get( descriptor.getPropertyKeyId() ); + if ( labelNodes != null ) + { + int labelId = descriptor.getLabelId(); + PrimitiveLongSet nodes = labelNodes.get( labelId ); + if ( nodes != null ) + { + long[] labels = new long[labelId]; + PrimitiveLongIterator nodeIterator = nodes.iterator(); + NodeRecord nodeRecord = new NodeRecord( -1 ); + try ( IndexUpdater updater = indexPopulator.newPopulatingUpdater( storeView ) ) + { + while ( nodeIterator.hasNext() ) + { + long nodeId = nodeIterator.next(); + NodeRecord record = nodeStore.getRecord( nodeId, nodeRecord, RecordLoad.FORCE ); + + int propertyKeyId = descriptor.getPropertyKeyId(); + + updater.process( + NodePropertyUpdate.remove( nodeId, propertyKeyId, StringUtils.EMPTY, labels ) ); + if ( record.inUse() ) + { + Property property = storeView.getProperty( nodeId, propertyKeyId ); + if ( property.isDefined() ) + { + Object propertyValue = property.value(); + updater.process( + NodePropertyUpdate.change( nodeId, propertyKeyId, StringUtils.EMPTY, + labels, propertyValue, labels ) ); + } + } + } + } + } + } + } + + /** + * Checks if provided node property update is applicable for current store view. + * In case if we use label store view - updates are always applicable, otherwise fallback to default behaviour of + * neo store view. + * + * @param updater updater that should process updates in required + * @param update node property update + * @param currentlyIndexedNodeId currently indexed node id + */ + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + long[] labels; + switch ( update.getUpdateMode() ) + { + case ADDED: + labels = update.getLabelsAfter(); + break; + case CHANGED: + //TODO: should it be like that???? + labels = Arrays.stream( update.getLabelsBefore() ) + .filter( item -> Arrays.binarySearch( update.getLabelsAfter(), item ) >= 0 ) + .toArray(); + break; + case REMOVED: + labels = update.getLabelsBefore(); + break; + default: + throw new UnsupportedOperationException( "unsupported update mode" ); + } + int property = update.getPropertyKeyId(); + long nodeId = update.getNodeId(); + for ( long label : labels ) + { + // TODO: put if absent ? + PrimitiveLongObjectMap labelsNodes = propertyLabelNodes.get( property ); + if ( labelsNodes == null) + { + labelsNodes = Primitive.longObjectMap(); + propertyLabelNodes.put( property, labelsNodes ); + } + PrimitiveLongSet nodes = labelsNodes.get( label ); + if (nodes == null) + { + nodes = Primitive.longSet(); + labelsNodes.put( label, nodes ); + } + nodes.add( nodeId ); + } + } + private class LabelScanViewIdIterator implements PrimitiveLongResourceIterator { private final LabelScanReader labelScanReader; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/NeoStoreIndexStoreView.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/NeoStoreIndexStoreView.java index d1595e284840..1cb64652d29a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/NeoStoreIndexStoreView.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/NeoStoreIndexStoreView.java @@ -19,22 +19,17 @@ */ package org.neo4j.kernel.impl.transaction.state.storeview; -import java.io.IOException; import java.util.Collection; import java.util.function.IntPredicate; import org.neo4j.helpers.collection.Visitor; import org.neo4j.kernel.api.exceptions.EntityNotFoundException; -import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; -import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.index.IndexDescriptor; -import org.neo4j.kernel.api.index.IndexPopulator; import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; import org.neo4j.kernel.api.properties.Property; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.api.index.IndexStoreView; -import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.NodePropertyUpdates; import org.neo4j.kernel.impl.api.index.StoreScan; import org.neo4j.kernel.impl.locking.LockService; @@ -99,37 +94,6 @@ public void incrementIndexUpdates( IndexDescriptor descriptor, long updatesDelta } } - /** - * Accept updates in cases if updated node id is bellow or equal to currently indexed node. - * - * @param updater - * @param update update to check - * @param currentlyIndexedNodeId id of currently indexed node - * @return true if update is applicable - */ - @Override - public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, - long currentlyIndexedNodeId ) - { - if ( update.getNodeId() <= currentlyIndexedNodeId ) - { - updater.process( update ); - } - } - - @Override - public void complete( IndexPopulator populator, IndexDescriptor descriptor ) - throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException - { - // no-op - } - - @Override - public boolean isFullScan() - { - return true; - } - @Override public DoubleLongRegister indexSample( IndexDescriptor descriptor, DoubleLongRegister output ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java index 9268baa27ce2..d96c1106581d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java @@ -21,17 +21,23 @@ import org.apache.commons.lang3.ArrayUtils; +import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.function.IntPredicate; -import org.neo4j.collection.primitive.PrimitiveLongCollections; -import org.neo4j.collection.primitive.PrimitiveLongIterator; import org.neo4j.collection.primitive.PrimitiveLongResourceIterator; import org.neo4j.helpers.collection.Iterators; import org.neo4j.helpers.collection.PrefetchingIterator; import org.neo4j.helpers.collection.Visitor; -import org.neo4j.kernel.api.labelscan.LabelScanStore; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexDescriptor; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; +import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.NodePropertyUpdates; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NodeStore; @@ -40,7 +46,6 @@ import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; -import org.neo4j.storageengine.api.schema.LabelScanReader; import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY; import static org.neo4j.kernel.api.labelscan.NodeLabelUpdate.labelChanges; @@ -139,6 +144,29 @@ private static boolean containsAnyLabel( int[] labelIdFilter, long[] labels ) return false; } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + // no-op + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + if ( update.getNodeId() <= currentlyIndexedNodeId ) + { + updater.process( update ); + } + } + + @Override + public void configure( List populations ) + { + populations.forEach( population -> population.populator.configureSampling( true ) ); + } + private class PropertyBlockIterator extends PrefetchingIterator { private final Iterator records; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/api/index/NonUniqueIndexPopulatorCompatibility.java b/community/kernel/src/test/java/org/neo4j/kernel/api/index/NonUniqueIndexPopulatorCompatibility.java index f9184f36a198..da02cb577077 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/api/index/NonUniqueIndexPopulatorCompatibility.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/api/index/NonUniqueIndexPopulatorCompatibility.java @@ -129,6 +129,7 @@ public void shouldApplyUpdatesIdempotently() throws Exception IndexSamplingConfig indexSamplingConfig = new IndexSamplingConfig( Config.empty() ); IndexPopulator populator = indexProvider.getPopulator( 17, descriptor, config, indexSamplingConfig ); populator.create(); + populator.configureSampling( true ); long nodeId = 1; int propertyKeyId = 10, labelId = 11; // Can we just use arbitrary ids here? final String propertyValue = "value1"; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java index 4d4a234b52c1..cb7680d058b9 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/BatchingMultipleIndexPopulatorTest.java @@ -22,12 +22,17 @@ import org.junit.After; import org.junit.Test; +import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.neo4j.helpers.collection.Visitor; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException; import org.neo4j.kernel.api.index.IndexConfiguration; import org.neo4j.kernel.api.index.IndexDescriptor; @@ -37,6 +42,7 @@ import org.neo4j.kernel.api.index.SchemaIndexProvider; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NeoStores; +import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView; import org.neo4j.logging.NullLogProvider; import org.neo4j.storageengine.api.schema.PopulationProgress; @@ -98,8 +104,12 @@ public void populateFromQueuePopulatesWhenThresholdReached() throws Exception { setProperty( QUEUE_THRESHOLD_NAME, 2 ); + NeoStores neoStores = mock( NeoStores.class ); + NodeStore nodeStore = mock( NodeStore.class ); + when( neoStores.getNodeStore() ).thenReturn( nodeStore ); + NeoStoreIndexStoreView storeView = - new NeoStoreIndexStoreView( LockService.NO_LOCK_SERVICE, mock( NeoStores.class ) ); + new NeoStoreIndexStoreView( LockService.NO_LOCK_SERVICE, neoStores ); BatchingMultipleIndexPopulator batchingPopulator = new BatchingMultipleIndexPopulator( storeView, mock( ExecutorService.class ), NullLogProvider.getInstance() ); @@ -112,6 +122,7 @@ public void populateFromQueuePopulatesWhenThresholdReached() throws Exception IndexUpdater updater2 = mock( IndexUpdater.class ); when( populator2.newPopulatingUpdater( any() ) ).thenReturn( updater2 ); + batchingPopulator.indexAllNodes(); NodePropertyUpdate update1 = NodePropertyUpdate.add( 1, 1, "foo", new long[]{1} ); NodePropertyUpdate update2 = NodePropertyUpdate.add( 2, 2, "bar", new long[]{2} ); NodePropertyUpdate update3 = NodePropertyUpdate.add( 3, 1, "baz", new long[]{1} ); @@ -369,10 +380,30 @@ public void stop() stop = true; } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + + } + @Override public PopulationProgress getProgress() { return PopulationProgress.NONE; } + + @Override + public void configure( List populations ) + { + + } } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java index 2f5e61e4fad3..869c080df832 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexPopulationJobTest.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; @@ -45,7 +46,9 @@ import org.neo4j.kernel.api.KernelAPI; import org.neo4j.kernel.api.KernelTransaction; import org.neo4j.kernel.api.Statement; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; import org.neo4j.kernel.api.exceptions.KernelException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; import org.neo4j.kernel.api.exceptions.TransactionFailureException; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.index.IndexConfiguration; @@ -102,6 +105,46 @@ public class IndexPopulationJobTest { + + @Rule + public final CleanupRule cleanup = new CleanupRule(); + + private GraphDatabaseAPI db; + + private final Label FIRST = Label.label( "FIRST" ); + private final Label SECOND = Label.label( "SECOND" ); + private final String name = "name"; + private final String age = "age"; + + private KernelAPI kernel; + private IndexStoreView indexStoreView; + private KernelSchemaStateStore stateHolder; + + private int labelId; + + @Before + public void before() throws Exception + { + db = (GraphDatabaseAPI) new TestGraphDatabaseFactory().newImpermanentDatabase(); + kernel = db.getDependencyResolver().resolveDependency( KernelAPI.class ); + stateHolder = new KernelSchemaStateStore( NullLogProvider.getInstance() ); + indexStoreView = indexStoreView(); + + try ( KernelTransaction tx = kernel.newTransaction( KernelTransaction.Type.implicit, AccessMode.Static.FULL ); + Statement statement = tx.acquireStatement() ) + { + labelId = statement.schemaWriteOperations().labelGetOrCreateForName( FIRST.name() ); + statement.schemaWriteOperations().labelGetOrCreateForName( SECOND.name() ); + tx.success(); + } + } + + @After + public void after() throws Exception + { + db.shutdown(); + } + @Test public void shouldPopulateIndexWithOneNode() throws Exception { @@ -428,11 +471,31 @@ public void stop() latch.finish(); } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + // no-op + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + // no-op + } + @Override public PopulationProgress getProgress() { return new PopulationProgress( 42, 100 ); } + + @Override + public void configure( List populations ) + { + // no-op + } } private class NodeChangingWriter extends IndexPopulator.Adapter @@ -587,43 +650,6 @@ private IndexPopulator inMemoryPopulator( boolean constraint ) throws Transactio return new InMemoryIndexProvider().getPopulator( 21, descriptor, indexConfig, samplingConfig ); } - private GraphDatabaseAPI db; - - private final Label FIRST = Label.label( "FIRST" ); - private final Label SECOND = Label.label( "SECOND" ); - private final String name = "name"; - private final String age = "age"; - - private KernelAPI kernel; - private IndexStoreView indexStoreView; - private KernelSchemaStateStore stateHolder; - - private int labelId; - public final @Rule CleanupRule cleanup = new CleanupRule(); - - @Before - public void before() throws Exception - { - db = (GraphDatabaseAPI) new TestGraphDatabaseFactory().newImpermanentDatabase(); - kernel = db.getDependencyResolver().resolveDependency( KernelAPI.class ); - stateHolder = new KernelSchemaStateStore( NullLogProvider.getInstance() ); - indexStoreView = indexStoreView(); - - try ( KernelTransaction tx = kernel.newTransaction( KernelTransaction.Type.implicit, AccessMode.Static.FULL ); - Statement statement = tx.acquireStatement() ) - { - labelId = statement.schemaWriteOperations().labelGetOrCreateForName( FIRST.name() ); - statement.schemaWriteOperations().labelGetOrCreateForName( SECOND.name() ); - tx.success(); - } - } - - @After - public void after() throws Exception - { - db.shutdown(); - } - private IndexPopulationJob newIndexPopulationJob( Label label, String propertyKey, IndexPopulator populator, FlippableIndexProxy flipper, boolean constraint ) throws TransactionFailureException diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java index ce52d6b03049..00f043a4c1aa 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/IndexingServiceTest.java @@ -56,6 +56,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.api.TokenNameLookup; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; import org.neo4j.kernel.api.exceptions.index.IndexNotFoundKernelException; import org.neo4j.kernel.api.exceptions.index.IndexPopulationFailedKernelException; @@ -241,7 +243,6 @@ public void shouldDeliverUpdatesThatOccurDuringPopulationToPopulator() throws Ex waitForIndexesToComeOnline( indexingService, 0 ); verify( populator, timeout( 1000 ) ).close( true ); - verify( populator ).configureSampling( false ); // then assertEquals( InternalIndexState.ONLINE, proxy.getState() ); @@ -1080,11 +1081,33 @@ public void stop() throw new UnsupportedOperationException(); } + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, + IndexEntryConflictException + { + // no-op + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, + NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + // no-op + } + @Override public PopulationProgress getProgress() { return new PopulationProgress( 42, 100 ); } + + @Override + public void configure( List populations ) + { + // no-op + } }; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorTest.java index 17b01c2cf39a..ef60cdf307fc 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorTest.java @@ -19,6 +19,7 @@ */ package org.neo4j.kernel.impl.api.index; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Answers; @@ -64,11 +65,20 @@ public class MultipleIndexPopulatorTest @Mock private IndexStoreView indexStoreView; + @Mock + private StoreScan storeScan; @Mock( answer = Answers.RETURNS_MOCKS ) private LogProvider logProvider; @InjectMocks private MultipleIndexPopulator multipleIndexPopulator; + @Before + public void setUp() + { + when( indexStoreView.visitNodes( any( int[].class ), any( IntPredicate.class ), any(Visitor.class), + any(Visitor.class) )).thenReturn( storeScan ); + } + @Test public void testMultiplePopulatorsCreation() throws Exception { @@ -300,6 +310,7 @@ public void testIndexFlip() throws IOException when( indexPopulator1.sampleResult() ).thenThrow( getSampleError() ); + multipleIndexPopulator.indexAllNodes(); multipleIndexPopulator.flipAfterPopulation(); verify( indexPopulator1 ).close( false ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorUpdatesTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorUpdatesTest.java index 5a2b12e38edf..87c8fe497178 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorUpdatesTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/index/MultipleIndexPopulatorUpdatesTest.java @@ -41,7 +41,6 @@ import org.neo4j.kernel.api.index.IndexUpdater; import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.index.SchemaIndexProvider; -import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.api.labelscan.NodeLabelUpdate; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.InlineNodeLabels; @@ -53,8 +52,6 @@ import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.RecordLoad; -import org.neo4j.kernel.impl.transaction.state.storeview.AdaptableIndexStoreView; -import org.neo4j.kernel.impl.transaction.state.storeview.LabelScanViewNodeStoreScan; import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView; import org.neo4j.kernel.impl.transaction.state.storeview.StoreViewNodeStoreScan; import org.neo4j.kernel.impl.util.Listener; @@ -205,36 +202,6 @@ public void receive( NodeRecord nodeRecord ) } } - private class ProcessListenableAdaptableIndexView extends AdaptableIndexStoreView - { - - private LabelScanStore labelScanStore; - private Listener processListener; - - public ProcessListenableAdaptableIndexView( LabelScanStore labelScanStore, LockService locks, - NeoStores neoStores ) - { - super( labelScanStore, locks, neoStores ); - this.labelScanStore = labelScanStore; - } - - @Override - public StoreScan visitNodes( int[] labelIds, - IntPredicate propertyKeyIdFilter, - Visitor propertyUpdatesVisitor, - Visitor labelUpdateVisitor ) - { - - return new ListenableLabelScanViewNodeStoreScan<>( nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, - propertyUpdatesVisitor, labelIds, propertyKeyIdFilter, processListener ); - } - - public void setProcessListener( Listener processListener ) - { - this.processListener = processListener; - } - } - private class ProcessListenableNeoStoreIndexView extends NeoStoreIndexStoreView { @@ -262,31 +229,6 @@ public void setProcessListener( Listener processListener ) } } - private class ListenableLabelScanViewNodeStoreScan extends LabelScanViewNodeStoreScan - { - - private Listener processListener; - - ListenableLabelScanViewNodeStoreScan( NodeStore nodeStore, LockService locks, - PropertyStore propertyStore, LabelScanStore labelScanStore, - Visitor labelUpdateVisitor, - Visitor propertyUpdatesVisitor, int[] labelIds, - IntPredicate propertyKeyIdFilter, Listener processListener ) - { - super( nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, propertyUpdatesVisitor, - labelIds, propertyKeyIdFilter ); - this.processListener = processListener; - } - - - @Override - public void process( NodeRecord nodeRecord) throws FAILURE - { - processListener.receive( nodeRecord ); - super.process( nodeRecord ); - } - } - private class ListenableNodeScanViewNodeStoreScan extends StoreViewNodeStoreScan { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeStoreScanTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeStoreScanTest.java index 674d66d4f1aa..21e7d569c8c9 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeStoreScanTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeStoreScanTest.java @@ -21,8 +21,17 @@ import org.junit.Test; +import java.io.IOException; +import java.util.List; import java.util.function.Supplier; +import org.neo4j.kernel.api.exceptions.EntityNotFoundException; +import org.neo4j.kernel.api.exceptions.PropertyNotFoundException; +import org.neo4j.kernel.api.exceptions.index.IndexEntryConflictException; +import org.neo4j.kernel.api.index.IndexDescriptor; +import org.neo4j.kernel.api.index.IndexPopulator; +import org.neo4j.kernel.api.index.NodePropertyUpdate; +import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.StoreScan; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NodeStore; @@ -62,6 +71,26 @@ public void shouldGiveBackCompletionPercentage() throws Throwable { private int read = 0; + @Override + public void complete( IndexPopulator indexPopulator, IndexDescriptor descriptor ) + throws EntityNotFoundException, PropertyNotFoundException, IOException, IndexEntryConflictException + { + // no-op + } + + @Override + public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, + long currentlyIndexedNodeId ) + { + // no-op + } + + @Override + public void configure( List list ) + { + // no-op + } + @Override public void process( NodeRecord node ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScanTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScanTest.java index c07fc3a867d5..414fdbeac4c7 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScanTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/storeview/LabelScanViewNodeStoreScanTest.java @@ -43,6 +43,7 @@ public class LabelScanViewNodeStoreScanTest { + private NeoStoreIndexStoreView storeView = mock( NeoStoreIndexStoreView.class ); private NodeStore nodeStore = mock( NodeStore.class ); private PropertyStore propertyStore = mock( PropertyStore.class ); private LabelScanStore labelScanStore = mock( LabelScanStore.class ); @@ -56,6 +57,7 @@ public void iterateOverLabeledNodeIds() throws Exception { PrimitiveLongIterator labeledNodes = PrimitiveLongCollections.iterator( 1, 2, 4, 8 ); + when( labelScanStore.newReader() ).thenReturn( labelScanReader ); when( nodeStore.getHighId() ).thenReturn( 15L ); when( labelScanReader.nodesWithAnyOfLabels( 1, 2 ) ).thenReturn( labeledNodes ); @@ -72,8 +74,8 @@ public void iterateOverLabeledNodeIds() throws Exception private LabelScanViewNodeStoreScan getLabelScanViewStoreScan( int[] labelIds ) { - return new LabelScanViewNodeStoreScan<>( nodeStore, LockService.NO_LOCK_SERVICE, propertyStore, labelScanStore, - labelUpdateVisitor, propertyUpdateVisitor, labelIds, propertyKeyIdFilter ); + return new LabelScanViewNodeStoreScan<>( storeView, nodeStore, LockService.NO_LOCK_SERVICE, propertyStore, + labelScanStore, labelUpdateVisitor, propertyUpdateVisitor, labelIds, propertyKeyIdFilter ); } -} \ No newline at end of file +} diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java index d4333154f32a..939772373d8e 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/DirectNonUniqueIndexSampler.java @@ -28,6 +28,10 @@ import org.neo4j.storageengine.api.schema.IndexSample; import org.neo4j.storageengine.api.schema.IndexSampler; +/** + * Non unique index sampler that ignores all include/exclude calls and builds + * sample based on values obtained directly from targeted index. + */ public class DirectNonUniqueIndexSampler implements NonUniqueIndexSampler { @@ -41,25 +45,25 @@ public DirectNonUniqueIndexSampler( SchemaIndex luceneIndex ) @Override public void include( String value ) { - + // no-op } @Override public void include( String value, long increment ) { - + // no-op } @Override public void exclude( String value ) { - + // no-op } @Override public void exclude( String value, long decrement ) { - + // no-op } @Override diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java index 0df76c95e9db..18d4aafd026a 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/populator/NonUniqueLuceneIndexPopulator.java @@ -56,11 +56,6 @@ public void configureSampling( boolean fullIndexSampling ) : new DirectNonUniqueIndexSampler( luceneIndex ); } - private DefaultNonUniqueIndexSampler createDefaultSampler() - { - return new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() ); - } - @Override public void verifyDeferredConstraints( PropertyAccessor accessor ) throws IndexEntryConflictException, IOException { @@ -77,25 +72,30 @@ public IndexUpdater newPopulatingUpdater( PropertyAccessor propertyAccessor ) th @Override public void includeSample( NodePropertyUpdate update ) { - if (updateSampling) + if ( updateSampling ) { checkSampler(); sampler.include( LuceneDocumentStructure.encodedStringValue( update.getValueAfter() ) ); } } - private void checkSampler() - { - if (sampler == null) - { - sampler = createDefaultSampler(); - } - } - @Override public IndexSample sampleResult() { checkSampler(); return sampler.result(); } + + private DefaultNonUniqueIndexSampler createDefaultSampler() + { + return new DefaultNonUniqueIndexSampler( samplingConfig.sampleSizeLimit() ); + } + + private void checkSampler() + { + if (sampler == null) + { + throw new IllegalStateException( "Please configure populator sampler before using it." ); + } + } } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexPopulatorTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexPopulatorTest.java index 6963e2c15aa3..00d723c14db2 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexPopulatorTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneSchemaIndexPopulatorTest.java @@ -70,7 +70,7 @@ public class LuceneSchemaIndexPopulatorTest private IndexStoreView indexStoreView; private LuceneSchemaIndexProvider provider; private Directory directory; - private IndexPopulator index; + private IndexPopulator indexPopulator; private IndexReader reader; private IndexSearcher searcher; private final long indexId = 0; @@ -88,8 +88,9 @@ public void before() throws Exception indexStoreView = mock( IndexStoreView.class ); IndexConfiguration indexConfig = IndexConfiguration.NON_UNIQUE; IndexSamplingConfig samplingConfig = new IndexSamplingConfig( Config.empty() ); - index = provider.getPopulator( indexId, indexDescriptor, indexConfig, samplingConfig ); - index.create(); + indexPopulator = provider.getPopulator( indexId, indexDescriptor, indexConfig, samplingConfig ); + indexPopulator.create(); + indexPopulator.configureSampling( true ); } @After @@ -106,14 +107,14 @@ public void after() throws Exception public void addingValuesShouldPersistThem() throws Exception { // WHEN - addUpdate( index, 1, "First" ); - addUpdate( index, 2, "Second" ); - addUpdate( index, 3, (byte) 1 ); - addUpdate( index, 4, (short) 2 ); - addUpdate( index, 5, 3 ); - addUpdate( index, 6, 4L ); - addUpdate( index, 7, 5F ); - addUpdate( index, 8, 6D ); + addUpdate( indexPopulator, 1, "First" ); + addUpdate( indexPopulator, 2, "Second" ); + addUpdate( indexPopulator, 3, (byte) 1 ); + addUpdate( indexPopulator, 4, (short) 2 ); + addUpdate( indexPopulator, 5, 3 ); + addUpdate( indexPopulator, 6, 4L ); + addUpdate( indexPopulator, 7, 5F ); + addUpdate( indexPopulator, 8, 6D ); // THEN assertIndexedValues( @@ -131,9 +132,9 @@ public void addingValuesShouldPersistThem() throws Exception public void multipleEqualValues() throws Exception { // WHEN - addUpdate( index, 1, "value" ); - addUpdate( index, 2, "value" ); - addUpdate( index, 3, "value" ); + addUpdate( indexPopulator, 1, "value" ); + addUpdate( indexPopulator, 2, "value" ); + addUpdate( indexPopulator, 3, "value" ); // THEN assertIndexedValues( @@ -144,10 +145,10 @@ public void multipleEqualValues() throws Exception public void multipleEqualValuesWithUpdateThatRemovesOne() throws Exception { // WHEN - addUpdate( index, 1, "value" ); - addUpdate( index, 2, "value" ); - addUpdate( index, 3, "value" ); - updatePopulator( index, asList( remove( 2, "value" ) ), indexStoreView ); + addUpdate( indexPopulator, 1, "value" ); + addUpdate( indexPopulator, 2, "value" ); + addUpdate( indexPopulator, 3, "value" ); + updatePopulator( indexPopulator, asList( remove( 2, "value" ) ), indexStoreView ); // THEN assertIndexedValues( @@ -158,10 +159,10 @@ public void multipleEqualValuesWithUpdateThatRemovesOne() throws Exception public void changeUpdatesInterleavedWithAdds() throws Exception { // WHEN - addUpdate( index, 1, "1" ); - addUpdate( index, 2, "2" ); - updatePopulator( index, asList( change( 1, "1", "1a" ) ), indexStoreView ); - addUpdate( index, 3, "3" ); + addUpdate( indexPopulator, 1, "1" ); + addUpdate( indexPopulator, 2, "2" ); + updatePopulator( indexPopulator, asList( change( 1, "1", "1a" ) ), indexStoreView ); + addUpdate( indexPopulator, 3, "3" ); // THEN assertIndexedValues( @@ -175,10 +176,10 @@ public void changeUpdatesInterleavedWithAdds() throws Exception public void addUpdatesInterleavedWithAdds() throws Exception { // WHEN - addUpdate( index, 1, "1" ); - addUpdate( index, 2, "2" ); - updatePopulator( index, asList( remove( 1, "1" ), add( 1, "1a" ) ), indexStoreView ); - addUpdate( index, 3, "3" ); + addUpdate( indexPopulator, 1, "1" ); + addUpdate( indexPopulator, 2, "2" ); + updatePopulator( indexPopulator, asList( remove( 1, "1" ), add( 1, "1a" ) ), indexStoreView ); + addUpdate( indexPopulator, 3, "3" ); // THEN assertIndexedValues( @@ -192,10 +193,10 @@ public void addUpdatesInterleavedWithAdds() throws Exception public void removeUpdatesInterleavedWithAdds() throws Exception { // WHEN - addUpdate( index, 1, "1" ); - addUpdate( index, 2, "2" ); - updatePopulator( index, asList( remove( 2, "2" ) ), indexStoreView ); - addUpdate( index, 3, "3" ); + addUpdate( indexPopulator, 1, "1" ); + addUpdate( indexPopulator, 2, "2" ); + updatePopulator( indexPopulator, asList( remove( 2, "2" ) ), indexStoreView ); + addUpdate( indexPopulator, 3, "3" ); // THEN assertIndexedValues( @@ -208,12 +209,12 @@ public void removeUpdatesInterleavedWithAdds() throws Exception public void multipleInterleaves() throws Exception { // WHEN - addUpdate( index, 1, "1" ); - addUpdate( index, 2, "2" ); - updatePopulator( index, asList( change( 1, "1", "1a" ), change( 2, "2", "2a" ) ), indexStoreView ); - addUpdate( index, 3, "3" ); - addUpdate( index, 4, "4" ); - updatePopulator( index, asList( change( 1, "1a", "1b" ), change( 4, "4", "4a" ) ), indexStoreView ); + addUpdate( indexPopulator, 1, "1" ); + addUpdate( indexPopulator, 2, "2" ); + updatePopulator( indexPopulator, asList( change( 1, "1", "1a" ), change( 2, "2", "2a" ) ), indexStoreView ); + addUpdate( indexPopulator, 3, "3" ); + addUpdate( indexPopulator, 4, "4" ); + updatePopulator( indexPopulator, asList( change( 1, "1a", "1b" ), change( 4, "4", "4a" ) ), indexStoreView ); // THEN assertIndexedValues( @@ -289,7 +290,7 @@ private void assertIndexedValues( Hit... expectedHits ) throws IOException private void switchToVerification() throws IOException { - index.close( true ); + indexPopulator.close( true ); assertEquals( InternalIndexState.ONLINE, provider.getInitialState( indexId ) ); reader = DirectoryReader.open( directory ); searcher = new IndexSearcher( reader ); diff --git a/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java b/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java index 8c3837cfca22..a3f7661e1904 100644 --- a/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java +++ b/community/neo4j/src/test/java/schema/MultiIndexPopulationConcurrentUpdatesIT.java @@ -51,7 +51,6 @@ import org.neo4j.kernel.impl.api.index.IndexProxy; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.api.index.IndexingServiceFactory; -import org.neo4j.kernel.impl.api.index.MultipleIndexPopulator; import org.neo4j.kernel.impl.api.index.NodePropertyUpdates; import org.neo4j.kernel.impl.api.index.SchemaIndexProviderMap; import org.neo4j.kernel.impl.api.index.StoreScan; @@ -68,6 +67,7 @@ import org.neo4j.kernel.impl.transaction.state.DefaultSchemaIndexProviderMap; import org.neo4j.kernel.impl.transaction.state.storeview.AdaptableIndexStoreView; import org.neo4j.kernel.impl.transaction.state.storeview.LabelScanViewNodeStoreScan; +import org.neo4j.kernel.impl.transaction.state.storeview.NeoStoreIndexStoreView; import org.neo4j.kernel.impl.util.JobScheduler; import org.neo4j.logging.NullLogProvider; import org.neo4j.storageengine.api.schema.IndexReader; @@ -382,17 +382,11 @@ public StoreScan visitNodes( int[] labelIds { StoreScan failureStoreScan = super.visitNodes( labelIds, propertyKeyIdFilter, propertyUpdatesVisitor, labelUpdateVisitor ); - return new LabelScanViewNodeStoreWrapper( nodeStore, locks, propertyStore, getLabelScanStore(), + return new LabelScanViewNodeStoreWrapper( this, nodeStore, locks, propertyStore, getLabelScanStore(), element -> false, propertyUpdatesVisitor, labelIds, propertyKeyIdFilter, (LabelScanViewNodeStoreScan) failureStoreScan, this, updates); } - @Override - public void acceptUpdate( MultipleIndexPopulator.MultipleIndexUpdater updater, NodePropertyUpdate update, - long currentlyIndexedNodeId ) - { - super.acceptUpdate( updater, update, currentlyIndexedNodeId ); - } } private class LabelScanViewNodeStoreWrapper extends LabelScanViewNodeStoreScan @@ -401,7 +395,7 @@ private class LabelScanViewNodeStoreWrapper extends LabelScanViewNodeStoreScan private AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper; private List updates; - public LabelScanViewNodeStoreWrapper( NodeStore nodeStore, LockService locks, + public LabelScanViewNodeStoreWrapper( NeoStoreIndexStoreView storeView, NodeStore nodeStore, LockService locks, PropertyStore propertyStore, LabelScanStore labelScanStore, Visitor labelUpdateVisitor, Visitor propertyUpdatesVisitor, int[] labelIds, IntPredicate propertyKeyIdFilter, @@ -409,9 +403,8 @@ public LabelScanViewNodeStoreWrapper( NodeStore nodeStore, LockService locks, AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper, List updates ) { - super( nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, propertyUpdatesVisitor, - labelIds, - propertyKeyIdFilter ); + super( storeView, nodeStore, locks, propertyStore, labelScanStore, labelUpdateVisitor, + propertyUpdatesVisitor, labelIds, propertyKeyIdFilter ); this.delegate = delegate; this.adaptableIndexStoreViewWrapper = adaptableIndexStoreViewWrapper; this.updates = updates; @@ -427,7 +420,7 @@ public void process( NodeRecord loaded ) throws Exception public PrimitiveLongResourceIterator getNodeIdIterator() { PrimitiveLongResourceIterator originalIterator = delegate.getNodeIdIterator(); - return new DelegatingPrimitiveLongResourceIterator( originalIterator, adaptableIndexStoreViewWrapper, + return new DelegatingPrimitiveLongResourceIterator( this, originalIterator, adaptableIndexStoreViewWrapper, updates ); } } @@ -437,12 +430,16 @@ private class DelegatingPrimitiveLongResourceIterator implements PrimitiveLongRe private AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper; private List updates; + private LabelScanViewNodeStoreWrapper storeScan; private PrimitiveLongResourceIterator delegate; - DelegatingPrimitiveLongResourceIterator( PrimitiveLongResourceIterator delegate, + DelegatingPrimitiveLongResourceIterator( + LabelScanViewNodeStoreWrapper storeScan, + PrimitiveLongResourceIterator delegate, AdaptableIndexStoreViewWrapper adaptableIndexStoreViewWrapper, List updates ) { + this.storeScan = storeScan; this.delegate = delegate; this.adaptableIndexStoreViewWrapper = adaptableIndexStoreViewWrapper; this.updates = updates; @@ -462,7 +459,7 @@ public long next() { for ( NodePropertyUpdate update : updates ) { - adaptableIndexStoreViewWrapper.acceptUpdate( null, update, Long.MAX_VALUE ); + storeScan.acceptUpdate( null, update, Long.MAX_VALUE ); } for ( NodePropertyUpdate update : updates ) diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java index 9c2f86d1e8bf..5def1415044d 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java @@ -480,7 +480,7 @@ public void includeSample( NodePropertyUpdate update ) @Override public void configureSampling( boolean fullIndexSampling ) { - //nothing + delegate.configureSampling( fullIndexSampling ); } @Override