From aad15dafd43c8349da4c5cf490d6a2f4990599a7 Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Tue, 9 May 2017 15:20:01 +0200 Subject: [PATCH] Small comment fixes in NeoStoreRule and some importer steps --- .../checking/full/StoreProcessorTest.java | 5 +- .../unsafe/impl/batchimport/HighestId.java | 2 +- .../batchimport/ParallelBatchImporter.java | 2 +- .../impl/batchimport/PropertyEncoderStep.java | 2 - .../batchimport/RelationshipGroupStage.java | 5 ++ ...SparseNodeFirstRelationshipProcessor.java} | 14 ++- ... => SparseNodeFirstRelationshipStage.java} | 14 ++- .../batchimport/staging/AbstractStep.java | 6 -- .../unsafe/impl/batchimport/staging/Step.java | 2 - .../command/HighIdTransactionApplierTest.java | 4 +- .../state/TransactionRecordStateTest.java | 74 +++++++++------- .../org/neo4j/test/rule/NeoStoresRule.java | 88 ++++++++++--------- .../batchimport/staging/ControlledStep.java | 6 -- .../staging/ProcessorStepTest.java | 1 - 14 files changed, 114 insertions(+), 111 deletions(-) rename community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/{NodeFirstRelationshipProcessor.java => SparseNodeFirstRelationshipProcessor.java} (70%) rename community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/{NodeFirstRelationshipStage.java => SparseNodeFirstRelationshipStage.java} (74%) diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java index d8fbc8792793..7b42f0461332 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java @@ -47,7 +47,7 @@ public class StoreProcessorTest public void shouldProcessAllTheRecordsInAStore() throws Exception { // given - RecordStore nodeStore = stores.open().getNodeStore(); + RecordStore nodeStore = stores.builder().build().getNodeStore(); ConsistencyReport.Reporter reporter = mock( ConsistencyReport.Reporter.class ); StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, reporter, Stage.SEQUENTIAL_FORWARD, CacheAccess.EMPTY ); @@ -76,7 +76,8 @@ public void shouldStopProcessingRecordsWhenSignalledToStop() throws Exception ConsistencyReport.Reporter reporter = mock( ConsistencyReport.Reporter.class ); StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, reporter, Stage.SEQUENTIAL_FORWARD, CacheAccess.EMPTY ); - RecordStore nodeStore = new RecordStore.Delegator( stores.open().getNodeStore() ) + RecordStore nodeStore = new RecordStore.Delegator( + stores.builder().build().getNodeStore() ) { @Override public RecordCursor newRecordCursor( NodeRecord record ) diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/HighestId.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/HighestId.java index aadac970f2ac..c5d7927bb673 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/HighestId.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/HighestId.java @@ -34,7 +34,7 @@ public void offer( long candidate ) do { currentHighest = highestId.get(); - if ( candidate < currentHighest ) + if ( candidate <= currentHighest ) { return; } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java index 2c278da9c123..c283d4ed4982 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporter.java @@ -343,7 +343,7 @@ private void importRelationships( NodeRelationshipCache nodeRelationshipCache, executeStage( new RelationshipGroupStage( topic, groupConfig, neoStore.getTemporaryRelationshipGroupStore(), nodeRelationshipCache ) ); // Set node nextRel fields - executeStage( new NodeFirstRelationshipStage( topic, nodeConfig, neoStore.getNodeStore(), + executeStage( new SparseNodeFirstRelationshipStage( topic, nodeConfig, neoStore.getNodeStore(), nodeRelationshipCache ) ); // Link relationship chains together for dense nodes diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/PropertyEncoderStep.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/PropertyEncoderStep.java index 0590d8adf024..2e303ed3730e 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/PropertyEncoderStep.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/PropertyEncoderStep.java @@ -48,8 +48,6 @@ * Encodes property data into {@link PropertyRecord property records}, attaching them to each * {@link Batch}. This step is designed to handle multiple threads doing the property encoding, * since property encoding is potentially the most costly step in this {@link Stage}. - * The delivered {@link PropertyRecord property records} all have local ids and so reassignment of those - * ids will have to be done later. */ public class PropertyEncoderStep extends ProcessorStep> diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupStage.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupStage.java index bce7ae38a7d4..27fe4060e79d 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupStage.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupStage.java @@ -20,10 +20,15 @@ package org.neo4j.unsafe.impl.batchimport; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache; import org.neo4j.unsafe.impl.batchimport.staging.Stage; +/** + * Takes information about relationship groups in the {@link NodeRelationshipCache}, which is produced + * as a side-effect of linking relationships together, and writes them out into {@link RelationshipGroupStore}. + */ public class RelationshipGroupStage extends Stage { public RelationshipGroupStage( String topic, Configuration config, diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipProcessor.java similarity index 70% rename from community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java rename to community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipProcessor.java index 12e23326e464..31d2cfdba8b0 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipProcessor.java @@ -20,22 +20,18 @@ package org.neo4j.unsafe.impl.batchimport; import org.neo4j.kernel.impl.store.record.NodeRecord; -import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache; /** - * Sets the {@link NodeRecord#setNextRel(long) relationship field} on all {@link NodeRecord nodes}. - * This is done after all relationships have been imported and the {@link NodeRelationshipCache node cache} - * points to the first relationship for each node. - * - * This step also creates {@link RelationshipGroupRecord group records} for the dense nodes as it encounters - * dense nodes, where it gets all relationship group information from {@link NodeRelationshipCache}. + * Sets the {@link NodeRecord#setNextRel(long) relationship field} on sparse nodes. + * This is done after all sparse node relationship links have been done and the {@link NodeRelationshipCache node cache} + * points to the first relationship for sparse each node. */ -public class NodeFirstRelationshipProcessor implements RecordProcessor +public class SparseNodeFirstRelationshipProcessor implements RecordProcessor { private final NodeRelationshipCache cache; - public NodeFirstRelationshipProcessor( NodeRelationshipCache cache ) + public SparseNodeFirstRelationshipProcessor( NodeRelationshipCache cache ) { this.cache = cache; } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipStage.java similarity index 74% rename from community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java rename to community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipStage.java index 380e5b535f0e..9b78e9c2eb2b 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/SparseNodeFirstRelationshipStage.java @@ -21,7 +21,6 @@ import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.record.NodeRecord; -import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache; import org.neo4j.unsafe.impl.batchimport.cache.NodeType; import org.neo4j.unsafe.impl.batchimport.staging.ReadRecordsStep; @@ -30,27 +29,26 @@ import static org.neo4j.unsafe.impl.batchimport.staging.Step.ORDER_SEND_DOWNSTREAM; /** - * Updates {@link NodeRecord node records} with relationship/group chain heads after relationship import. Steps: + * Updates sparse {@link NodeRecord node records} with relationship heads after relationship linking. Steps: * *
    *
  1. {@link ReadNodeIdsByCacheStep} looks at {@link NodeRelationshipCache} for which nodes have had * relationships imported and loads those {@link NodeRecord records} from store.
    2. - *
  2. {@link RecordProcessorStep} / {@link NodeFirstRelationshipProcessor} uses {@link NodeRelationshipCache} - * to update each {@link NodeRecord#setNextRel(long)}. For dense nodes {@link RelationshipGroupRecord group records} - * are created and set as {@link NodeRecord#setNextRel(long)}.
    3. + *
  3. {@link RecordProcessorStep} / {@link SparseNodeFirstRelationshipProcessor} uses {@link NodeRelationshipCache} + * to update each {@link NodeRecord#setNextRel(long)}. *
  4. {@link UpdateRecordsStep} writes the updated records back into store.
    5. *
*/ -public class NodeFirstRelationshipStage extends Stage +public class SparseNodeFirstRelationshipStage extends Stage { - public NodeFirstRelationshipStage( String topic, Configuration config, NodeStore nodeStore, + public SparseNodeFirstRelationshipStage( String topic, Configuration config, NodeStore nodeStore, NodeRelationshipCache cache ) { super( "Node --> Relationship" + topic, config, ORDER_SEND_DOWNSTREAM ); add( new ReadNodeIdsByCacheStep( control(), config, cache, NodeType.NODE_TYPE_SPARSE ) ); add( new ReadRecordsStep<>( control(), config, true, nodeStore ) ); add( new RecordProcessorStep<>( control(), "LINK", config, - new NodeFirstRelationshipProcessor( cache ), false ) ); + new SparseNodeFirstRelationshipProcessor( cache ), false ) ); add( new UpdateRecordsStep<>( control(), config, nodeStore ) ); } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/AbstractStep.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/AbstractStep.java index 026d96fb4b88..cb96ca39dd4d 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/AbstractStep.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/AbstractStep.java @@ -245,10 +245,4 @@ public String toString() return format( "%s[%s, processors:%d, batches:%d", getClass().getSimpleName(), name, processors( 0 ), doneBatches.get() ); } - - @Override - public long doneBatches() - { - return doneBatches.get(); - } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Step.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Step.java index 98f5d1aacabe..f8dd423e5bd1 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Step.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/Step.java @@ -93,6 +93,4 @@ public interface Step extends Parallelizable, AutoCloseable, Panicable */ @Override void close() throws Exception; - - long doneBatches(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplierTest.java index 0eeaa00bf530..9b83e7fa5473 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplierTest.java @@ -45,7 +45,7 @@ public class HighIdTransactionApplierTest public void shouldUpdateHighIdsOnExternalTransaction() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); HighIdTransactionApplier tracker = new HighIdTransactionApplier( neoStores ); // WHEN @@ -108,7 +108,7 @@ public void shouldUpdateHighIdsOnExternalTransaction() throws Exception public void shouldTrackSecondaryUnitIdsAsWell() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); HighIdTransactionApplier tracker = new HighIdTransactionApplier( neoStores ); NodeRecord node = new NodeRecord( 5 ).initialize( true, 123, true, 456, 0 ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java index 299c23c17d41..7167f5709ff0 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java @@ -119,9 +119,9 @@ public class TransactionRecordStateTest private static final String value1 = "first"; private static final int value2 = 4; private static final long[] noLabels = new long[0]; - private long[] oneLabelId = new long[]{3}; - private long[] secondLabelId = new long[]{4}; - private long[] bothLabelIds = new long[]{3, 4}; + private final long[] oneLabelId = new long[]{3}; + private final long[] secondLabelId = new long[]{4}; + private final long[] bothLabelIds = new long[]{3, 4}; public static void assertRelationshipGroupDoesNotExist( RecordChangeSet recordChangeSet, NodeRecord node, int type ) @@ -203,7 +203,7 @@ public void shouldCreateEqualNodePropertyUpdatesOnRecoveryOfCreatedNode() throws * was created resulted in two exact copies of NodePropertyUpdates. */ // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; int labelId = 5, propertyKeyId = 7; @@ -254,7 +254,7 @@ public void shouldWriteProperPropertyRecordsWhenOnlyChangingLinkage() throws Exc */ // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); int nodeId = 0; recordState.nodeCreate( nodeId ); @@ -297,7 +297,7 @@ private void verifyPropertyRecord( PropertyRecord record ) public void shouldConvertLabelAdditionToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); Object value1 = LONG_STRING, value2 = LONG_STRING.getBytes(); @@ -320,7 +320,7 @@ public void shouldConvertLabelAdditionToNodePropertyUpdates() throws Exception public void shouldConvertMixedLabelAdditionAndSetPropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -346,7 +346,7 @@ public void shouldConvertMixedLabelAdditionAndSetPropertyToNodePropertyUpdates() public void shouldConvertLabelRemovalToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -369,7 +369,7 @@ public void shouldConvertLabelRemovalToNodePropertyUpdates() throws Exception public void shouldConvertMixedLabelRemovalAndRemovePropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -395,7 +395,7 @@ public void shouldConvertMixedLabelRemovalAndRemovePropertyToNodePropertyUpdates public void shouldConvertMixedLabelRemovalAndAddPropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -421,7 +421,7 @@ public void shouldConvertMixedLabelRemovalAndAddPropertyToNodePropertyUpdates() public void shouldConvertChangedPropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); int nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -449,7 +449,7 @@ public void shouldConvertChangedPropertyToNodePropertyUpdates() throws Exception public void shouldConvertRemovedPropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); int nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); recordState.nodeCreate( nodeId ); @@ -477,7 +477,7 @@ public void shouldConvertRemovedPropertyToNodePropertyUpdates() throws Exception public void shouldDeleteDynamicLabelsForDeletedNode() throws Throwable { // GIVEN a store that has got a node with a dynamic label record - NeoStores store = neoStoresRule.open(); + NeoStores store = neoStoresRule.builder().build(); BatchTransactionApplier applier = new NeoStoreBatchTransactionApplier( store, mock( CacheAccessBackDoor.class ), LockService.NO_LOCK_SERVICE ); AtomicLong nodeId = new AtomicLong(); @@ -496,7 +496,7 @@ public void shouldDeleteDynamicLabelsForDeletedNode() throws Throwable public void shouldDeleteDynamicLabelsForDeletedNodeForRecoveredTransaction() throws Throwable { // GIVEN a store that has got a node with a dynamic label record - NeoStores store = neoStoresRule.open(); + NeoStores store = neoStoresRule.builder().build(); BatchTransactionApplier applier = new NeoStoreBatchTransactionApplier( store, mock( CacheAccessBackDoor.class ), LockService.NO_LOCK_SERVICE ); AtomicLong nodeId = new AtomicLong(); @@ -520,7 +520,8 @@ public void shouldDeleteDynamicLabelsForDeletedNodeForRecoveredTransaction() thr public void shouldExtractCreatedCommandsInCorrectOrder() throws Throwable { // GIVEN - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); long nodeId = 0, relId = 1; recordState.nodeCreate( nodeId ); @@ -547,7 +548,8 @@ public void shouldExtractCreatedCommandsInCorrectOrder() throws Throwable public void shouldExtractUpdateCommandsInCorrectOrder() throws Throwable { // GIVEN - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); long nodeId = 0, relId1 = 1, relId2 = 2, relId3 = 3; recordState.nodeCreate( nodeId ); @@ -599,7 +601,8 @@ public void shouldIgnoreRelationshipGroupCommandsForGroupThatIsCreatedAndDeleted // Given: // - dense node threshold of 5 // - node with 4 rels of type A and 1 rel of type B - NeoStores neoStore = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "5" ); + NeoStores neoStore = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "5" ).build(); int A = 0, B = 1; TransactionRecordState state = newTransactionRecordState( neoStore ); state.nodeCreate( 0 ); @@ -626,7 +629,8 @@ public void shouldIgnoreRelationshipGroupCommandsForGroupThatIsCreatedAndDeleted public void shouldExtractDeleteCommandsInCorrectOrder() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); long nodeId1 = 0, nodeId2 = 1, relId1 = 1, relId2 = 2, relId4 = 10; recordState.nodeCreate( nodeId1 ); @@ -667,7 +671,7 @@ public void shouldExtractDeleteCommandsInCorrectOrder() throws Exception public void shouldValidateConstraintIndexAsPartOfExtraction() throws Throwable { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); final long indexId = neoStores.getSchemaStore().nextId(); @@ -686,7 +690,7 @@ public void shouldValidateConstraintIndexAsPartOfExtraction() throws Throwable public void shouldCreateProperBeforeAndAfterPropertyCommandsWhenAddingProperty() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); TransactionRecordState recordState = newTransactionRecordState( neoStores ); int nodeId = 1; @@ -714,7 +718,7 @@ public void shouldCreateProperBeforeAndAfterPropertyCommandsWhenAddingProperty() public void shouldConvertAddedPropertyToNodePropertyUpdates() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); long nodeId = 0; TransactionRecordState recordState = newTransactionRecordState( neoStores ); @@ -751,7 +755,7 @@ public synchronized Object answer( final InvocationOnMock invocation ) throws Th return null; } } ); - NeoStores neoStores = neoStoresRule.open(); + NeoStores neoStores = neoStoresRule.builder().build(); NodeStore nodeStore = neoStores.getNodeStore(); long[] nodes = { // allocate ids nodeStore.nextId(), nodeStore.nextId(), nodeStore.nextId(), nodeStore.nextId(), nodeStore.nextId(), @@ -810,7 +814,8 @@ public synchronized Object answer( final InvocationOnMock invocation ) throws Th public void movingBilaterallyOfTheDenseNodeThresholdIsConsistent() throws Exception { // GIVEN - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "10" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "10" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); long nodeId = neoStores.getNodeStore().nextId(); @@ -871,7 +876,8 @@ public boolean visitRelationshipGroupCommand( Command.RelationshipGroupCommand c public void shouldConvertToDenseNodeRepresentationWhenHittingThresholdWithDifferentTypes() throws Exception { // GIVEN a node with a total of denseNodeThreshold-1 relationships - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "50" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "50" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); long nodeId = neoStores.getNodeStore().nextId(); int typeA = 0, typeB = 1, typeC = 2; @@ -905,7 +911,8 @@ public void shouldConvertToDenseNodeRepresentationWhenHittingThresholdWithTheSam throws Exception { // GIVEN a node with a total of denseNodeThreshold-1 relationships - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "49" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "49" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); long nodeId = neoStores.getNodeStore().nextId(); int typeA = 0; @@ -930,7 +937,8 @@ public void shouldConvertToDenseNodeRepresentationWhenHittingThresholdWithTheSam throws Exception { // GIVEN a node with a total of denseNodeThreshold-1 relationships - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "8" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "8" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); long nodeId = neoStores.getNodeStore().nextId(); int typeA = 0; @@ -953,7 +961,8 @@ public void shouldConvertToDenseNodeRepresentationWhenHittingThresholdWithTheSam public void shouldMaintainCorrectDataWhenDeletingFromDenseNodeWithOneType() throws Exception { // GIVEN a node with a total of denseNodeThreshold-1 relationships - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "13" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "13" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); int nodeId = (int) neoStores.getNodeStore().nextId(), typeA = 0; tx.nodeCreate( nodeId ); @@ -971,7 +980,8 @@ public void shouldMaintainCorrectDataWhenDeletingFromDenseNodeWithOneType() thro public void shouldMaintainCorrectDataWhenDeletingFromDenseNodeWithManyTypes() throws Exception { // GIVEN a node with a total of denseNodeThreshold-1 relationships - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); TransactionRecordState tx = newTransactionRecordState( neoStores ); long nodeId = neoStores.getNodeStore().nextId(); int typeA = 0, typeB = 12, typeC = 600; @@ -1051,7 +1061,8 @@ public void shouldSortRelationshipGroups() throws Throwable { // GIVEN int type5 = 5, type10 = 10, type15 = 15; - NeoStores neoStores = neoStoresRule.open( GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); { TransactionRecordState recordState = newTransactionRecordState( neoStores ); neoStores.getRelationshipTypeTokenStore().setHighId( 16 ); @@ -1118,8 +1129,9 @@ public void shouldPrepareRelevantRecords() throws Exception { // GIVEN PrepareTrackingRecordFormats format = new PrepareTrackingRecordFormats( Standard.LATEST_RECORD_FORMATS ); - NeoStores neoStores = neoStoresRule.open( format, - GraphDatabaseSettings.dense_node_threshold.name(), "1" ); + NeoStores neoStores = neoStoresRule.builder() + .with( format ) + .with( GraphDatabaseSettings.dense_node_threshold.name(), "1" ).build(); // WHEN TransactionRecordState state = newTransactionRecordState( neoStores ); diff --git a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java index 07dbb0f9ac13..8e43d756ce52 100644 --- a/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java +++ b/community/kernel/src/test/java/org/neo4j/test/rule/NeoStoresRule.java @@ -46,15 +46,18 @@ /** * Rule for opening a {@link NeoStores}, either via {@link #open(String...)}, which just uses an in-memory - * file system, or via {@link #open(FileSystemAbstraction, PageCache, RecordFormats, String...)} which is suitable in an - * environment where you already have an fs and page cache available. + * file system, or via {@link #open(FileSystemAbstraction, PageCache, RecordFormats, Function, String...)} + * which is suitable in an environment where you already have an fs and page cache available. */ public class NeoStoresRule extends ExternalResource { private final Class testClass; private NeoStores neoStores; - private EphemeralFileSystemAbstraction efs; - private PageCache pageCache; + + // Custom components which are managed by this rule if user doesn't supply them + private EphemeralFileSystemAbstraction ruleFs; + private PageCache rulePageCache; + private final StoreType[] stores; public NeoStoresRule( Class testClass, StoreType... stores ) @@ -68,45 +71,14 @@ public Builder builder() return new Builder(); } - public NeoStores open( String... config ) throws IOException - { - Config configuration = Config.embeddedDefaults( stringMap( config ) ); - RecordFormats formats = RecordFormatSelector.selectForConfig( configuration, NullLogProvider.getInstance() ); - return open( formats, config ); - } - - public NeoStores open( RecordFormats format, String... config ) throws IOException - { - efs = new EphemeralFileSystemAbstraction(); - Config conf = Config.embeddedDefaults( stringMap( config ) ); - pageCache = getOrCreatePageCache( conf, efs ); - return open( efs, pageCache, format, fs -> new DefaultIdGeneratorFactory( fs ), config ); - } - - public NeoStores open( FileSystemAbstraction fs, PageCache pageCache, RecordFormats format, + private NeoStores open( FileSystemAbstraction fs, PageCache pageCache, RecordFormats format, Function idGeneratorFactory, String... config ) throws IOException { assert neoStores == null : "Already opened"; - if ( fs == null ) - { - fs = efs = new EphemeralFileSystemAbstraction(); - } TestDirectory testDirectory = TestDirectory.testDirectory( testClass, fs ); File storeDir = testDirectory.makeGraphDbDir(); - if ( config == null ) - { - config = new String[0]; - } - Config configuration = Config.embeddedDefaults( stringMap( config ) ); - if ( pageCache == null ) - { - pageCache = this.pageCache = getOrCreatePageCache( configuration, fs ); - } - if ( format == null ) - { - format = RecordFormatSelector.defaultFormat(); - } + Config configuration = configOf( config ); StoreFactory storeFactory = new StoreFactory( storeDir, configuration, idGeneratorFactory.apply( fs ), pageCache, fs, format, NullLogProvider.getInstance() ); return neoStores = stores.length == 0 @@ -114,13 +86,18 @@ public NeoStores open( FileSystemAbstraction fs, PageCache pageCache, RecordForm : storeFactory.openNeoStores( true, stores ); } + private static Config configOf( String... config ) + { + return Config.embeddedDefaults( stringMap( config ) ); + } + @Override protected void after( boolean successful ) throws Throwable { - IOUtils.closeAll( neoStores, pageCache ); - if ( efs != null ) + IOUtils.closeAll( neoStores, rulePageCache ); + if ( ruleFs!= null ) { - efs.close(); + ruleFs.close(); } } @@ -172,7 +149,38 @@ public Builder with( Function idGenera public NeoStores build() throws IOException { + if ( fs == null ) + { + fs = ruleFs(); + } + if ( config == null ) + { + config = new String[0]; + } + Config dbConfig = configOf( config ); + if ( pageCache == null ) + { + pageCache = rulePageCache( dbConfig, fs ); + } + if ( format == null ) + { + format = RecordFormatSelector.selectForConfig( dbConfig, NullLogProvider.getInstance() ); + } + if ( idGeneratorFactory == null ) + { + idGeneratorFactory = fs -> new DefaultIdGeneratorFactory( fs ); + } return open( fs, pageCache, format, idGeneratorFactory, config ); } } + + private PageCache rulePageCache( Config dbConfig, FileSystemAbstraction fs ) + { + return rulePageCache = getOrCreatePageCache( dbConfig, fs ); + } + + private EphemeralFileSystemAbstraction ruleFs() + { + return ruleFs = new EphemeralFileSystemAbstraction(); + } } diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ControlledStep.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ControlledStep.java index c668b4db7076..a23520898265 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ControlledStep.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ControlledStep.java @@ -172,12 +172,6 @@ public void complete() completed = true; } - @Override - public long doneBatches() - { - return stats.get( Keys.done_batches ).value; - } - private static class ControlledStat implements Stat { private final long value; diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ProcessorStepTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ProcessorStepTest.java index d40eb94af64b..aafb36e8f7cb 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ProcessorStepTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/staging/ProcessorStepTest.java @@ -115,7 +115,6 @@ private static class BlockingProcessorStep extends ProcessorStep @Override protected void process( Void batch, BatchSender sender ) throws Throwable { - System.out.println( Thread.currentThread().getName() + " process" ); latch.await(); } }