diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/AbstractStoreProcessor.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/AbstractStoreProcessor.java index d26a5366b4f8d..bdafc7771b4e0 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/AbstractStoreProcessor.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/AbstractStoreProcessor.java @@ -55,11 +55,6 @@ public abstract class AbstractStoreProcessor extends RecordStore.Processor labelTokenChecker; private final RecordCheck relationshipGroupChecker; - public AbstractStoreProcessor() - { - this( CheckDecorator.NONE ); - } - public AbstractStoreProcessor( CheckDecorator decorator ) { this.sparseNodeChecker = decorator.decorateNodeChecker( NodeRecordCheck.forSparseNodes() ); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java index 668b6cc4f362c..0f0de36392adf 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java @@ -76,11 +76,6 @@ public Stage getStage() return stage; } - public int getStageIndex() - { - return stage.ordinal(); - } - @Override public void processNode( RecordStore store, NodeRecord node ) { diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java index eaf503d6b2d31..20cc3bd7cebe2 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java @@ -26,10 +26,12 @@ import org.neo4j.consistency.checking.RecordCheck; import org.neo4j.consistency.checking.cache.CacheAccess; import org.neo4j.consistency.report.ConsistencyReport; -import org.neo4j.kernel.impl.store.RecordCursor; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.test.rule.NeoStoresRule; import static org.mockito.ArgumentMatchers.any; @@ -80,20 +82,13 @@ public void shouldStopProcessingRecordsWhenSignalledToStop() throws Exception stores.builder().build().getNodeStore() ) { @Override - public RecordCursor newRecordCursor( NodeRecord record ) + public void getRecordByCursor( long id, NodeRecord target, RecordLoad mode, PageCursor cursor ) throws InvalidRecordException { - return new RecordCursor.Delegator( super.newRecordCursor( record ) ) + if ( id == 3 ) { - @Override - public boolean next( long id ) - { - if ( id == 3 ) - { - processor.stop(); - } - return super.next( id ); - } - }; + processor.stop(); + } + super.getRecordByCursor( id, target, mode, cursor ); } }; nodeStore.updateRecord( node( 0, false, 0, 0 ) ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordNodeCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordNodeCursor.java index 6bca96d76ea7e..d4507592c1ded 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordNodeCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordNodeCursor.java @@ -23,8 +23,6 @@ import org.neo4j.kernel.impl.newapi.RelationshipReferenceEncoding; import org.neo4j.kernel.impl.store.NodeLabelsField; import org.neo4j.kernel.impl.store.NodeStore; -import org.neo4j.kernel.impl.store.RecordCursor; -import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.storageengine.api.StorageNodeCursor; @@ -32,7 +30,6 @@ public class RecordNodeCursor extends NodeRecord implements StorageNodeCursor { private NodeStore read; - private RecordCursor labelCursor; private PageCursor pageCursor; private long next; private long highMark; @@ -89,14 +86,14 @@ public long nodeReference() @Override public long[] labels() { - return NodeLabelsField.get( this, labelCursor() ); + return NodeLabelsField.get( this, read ); } @Override public boolean hasLabel( int label ) { //Get labels from store and put in intSet, unfortunately we get longs back - long[] longs = NodeLabelsField.get( this, labelCursor() ); + long[] longs = NodeLabelsField.get( this, read ); for ( long labelToken : longs ) { if ( labelToken == label ) @@ -204,15 +201,6 @@ private void reset() clear(); } - private RecordCursor labelCursor() - { - if ( labelCursor == null ) - { - labelCursor = read.newLabelCursor(); - } - return labelCursor; - } - private boolean isSingle() { return highMark == NO_ID; @@ -237,12 +225,6 @@ public String toString() @Override public void release() { - if ( labelCursor != null ) - { - labelCursor.close(); - labelCursor = null; - } - if ( pageCursor != null ) { pageCursor.close(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java index cd9f13019bde5..8e34005449352 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java @@ -186,7 +186,7 @@ public DynamicRecord nextRecord() return StandardDynamicRecordAllocator.allocateRecord( nextId() ); } - public void allocateRecordsFromBytes( Collection target, byte[] src ) + void allocateRecordsFromBytes( Collection target, byte[] src ) { allocateRecordsFromBytes( target, src, this ); } @@ -198,8 +198,7 @@ public String toString() ", blockSize:" + getRecordDataSize() + "]"; } - public Pair readFullByteArray( - Iterable records, PropertyType propertyType ) + Pair readFullByteArray( Iterable records, PropertyType propertyType ) { for ( DynamicRecord record : records ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java index 0c85d7aeb352f..01f9d90ce62e8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java @@ -24,6 +24,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.OpenOption; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.Collection; import org.neo4j.graphdb.config.Setting; @@ -415,6 +416,7 @@ public boolean isInUse( long id ) /** * DANGER: make sure to always close this cursor. */ + @Override public PageCursor openPageCursorForReading( long id ) { try @@ -1159,15 +1161,16 @@ public void prepareForCommit( RECORD record, IdSequence idSequence ) @Override public void scanAllRecords( Visitor visitor ) throws EXCEPTION { - try ( RecordCursor cursor = newRecordCursor( newRecord() ) ) + try ( PageCursor cursor = openPageCursorForReading( 0 ) ) { + RECORD record = newRecord(); long highId = getHighId(); - cursor.acquire( getNumberOfReservedLowIds(), CHECK ); for ( long id = getNumberOfReservedLowIds(); id < highId; id++ ) { - if ( cursor.next( id ) ) + getRecordByCursor( id, record, CHECK, cursor ); + if ( record.inUse() ) { - visitor.visit( cursor.get() ); + visitor.visit( record ); } } } @@ -1176,11 +1179,24 @@ public void scanAllRecords( Visitor getRecords( long firstId, RecordLoad mode ) { - try ( RecordCursor cursor = newRecordCursor( newRecord() ) ) + Collection records = new ArrayList<>(); + long id = firstId; + try ( PageCursor cursor = openPageCursorForReading( firstId ) ) { - cursor.acquire( firstId, mode ); - return cursor.getAll(); + RECORD record; + do + { + record = newRecord(); + getRecordByCursor( id, record, mode, cursor ); + if ( record.inUse() ) + { + records.add( record ); + } + id = getNextRecordReference( record ); + } + while ( record.inUse() && !Record.NULL_REFERENCE.is( id ) ); } + return records; } @Override @@ -1193,7 +1209,6 @@ private void verifyAfterNotRead( RECORD record, RecordLoad mode ) { record.clear(); mode.verify( record ); - } final void checkForDecodingErrors( PageCursor cursor, long recordId, RecordLoad mode ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java index e452fec2981a8..57a42da7b8ba9 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java @@ -64,15 +64,6 @@ public static long[] get( NodeRecord node, NodeStore nodeStore ) return getDynamicLabelsArray( node.getUsedDynamicLabelRecords(), nodeStore.getDynamicLabelStore() ); } - public static long[] get( NodeRecord node, RecordCursor dynamicLabelCursor ) - { - if ( node.isLight() ) - { - NodeStore.ensureHeavy( node, dynamicLabelCursor ); - } - return getDynamicLabelsArrayFromHeavyRecords( node.getUsedDynamicLabelRecords() ); - } - @Override public long[] getIfLoaded() { @@ -91,8 +82,7 @@ public Collection put( long[] labelIds, NodeStore nodeStore, Dyna return putSorted( node, labelIds, nodeStore, allocator ); } - public static Collection putSorted( NodeRecord node, long[] labelIds, NodeStore nodeStore, - DynamicRecordAllocator allocator ) + static Collection putSorted( NodeRecord node, long[] labelIds, NodeStore nodeStore, DynamicRecordAllocator allocator ) { long existingLabelsField = node.getLabelField(); long existingLabelsBits = parseLabelsBody( existingLabelsField ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeLabelsField.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeLabelsField.java index 1c25e9802d185..9f5f7e58adad1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeLabelsField.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeLabelsField.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.impl.store; -import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; /** @@ -59,13 +58,6 @@ public static long[] get( NodeRecord node, NodeStore nodeStore ) : InlineNodeLabels.get( node ); } - public static long[] get( NodeRecord node, RecordCursor dynamicLabelCursor ) - { - return fieldPointsToDynamicRecordOfLabels( node.getLabelField() ) - ? DynamicNodeLabels.get( node, dynamicLabelCursor ) - : InlineNodeLabels.get( node ); - } - public static boolean fieldPointsToDynamicRecordOfLabels( long labelField ) { return (labelField & 0x8000000000L) != 0; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java index 48700b38f57a2..1ebead7d964bc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java @@ -22,7 +22,6 @@ import java.io.File; import java.nio.file.OpenOption; import java.util.Arrays; -import java.util.List; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; @@ -107,14 +106,6 @@ public void ensureHeavy( NodeRecord node, long firstDynamicLabelRecord ) node.setLabelField( node.getLabelField(), dynamicLabelStore.getRecords( firstDynamicLabelRecord, RecordLoad.NORMAL ) ); } - public static void ensureHeavy( NodeRecord node, RecordCursor dynamicLabelCursor ) - { - long firstDynamicLabelId = NodeLabelsField.firstDynamicLabelRecordId( node.getLabelField() ); - dynamicLabelCursor.placeAt( firstDynamicLabelId, RecordLoad.NORMAL ); - List dynamicLabelRecords = dynamicLabelCursor.getAll(); - node.setLabelField( node.getLabelField(), dynamicLabelRecords ); - } - @Override public void updateRecord( NodeRecord record ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java index 678662b27e204..24bdef1f1974b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java @@ -115,6 +115,16 @@ public interface RecordStore extends IdSequen */ RECORD getRecord( long id, RECORD target, RecordLoad mode ) throws InvalidRecordException; + /** + * Opens a {@link PageCursor} on this store, capable of reading records using + * {@link #getRecordByCursor(long, AbstractBaseRecord, RecordLoad, PageCursor)}. + * The caller is responsible for closing it when done with it. + * + * @param id cursor will initially be placed at the page containing this record id. + * @return PageCursor for reading records. + */ + PageCursor openPageCursorForReading( long id ); + /** * Reads a record from the store into {@code target}, see * {@link RecordStore#getRecord(long, AbstractBaseRecord, RecordLoad)}. @@ -304,6 +314,12 @@ public R getRecord( long id, R target, RecordLoad mode ) throws InvalidRecordExc return actual.getRecord( id, target, mode ); } + @Override + public PageCursor openPageCursorForReading( long id ) + { + return actual.openPageCursorForReading( id ); + } + @Override public void getRecordByCursor( long id, R target, RecordLoad mode, PageCursor cursor ) throws InvalidRecordException { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/Scanner.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/Scanner.java index 336deabd1a43d..a3d5f28e8b4b4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/Scanner.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/Scanner.java @@ -26,6 +26,7 @@ import org.neo4j.graphdb.ResourceIterable; import org.neo4j.graphdb.ResourceIterator; import org.neo4j.helpers.collection.PrefetchingResourceIterator; +import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.RecordLoad; @@ -57,15 +58,18 @@ public static ResourceIterable scan( final Rec private static class Scan extends PrefetchingResourceIterator { private final LongIterator ids; - private final RecordCursor cursor; + private final RecordStore store; + private final PageCursor cursor; + private final R record; private final Predicate[] filters; Scan( RecordStore store, boolean forward, final Predicate... filters ) { this.filters = filters; this.ids = new StoreIdIterator( store, forward ); - this.cursor = store.newRecordCursor( store.newRecord() ); - cursor.acquire( 0, RecordLoad.CHECK ); + this.store = store; + this.cursor = store.openPageCursorForReading( 0 ); + this.record = store.newRecord(); } @Override @@ -73,9 +77,9 @@ protected R fetchNextOrNull() { while ( ids.hasNext() ) { - if ( cursor.next( ids.next() ) ) + store.getRecordByCursor( ids.next(), record, RecordLoad.CHECK, cursor ); + if ( record.inUse() ) { - R record = cursor.get(); if ( passesFilters( record ) ) { return record; diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/DeleteDuplicateNodesStep.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/DeleteDuplicateNodesStep.java index 47e7797eecc4c..09f19a27f24cc 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/DeleteDuplicateNodesStep.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/DeleteDuplicateNodesStep.java @@ -21,6 +21,7 @@ import org.eclipse.collections.api.iterator.LongIterator; +import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.RecordCursor; @@ -59,13 +60,13 @@ protected void process() { NodeRecord nodeRecord = nodeStore.newRecord(); PropertyRecord propertyRecord = propertyStore.newRecord(); - try ( RecordCursor cursor = nodeStore.newRecordCursor( nodeRecord ).acquire( 0, NORMAL ); - RecordCursor propertyCursor = propertyStore.newRecordCursor( propertyRecord ).acquire( 0, NORMAL ) ) + try ( PageCursor cursor = nodeStore.openPageCursorForReading( 0 ); + PageCursor propertyCursor = propertyStore.openPageCursorForReading( 0 ) ) { while ( nodeIds.hasNext() ) { long duplicateNodeId = nodeIds.next(); - cursor.next( duplicateNodeId ); + nodeStore.getRecordByCursor( duplicateNodeId, nodeRecord, NORMAL, cursor ); assert nodeRecord.inUse() : nodeRecord; // Ensure heavy so that the dynamic label records gets loaded (and then deleted) too nodeStore.ensureHeavy( nodeRecord ); @@ -74,7 +75,7 @@ protected void process() long nextProp = nodeRecord.getNextProp(); while ( !Record.NULL_REFERENCE.is( nextProp ) ) { - propertyCursor.next( nextProp ); + propertyStore.getRecordByCursor( nextProp, propertyRecord, NORMAL, propertyCursor ); assert propertyRecord.inUse() : propertyRecord + " for " + nodeRecord; propertyStore.ensureHeavy( propertyRecord ); propertiesRemoved += propertyRecord.numberOfProperties(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstGroupStage.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstGroupStage.java index fa2f329d2e8c5..3cd61f643af30 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstGroupStage.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstGroupStage.java @@ -19,6 +19,7 @@ */ package org.neo4j.unsafe.impl.batchimport; +import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; @@ -39,7 +40,7 @@ public class NodeFirstGroupStage extends Stage public static final String NAME = "Node --> Group"; public NodeFirstGroupStage( Configuration config, RecordStore groupStore, - RecordStore nodeStore, ByteArray cache ) + NodeStore nodeStore, ByteArray cache ) { super( NAME, null, config, 0 ); add( new BatchFeedStep( control(), config, allIn( groupStore, config ), groupStore.getRecordSize() ) ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeSetFirstGroupStep.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeSetFirstGroupStep.java index 37b2fa736dd33..1291aab1bdc4f 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeSetFirstGroupStep.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeSetFirstGroupStep.java @@ -19,8 +19,8 @@ */ package org.neo4j.unsafe.impl.batchimport; -import org.neo4j.kernel.impl.store.RecordCursor; -import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.ByteArray; @@ -37,25 +37,25 @@ public class NodeSetFirstGroupStep extends ProcessorStep nodeRecordCursor; + private final NodeStore nodeStore; + private final PageCursor nodeCursor; private NodeRecord[] current; private int cursor; - public NodeSetFirstGroupStep( StageControl control, Configuration config, - RecordStore nodeStore, ByteArray cache ) + NodeSetFirstGroupStep( StageControl control, Configuration config, NodeStore nodeStore, ByteArray cache ) { super( control, "FIRST", config, 1 ); this.cache = cache; this.batchSize = config.batchSize(); - this.nodeRecordCursor = nodeStore.newRecordCursor( nodeStore.newRecord() ); + this.nodeStore = nodeStore; + this.nodeCursor = nodeStore.openPageCursorForReading( 0 ); newBatch(); } @Override public void start( int orderingGuarantees ) { - nodeRecordCursor.acquire( 0, NORMAL ); super.start( orderingGuarantees ); } @@ -79,12 +79,12 @@ protected void process( RelationshipGroupRecord[] batch, BatchSender sender ) if ( cache.getByte( nodeId, 0 ) == 0 ) { cache.setByte( nodeId, 0, (byte) 1 ); - nodeRecordCursor.next( nodeId ); - NodeRecord node = nodeRecordCursor.get().clone(); - node.setNextRel( group.getId() ); - node.setDense( true ); + NodeRecord nodeRecord = nodeStore.newRecord(); + nodeStore.getRecordByCursor( nodeId, nodeRecord, NORMAL, nodeCursor ); + nodeRecord.setNextRel( group.getId() ); + nodeRecord.setDense( true ); - current[cursor++] = node; + current[cursor++] = nodeRecord; if ( cursor == batchSize ) { sender.send( current ); @@ -107,7 +107,7 @@ protected void lastCallForEmittingOutstandingBatches( BatchSender sender ) @Override public void close() throws Exception { - nodeRecordCursor.close(); + nodeCursor.close(); super.close(); } } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/ReadRecordsStep.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/ReadRecordsStep.java index 394347f93671e..aa546dacd06ae 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/ReadRecordsStep.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/ReadRecordsStep.java @@ -22,10 +22,8 @@ import org.eclipse.collections.api.iterator.LongIterator; import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.kernel.impl.store.RecordCursor; import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; -import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.unsafe.impl.batchimport.Configuration; /** @@ -80,12 +78,12 @@ protected void process( LongIterator idRange, BatchSender sender ) int i = 0; // Just use the first record in the batch here to satisfy the record cursor. // The truth is that we'll be using the read method which accepts an external record anyway so it doesn't matter. - try ( RecordCursor cursor = store.newRecordCursor( batch[0] ).acquire( id, RecordLoad.CHECK ) ) + try ( PageCursor cursor = store.openPageCursorForReading( id ) ) { boolean hasNext = true; while ( hasNext ) { - if ( assembler.append( cursor, batch, id, i ) ) + if ( assembler.append( store, cursor, batch, id, i ) ) { i++; } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/RecordDataAssembler.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/RecordDataAssembler.java index 583966708cc58..b279b2bf911bd 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/RecordDataAssembler.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/staging/RecordDataAssembler.java @@ -24,7 +24,8 @@ import java.util.function.Supplier; import org.neo4j.function.Predicates; -import org.neo4j.kernel.impl.store.RecordCursor; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.RecordLoad; @@ -63,10 +64,11 @@ public RECORD[] newBatchObject( int batchSize ) return (RECORD[]) array; } - public boolean append( RecordCursor cursor, RECORD[] array, long id, int index ) + public boolean append( RecordStore store, PageCursor cursor, RECORD[] array, long id, int index ) { RECORD record = array[index]; - return cursor.next( id, record, RecordLoad.CHECK ) && filter.test( record ); + store.getRecordByCursor( id, record, RecordLoad.CHECK, cursor ); + return record.inUse() && filter.test( record ); } public RECORD[] cutOffAt( RECORD[] array, int length ) diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index f37dbc7bf945d..3df976879dfb2 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -46,6 +46,7 @@ import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.StoreFactory; import org.neo4j.kernel.impl.store.StoreType; @@ -345,7 +346,7 @@ public RelationshipStore getRelationshipStore() return neoStores.getRelationshipStore(); } - public RecordStore getRelationshipGroupStore() + public RelationshipGroupStore getRelationshipGroupStore() { return neoStores.getRelationshipGroupStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/AbstractDynamicStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/AbstractDynamicStoreTest.java index 041e51f04e411..0e2b83107a63f 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/AbstractDynamicStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/AbstractDynamicStoreTest.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Iterator; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.StoreChannel; @@ -90,14 +91,14 @@ public void dynamicRecordCursorReadsInUseRecords() second.setNextBlock( third.getId() ); store.updateRecord( second ); - RecordCursor recordsCursor = store.newRecordCursor( store.newRecord() ).acquire( 1, NORMAL ); - assertTrue( recordsCursor.next() ); - assertEquals( first, recordsCursor.get() ); - assertTrue( recordsCursor.next() ); - assertEquals( second, recordsCursor.get() ); - assertTrue( recordsCursor.next() ); - assertEquals( third, recordsCursor.get() ); - assertFalse( recordsCursor.next() ); + Iterator records = store.getRecords( 1, NORMAL ).iterator(); + assertTrue( records.hasNext() ); + assertEquals( first, records.next() ); + assertTrue( records.hasNext() ); + assertEquals( second, records.next() ); + assertTrue( records.hasNext() ); + assertEquals( third, records.next() ); + assertFalse( records.hasNext() ); } } @@ -118,15 +119,15 @@ public void dynamicRecordCursorReadsNotInUseRecords() second.setInUse( false ); store.updateRecord( second ); - RecordCursor recordsCursor = store.newRecordCursor( store.newRecord() ).acquire( 1, FORCE ); - assertTrue( recordsCursor.next() ); - assertEquals( first, recordsCursor.get() ); - assertFalse( recordsCursor.next() ); - assertEquals( second, recordsCursor.get() ); + Iterator records = store.getRecords( 1, FORCE ).iterator(); + assertTrue( records.hasNext() ); + assertEquals( first, records.next() ); + assertFalse( records.hasNext() ); + assertEquals( second, records.next() ); // because mode == FORCE we can still move through the chain - assertTrue( recordsCursor.next() ); - assertEquals( third, recordsCursor.get() ); - assertFalse( recordsCursor.next() ); + assertTrue( records.hasNext() ); + assertEquals( third, records.next() ); + assertFalse( records.hasNext() ); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java index 999215f3f1bd0..64eb9c0525417 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeStoreTest.java @@ -31,16 +31,13 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.concurrent.ThreadLocalRandom; import java.util.function.LongSupplier; -import java.util.stream.LongStream; import org.neo4j.graphdb.mockfs.DelegatingFileSystemAbstraction; import org.neo4j.graphdb.mockfs.DelegatingStoreChannel; import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; -import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.Visitor; import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.fs.OpenMode; @@ -60,18 +57,13 @@ import org.neo4j.test.rule.fs.EphemeralFileSystemRule; import static java.util.Arrays.asList; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import static org.neo4j.helpers.Exceptions.contains; import static org.neo4j.kernel.impl.store.DynamicArrayStore.allocateFromNumbers; import static org.neo4j.kernel.impl.store.NodeStore.readOwnerFromDynamicLabelsRecord; @@ -346,24 +338,6 @@ public void shouldFreeSecondaryUnitIdOfShrunkRecord() throws Exception verify( idGenerator ).freeId( 10L ); } - @Test - @SuppressWarnings( "unchecked" ) - public void ensureHeavy() - { - long[] labels = LongStream.range( 1, 1000 ).toArray(); - NodeRecord node = new NodeRecord( 5 ); - node.setLabelField( 10, Collections.emptyList() ); - Collection dynamicLabelRecords = DynamicNodeLabels.putSorted( node, labels, - mock( NodeStore.class ), new StandaloneDynamicRecordAllocator() ); - assertThat( dynamicLabelRecords, not( empty() ) ); - RecordCursor dynamicLabelCursor = mock( RecordCursor.class ); - when( dynamicLabelCursor.getAll() ).thenReturn( Iterables.asList( dynamicLabelRecords ) ); - - NodeStore.ensureHeavy( node, dynamicLabelCursor ); - - assertEquals( dynamicLabelRecords, node.getDynamicLabelRecords() ); - } - private NodeStore newNodeStore( FileSystemAbstraction fs ) throws IOException { return newNodeStore( fs, pageCacheRule.getPageCache( fs ) ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java index 2bc51eb4a912c..d2ac0360cb59d 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/RelationshipGroupDefragmenterTest.java @@ -33,10 +33,11 @@ import java.util.BitSet; import java.util.Collection; +import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.NullLogService; -import org.neo4j.kernel.impl.store.RecordCursor; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.format.ForcedSecondaryUnitRecordFormats; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.format.standard.Standard; @@ -208,12 +209,11 @@ private void defrag( int nodeCount, RecordStore groupSt private void verifyGroupsAreSequentiallyOrderedByNode() { - RecordStore store = stores.getRelationshipGroupStore(); + RelationshipGroupStore store = stores.getRelationshipGroupStore(); long firstId = store.getNumberOfReservedLowIds(); long groupCount = store.getHighId() - firstId; RelationshipGroupRecord groupRecord = store.newRecord(); - RecordCursor groupCursor = - store.newRecordCursor( groupRecord ).acquire( firstId, CHECK ); + PageCursor groupCursor = store.openPageCursorForReading( firstId ); long highGroupId = store.getHighId(); long currentNodeId = -1; int currentTypeId = -1; @@ -221,7 +221,8 @@ private void verifyGroupsAreSequentiallyOrderedByNode() int currentGroupLength = 0; for ( long id = firstId; id < highGroupId; id++, newGroupCount++ ) { - if ( !groupCursor.next( id ) ) + store.getRecordByCursor( id, groupRecord, CHECK, groupCursor ); + if ( !groupRecord.inUse() ) { // This will be the case if we have double record units, just assert that fact assertTrue( units > 1 ); diff --git a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputEstimateCalculationIT.java b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputEstimateCalculationIT.java index 61d937ea96a61..fecab3c313c29 100644 --- a/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputEstimateCalculationIT.java +++ b/community/kernel/src/test/java/org/neo4j/unsafe/impl/batchimport/input/csv/CsvInputEstimateCalculationIT.java @@ -44,17 +44,16 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.logging.NullLogService; import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory; +import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageReader; import org.neo4j.kernel.impl.store.NeoStores; -import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.PropertyValueRecordSizeCalculator; -import org.neo4j.kernel.impl.store.RecordCursor; import org.neo4j.kernel.impl.store.StoreFactory; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.format.RecordFormats; import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory; -import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.logging.NullLog; import org.neo4j.logging.NullLogProvider; +import org.neo4j.storageengine.api.StoragePropertyCursor; import org.neo4j.test.rule.RandomRule; import org.neo4j.test.rule.TestDirectory; import org.neo4j.unsafe.impl.batchimport.AdditionalInitialIds; @@ -79,11 +78,9 @@ import static org.junit.Assert.assertThat; import static org.neo4j.csv.reader.CharSeekers.charSeeker; import static org.neo4j.csv.reader.Readables.wrap; -import static org.neo4j.helpers.collection.Iterables.count; import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME; import static org.neo4j.kernel.impl.store.NoStoreHeader.NO_STORE_HEADER; import static org.neo4j.kernel.impl.store.format.standard.Standard.LATEST_RECORD_FORMATS; -import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; import static org.neo4j.unsafe.impl.batchimport.ImportLogic.NO_MONITOR; import static org.neo4j.unsafe.impl.batchimport.input.RandomEntityDataGenerator.convert; import static org.neo4j.unsafe.impl.batchimport.input.csv.Configuration.COMMAS; @@ -131,7 +128,7 @@ public void shouldCalculateCorrectEstimates() throws Exception assertRoughlyEqual( estimates.numberOfNodes(), stores.getNodeStore().getNumberOfIdsInUse() ); assertRoughlyEqual( estimates.numberOfRelationships(), stores.getRelationshipStore().getNumberOfIdsInUse() ); assertRoughlyEqual( estimates.numberOfNodeProperties() + estimates.numberOfRelationshipProperties(), - calculateNumberOfProperties( stores.getPropertyStore() ) ); + calculateNumberOfProperties( stores ) ); } assertRoughlyEqual( propertyStorageSize(), estimates.sizeOfNodeProperties() + estimates.sizeOfRelationshipProperties() ); } @@ -192,17 +189,19 @@ private Input generateData() throws IOException IdType.INTEGER, COMMAS, Collector.EMPTY, groups ); } - private long calculateNumberOfProperties( PropertyStore propertyStore ) + private long calculateNumberOfProperties( NeoStores stores ) { long count = 0; - try ( RecordCursor cursor = propertyStore.newRecordCursor( propertyStore.newRecord() ).acquire( 0, CHECK ) ) + try ( RecordStorageReader reader = new RecordStorageReader( stores ); + StoragePropertyCursor cursor = reader.allocatePropertyCursor() ) { - long highId = propertyStore.getHighId(); + long highId = stores.getPropertyStore().getHighId(); for ( long id = 0; id < highId; id++ ) { - if ( cursor.next( id ) ) + cursor.init( id ); + while ( cursor.next() ) { - count += count( cursor.get() ); + count++; } } }