From cab9968df300b2cc5b4c55ecfd3b37a4082ddeee Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Fri, 26 Jun 2015 10:17:21 +0200 Subject: [PATCH] Simplified store classes - RecordStore changed to only have one method for getting records, accepts a record instance to write into. RecordStore#newRecord() was added for creating records and to give clients full control over instantiation of those. - RecordStore#newRecordCursor() added for allowing client to use one PageCursor over multiple records. - CommonAbstractStore/AbstractStore was collapsed into AbstractRecordStore. - AbstractRecordStore implements #getRecord(), no other store does, giving consistent behaviour over all stores. - Each store mostly implements its format in read/WriteRecord, this could even be injected later on. --- .../checking/SchemaRecordCheck.java | 6 +- .../checking/full/CountsBuilderDecorator.java | 3 +- .../consistency/checking/full/FullCheck.java | 4 +- .../checking/full/PropertyReader.java | 12 +- .../repair/OwningNodeRelationshipChain.java | 6 +- .../repair/RelationshipChainExplorer.java | 6 +- .../consistency/store/DirectRecordAccess.java | 4 +- .../store/synthetic/CountsEntry.java | 18 +- .../store/synthetic/IndexEntry.java | 8 +- .../store/synthetic/LabelScanDocument.java | 11 +- .../checking/DynamicRecordCheckTest.java | 4 +- .../checking/NodeRecordCheckTest.java | 2 +- .../full/FullCheckIntegrationTest.java | 21 +- .../checking/full/StoreProcessorTest.java | 45 +- .../OwningNodeRelationshipChainTest.java | 26 +- .../consistency/repair/RecordSetTest.java | 1 - .../repair/RelationshipChainExplorerTest.java | 6 +- .../report/ConsistencyReporterTest.java | 4 +- .../neo4j/consistency/checking/NodeField.java | 2 +- .../checking/SchemaRecordCheck.java | 6 +- .../consistency/checking/cache/CacheTask.java | 7 +- .../checking/full/CloningRecordIterable.java | 55 +++ .../checking/full/CountsBuilderDecorator.java | 15 +- .../consistency/checking/full/FullCheck.java | 3 +- .../checking/full/IterableStore.java | 16 +- .../checking/full/NodeLabelReader.java | 4 +- .../checking/full/PropertyReader.java | 15 +- .../checking/full/StoreProcessor.java | 3 +- .../repair/OwningNodeRelationshipChain.java | 7 +- .../repair/RelationshipChainExplorer.java | 12 +- .../statistics/AccessStatistics.java | 5 +- .../AccessStatsKeepingStoreAccess.java | 12 +- .../consistency/store/DirectRecordAccess.java | 5 +- .../store/synthetic/CountsEntry.java | 32 +- .../store/synthetic/IndexEntry.java | 8 +- .../store/synthetic/LabelScanDocument.java | 11 +- .../checking/DynamicRecordCheckTest.java | 4 +- .../full/FullCheckIntegrationTest.java | 20 +- .../full/NodeCorrectlyIndexedCheckTest.java | 4 +- .../checking/full/StoreProcessorTest.java | 38 +- .../OwningNodeRelationshipChainTest.java | 21 +- .../repair/RelationshipChainExplorerTest.java | 6 +- .../pagecache/AdversarialReadPageCursor.java | 12 +- .../kernel/impl/api/store/DiskLayer.java | 19 +- .../impl/api/store/PropertyRecordCursor.java | 17 +- .../api/store/StoreAbstractNodeCursor.java | 22 +- .../StoreAbstractRelationshipCursor.java | 5 +- .../api/store/StoreIteratorNodeCursor.java | 5 +- .../StoreIteratorRelationshipCursor.java | 3 +- .../store/StoreNodeRelationshipCursor.java | 31 +- .../api/store/StorePropertyPayloadCursor.java | 41 +- .../impl/api/store/StoreSingleNodeCursor.java | 6 +- .../store/StoreSingleRelationshipCursor.java | 2 +- .../impl/store/AbstractDynamicStore.java | 447 +++-------------- .../impl/store/AbstractRecordStore.java | 59 --- .../kernel/impl/store/AbstractStore.java | 118 ----- .../impl/store/CommonAbstractStore.java | 461 ++++++++++++++++-- .../impl/store/DelegatingRecordStore.java | 137 ------ .../kernel/impl/store/DynamicArrayStore.java | 11 +- .../kernel/impl/store/DynamicNodeLabels.java | 13 +- .../kernel/impl/store/DynamicStringStore.java | 11 +- .../store/ExistingThenNewRecordAllocator.java | 9 +- .../kernel/impl/store/LabelTokenStore.java | 18 +- .../kernel/impl/store/MetaDataStore.java | 88 +++- .../neo4j/kernel/impl/store/NodeStore.java | 237 ++------- .../impl/store/PropertyKeyTokenStore.java | 24 +- .../kernel/impl/store/PropertyStore.java | 295 ++++------- ...ynamicBlockSize.java => RecordCursor.java} | 11 +- .../neo4j/kernel/impl/store/RecordStore.java | 186 +++++-- .../impl/store/RelationshipGroupStore.java | 208 ++------ .../kernel/impl/store/RelationshipStore.java | 250 ++-------- .../store/RelationshipTypeTokenStore.java | 14 +- .../kernel/impl/store/SchemaStorage.java | 6 +- .../neo4j/kernel/impl/store/SchemaStore.java | 17 +- .../neo4j/kernel/impl/store/StoreAccess.java | 8 +- .../neo4j/kernel/impl/store/TokenStore.java | 154 +----- .../impl/store/counts/CountsTracker.java | 1 + .../store/record/Abstract64BitRecord.java | 16 +- .../impl/store/record/AbstractBaseRecord.java | 35 +- .../impl/store/record/AbstractRecord.java | 11 +- .../impl/store/record/DynamicRecord.java | 118 ++--- .../impl/store/record/LabelTokenRecord.java | 9 +- .../store/record/NeoStoreActualRecord.java | 48 ++ .../impl/store/record/NeoStoreRecord.java | 16 +- .../kernel/impl/store/record/NodeRecord.java | 37 +- .../impl/store/record/PrimitiveRecord.java | 22 +- .../store/record/PropertyKeyTokenRecord.java | 24 +- .../impl/store/record/PropertyRecord.java | 44 +- .../kernel/impl/store/record/RecordLoad.java | 92 +++- .../store/record/RelationshipGroupRecord.java | 50 +- .../impl/store/record/RelationshipRecord.java | 55 ++- .../record/RelationshipTypeTokenRecord.java | 8 +- .../impl/store/record/SchemaRecord.java | 84 ++++ .../kernel/impl/store/record/TokenRecord.java | 24 +- .../kernel/impl/storemigration/StoreFile.java | 1 - .../LegacyRelationshipStoreReader.java | 12 +- .../v19/Legacy19NodeStoreReader.java | 8 +- .../v20/Legacy20NodeStoreReader.java | 10 +- .../DuplicatePropertyRemover.java | 16 +- .../IndexConsultedPropertyBlockSweeper.java | 14 +- .../NonIndexedConflictResolver.java | 5 +- .../PropertyDeduplicator.java | 14 +- .../impl/transaction/command/Command.java | 30 +- .../command/PhysicalLogCommandReaderV1_9.java | 8 +- .../command/PhysicalLogCommandReaderV2_0.java | 6 +- .../command/PhysicalLogCommandReaderV2_1.java | 7 +- .../command/PhysicalLogCommandReaderV2_2.java | 3 +- .../PhysicalLogCommandReaderV2_2_4.java | 3 +- .../command/PhysicalLogCommandReaderV3_0.java | 3 +- .../impl/transaction/state/Loaders.java | 58 +-- .../state/NeoStoreIndexStoreView.java | 17 +- .../transaction/state/OnlineIndexUpdates.java | 16 +- .../transaction/state/PropertyLoader.java | 6 +- .../transaction/state/RecordAccessSet.java | 6 +- .../transaction/state/RecordChangeSet.java | 10 +- .../state/TransactionRecordState.java | 15 +- .../unsafe/batchinsert/BatchInserterImpl.java | 7 +- .../BatchRelationshipIterable.java | 10 +- .../unsafe/batchinsert/BatchTokenHolder.java | 20 - .../batchinsert/DirectRecordAccess.java | 6 +- .../batchinsert/DirectRecordAccessSet.java | 7 +- .../batchimport/EntityStoreUpdaterStep.java | 6 +- .../impl/batchimport/NodeEncoderStep.java | 6 +- .../NodeFirstRelationshipProcessor.java | 3 +- .../impl/batchimport/ReadNodeRecordsStep.java | 7 +- .../ReadRelationshipCountsDataStep.java | 2 +- .../ReadRelationshipRecordsBackwardsStep.java | 2 +- .../store/BatchingDynamicRecordAllocator.java | 4 +- .../kernel/impl/AbstractNeo4jTestCase.java | 4 +- .../impl/api/KernelTransactionsTest.java | 1 + .../kernel/impl/api/store/CacheLayerTest.java | 1 + .../StoreNodeRelationshipCursorTest.java | 2 +- .../store/StorePropertyPayloadCursorTest.java | 14 +- .../core/JumpingFileSystemAbstraction.java | 2 +- .../impl/core/TestShortStringProperties.java | 2 +- .../impl/store/AbstractDynamicStoreTest.java | 8 +- .../impl/store/CommonAbstractStoreTest.java | 35 +- .../ExistingThenNewRecordAllocatorTest.java | 13 +- .../impl/store/LabelTokenStoreTest.java | 32 +- .../kernel/impl/store/NeoStoresTest.java | 26 +- .../kernel/impl/store/NodeRecordTest.java | 4 +- .../kernel/impl/store/NodeStoreTest.java | 13 +- .../kernel/impl/store/PropertyStoreTest.java | 4 +- .../kernel/impl/store/RecordSizesDocTest.java | 2 +- .../store/RecordStoreConsistentReadTest.java | 69 +-- .../kernel/impl/store/RecordStoreUtil.java | 68 +++ .../store/RelationshipGroupStoreTest.java | 35 +- .../kernel/impl/store/TestDynamicStore.java | 40 +- .../store/TestGrowingFileMemoryMapping.java | 5 +- .../kernel/impl/store/UpgradeStoreIT.java | 2 +- .../storemigration/MigrationTestUtils.java | 1 + .../legacylogs/LogEntrySortingCursorTest.java | 1 + ...DeferredIndexedConflictResolutionTest.java | 3 +- .../DuplicatePropertyRemoverTest.java | 26 +- ...ndexConsultedPropertyBlockSweeperTest.java | 10 +- .../NonIndexedConflictResolverTest.java | 15 +- .../PropertyDeduplicatorTestUtil.java | 4 +- .../impl/transaction/command/Commands.java | 2 +- .../log/BatchingTransactionAppenderTest.java | 10 +- .../entry/VersionAwareLogEntryReaderTest.java | 4 +- .../state/ApplyRecoveredTransactionsTest.java | 1 - .../transaction/state/ControlledLoaders.java | 5 +- .../state/IntegrityValidatorTest.java | 7 +- .../transaction/state/NodeStoreScanTest.java | 5 +- .../transaction/state/PropertyLoaderTest.java | 59 ++- .../state/RelationshipCreatorTest.java | 9 +- .../state/RelationshipGroupGetterTest.java | 90 ++-- .../state/SchemaRuleCommandTest.java | 35 +- .../state/TransactionRecordStateTest.java | 55 +-- .../WriteTransactionCommandOrderingTest.java | 14 +- .../unsafe/batchinsert/BatchInsertTest.java | 11 +- .../upgrade/StoreUpgradeOnStartupTest.java | 1 + .../StoreUpgraderInterruptionTestIT.java | 1 + .../TestMigrateToDenseNodeSupport.java | 1 + .../tools/applytx/DumpRecordsCommand.java | 24 +- .../java/org/neo4j/tools/dump/DumpStore.java | 10 +- .../org/neo4j/tools/dump/DumpStoreChain.java | 9 +- .../neo4j/tools/rawstorereader/RsdrMain.java | 8 +- 178 files changed, 2599 insertions(+), 2689 deletions(-) create mode 100644 community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CloningRecordIterable.java delete mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractRecordStore.java delete mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractStore.java delete mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/store/DelegatingRecordStore.java rename community/kernel/src/main/java/org/neo4j/kernel/impl/store/{DynamicBlockSize.java => RecordCursor.java} (69%) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreActualRecord.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/SchemaRecord.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/store/RecordStoreUtil.java diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/SchemaRecordCheck.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/SchemaRecordCheck.java index a4436b477e128..85d2e7c52b52d 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/SchemaRecordCheck.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/SchemaRecordCheck.java @@ -165,7 +165,7 @@ public void checkIndexRule( IndexRule rule, DynamicRecord record, RecordAccess r if ( rule.isConstraintIndex() && rule.getOwningConstraint() != null ) { - DynamicRecord previousObligation = constraintObligations.put( rule.getOwningConstraint(), record ); + DynamicRecord previousObligation = constraintObligations.put( rule.getOwningConstraint(), record.clone() ); if ( previousObligation != null ) { engine.report().duplicateObligation( previousObligation ); @@ -179,7 +179,7 @@ public void checkUniquenessConstraintRule( UniquePropertyConstraintRule rule, Dy { checkLabelAndPropertyRule( rule, rule.getPropertyKey(), record, records, engine ); - DynamicRecord previousObligation = indexObligations.put( rule.getOwnedIndex(), record ); + DynamicRecord previousObligation = indexObligations.put( rule.getOwnedIndex(), record.clone() ); if ( previousObligation != null ) { engine.report().duplicateObligation( previousObligation ); @@ -283,7 +283,7 @@ private void checkRelTypeAndPropertyRule( SchemaRule rule, int propertyKey, Dyna private void checkForDuplicates( SchemaRule rule, DynamicRecord record, CheckerEngine engine ) { - DynamicRecord previousContentRecord = verifiedRulesWithRecords.put( rule, record ); + DynamicRecord previousContentRecord = verifiedRulesWithRecords.put( rule, record.clone() ); if ( previousContentRecord != null ) { engine.report().duplicateRuleContent( previousContentRecord ); diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/CountsBuilderDecorator.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/CountsBuilderDecorator.java index b89786b8fae7a..bfded8752058c 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/CountsBuilderDecorator.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/CountsBuilderDecorator.java @@ -48,6 +48,7 @@ import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.nodeKey; import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.relationshipKey; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; import static org.neo4j.legacy.consistency.checking.full.NodeLabelReader.getListOfLabels; class CountsBuilderDecorator extends CheckDecorator.Adapter @@ -338,6 +339,6 @@ private static Set labelsFor( NodeStore nodeStore, RecordAccess recordAccess, long nodeId ) { - return getListOfLabels( nodeStore.forceGetRecord( nodeId ), recordAccess, engine ); + return getListOfLabels( nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ), recordAccess, engine ); } } diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/FullCheck.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/FullCheck.java index 250f97799ffd2..318cf5819f735 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/FullCheck.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/FullCheck.java @@ -47,6 +47,8 @@ import org.neo4j.legacy.consistency.store.DirectRecordAccess; import org.neo4j.logging.Log; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class FullCheck { private final boolean checkPropertyOwners; @@ -161,7 +163,7 @@ private static T[] readAllRecords( Class type, T[] records = (T[]) Array.newInstance( type, (int) store.getHighId() ); for ( int i = 0; i < records.length; i++ ) { - records[i] = store.forceGetRecord( i ); + records[i] = store.getRecord( i, store.newRecord(), FORCE ); } return records; } diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/PropertyReader.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/PropertyReader.java index ce04165c92fec..9511f86213b02 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/PropertyReader.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/checking/full/PropertyReader.java @@ -27,13 +27,14 @@ import org.neo4j.kernel.api.index.PropertyAccessor; import org.neo4j.kernel.api.properties.DefinedProperty; import org.neo4j.kernel.api.properties.Property; -import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class PropertyReader implements PropertyAccessor { private final PropertyStore propertyStore; @@ -67,9 +68,9 @@ public DefinedProperty propertyValue( PropertyBlock block ) @Override public Property getProperty( long nodeId, int propertyKeyId ) { - try + NodeRecord nodeRecord = nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ); + if ( nodeRecord.inUse() ) { - NodeRecord nodeRecord = nodeStore.getRecord( nodeId ); for ( PropertyBlock block : propertyBlocks( nodeRecord ) ) { if ( block.getKeyIndexId() == propertyKeyId ) @@ -78,11 +79,6 @@ public Property getProperty( long nodeId, int propertyKeyId ) } } } - catch ( InvalidRecordException e ) - { - // Fine, we'll just return an empty property below - } - return Property.noNodeProperty( nodeId, propertyKeyId ); } } diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChain.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChain.java index 073ffcd0ef905..a45eb787a62b4 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChain.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChain.java @@ -23,6 +23,8 @@ import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class OwningNodeRelationshipChain { private final RelationshipChainExplorer relationshipChainExplorer; @@ -42,11 +44,9 @@ public RecordSet findRelationshipChainsThatThisRecordShouldB for ( RelationshipNodeField field : RelationshipNodeField.values() ) { long nodeId = field.get( relationship ); - NodeRecord nodeRecord = nodeStore.forceGetRecord( nodeId ); + NodeRecord nodeRecord = nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ); records.addAll( relationshipChainExplorer.followChainFromNode( nodeId, nodeRecord.getNextRel() ) ); } return records; } - - } diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorer.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorer.java index f733897e6c629..77509eac659d6 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorer.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorer.java @@ -22,6 +22,8 @@ import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.RelationshipRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; import static org.neo4j.legacy.consistency.repair.RelationshipChainDirection.NEXT; import static org.neo4j.legacy.consistency.repair.RelationshipChainDirection.PREV; @@ -63,7 +65,7 @@ private RecordSet expandChainInBothDirections( RelationshipR protected RecordSet followChainFromNode(long nodeId, long relationshipId ) { - RelationshipRecord record = recordStore.getRecord( relationshipId ); + RelationshipRecord record = recordStore.getRecord( relationshipId, recordStore.newRecord(), NORMAL ); return expandChain( record, nodeId, NEXT ); } @@ -76,7 +78,7 @@ private RecordSet expandChain( RelationshipRecord record, lo long nextRelId = direction.fieldFor( nodeId, currentRecord ).relOf( currentRecord ); while ( currentRecord.inUse() && !direction.fieldFor( nodeId, currentRecord ).endOfChain( currentRecord ) ) { - currentRecord = recordStore.forceGetRecord( nextRelId ); + currentRecord = recordStore.getRecord( nextRelId, recordStore.newRecord(), FORCE ); chain.add( currentRecord ); nextRelId = direction.fieldFor( nodeId, currentRecord ).relOf( currentRecord ); } diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/DirectRecordAccess.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/DirectRecordAccess.java index 2c6f0e74bfacf..40e9dcfa45ba3 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/DirectRecordAccess.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/DirectRecordAccess.java @@ -32,6 +32,8 @@ import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class DirectRecordAccess implements DiffRecordAccess { final StoreAccess access; @@ -127,7 +129,7 @@ public RecordReference propertyKeyName( int id ) RecordReference referenceTo( RecordStore store, long id ) { - return new DirectRecordReference<>( store.forceGetRecord( id ), this ); + return new DirectRecordReference<>( store.getRecord( id, store.newRecord(), FORCE ), this ); } @Override diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/CountsEntry.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/CountsEntry.java index 94849df3cd619..0ea69544e17e8 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/CountsEntry.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/CountsEntry.java @@ -32,8 +32,8 @@ */ public class CountsEntry extends AbstractBaseRecord { - private final CountsKey key; - private final long count; + private CountsKey key; + private long count; public CountsEntry( CountsKey key, long count ) { @@ -42,6 +42,14 @@ public CountsEntry( CountsKey key, long count ) setInUse( true ); } + @Override + public void clear() + { + super.clear(); + key = null; + count = 0; + } + @Override public String toString() { @@ -54,6 +62,12 @@ public long getLongId() throw new UnsupportedOperationException(); } + @Override + public void setId( long id ) + { + throw new UnsupportedOperationException(); + } + public CountsKey getCountsKey() { return key; diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/IndexEntry.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/IndexEntry.java index d26980f31fe44..ee4c9fbd260e2 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/IndexEntry.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/IndexEntry.java @@ -30,7 +30,13 @@ public class IndexEntry extends Abstract64BitRecord public IndexEntry( long nodeId ) { super( nodeId ); - setInUse( true ); + initialize( true ); + } + + @Override + public void clear() + { + initialize( false ); } @Override diff --git a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/LabelScanDocument.java b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/LabelScanDocument.java index 6ba12de380d19..d21d762cb503e 100644 --- a/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/LabelScanDocument.java +++ b/community/consistency-check-legacy/src/main/java/org/neo4j/legacy/consistency/store/synthetic/LabelScanDocument.java @@ -28,13 +28,20 @@ */ public class LabelScanDocument extends Abstract64BitRecord { - private final NodeLabelRange nodeLabelRange; + private NodeLabelRange nodeLabelRange; public LabelScanDocument( NodeLabelRange nodeLabelRange ) { super( nodeLabelRange.id() ); + initialize( true ); this.nodeLabelRange = nodeLabelRange; - setInUse( true ); + } + + @Override + public void clear() + { + initialize( false ); + this.nodeLabelRange = null; } public NodeLabelRange getNodeLabelRange() diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/DynamicRecordCheckTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/DynamicRecordCheckTest.java index 9677f2b8df5fa..5eddff12f1aec 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/DynamicRecordCheckTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/DynamicRecordCheckTest.java @@ -330,8 +330,8 @@ public static RecordStore configureDynamicStore( int blockSize ) { @SuppressWarnings( "unchecked" ) RecordStore mock = mock( RecordStore.class ); - when( mock.getRecordSize() ).thenReturn( blockSize + AbstractDynamicStore.BLOCK_HEADER_SIZE ); - when( mock.getRecordHeaderSize() ).thenReturn( AbstractDynamicStore.BLOCK_HEADER_SIZE ); + when( mock.getRecordSize() ).thenReturn( blockSize + AbstractDynamicStore.RECORD_HEADER_SIZE ); + when( mock.getRecordHeaderSize() ).thenReturn( AbstractDynamicStore.RECORD_HEADER_SIZE ); return mock; } } diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/NodeRecordCheckTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/NodeRecordCheckTest.java index 237f8cc3c75ce..a29e5776d8685 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/NodeRecordCheckTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/NodeRecordCheckTest.java @@ -99,7 +99,7 @@ public void shouldReportRelationshipNotInUse() throws Exception { // given NodeRecord node = inUse( new NodeRecord( 42, false, 7, 11 ) ); - RelationshipRecord relationship = add( notInUse( new RelationshipRecord( 7, 0, 0, 0 ) ) ); + RelationshipRecord relationship = add( new RelationshipRecord( 7, 0, 0, 0 ) ); add( inUse( new PropertyRecord( 11 ) ) ); // when diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/FullCheckIntegrationTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/FullCheckIntegrationTest.java index 4e01e0813d06a..3566f97766b0a 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/FullCheckIntegrationTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/FullCheckIntegrationTest.java @@ -122,6 +122,8 @@ import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_PROPERTY; import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP; import static org.neo4j.kernel.impl.store.record.Record.NO_PREV_RELATIONSHIP; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; import static org.neo4j.kernel.impl.store.record.RelationshipPropertyExistenceConstraintRule.relPropertyExistenceConstraintRule; import static org.neo4j.kernel.impl.util.Bits.bits; import static org.neo4j.legacy.consistency.checking.RecordCheckTestBase.inUse; @@ -402,7 +404,7 @@ public void shouldReportIndexInconsistencies() throws Exception // given for ( Long indexedNodeId : indexedNodes ) { - fixture.directStoreAccess().nativeStores().getNodeStore().forceUpdateRecord( + fixture.directStoreAccess().nativeStores().getNodeStore().updateRecord( notInUse( new NodeRecord( indexedNodeId, false, -1, -1 ) ) ); } @@ -442,7 +444,7 @@ public void shouldNotReportIndexInconsistenciesIfIndexIsFailed() throws Exceptio for ( Long indexedNodeId : indexedNodes ) { - storeAccess.nativeStores().getNodeStore().forceUpdateRecord( + storeAccess.nativeStores().getNodeStore().updateRecord( notInUse( new NodeRecord( indexedNodeId, false, -1, -1 ) ) ); } @@ -1100,7 +1102,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getRelationshipTypeNameStore().forceGetRecord( inconsistentName.get() ); + DynamicRecord record = access.getRelationshipTypeNameStore().getRecord( inconsistentName.get(), + access.getRelationshipTypeNameStore().newRecord(), FORCE ); record.setNextBlock( record.getId() ); access.getRelationshipTypeNameStore().updateRecord( record ); @@ -1128,7 +1131,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getPropertyKeyNameStore().forceGetRecord( inconsistentName.get()+1 ); + DynamicRecord record = access.getPropertyKeyNameStore().getRecord( inconsistentName.get()+1, + access.getPropertyKeyNameStore().newRecord(), NORMAL ); record.setNextBlock( record.getId() ); access.getPropertyKeyNameStore().updateRecord( record ); @@ -1146,7 +1150,8 @@ public void shouldReportRelationshipTypeInconsistencies() throws Exception // given StoreAccess access = fixture.directStoreAccess().nativeStores(); RecordStore relTypeStore = access.getRelationshipTypeTokenStore(); - RelationshipTypeTokenRecord record = relTypeStore.forceGetRecord( (int) relTypeStore.nextId() ); + RelationshipTypeTokenRecord record = relTypeStore.getRecord( (int) relTypeStore.nextId(), + relTypeStore.newRecord(), FORCE ); record.setNameId( 20 ); record.setInUse( true ); relTypeStore.updateRecord( record ); @@ -1165,7 +1170,8 @@ public void shouldReportLabelInconsistencies() throws Exception { // given StoreAccess access = fixture.directStoreAccess().nativeStores(); - LabelTokenRecord record = access.getLabelTokenStore().forceGetRecord( 1 ); + LabelTokenRecord record = access.getLabelTokenStore().getRecord( 1, + access.getLabelTokenStore().newRecord(), FORCE ); record.setNameId( 20 ); record.setInUse( true ); access.getLabelTokenStore().updateRecord( record ); @@ -1194,7 +1200,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getPropertyKeyNameStore().forceGetRecord( inconsistentKey.get()+1 ); + DynamicRecord record = access.getPropertyKeyNameStore().getRecord( inconsistentKey.get()+1, + access.getPropertyKeyNameStore().newRecord(), FORCE ); record.setInUse( false ); access.getPropertyKeyNameStore().updateRecord( record ); diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/StoreProcessorTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/StoreProcessorTest.java index c15dae1103c37..dcf4a9359d542 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/StoreProcessorTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/checking/full/StoreProcessorTest.java @@ -21,14 +21,18 @@ import org.junit.Test; import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RecordStoreUtil; +import org.neo4j.kernel.impl.store.RecordStoreUtil.ReadNodeAnswer; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.legacy.consistency.checking.CheckDecorator; import org.neo4j.legacy.consistency.report.ConsistencyReport; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -41,19 +45,22 @@ public class StoreProcessorTest public void shouldProcessAllTheRecordsInAStore() throws Exception { // given - StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, mock( ConsistencyReport.Reporter.class ) ); + StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, + mock( ConsistencyReport.Reporter.class ) ); RecordStore recordStore = mock( RecordStore.class ); when( recordStore.getHighId() ).thenReturn( 3L ); - when( recordStore.forceGetRecord( any( Long.class ) ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); + when( recordStore.newRecord() ).thenAnswer( new RecordStoreUtil.NewNodeRecordAnswer() ); + when( recordStore.getRecord( anyLong(), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); // when processor.applyFiltered( recordStore ); // then - verify( recordStore ).forceGetRecord( 0 ); - verify( recordStore ).forceGetRecord( 1 ); - verify( recordStore ).forceGetRecord( 2 ); - verify( recordStore, never() ).forceGetRecord( 3 ); + verify( recordStore ).getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore, never() ).getRecord( eq( 3L ), any( NodeRecord.class ), any( RecordLoad.class ) ); } @SuppressWarnings("unchecked") @@ -61,18 +68,23 @@ public void shouldProcessAllTheRecordsInAStore() throws Exception public void shouldStopProcessingRecordsWhenSignalledToStop() throws Exception { // given - final StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, mock( ConsistencyReport.Reporter.class ) ); + final StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, + mock( ConsistencyReport.Reporter.class ) ); RecordStore recordStore = mock( RecordStore.class ); when( recordStore.getHighId() ).thenReturn( 4L ); - when( recordStore.forceGetRecord( 0L ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); - when( recordStore.forceGetRecord( 1L ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); - when( recordStore.forceGetRecord( 2L ) ).thenAnswer( new Answer() + when( recordStore.newRecord() ).thenAnswer( new RecordStoreUtil.NewNodeRecordAnswer() ); + when( recordStore.getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); + when( recordStore.getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); + when( recordStore.getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) { @Override public NodeRecord answer( InvocationOnMock invocation ) throws Throwable { processor.stop(); - return new NodeRecord( 0, false, 0, 0 ); + return super.answer( invocation ); } } ); @@ -80,9 +92,10 @@ public NodeRecord answer( InvocationOnMock invocation ) throws Throwable processor.applyFiltered( recordStore ); // then - verify( recordStore ).forceGetRecord( 0 ); - verify( recordStore ).forceGetRecord( 1 ); - verify( recordStore ).forceGetRecord( 2 ); - verify( recordStore, never() ).forceGetRecord( 3 ); + verify( recordStore ).getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore, never() ).getRecord( eq( 3L ), any( NodeRecord.class ), any( RecordLoad.class ) ); } } + diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChainTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChainTest.java index 026ed6615a601..942e29ce8af8c 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChainTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/OwningNodeRelationshipChainTest.java @@ -23,15 +23,16 @@ import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; import org.junit.Test; - import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RecordStoreUtil; +import org.neo4j.kernel.impl.store.RecordStoreUtil.ReadNodeAnswer; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipRecord; -import org.neo4j.legacy.consistency.repair.OwningNodeRelationshipChain; -import org.neo4j.legacy.consistency.repair.RecordSet; -import org.neo4j.legacy.consistency.repair.RelationshipChainExplorer; import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -43,9 +44,9 @@ public class OwningNodeRelationshipChainTest public void shouldFindBothChainsThatTheRelationshipRecordShouldBelongTo() throws Exception { // given - int node1 = 101, node1Rel = 1001; - int node2 = 201, node2Rel = 2001; - int sharedRel = 1000; + long node1 = 101, node1Rel = 1001; + long node2 = 201, node2Rel = 2001; + long sharedRel = 1000; int relType = 0; RecordSet node1RelChain = RecordSet.asSet( @@ -59,10 +60,11 @@ public void shouldFindBothChainsThatTheRelationshipRecordShouldBelongTo() throws @SuppressWarnings("unchecked") RecordStore recordStore = mock( RecordStore.class ); - when( recordStore.forceGetRecord( node1 ) ).thenReturn( - new NodeRecord( node1, false, node1Rel, NO_NEXT_PROPERTY.intValue() ) ); - when( recordStore.forceGetRecord( node2 ) ).thenReturn( - new NodeRecord( node2, false, node2Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.getRecord( eq( node1 ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, node1Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.getRecord( eq( node2 ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, node2Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.newRecord() ).thenAnswer( new RecordStoreUtil.NewNodeRecordAnswer() ); RelationshipChainExplorer relationshipChainExplorer = mock( RelationshipChainExplorer.class ); when( relationshipChainExplorer.followChainFromNode( node1, node1Rel ) ).thenReturn( node1RelChain ); @@ -98,6 +100,4 @@ public void describeTo( Description description ) } }; } - - } diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RecordSetTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RecordSetTest.java index c57dc99cbf67f..408bfa94de1c5 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RecordSetTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RecordSetTest.java @@ -22,7 +22,6 @@ import org.junit.Test; import org.neo4j.kernel.impl.store.record.NodeRecord; -import org.neo4j.legacy.consistency.repair.RecordSet; import static org.junit.Assert.assertEquals; diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorerTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorerTest.java index d3cabe7ad134d..2db2cf43ecb06 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorerTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/repair/RelationshipChainExplorerTest.java @@ -42,6 +42,8 @@ import static org.junit.Assert.assertEquals; +import static org.neo4j.kernel.impl.store.CommonAbstractStore.getRecord; + public class RelationshipChainExplorerTest { private static final int NDegreeTwoNodes = 10; @@ -74,7 +76,7 @@ public void shouldLoadAllConnectedRelationshipRecordsAndTheirFullChainsOfRelatio int relationshipIdInMiddleOfChain = 10; RecordSet records = new RelationshipChainExplorer( relationshipStore ) .exploreRelationshipRecordChainsToDepthTwo( - relationshipStore.getRecord( relationshipIdInMiddleOfChain ) ); + getRecord( relationshipStore, relationshipIdInMiddleOfChain ) ); // then assertEquals( NDegreeTwoNodes * 2, records.size() ); @@ -91,7 +93,7 @@ public void shouldCopeWithAChainThatReferencesNotInUseZeroValueRecords() throws int relationshipIdInMiddleOfChain = 10; RecordSet records = new RelationshipChainExplorer( relationshipStore ) .exploreRelationshipRecordChainsToDepthTwo( - relationshipStore.getRecord( relationshipIdInMiddleOfChain ) ); + getRecord( relationshipStore, relationshipIdInMiddleOfChain ) ); // then int recordsInaccessibleBecauseOfBrokenChain = 3; diff --git a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/report/ConsistencyReporterTest.java b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/report/ConsistencyReporterTest.java index 58f5ce6cd73d3..6b536bdc10cd8 100644 --- a/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/report/ConsistencyReporterTest.java +++ b/community/consistency-check-legacy/src/test/java/org/neo4j/legacy/consistency/report/ConsistencyReporterTest.java @@ -65,7 +65,6 @@ import org.neo4j.legacy.consistency.store.synthetic.LabelScanDocument; import org.neo4j.storageengine.api.schema.SchemaRule; -import static java.lang.String.format; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.doAnswer; @@ -73,6 +72,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.verifyZeroInteractions; + +import static java.lang.String.format; + import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.nodeKey; @RunWith(Suite.class) diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/NodeField.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/NodeField.java index 8cf3542e0ab43..ec89c9fc078d9 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/NodeField.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/NodeField.java @@ -198,7 +198,7 @@ public void checkConsistency( RelationshipRecord relationship, else { // build the node record from cached values with only valid fields as id, inUse, and nextRel. - NodeRecord node = new NodeRecord(valueFrom( relationship )); + NodeRecord node = new NodeRecord( valueFrom( relationship ) ); CacheAccess.Client client = records.cacheAccess().client(); node.setInUse( client.getFromCache( node.getId(), SLOT_SOURCE_OR_TARGET ) != RelationshipLink.SOURCE ); node.setNextRel( client.getFromCache( node.getId(), SLOT_RELATIONSHIP_ID ) ); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/SchemaRecordCheck.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/SchemaRecordCheck.java index 47ac6cf90391a..c4bfd8deb60d3 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/SchemaRecordCheck.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/SchemaRecordCheck.java @@ -157,7 +157,7 @@ public void checkIndexRule( IndexRule rule, DynamicRecord record, RecordAccess r if ( rule.isConstraintIndex() && rule.getOwningConstraint() != null ) { - DynamicRecord previousObligation = constraintObligations.put( rule.getOwningConstraint(), record ); + DynamicRecord previousObligation = constraintObligations.put( rule.getOwningConstraint(), record.clone() ); if ( previousObligation != null ) { engine.report().duplicateObligation( previousObligation ); @@ -171,7 +171,7 @@ public void checkUniquenessConstraintRule( UniquePropertyConstraintRule rule, Dy { checkLabelAndPropertyRule( rule, rule.getPropertyKey(), record, records, engine ); - DynamicRecord previousObligation = indexObligations.put( rule.getOwnedIndex(), record ); + DynamicRecord previousObligation = indexObligations.put( rule.getOwnedIndex(), record.clone() ); if ( previousObligation != null ) { engine.report().duplicateObligation( previousObligation ); @@ -275,7 +275,7 @@ private void checkRelTypeAndPropertyRule( SchemaRule rule, int propertyKey, Dyna private void checkForDuplicates( SchemaRule rule, DynamicRecord record, CheckerEngine engine ) { - DynamicRecord previousContentRecord = verifiedRulesWithRecords.put( rule, record ); + DynamicRecord previousContentRecord = verifiedRulesWithRecords.put( rule, record.clone() ); if ( previousContentRecord != null ) { engine.report().duplicateRuleContent( previousContentRecord ); diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/CacheTask.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/CacheTask.java index f364cc698c681..3cff631c4590e 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/CacheTask.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/cache/CacheTask.java @@ -27,6 +27,8 @@ import org.neo4j.kernel.impl.store.StoreAccess; import org.neo4j.kernel.impl.store.record.NodeRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + /** * Action to be manipulate the {@link CacheAccess} in some way. */ @@ -103,8 +105,9 @@ protected void processCache() { if ( client.getFromCache( nodeId, CacheSlots.NextRelationhip.SLOT_FIRST_IN_TARGET ) == 0 ) { - NodeRecord node = nodeStore.forceGetRecord( nodeId ); - if ( !node.isDense() ) + // TODO reuse record instances? + NodeRecord node = nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ); + if ( node.inUse() && !node.isDense() ) { storeProcessor.processNode( nodeStore, node ); } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CloningRecordIterable.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CloningRecordIterable.java new file mode 100644 index 0000000000000..d16d807db4986 --- /dev/null +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CloningRecordIterable.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.consistency.checking.full; + +import java.util.Iterator; + +import org.neo4j.helpers.collection.PrefetchingIterator; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; + +public class CloningRecordIterable implements Iterable +{ + private final Iterable actual; + + public CloningRecordIterable( Iterable actual ) + { + this.actual = actual; + } + + @Override + public Iterator iterator() + { + return new PrefetchingIterator() + { + private final Iterator actualIterator = actual.iterator(); + + @Override + protected R fetchNextOrNull() + { + return actualIterator.hasNext() ? (R) actualIterator.next().clone() : null; + } + }; + } + + public static Iterable cloned( Iterable actual ) + { + return new CloningRecordIterable<>( actual ); + } +} diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CountsBuilderDecorator.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CountsBuilderDecorator.java index f4875914de574..07202411e8317 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CountsBuilderDecorator.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/CountsBuilderDecorator.java @@ -55,6 +55,7 @@ import static org.neo4j.consistency.checking.full.NodeLabelReader.getListOfLabels; import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.nodeKey; import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.relationshipKey; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; class CountsBuilderDecorator extends CheckDecorator.Adapter { @@ -162,7 +163,7 @@ public void checkCounts( CountsAccessor counts, final ConsistencyReporter report public void visitNodeCount( int labelId, long count ) { nodeEntries.incrementAndGet(); - reporter.forCounts( new CountsEntry( nodeKey( labelId ), count ), CHECK_NODE_COUNT ); + reporter.forCounts( new CountsEntry().initialize( nodeKey( labelId ), count ), CHECK_NODE_COUNT ); listener.add( 1 ); } @@ -170,15 +171,17 @@ public void visitNodeCount( int labelId, long count ) public void visitRelationshipCount( int startLabelId, int relTypeId, int endLabelId, long count ) { relationshipEntries.incrementAndGet(); - reporter.forCounts( new CountsEntry( relationshipKey( startLabelId, relTypeId, endLabelId ), count ), + reporter.forCounts( + new CountsEntry().initialize( relationshipKey( startLabelId, relTypeId, endLabelId ), count ), CHECK_RELATIONSHIP_COUNT ); listener.add( 1 ); } } ); - reporter.forCounts( new CountsEntry( nodeKey( WILDCARD ), nodeEntries.get() ), CHECK_NODE_KEY_COUNT ); reporter.forCounts( - new CountsEntry( relationshipKey( WILDCARD, WILDCARD, WILDCARD ), relationshipEntries.get() ), - CHECK_RELATIONSHIP_KEY_COUNT ); + new CountsEntry().initialize( nodeKey( WILDCARD ), nodeEntries.get() ), CHECK_NODE_KEY_COUNT ); + reporter.forCounts( + new CountsEntry().initialize( relationshipKey( WILDCARD, WILDCARD, WILDCARD ), + relationshipEntries.get() ), CHECK_RELATIONSHIP_KEY_COUNT ); listener.done(); } @@ -367,6 +370,6 @@ private static Set labelsFor( RecordStore nodeStore, RecordAccess recordAccess, long nodeId ) { - return getListOfLabels( nodeStore.forceGetRecord( nodeId ), recordAccess, engine ); + return getListOfLabels( nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ), recordAccess, engine ); } } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/FullCheck.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/FullCheck.java index 1a8b8eae94e46..d5f479b2effd0 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/FullCheck.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/FullCheck.java @@ -51,6 +51,7 @@ import org.neo4j.logging.Log; import static org.neo4j.consistency.report.ConsistencyReporter.NO_MONITOR; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; public class FullCheck { @@ -165,7 +166,7 @@ private static T[] readAllRecords( Class type, T[] records = (T[]) Array.newInstance( type, (int) store.getHighId() ); for ( int i = 0; i < records.length; i++ ) { - records[i] = store.forceGetRecord( i ); + records[i] = store.getRecord( i, store.newRecord(), FORCE ); } return records; } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/IterableStore.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/IterableStore.java index ffb5779f6778d..5998b30ce1d1e 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/IterableStore.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/IterableStore.java @@ -24,10 +24,12 @@ import org.neo4j.helpers.collection.BoundedIterable; import org.neo4j.kernel.impl.store.RecordStore; -import org.neo4j.kernel.impl.store.RecordStore.Scanner; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import static org.neo4j.consistency.checking.full.CloningRecordIterable.cloned; import static org.neo4j.kernel.impl.store.RecordStore.IN_USE; +import static org.neo4j.kernel.impl.store.RecordStore.Scanner.scan; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; public class IterableStore implements BoundedIterable { @@ -54,7 +56,7 @@ public void close() throws IOException @Override public Iterator iterator() { - return Scanner.scan( store, forward, IN_USE ).iterator(); + return cloned( scan( store, forward, IN_USE ) ).iterator(); } public void warmUpCache() @@ -62,16 +64,10 @@ public void warmUpCache() int recordsPerPage = store.getRecordsPerPage(); long id = 0; long half = store.getHighId() / 2; + RECORD record = store.newRecord(); while ( id < half ) { - try - { - store.getRecord( id ); - } - catch ( Exception e ) - { - // ignore and continue - } + store.getRecord( id, record, FORCE ); id += (recordsPerPage-1); } } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/NodeLabelReader.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/NodeLabelReader.java index 74c2b7566e939..ae432229ed9dd 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/NodeLabelReader.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/NodeLabelReader.java @@ -42,6 +42,8 @@ import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.Record; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class NodeLabelReader { public static Set getListOfLabels( @@ -99,7 +101,7 @@ public static long[] getListOfLabels( NodeRecord nodeRecord, RecordStore getPropertyRecordChain( NodeRecord nodeRecord ) @@ -57,7 +61,7 @@ public Collection getPropertyRecordChain( long firstId ) List toReturn = new LinkedList<>(); while ( nextProp != Record.NO_NEXT_PROPERTY.intValue() ) { - PropertyRecord propRecord = storeAccess.getPropertyStore().forceGetRecord( nextProp ); + PropertyRecord propRecord = propertyStore.getRecord( nextProp, propertyStore.newRecord(), FORCE ); toReturn.add( propRecord ); nextProp = propRecord.getNextProp(); } @@ -99,8 +103,7 @@ public DefinedProperty propertyValue( PropertyBlock block ) @Override public Property getProperty( long nodeId, int propertyKeyId ) { - NodeRecord nodeRecord = storeAccess.getNodeStore().forceGetRecord( nodeId ); - if ( nodeRecord != null ) + if ( nodeStore.getRecord( nodeId, nodeRecord, FORCE ).inUse() ) { for ( PropertyBlock block : propertyBlocks( nodeRecord ) ) { diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java index cf48cc6e42ab1..7ef75557bbc17 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/checking/full/StoreProcessor.java @@ -44,6 +44,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; import static org.neo4j.consistency.checking.cache.DefaultCacheAccess.DEFAULT_QUEUE_SIZE; +import static org.neo4j.consistency.checking.full.CloningRecordIterable.cloned; import static org.neo4j.consistency.checking.full.RecordDistributor.distributeRecords; import static org.neo4j.kernel.impl.store.RecordStore.Scanner.scan; @@ -224,6 +225,6 @@ public void close() } }; distributeRecords( numberOfThreads, getClass().getSimpleName(), qSize, - scan( store, stage.isForward(), filters ), progressListener, processor ); + cloned( scan( store, stage.isForward(), filters ) ), progressListener, processor ); } } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/repair/OwningNodeRelationshipChain.java b/community/consistency-check/src/main/java/org/neo4j/consistency/repair/OwningNodeRelationshipChain.java index 906aea20bd43a..e7eaa2afbd6d4 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/repair/OwningNodeRelationshipChain.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/repair/OwningNodeRelationshipChain.java @@ -21,18 +21,21 @@ import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipRecord; public class OwningNodeRelationshipChain { private final RelationshipChainExplorer relationshipChainExplorer; private final RecordStore nodeStore; + private final NodeRecord nodeRecord; public OwningNodeRelationshipChain( RelationshipChainExplorer relationshipChainExplorer, RecordStore nodeStore ) { this.relationshipChainExplorer = relationshipChainExplorer; this.nodeStore = nodeStore; + this.nodeRecord = nodeStore.newRecord(); } public RecordSet findRelationshipChainsThatThisRecordShouldBelongTo( @@ -42,11 +45,9 @@ public RecordSet findRelationshipChainsThatThisRecordShouldB for ( RelationshipNodeField field : RelationshipNodeField.values() ) { long nodeId = field.get( relationship ); - NodeRecord nodeRecord = nodeStore.forceGetRecord( nodeId ); + nodeStore.getRecord( nodeId, nodeRecord, RecordLoad.FORCE ); records.addAll( relationshipChainExplorer.followChainFromNode( nodeId, nodeRecord.getNextRel() ) ); } return records; } - - } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/repair/RelationshipChainExplorer.java b/community/consistency-check/src/main/java/org/neo4j/consistency/repair/RelationshipChainExplorer.java index e79f92895ba12..b44be8720b943 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/repair/RelationshipChainExplorer.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/repair/RelationshipChainExplorer.java @@ -24,6 +24,8 @@ import static org.neo4j.consistency.repair.RelationshipChainDirection.NEXT; import static org.neo4j.consistency.repair.RelationshipChainDirection.PREV; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; public class RelationshipChainExplorer { @@ -61,26 +63,24 @@ private RecordSet expandChainInBothDirections( RelationshipR return expandChain( record, nodeId, PREV ).union( expandChain( record, nodeId, NEXT ) ); } - protected RecordSet followChainFromNode(long nodeId, long relationshipId ) + protected RecordSet followChainFromNode( long nodeId, long relationshipId ) { - RelationshipRecord record = recordStore.getRecord( relationshipId ); - return expandChain( record, nodeId, NEXT ); + return expandChain( recordStore.getRecord( relationshipId, recordStore.newRecord(), NORMAL ), nodeId, NEXT ); } private RecordSet expandChain( RelationshipRecord record, long nodeId, RelationshipChainDirection direction ) { - RecordSet chain = new RecordSet(); + RecordSet chain = new RecordSet<>(); chain.add( record ); RelationshipRecord currentRecord = record; long nextRelId = direction.fieldFor( nodeId, currentRecord ).relOf( currentRecord ); while ( currentRecord.inUse() && !direction.fieldFor( nodeId, currentRecord ).endOfChain( currentRecord ) ) { - currentRecord = recordStore.forceGetRecord( nextRelId ); + currentRecord = recordStore.getRecord( nextRelId, recordStore.newRecord(), FORCE ); chain.add( currentRecord ); nextRelId = direction.fieldFor( nodeId, currentRecord ).relOf( currentRecord ); } return chain; } - } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatistics.java b/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatistics.java index f664e3d7d8198..5068918950fa5 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatistics.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatistics.java @@ -24,12 +24,13 @@ import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import static java.lang.String.format; /** - * Keeps access statistics about a store, i.e. identifying {@link RecordStore#getRecord(long)} patterns - * and how random the access is. + * Keeps access statistics about a store, i.e. identifying + * {@link RecordStore#getRecord(long, AbstractBaseRecord, RecordLoad)} patterns and how random the access is. */ public class AccessStatistics { diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatsKeepingStoreAccess.java b/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatsKeepingStoreAccess.java index 4913ac737fd2c..f49f998ce540e 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatsKeepingStoreAccess.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/statistics/AccessStatsKeepingStoreAccess.java @@ -24,6 +24,7 @@ import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.StoreAccess; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; /** * {@link StoreAccess} that decorates each store, feeding stats about access into {@link AccessStatistics}. @@ -63,17 +64,10 @@ protected AccessStats getAccessStats() } @Override - public RECORD getRecord( long id ) + public RECORD getRecord( long id, RECORD record, RecordLoad load ) { accessStats.upRead( id ); - return super.getRecord( id ); - } - - @Override - public RECORD forceGetRecord( long id ) - { - accessStats.upRead( id ); - return super.forceGetRecord( id ); + return super.getRecord( id, record, load ); } } } diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/store/DirectRecordAccess.java b/community/consistency-check/src/main/java/org/neo4j/consistency/store/DirectRecordAccess.java index fecde5f1d0d5f..d00fdd404be5f 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/store/DirectRecordAccess.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/store/DirectRecordAccess.java @@ -38,6 +38,8 @@ import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class DirectRecordAccess implements RecordAccess { final StoreAccess access; @@ -155,7 +157,6 @@ public RecordReference propertyKeyName( int id ) return referenceTo( access.getPropertyKeyNameStore(), id ); } - @Override public RecordReference graph() { @@ -170,7 +171,7 @@ public boolean shouldCheck( long id, MultiPassStore store ) DirectRecordReference referenceTo( RecordStore store, long id ) { - return new DirectRecordReference<>( store.forceGetRecord( id ), this); + return new DirectRecordReference<>( store.getRecord( id, store.newRecord(), FORCE ), this); } @Override diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/CountsEntry.java b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/CountsEntry.java index e117a5da5a142..f8ee22f6c19ca 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/CountsEntry.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/CountsEntry.java @@ -30,14 +30,34 @@ */ public class CountsEntry extends AbstractBaseRecord { - private final CountsKey key; - private final long count; + private CountsKey key; + private long count; + + // ========= TRANSITIONAL CONSTRUCTORS ============= + public CountsEntry() + { + } public CountsEntry( CountsKey key, long count ) { + initialize( key, count ); + } + // ========= TRANSITIONAL CONSTRUCTORS ============= + + public CountsEntry initialize( CountsKey key, long count ) + { + super.initialize( true ); this.key = key; this.count = count; - setInUse( true ); + return this; + } + + @Override + public void clear() + { + super.initialize( false ); + this.key = null; + this.count = 0; } @Override @@ -46,6 +66,12 @@ public String toString() return "CountsEntry[" + key + ": " + count + "]"; } + @Override + public void setId( long id ) + { + throw new UnsupportedOperationException(); + } + @Override public long getLongId() { diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/IndexEntry.java b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/IndexEntry.java index b58cc60d15438..3ab2a3c3e7dc3 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/IndexEntry.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/IndexEntry.java @@ -30,7 +30,13 @@ public class IndexEntry extends Abstract64BitRecord public IndexEntry( long nodeId ) { super( nodeId ); - setInUse( true ); + initialize( true ); + } + + @Override + public void clear() + { + initialize( false ); } @Override diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/LabelScanDocument.java b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/LabelScanDocument.java index 16ac0a0748674..af2e41bc382d6 100644 --- a/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/LabelScanDocument.java +++ b/community/consistency-check/src/main/java/org/neo4j/consistency/store/synthetic/LabelScanDocument.java @@ -28,13 +28,20 @@ */ public class LabelScanDocument extends Abstract64BitRecord { - private final NodeLabelRange nodeLabelRange; + private NodeLabelRange nodeLabelRange; public LabelScanDocument( NodeLabelRange nodeLabelRange ) { super( nodeLabelRange.id() ); + initialize( true ); this.nodeLabelRange = nodeLabelRange; - setInUse( true ); + } + + @Override + public void clear() + { + initialize( false ); + this.nodeLabelRange = null; } public NodeLabelRange getNodeLabelRange() diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/DynamicRecordCheckTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/DynamicRecordCheckTest.java index d7c8c23fd0e40..5425c494fb401 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/DynamicRecordCheckTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/DynamicRecordCheckTest.java @@ -271,8 +271,8 @@ public static RecordStore configureDynamicStore( int blockSize ) { @SuppressWarnings( "unchecked" ) RecordStore mock = mock( RecordStore.class ); - when( mock.getRecordSize() ).thenReturn( blockSize + AbstractDynamicStore.BLOCK_HEADER_SIZE ); - when( mock.getRecordHeaderSize() ).thenReturn( AbstractDynamicStore.BLOCK_HEADER_SIZE ); + when( mock.getRecordSize() ).thenReturn( blockSize + AbstractDynamicStore.RECORD_HEADER_SIZE ); + when( mock.getRecordHeaderSize() ).thenReturn( AbstractDynamicStore.RECORD_HEADER_SIZE ); return mock; } } diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java index 299115d6b9dc8..14e2945372cd2 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/FullCheckIntegrationTest.java @@ -136,6 +136,7 @@ import static org.neo4j.kernel.impl.store.record.Record.NO_PREV_RELATIONSHIP; import static org.neo4j.kernel.impl.store.record.RelationshipPropertyExistenceConstraintRule .relPropertyExistenceConstraintRule; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; import static org.neo4j.kernel.impl.util.Bits.bits; import static org.neo4j.test.Property.property; import static org.neo4j.test.Property.set; @@ -444,7 +445,7 @@ public void shouldReportIndexInconsistencies() throws Exception // given for ( Long indexedNodeId : indexedNodes ) { - fixture.directStoreAccess().nativeStores().getNodeStore().forceUpdateRecord( + fixture.directStoreAccess().nativeStores().getNodeStore().updateRecord( notInUse( new NodeRecord( indexedNodeId, false, -1, -1 ) ) ); } @@ -484,7 +485,7 @@ public void shouldNotReportIndexInconsistenciesIfIndexIsFailed() throws Exceptio for ( Long indexedNodeId : indexedNodes ) { - storeAccess.nativeStores().getNodeStore().forceUpdateRecord( + storeAccess.nativeStores().getNodeStore().updateRecord( notInUse( new NodeRecord( indexedNodeId, false, -1, -1 ) ) ); } @@ -1178,7 +1179,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getRelationshipTypeNameStore().forceGetRecord( inconsistentName.get() ); + DynamicRecord record = access.getRelationshipTypeNameStore().getRecord( inconsistentName.get(), + access.getRelationshipTypeNameStore().newRecord(), FORCE ); record.setNextBlock( record.getId() ); access.getRelationshipTypeNameStore().updateRecord( record ); @@ -1206,7 +1208,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getPropertyKeyNameStore().forceGetRecord( inconsistentName.get()+1 ); + DynamicRecord record = access.getPropertyKeyNameStore().getRecord( inconsistentName.get()+1, + access.getPropertyKeyNameStore().newRecord(), FORCE ); record.setNextBlock( record.getId() ); access.getPropertyKeyNameStore().updateRecord( record ); @@ -1224,7 +1227,8 @@ public void shouldReportRelationshipTypeInconsistencies() throws Exception // given StoreAccess access = fixture.directStoreAccess().nativeStores(); RecordStore relTypeStore = access.getRelationshipTypeTokenStore(); - RelationshipTypeTokenRecord record = relTypeStore.forceGetRecord( (int) relTypeStore.nextId() ); + RelationshipTypeTokenRecord record = relTypeStore.getRecord( (int) relTypeStore.nextId(), + relTypeStore.newRecord(), FORCE ); record.setNameId( 20 ); record.setInUse( true ); relTypeStore.updateRecord( record ); @@ -1243,7 +1247,8 @@ public void shouldReportLabelInconsistencies() throws Exception { // given StoreAccess access = fixture.directStoreAccess().nativeStores(); - LabelTokenRecord record = access.getLabelTokenStore().forceGetRecord( 1 ); + LabelTokenRecord record = access.getLabelTokenStore().getRecord( 1, + access.getLabelTokenStore().newRecord(), FORCE ); record.setNameId( 20 ); record.setInUse( true ); access.getLabelTokenStore().updateRecord( record ); @@ -1272,7 +1277,8 @@ protected void transactionData( GraphStoreFixture.TransactionDataBuilder tx, } } ); StoreAccess access = fixture.directStoreAccess().nativeStores(); - DynamicRecord record = access.getPropertyKeyNameStore().forceGetRecord( inconsistentKey.get()+1 ); + DynamicRecord record = access.getPropertyKeyNameStore().getRecord( inconsistentKey.get()+1, + access.getPropertyKeyNameStore().newRecord(), FORCE ); record.setInUse( false ); access.getPropertyKeyNameStore().updateRecord( record ); diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java index b3597afb56337..4cc5df6c401d7 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeCorrectlyIndexedCheckTest.java @@ -48,12 +48,14 @@ import org.neo4j.storageengine.api.schema.IndexReader; import org.neo4j.storageengine.api.schema.IndexSampler; -import static java.util.Arrays.asList; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; + +import static java.util.Arrays.asList; + import static org.neo4j.collection.primitive.PrimitiveLongCollections.emptyIterator; import static org.neo4j.kernel.api.properties.Property.stringProperty; import static org.neo4j.kernel.impl.store.record.IndexRule.constraintIndexRule; diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java index a0da8ba4312e8..9937abda07a58 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/StoreProcessorTest.java @@ -21,15 +21,19 @@ import org.junit.Test; import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.neo4j.consistency.checking.CheckDecorator; import org.neo4j.consistency.checking.cache.CacheAccess; import org.neo4j.consistency.report.ConsistencyReport; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RecordStoreUtil.NewNodeRecordAnswer; +import org.neo4j.kernel.impl.store.RecordStoreUtil.ReadNodeAnswer; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -45,17 +49,19 @@ public void shouldProcessAllTheRecordsInAStore() throws Exception StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, mock( ConsistencyReport.Reporter.class ), Stage.SEQUENTIAL_FORWARD, CacheAccess.EMPTY ); RecordStore recordStore = mock( RecordStore.class ); + when( recordStore.newRecord() ).thenAnswer( new NewNodeRecordAnswer() ); when( recordStore.getHighId() ).thenReturn( 3L ); - when( recordStore.forceGetRecord( any( Long.class ) ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); + when( recordStore.getRecord( anyLong(), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); // when processor.applyFiltered( recordStore ); // then - verify( recordStore ).forceGetRecord( 0 ); - verify( recordStore ).forceGetRecord( 1 ); - verify( recordStore ).forceGetRecord( 2 ); - verify( recordStore, never() ).forceGetRecord( 3 ); + verify( recordStore ).getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore, never() ).getRecord( eq( 3L ), any( NodeRecord.class ), any( RecordLoad.class ) ); } @SuppressWarnings("unchecked") @@ -66,16 +72,20 @@ public void shouldStopProcessingRecordsWhenSignalledToStop() throws Exception final StoreProcessor processor = new StoreProcessor( CheckDecorator.NONE, mock( ConsistencyReport.Reporter.class ), Stage.SEQUENTIAL_FORWARD, CacheAccess.EMPTY ); RecordStore recordStore = mock( RecordStore.class ); + when( recordStore.newRecord() ).thenAnswer( new NewNodeRecordAnswer() ); when( recordStore.getHighId() ).thenReturn( 4L ); - when( recordStore.forceGetRecord( 0L ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); - when( recordStore.forceGetRecord( 1L ) ).thenReturn( new NodeRecord( 0, false, 0, 0 ) ); - when( recordStore.forceGetRecord( 2L ) ).thenAnswer( new Answer() + when( recordStore.getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); + when( recordStore.getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) ); + when( recordStore.getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, 0, 0 ) { @Override public NodeRecord answer( InvocationOnMock invocation ) throws Throwable { processor.stop(); - return new NodeRecord( 0, false, 0, 0 ); + return super.answer( invocation ); } } ); @@ -83,9 +93,9 @@ public NodeRecord answer( InvocationOnMock invocation ) throws Throwable processor.applyFiltered( recordStore ); // then - verify( recordStore ).forceGetRecord( 0 ); - verify( recordStore ).forceGetRecord( 1 ); - verify( recordStore ).forceGetRecord( 2 ); - verify( recordStore, never() ).forceGetRecord( 3 ); + verify( recordStore ).getRecord( eq( 0L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 1L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore ).getRecord( eq( 2L ), any( NodeRecord.class ), any( RecordLoad.class ) ); + verify( recordStore, never() ).getRecord( eq( 3L ), any( NodeRecord.class ), any( RecordLoad.class ) ); } } diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/repair/OwningNodeRelationshipChainTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/repair/OwningNodeRelationshipChainTest.java index a9a2ed9cc6ec5..9bec79e6774f8 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/repair/OwningNodeRelationshipChainTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/repair/OwningNodeRelationshipChainTest.java @@ -25,10 +25,14 @@ import org.junit.Test; import org.neo4j.kernel.impl.store.RecordStore; +import org.neo4j.kernel.impl.store.RecordStoreUtil.ReadNodeAnswer; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import static org.junit.Assert.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,9 +44,9 @@ public class OwningNodeRelationshipChainTest public void shouldFindBothChainsThatTheRelationshipRecordShouldBelongTo() throws Exception { // given - int node1 = 101, node1Rel = 1001; - int node2 = 201, node2Rel = 2001; - int sharedRel = 1000; + long node1 = 101, node1Rel = 1001; + long node2 = 201, node2Rel = 2001; + long sharedRel = 1000; int relType = 0; RecordSet node1RelChain = RecordSet.asSet( @@ -56,10 +60,11 @@ public void shouldFindBothChainsThatTheRelationshipRecordShouldBelongTo() throws @SuppressWarnings("unchecked") RecordStore recordStore = mock( RecordStore.class ); - when( recordStore.forceGetRecord( node1 ) ).thenReturn( - new NodeRecord( node1, false, node1Rel, NO_NEXT_PROPERTY.intValue() ) ); - when( recordStore.forceGetRecord( node2 ) ).thenReturn( - new NodeRecord( node2, false, node2Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.getRecord( eq( node1 ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, node1Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.getRecord( eq( node2 ), any( NodeRecord.class ), any( RecordLoad.class ) ) ) + .thenAnswer( new ReadNodeAnswer( false, node2Rel, NO_NEXT_PROPERTY.intValue() ) ); + when( recordStore.newRecord() ).thenReturn( new NodeRecord( -1 ) ); RelationshipChainExplorer relationshipChainExplorer = mock( RelationshipChainExplorer.class ); when( relationshipChainExplorer.followChainFromNode( node1, node1Rel ) ).thenReturn( node1RelChain ); @@ -95,6 +100,4 @@ public void describeTo( Description description ) } }; } - - } diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java b/community/consistency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java index d437db3d5355a..811fd25d03c63 100644 --- a/community/consistency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java +++ b/community/consistency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java @@ -42,6 +42,8 @@ import static org.junit.Assert.assertEquals; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + public class RelationshipChainExplorerTest { private static final int NDegreeTwoNodes = 10; @@ -74,7 +76,7 @@ public void shouldLoadAllConnectedRelationshipRecordsAndTheirFullChainsOfRelatio int relationshipIdInMiddleOfChain = 10; RecordSet records = new RelationshipChainExplorer( relationshipStore ) .exploreRelationshipRecordChainsToDepthTwo( - relationshipStore.getRecord( relationshipIdInMiddleOfChain ) ); + relationshipStore.getRecord( relationshipIdInMiddleOfChain, relationshipStore.newRecord(), NORMAL ) ); // then assertEquals( NDegreeTwoNodes * 2, records.size() ); @@ -91,7 +93,7 @@ public void shouldCopeWithAChainThatReferencesNotInUseZeroValueRecords() throws int relationshipIdInMiddleOfChain = 10; RecordSet records = new RelationshipChainExplorer( relationshipStore ) .exploreRelationshipRecordChainsToDepthTwo( - relationshipStore.getRecord( relationshipIdInMiddleOfChain ) ); + relationshipStore.getRecord( relationshipIdInMiddleOfChain, relationshipStore.newRecord(), NORMAL ) ); // then int recordsInaccessibleBecauseOfBrokenChain = 3; diff --git a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialReadPageCursor.java b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialReadPageCursor.java index 5902e9258cebd..6601aaf4e5e61 100644 --- a/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialReadPageCursor.java +++ b/community/io/src/test/java/org/neo4j/adversaries/pagecache/AdversarialReadPageCursor.java @@ -81,13 +81,15 @@ public void putByte( int offset, byte value ) @Override public long getLong() { - return currentReadIsInconsistent ? 0 : delegate.getLong(); + long result = delegate.getLong(); + return currentReadIsInconsistent ? 0 : result; } @Override public long getLong( int offset ) { - return currentReadIsInconsistent ? 0 : delegate.getLong( offset ); + long result = delegate.getLong( offset ); + return currentReadIsInconsistent ? 0 : result; } @Override @@ -105,13 +107,15 @@ public void putLong( int offset, long value ) @Override public int getInt() { - return currentReadIsInconsistent ? 0 : delegate.getInt(); + int result = delegate.getInt(); + return currentReadIsInconsistent ? 0 : result; } @Override public int getInt( int offset ) { - return currentReadIsInconsistent ? 0 : delegate.getInt( offset ); + int result = delegate.getInt( offset ); + return currentReadIsInconsistent ? 0 : result; } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/DiskLayer.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/DiskLayer.java index fcfe6d577f621..863f9fbe2ffa4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/DiskLayer.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/DiskLayer.java @@ -50,7 +50,6 @@ import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder; import org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder; import org.neo4j.kernel.impl.core.TokenNotFoundException; -import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.RelationshipStore; @@ -61,6 +60,7 @@ import org.neo4j.kernel.impl.store.record.NodePropertyConstraintRule; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PropertyConstraintRule; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipPropertyConstraintRule; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.transaction.state.PropertyLoader; @@ -78,6 +78,7 @@ import static org.neo4j.helpers.collection.Iterables.filter; import static org.neo4j.helpers.collection.Iterables.map; import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; import static org.neo4j.register.Registers.newDoubleLongRegister; /** @@ -426,12 +427,8 @@ public void relationshipVisit( long relationshipId RelationshipVisitor relationshipVisitor ) throws EntityNotFoundException, EXCEPTION { // TODO Please don't create a record for this, it's ridiculous - RelationshipRecord record; - try - { - record = relationshipStore.getRecord( relationshipId ); - } - catch ( InvalidRecordException e ) + RelationshipRecord record = relationshipStore.newRecord(); + if ( !relationshipStore.getRecord( relationshipId, record, FORCE ).inUse() ) { throw new EntityNotFoundException( EntityType.RELATIONSHIP, relationshipId ); } @@ -445,7 +442,7 @@ public PrimitiveLongIterator nodesGetAll() { private long highId = nodeStore.getHighestPossibleIdInUse(); private long currentId; - private final NodeRecord reusableNodeRecord = new NodeRecord( -1 ); // reused + private final NodeRecord record = new NodeRecord( -1 ); // reused @Override protected boolean fetchNext() @@ -456,8 +453,8 @@ protected boolean fetchNext() { try { - NodeRecord record = nodeStore.loadRecord( currentId, reusableNodeRecord ); - if ( record != null && record.inUse() ) + nodeStore.getRecord( currentId, record, RecordLoad.CHECK ); + if ( record.inUse() ) { return next( record.getId() ); } @@ -501,7 +498,7 @@ protected boolean fetchNext() { try { - if ( relationshipStore.fillRecord( currentId, reusableRecord, CHECK ) && reusableRecord.inUse() ) + if ( relationshipStore.getRecord( currentId, reusableRecord, CHECK ).inUse() ) { return next( reusableRecord.getId() ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/PropertyRecordCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/PropertyRecordCursor.java index d215993f6d4b8..3fae7bde5d5da 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/PropertyRecordCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/PropertyRecordCursor.java @@ -23,6 +23,8 @@ import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + /** * Cursor over {@link PropertyRecord} instances. * @@ -32,9 +34,8 @@ public class PropertyRecordCursor { private long currentRecordId; - private PropertyStore propertyStore; - - private PropertyRecord record; + private final PropertyStore propertyStore; + private final PropertyRecord record; public PropertyRecordCursor( PropertyRecord record, PropertyStore propertyStore ) { @@ -49,17 +50,13 @@ public void init( long firstRecordId) public boolean next() { - if (currentRecordId != Record.NO_NEXT_PROPERTY.intValue()) + if ( currentRecordId != Record.NO_NEXT_PROPERTY.intValue() ) { - record.setId( currentRecordId ); - propertyStore.getRecord( record ); + propertyStore.getRecord( currentRecordId, record, NORMAL ); currentRecordId = record.getNextProp(); - return true; - } else - { - return false; } + return false; } public PropertyRecord getRecord() diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java index d36e2498c9040..4462af36ad54a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java @@ -49,6 +49,8 @@ import static org.neo4j.kernel.impl.locking.LockService.NO_LOCK_SERVICE; import static org.neo4j.kernel.impl.store.NodeLabelsField.parseLabelsField; +import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; /** * Base cursor for nodes. @@ -103,7 +105,7 @@ protected StoreNodeRelationshipCursor create() { return new StoreNodeRelationshipCursor( new RelationshipRecord( -1 ), neoStores, - new RelationshipGroupRecord( -1, -1 ), storeStatement, this, lockService ); + new RelationshipGroupRecord( -1 ), storeStatement, this, lockService ); } }; singlePropertyCursor = new InstanceCache() @@ -157,8 +159,7 @@ private Lock shortLivedReadLock() try { // It's safer to re-read the node record here, specifically nextProp, after acquiring the lock - nodeStore.loadRecord( nodeRecord.getId(), nodeRecord ); - if ( !nodeRecord.inUse() ) + if ( !nodeStore.getRecord( nodeRecord.getId(), nodeRecord, CHECK ).inUse() ) { // So it looks like the node has been deleted. The current behavior of NodeStore#loadRecord // is to only set the inUse field on loading an unused record. This should (and will) @@ -215,6 +216,7 @@ public Cursor relationshipTypes() { private long groupId = nodeRecord.getNextRel(); private final IntValue value = new IntValue(); + private final RelationshipGroupRecord group = relationshipGroupStore.newRecord(); @Override public boolean next() @@ -224,7 +226,7 @@ public boolean next() return false; } - RelationshipGroupRecord group = relationshipGroupStore.getRecord( groupId ); + relationshipGroupStore.getRecord( groupId, group, NORMAL ); try { value.setValue( group.getType() ); @@ -293,9 +295,10 @@ public int degree( Direction direction ) { long groupId = nodeRecord.getNextRel(); long count = 0; + RelationshipGroupRecord group = relationshipGroupStore.newRecord(); while ( groupId != Record.NO_NEXT_RELATIONSHIP.intValue() ) { - RelationshipGroupRecord group = relationshipGroupStore.getRecord( groupId ); + relationshipGroupStore.getRecord( groupId, group, NORMAL ); count += nodeDegreeByDirection( group, direction ); groupId = group.getNext(); } @@ -321,9 +324,10 @@ public int degree( Direction direction, int relType ) if ( nodeRecord.isDense() ) { long groupId = nodeRecord.getNextRel(); + RelationshipGroupRecord group = relationshipGroupStore.newRecord(); while ( groupId != Record.NO_NEXT_RELATIONSHIP.intValue() ) { - RelationshipGroupRecord group = relationshipGroupStore.getRecord( groupId ); + relationshipGroupStore.getRecord( groupId, group, NORMAL ); if ( group.getType() == relType ) { return (int) nodeDegreeByDirection( group, direction ); @@ -408,7 +412,8 @@ private long countByFirstPrevPointer( long relationshipId ) { return 0; } - RelationshipRecord record = relationshipStore.getRecord( relationshipId ); + RelationshipRecord record = relationshipStore.getRecord( relationshipId, + relationshipStore.newRecord(), NORMAL ); if ( record.getFirstNode() == nodeRecord.getId() ) { return record.getFirstPrevRel(); @@ -508,6 +513,7 @@ private class DegreeItemDenseCursor implements Cursor, DegreeItem private int type; private long outgoing; private long incoming; + private final RelationshipGroupRecord group = relationshipGroupStore.newRecord(); public DegreeItemDenseCursor( long groupId ) { @@ -519,7 +525,7 @@ public boolean next() { if ( groupId != Record.NO_NEXT_RELATIONSHIP.intValue() ) { - RelationshipGroupRecord group = relationshipGroupStore.getRecord( groupId ); + relationshipGroupStore.getRecord( groupId, group, NORMAL ); this.type = group.getType(); long loop = countByFirstPrevPointer( group.getFirstLoop() ); outgoing = countByFirstPrevPointer( group.getFirstOut() ) + loop; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java index f685799ca1450..3c957c39d47d1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java @@ -33,7 +33,7 @@ import org.neo4j.storageengine.api.RelationshipItem; import static org.neo4j.kernel.impl.locking.LockService.NO_LOCK_SERVICE; -import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; /** * Base cursor for relationships. @@ -124,8 +124,7 @@ private Lock shortLivedReadLock() try { // It's safer to re-read the relationship record here, specifically nextProp, after acquiring the lock - relationshipStore.fillRecord( relationshipRecord.getId(), relationshipRecord, FORCE ); - if ( !relationshipRecord.inUse() ) + if ( !relationshipStore.getRecord( relationshipRecord.getId(), relationshipRecord, CHECK ).inUse() ) { // So it looks like the node has been deleted. The current behavior of RelationshipStore#fillRecord // w/ FORCE is to only set the inUse field on loading an unused record. This should (and will) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorNodeCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorNodeCursor.java index 7441fcc304571..2b5379e3da3ee 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorNodeCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorNodeCursor.java @@ -27,6 +27,8 @@ import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.record.NodeRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; + /** * Cursor for iterating a set of nodes. It is attached to an iterator, typically from * an index seek or similar. @@ -57,8 +59,7 @@ public boolean next() { while ( iterator != null && iterator.hasNext() ) { - NodeRecord record = nodeStore.loadRecord( iterator.next(), nodeRecord ); - if ( record != null && record.inUse() ) + if ( nodeStore.getRecord( iterator.next(), nodeRecord, CHECK ).inUse() ) { return true; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorRelationshipCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorRelationshipCursor.java index 9e89c21665bf0..cec54762c08a6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorRelationshipCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreIteratorRelationshipCursor.java @@ -56,8 +56,7 @@ public boolean next() { while ( iterator != null && iterator.hasNext() ) { - if ( relationshipStore.fillRecord( iterator.next(), relationshipRecord, - CHECK ) && relationshipRecord.inUse() ) + if ( relationshipStore.getRecord( iterator.next(), relationshipRecord, CHECK ).inUse() ) { return true; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursor.java index 157803ba030ef..d8e1fd1b5c454 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursor.java @@ -24,13 +24,15 @@ import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.NeoStores; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.record.Record; -import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.storageengine.api.Direction; +import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP; +import static org.neo4j.kernel.impl.store.record.Record.NULL_REFERENCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + /** * Cursor over the chain of relationships from one node. *

@@ -40,7 +42,6 @@ public class StoreNodeRelationshipCursor extends StoreAbstractRelationshipCursor { private final RelationshipGroupRecord groupRecord; private final Consumer instanceCache; - private boolean isDense; private long relationshipId; private long fromNodeId; @@ -48,7 +49,6 @@ public class StoreNodeRelationshipCursor extends StoreAbstractRelationshipCursor private int[] relTypes; private int groupChainIndex; private boolean end; - private final RelationshipGroupStore groupStore; public StoreNodeRelationshipCursor( RelationshipRecord relationshipRecord, NeoStores neoStores, @@ -58,10 +58,8 @@ public StoreNodeRelationshipCursor( RelationshipRecord relationshipRecord, LockService lockService ) { super( relationshipRecord, neoStores, storeStatement, lockService ); - this.groupRecord = groupRecord; this.instanceCache = instanceCache; - this.groupStore = neoStores.getRelationshipGroupStore(); } public StoreNodeRelationshipCursor init( boolean isDense, @@ -87,7 +85,7 @@ public StoreNodeRelationshipCursor init( boolean isDense, if ( isDense && relationshipId != Record.NO_NEXT_RELATIONSHIP.intValue() ) { - groupStore.forceGetRecord( relationshipId, groupRecord ); + relationshipGroupStore.getRecord( firstRelId, groupRecord, FORCE ); relationshipId = nextChainStart(); } else @@ -101,11 +99,9 @@ public StoreNodeRelationshipCursor init( boolean isDense, @Override public boolean next() { - while ( relationshipId != Record.NO_NEXT_RELATIONSHIP.intValue() ) + while ( relationshipId != NO_NEXT_RELATIONSHIP.intValue() ) { - relationshipRecord.setId( relationshipId ); - - relationshipStore.fillRecord( relationshipId, relationshipRecord, RecordLoad.FORCE ); + relationshipStore.getRecord( relationshipId, relationshipRecord, FORCE ); // If we end up on a relationship record that isn't in use there's a good chance there // have been a concurrent transaction deleting this record under or feet. Since we don't @@ -170,7 +166,7 @@ else if ( relationshipRecord.getSecondNode() == fromNodeId ) // If there are no more relationships, and this is from a dense node, then // traverse the next group - if ( relationshipId == Record.NO_NEXT_RELATIONSHIP.intValue() && isDense ) + if ( relationshipId == NO_NEXT_RELATIONSHIP.intValue() && isDense ) { relationshipId = nextChainStart(); } @@ -201,18 +197,17 @@ private long nextChainStart() { GroupChain groupChain = GROUP_CHAINS[groupChainIndex++]; long chainStart = groupChain.chainStart( groupRecord ); - if ( chainStart != Record.NO_NEXT_RELATIONSHIP.intValue() && - (direction == Direction.BOTH || groupChain.matchesDirection( direction )) ) + if ( !NULL_REFERENCE.is( chainStart ) + && (direction == Direction.BOTH || groupChain.matchesDirection( direction )) ) { return chainStart; } } } - // Go to the next group - if ( !Record.NULL_REFERENCE.is( groupRecord.getNext() ) ) + if ( !NULL_REFERENCE.is( groupRecord.getNext() ) ) { - groupStore.forceGetRecord( groupRecord.getNext(), groupRecord ); + relationshipGroupStore.getRecord( groupRecord.getNext(), groupRecord, FORCE ); } else { @@ -225,7 +220,7 @@ private long nextChainStart() { // Ignore - next line will ensure we're finished anyway } - return Record.NO_NEXT_RELATIONSHIP.intValue(); + return NULL_REFERENCE.intValue(); } private boolean checkType( int type ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StorePropertyPayloadCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StorePropertyPayloadCursor.java index 72bfc5e359996..84fb566d1bb6d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StorePropertyPayloadCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StorePropertyPayloadCursor.java @@ -19,12 +19,11 @@ */ package org.neo4j.kernel.impl.api.store; -import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -import org.neo4j.cursor.GenericCursor; +import org.neo4j.cursor.Cursor; import org.neo4j.helpers.UTF8; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.impl.store.AbstractDynamicStore; @@ -33,8 +32,8 @@ import org.neo4j.kernel.impl.store.LongerShortString; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.PropertyType; +import org.neo4j.kernel.impl.store.RecordCursor; import org.neo4j.kernel.impl.store.ShortArray; -import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.util.Bits; @@ -51,6 +50,7 @@ import static org.neo4j.kernel.impl.store.PropertyType.SHORT_ARRAY; import static org.neo4j.kernel.impl.store.PropertyType.SHORT_STRING; import static org.neo4j.kernel.impl.store.PropertyType.STRING; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; /** * Cursor that provides a view on property blocks of a particular property record. @@ -81,8 +81,8 @@ class StorePropertyPayloadCursor private final DynamicStringStore stringStore; private final DynamicArrayStore arrayStore; - private AbstractDynamicStore.DynamicRecordCursor stringRecordCursor; - private AbstractDynamicStore.DynamicRecordCursor arrayRecordCursor; + private RecordCursor stringRecordCursor; + private RecordCursor arrayRecordCursor; private ByteBuffer buffer = cachedBuffer; private final long[] data = new long[MAX_NUMBER_OF_PAYLOAD_LONG_ARRAY]; @@ -201,18 +201,11 @@ String shortStringValue() String stringValue() { assertOfType( STRING ); - try - { - if ( stringRecordCursor == null ) - { - stringRecordCursor = stringStore.newDynamicRecordCursor(); - } - readFromStore( stringStore, stringRecordCursor ); - } - catch ( IOException e ) + if ( stringRecordCursor == null ) { - throw new UnderlyingStorageException( "Unable to read string value", e ); + stringRecordCursor = stringStore.newRecordCursor( stringStore.newRecord() ); } + readFromStore( stringStore, stringRecordCursor ); buffer.flip(); return UTF8.decode( buffer.array(), 0, buffer.limit() ); } @@ -227,18 +220,11 @@ Object shortArrayValue() Object arrayValue() { assertOfType( ARRAY ); - try - { - if ( arrayRecordCursor == null ) - { - arrayRecordCursor = arrayStore.newDynamicRecordCursor(); - } - readFromStore( arrayStore, arrayRecordCursor ); - } - catch ( IOException e ) + if ( arrayRecordCursor == null ) { - throw new UnderlyingStorageException( "Unable to read array value", e ); + arrayRecordCursor = arrayStore.newRecordCursor( arrayStore.newRecord() ); } + readFromStore( arrayStore, arrayRecordCursor ); buffer.flip(); return readArrayFromBuffer( buffer ); } @@ -264,12 +250,11 @@ private Bits valueAsBits() return bits; } - private void readFromStore( AbstractDynamicStore store, AbstractDynamicStore.DynamicRecordCursor cursor ) - throws IOException + private void readFromStore( AbstractDynamicStore store, RecordCursor cursor ) { buffer.clear(); long startBlockId = PropertyBlock.fetchLong( currentHeader() ); - try ( GenericCursor records = store.getRecordsCursor( startBlockId, true, cursor ) ) + try ( Cursor records = store.placeRecordCursor( startBlockId, cursor, NORMAL ) ) { while ( records.next() ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleNodeCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleNodeCursor.java index 354eaa01c2c62..d19dd7a07825b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleNodeCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleNodeCursor.java @@ -26,6 +26,8 @@ import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.record.NodeRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; + /** * Cursor for a single node. */ @@ -57,9 +59,7 @@ public boolean next() { try { - nodeRecord.setId( nodeId ); - NodeRecord record = nodeStore.loadRecord( nodeId, this.nodeRecord ); - return record != null && record.inUse(); + return nodeStore.getRecord( nodeId, nodeRecord, CHECK ).inUse(); } finally { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursor.java index e1371a63fd640..16ae199bb4983 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreSingleRelationshipCursor.java @@ -56,7 +56,7 @@ public boolean next() try { relationshipRecord.setId( relationshipId ); - return relationshipStore.fillRecord( relationshipId, relationshipRecord, RecordLoad.CHECK ); + return relationshipStore.getRecord( relationshipId, relationshipRecord, RecordLoad.CHECK ).inUse(); } finally { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java index 6dcfa555a8bbb..173592c9e0b30 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java @@ -25,25 +25,22 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; -import org.neo4j.cursor.GenericCursor; import org.neo4j.helpers.collection.IteratorUtil; import org.neo4j.helpers.collection.Pair; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; -import org.neo4j.kernel.impl.store.id.IdType; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGenerator; +import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; +import org.neo4j.kernel.impl.store.id.IdType; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.logging.LogProvider; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; +import static java.lang.String.format; /** * An abstract representation of a dynamic store. The difference between a @@ -63,21 +60,14 @@ * Note, the first block of a dynamic store is reserved and contains information * about the store. */ -public abstract class AbstractDynamicStore extends CommonAbstractStore implements RecordStore, - DynamicBlockSize, DynamicRecordAllocator +public abstract class AbstractDynamicStore extends CommonAbstractStore + implements DynamicRecordAllocator { public static final byte[] NO_DATA = new byte[0]; // (in_use+next high)(1 byte)+nr_of_bytes(3 bytes)+next_block(int) - public static final int BLOCK_HEADER_SIZE = 1 + 3 + 4; // = 8 - - // Return signals for the readRecordHeader() method: - private static int hasDataSignal = 0; - private static int hasNoDataSignal = 1; - private static int notInUseSignal = 2; - private static int illegalSizeSignal = 3; + public static final int RECORD_HEADER_SIZE = 1 + 3 + 4; // = 8 - private final int blockSizeFromConfiguration; - private int blockSize; + private int recordSize; public AbstractDynamicStore( File fileName, @@ -86,10 +76,11 @@ public AbstractDynamicStore( IdGeneratorFactory idGeneratorFactory, PageCache pageCache, LogProvider logProvider, - int blockSizeFromConfiguration ) + String typeDescriptor, + int dataSizeFromConfiguration ) { - super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider ); - this.blockSizeFromConfiguration = blockSizeFromConfiguration; + super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider, typeDescriptor ); + this.recordSize = dataSizeFromConfiguration + RECORD_HEADER_SIZE; } /** @@ -100,38 +91,23 @@ public AbstractDynamicStore( */ public static int getRecordSize( int dataSize ) { - return dataSize + BLOCK_HEADER_SIZE; + return dataSize + RECORD_HEADER_SIZE; } @Override - protected void initialiseNewStoreFile( PagedFile file ) throws IOException + protected ByteBuffer createHeaderRecord() { - int blockSize = blockSizeFromConfiguration; - if ( blockSize < 1 || blockSize > 0xFFFF ) + if ( recordSize < 1 || recordSize > 0xFFFF ) { - throw new IllegalArgumentException( "Illegal block size[" + blockSize + "], limit is 65535" ); - } - - blockSize += BLOCK_HEADER_SIZE; - - try ( PageCursor pageCursor = file.io( 0, PagedFile.PF_SHARED_WRITE_LOCK ) ) - { - if ( pageCursor.next() ) - { - do - { - pageCursor.putInt( blockSize ); - } - while ( pageCursor.shouldRetry() ); - } + throw new IllegalArgumentException( "Illegal block size[" + recordSize + "], limit is 65535" ); } + return intHeaderData( recordSize ); + } - File idFileName = new File( storageFileName.getPath() + ".id" ); - idGeneratorFactory.create( idFileName, 0, true ); - // TODO highestIdInUse = 0 works now, but not when slave can create store files. - IdGenerator idGenerator = idGeneratorFactory.open( idFileName, idType.getGrabSize(), idType, 0 ); - idGenerator.nextId(); // reserve first for blockSize - idGenerator.close(); + @Override + public long getNextRecordReference( DynamicRecord record ) + { + return record.getNextBlock(); } public static void allocateRecordsFromBytes( @@ -164,7 +140,6 @@ public static void allocateRecordsFromBytes( record.setNextBlock( Record.NO_NEXT_BLOCK.intValue() ); } recordList.add( record ); - assert !record.isLight(); assert record.getData() != null; } while ( nextRecord != null ); @@ -255,19 +230,19 @@ record = new DynamicRecord( nextId() ); @Override public int dataSize() { - return getBlockSize() - AbstractDynamicStore.BLOCK_HEADER_SIZE; + return getRecordSize() - AbstractDynamicStore.RECORD_HEADER_SIZE; } @Override public int getRecordSize() { - return getBlockSize(); + return recordSize; } @Override public int getRecordHeaderSize() { - return BLOCK_HEADER_SIZE; + return RECORD_HEADER_SIZE; } /** @@ -282,51 +257,20 @@ public int getNumberOfReservedLowIds() } @Override - protected void readAndVerifyBlockSize() throws IOException + protected void readAndVerifyHeaderRecord() throws IOException { - blockSize = getHeaderRecord(); + recordSize = getHeaderRecord(); } - /** - * Returns the byte size of each block for this dynamic store - * - * @return The block size of this store - */ @Override - public int getBlockSize() + public DynamicRecord newRecord() { - return blockSize; + return new DynamicRecord( -1 ); } @Override - public void updateRecord( DynamicRecord record ) + protected void writeRecord( PageCursor cursor, DynamicRecord record ) { - long blockId = record.getId(); - long pageId = pageIdForRecord( blockId ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - writeRecord( cursor, record ); - } - while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - // [next][type][data] - - private void writeRecord( PageCursor cursor, DynamicRecord record ) - { - long recordId = record.getId(); - int offset = offsetForId( recordId ); - cursor.setOffset( offset ); if ( record.inUse() ) { long nextBlock = record.getNextBlock(); @@ -350,122 +294,21 @@ private void writeRecord( PageCursor cursor, DynamicRecord record ) cursor.putInt( firstInteger ); cursor.putInt( (int) nextBlock ); - if ( !record.isLight() ) - { - cursor.putBytes( record.getData() ); - } + cursor.putBytes( record.getData() ); } else { cursor.putByte( Record.NOT_IN_USE.byteValue() ); - freeId( recordId ); } } - @Override - public void forceUpdateRecord( DynamicRecord record ) - { - updateRecord( record ); - } - public void allocateRecordsFromBytes( Collection target, byte src[] ) { allocateRecordsFromBytes( target, src, IteratorUtil.emptyIterator(), this ); } - public Collection getLightRecords( long startBlockId ) - { - return getRecords( startBlockId, false ); - } - - private Collection getRecords( long startBlockId, boolean readBothHeaderAndData ) - { - // TODO we should instead be passed in a consumer of records, so we don't have to spend memory building up - // this list - List recordList = new LinkedList<>(); - long blockId = startBlockId; - int noNextBlock = Record.NO_NEXT_BLOCK.intValue(); - - try ( PageCursor cursor = storeFile.io( 0, PF_SHARED_READ_LOCK ) ) - { - while ( blockId != noNextBlock && cursor.next( pageIdForRecord( blockId ) ) ) - { - DynamicRecord record = new DynamicRecord( blockId ); - int headerReadResult; - do - { - cursor.setOffset( offsetForId( blockId ) ); - headerReadResult = readRecordHeader( cursor, record, false ); - if ( headerReadResult == hasDataSignal && readBothHeaderAndData ) - { - readRecordData( cursor, record ); - } - } - while ( cursor.shouldRetry() ); - - checkForInUse( headerReadResult, record ); - checkForIllegalSize( headerReadResult, record ); - recordList.add( record ); - blockId = record.getNextBlock(); - } - return recordList; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - public DynamicRecordCursor newDynamicRecordCursor() - { - return new DynamicRecordCursor(); - } - - public DynamicRecordCursor getRecordsCursor( final long startBlockId, - final boolean readBothHeaderAndData ) - { - return getRecordsCursor( startBlockId, readBothHeaderAndData, newDynamicRecordCursor() ); - } - - public DynamicRecordCursor getRecordsCursor( final long startBlockId, - final boolean readBothHeaderAndData, DynamicRecordCursor dynamicRecordCursor ) - { - try - { - final PageCursor cursor = storeFile.io( 0, PF_SHARED_READ_LOCK ); - - dynamicRecordCursor.init( startBlockId, cursor, readBothHeaderAndData ); - return dynamicRecordCursor; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - private void checkForInUse( int headerReadResult, DynamicRecord record ) - { - if ( headerReadResult == notInUseSignal ) - { - throw new InvalidRecordException( "DynamicRecord Not in use, blockId[" + record.getId() + "]" ); - } - } - - private void checkForIllegalSize( int headerReadResult, DynamicRecord record ) - { - if ( headerReadResult == illegalSizeSignal ) - { - int dataSize = getBlockSize() - AbstractDynamicStore.BLOCK_HEADER_SIZE; - throw new InvalidRecordException( "Next block set[" + record.getNextBlock() - + "] current block illegal size[" + record.getLength() + "/" + dataSize + - "]" ); - } - } - - /** - * Reads data from the cursor into the given record, and returns one of the signals specified above. - */ - private int readRecordHeader( PageCursor cursor, DynamicRecord record, boolean force ) + @Override + protected void readRecord( PageCursor cursor, DynamicRecord record, RecordLoad mode ) { /* * First 4b @@ -477,156 +320,45 @@ private int readRecordHeader( PageCursor cursor, DynamicRecord record, boolean f */ long firstInteger = cursor.getUnsignedInt(); boolean isStartRecord = (firstInteger & 0x80000000) == 0; - long maskedInteger = firstInteger & ~0x80000000; - int highNibbleInMaskedInteger = (int) ((maskedInteger) >> 28); - boolean inUse = highNibbleInMaskedInteger == Record.IN_USE.intValue(); - if ( !inUse && !force ) + boolean inUse = (firstInteger & 0x10000000) != 0; + if ( mode.shouldLoad( inUse ) ) { - return notInUseSignal; - } - int dataSize = getBlockSize() - AbstractDynamicStore.BLOCK_HEADER_SIZE; + int dataSize = getRecordSize() - AbstractDynamicStore.RECORD_HEADER_SIZE; + int nrOfBytes = (int) (firstInteger & 0xFFFFFF); - int nrOfBytes = (int) (firstInteger & 0xFFFFFF); + /* + * Pointer to next block 4b (low bits of the pointer) + */ + long nextBlock = cursor.getUnsignedInt(); + long nextModifier = (firstInteger & 0xF000000L) << 8; - /* - * Pointer to next block 4b (low bits of the pointer) - */ - long nextBlock = cursor.getUnsignedInt(); - long nextModifier = (firstInteger & 0xF000000L) << 8; - - long longNextBlock = CommonAbstractStore.longFromIntAndMod( nextBlock, nextModifier ); - boolean hasDataToRead = true; - record.setInUse( inUse ); - record.setStartRecord( isStartRecord ); - record.setLength( nrOfBytes ); - record.setNextBlock( longNextBlock ); - if ( longNextBlock != Record.NO_NEXT_BLOCK.intValue() - && nrOfBytes < dataSize || nrOfBytes > dataSize ) - { - hasDataToRead = false; - if ( !force ) + long longNextBlock = CommonAbstractStore.longFromIntAndMod( nextBlock, nextModifier ); + record.initialize( inUse, isStartRecord, longNextBlock, -1, nrOfBytes ); + if ( longNextBlock != Record.NO_NEXT_BLOCK.intValue() + && nrOfBytes < dataSize || nrOfBytes > dataSize ) { - return illegalSizeSignal; + int blockDataSize = getRecordSize() - getRecordHeaderSize(); + mode.report( format( "Next block set[%d] current block illegal size[%d/%d]", + record.getNextBlock(), record.getLength(), blockDataSize ) ); } - } - return hasDataToRead ? hasDataSignal : hasNoDataSignal; - } - - private void readRecordData( PageCursor cursor, DynamicRecord record ) - { - int len = record.getLength(); - byte[] data = record.getData(); - if ( data == null || data.length != len ) - { - data = new byte[len]; - } - cursor.getBytes( data ); - record.setData( data ); - } - public void ensureHeavy( DynamicRecord record ) - { - if ( !record.isLight() ) - { - return; - } - if ( record.getLength() == 0 ) // don't go though the trouble of acquiring the window if we would read nothing - { - record.setData( NO_DATA ); - return; - } - - long pageId = pageIdForRecord( record.getId() ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) + if ( record.getLength() == 0 ) // don't go though the trouble of acquiring the window if we would read nothing { - int offset = offsetForId( record.getId() ); - do - { - // Add the BLOCK_HEADER_SIZE to the offset, since we don't read it - cursor.setOffset( offset + BLOCK_HEADER_SIZE ); - readRecordData( cursor, record ); - } - while ( cursor.shouldRetry() ); + record.setData( NO_DATA ); + return; } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - @Override - public DynamicRecord getRecord( long id ) - { - DynamicRecord record = new DynamicRecord( id ); - long pageId = pageIdForRecord( id ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - int headerReadResult = notInUseSignal; - if ( cursor.next() ) + int len = record.getLength(); + byte[] data = record.getData(); + if ( data == null || data.length != len ) { - int offset = offsetForId( record.getId() ); - do - { - cursor.setOffset( offset ); - headerReadResult = readRecordHeader( cursor, record, false ); - if ( headerReadResult == hasDataSignal ) - { - readRecordData( cursor, record ); - } - } - while ( cursor.shouldRetry() ); + data = new byte[len]; } - - checkForInUse( headerReadResult, record ); - checkForIllegalSize( headerReadResult, record ); - return record; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); + cursor.getBytes( data ); + record.setData( data ); } } - @Override - public DynamicRecord forceGetRecord( long id ) - { - DynamicRecord record = new DynamicRecord( id ); - long pageId = pageIdForRecord( id ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) - { - int offset = offsetForId( record.getId() ); - int headerReadResult; - do - { - cursor.setOffset( offset ); - headerReadResult = readRecordHeader( cursor, record, true ); - if ( headerReadResult == hasDataSignal ) - { - readRecordData( cursor, record ); - } - } - while ( cursor.shouldRetry() ); - checkForIllegalSize( headerReadResult, record ); - } - return record; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - @Override - public Collection getRecords( long startBlockId ) - { - return getRecords( startBlockId, true ); - } - @Override protected boolean isInUse( byte inUseByte ) { @@ -650,67 +382,4 @@ public String toString() return readFullByteArrayFromHeavyRecords( records, propertyType ); } - - public class DynamicRecordCursor extends GenericCursor - { - private PageCursor cursor; - private boolean readBothHeaderAndData; - long blockId; - int noNextBlock; - - private final DynamicRecord record = new DynamicRecord( blockId ); - - public void init( long startBlockId, PageCursor cursor, boolean readBothHeaderAndData ) - { - this.cursor = cursor; - this.readBothHeaderAndData = readBothHeaderAndData; - blockId = startBlockId; - noNextBlock = Record.NO_NEXT_BLOCK.intValue(); - } - - @Override - public boolean next() - { - try - { - if ( blockId != noNextBlock && cursor.next( pageIdForRecord( blockId ) ) ) - { - record.setId( blockId ); - - int headerReadResult; - do - { - cursor.setOffset( offsetForId( blockId ) ); - headerReadResult = readRecordHeader( cursor, record, false ); - if ( headerReadResult == hasDataSignal && readBothHeaderAndData ) - { - readRecordData( cursor, record ); - } - } - while ( cursor.shouldRetry() ); - - checkForInUse( headerReadResult, record ); - checkForIllegalSize( headerReadResult, record ); - current = record; - blockId = record.getNextBlock(); - return true; - } - else - { - return false; - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - @Override - public void close() - { - cursor.close(); - cursor = null; - } - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractRecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractRecordStore.java deleted file mode 100644 index 5b6cc1c15204e..0000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractRecordStore.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.store; - -import java.io.File; -import java.util.Collection; - -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; -import org.neo4j.kernel.impl.store.id.IdType; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; -import org.neo4j.logging.LogProvider; - -import static java.util.Collections.singletonList; - -public abstract class AbstractRecordStore extends AbstractStore - implements RecordStore -{ - public AbstractRecordStore( - File fileName, - Config conf, - IdType idType, - IdGeneratorFactory idGeneratorFactory, - PageCache pageCache, - LogProvider logProvider ) - { - super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider ); - } - - @Override - public Collection getRecords( long id ) - { - return singletonList( getRecord( id ) ); - } - - @Override - public int getNumberOfReservedLowIds() - { - return 0; - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractStore.java deleted file mode 100644 index 4f98c20cb927f..0000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractStore.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.store; - -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; - -import org.neo4j.io.pagecache.PageCache; -import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; -import org.neo4j.kernel.impl.store.id.IdType; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.store.id.IdGenerator; -import org.neo4j.kernel.impl.store.record.Record; -import org.neo4j.logging.LogProvider; - -/** - * An abstract representation of a store. A store is a file that contains - * records. Each record has a fixed size (getRecordSize()) so - * the position for a record can be calculated by - * id * getRecordSize(). - *

- * A store has an {@link IdGenerator} managing the records that are free or in - * use. - */ -public abstract class AbstractStore extends CommonAbstractStore -{ - public AbstractStore( - File fileName, - Config conf, - IdType idType, - IdGeneratorFactory idGeneratorFactory, - PageCache pageCache, - LogProvider logProvider ) - { - super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider ); - } - - /** - * Returns the fixed size of each record in this store. - * - * @return The record size - */ - public abstract int getRecordSize(); - - @Override - protected void readAndVerifyBlockSize() throws IOException - { - // record size is fixed for non-dynamic stores, so nothing to do here - } - - @Override - protected boolean isInUse( byte inUseByte ) - { - return (inUseByte & 0x1) == Record.IN_USE.intValue(); - } - - @Override - protected void initialiseNewStoreFile( PagedFile file ) throws IOException - { - - ByteBuffer headerRecord = createHeaderRecord(); - if ( headerRecord != null ) - { - try ( PageCursor pageCursor = file.io( 0, PagedFile.PF_SHARED_WRITE_LOCK ) ) - { - if ( pageCursor.next() ) - { - do - { - pageCursor.setOffset( 0 ); - pageCursor.putBytes( headerRecord.array() ); - } - while ( pageCursor.shouldRetry() ); - } - } - - } - - File idFileName = new File( storageFileName.getPath() + ".id" ); - idGeneratorFactory.create( idFileName, 0, true ); - if ( headerRecord != null ) - { - IdGenerator idGenerator = idGeneratorFactory.open( idFileName, 1, idType, 0 ); - initialiseNewIdGenerator( idGenerator ); - idGenerator.close(); - } - } - - protected void initialiseNewIdGenerator( IdGenerator idGenerator ) - { - idGenerator.nextId(); // reserve first for blockSize - } - - protected ByteBuffer createHeaderRecord() - { - return null; - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java index 186a7b3a14ccc..a75b0d2fd5cf1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java @@ -21,8 +21,12 @@ import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; import java.nio.file.StandardOpenOption; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; import org.neo4j.graphdb.config.Setting; import org.neo4j.graphdb.factory.GraphDatabaseSettings; @@ -35,8 +39,9 @@ import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGenerator; import org.neo4j.kernel.impl.store.id.IdGeneratorImpl; -import org.neo4j.kernel.impl.store.id.IdSequence; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; import org.neo4j.logging.Logger; @@ -44,12 +49,16 @@ import static org.neo4j.helpers.Exceptions.launderedException; import static org.neo4j.io.pagecache.PagedFile.PF_READ_AHEAD; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; +import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; +import static org.neo4j.kernel.impl.store.record.Record.NULL_REFERENCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.CHECK; /** * Contains common implementation for {@link AbstractStore} and * {@link AbstractDynamicStore}. */ -public abstract class CommonAbstractStore implements IdSequence, AutoCloseable +public abstract class CommonAbstractStore + implements RecordStore, AutoCloseable { public static final String ALL_STORES_VERSION = "v0.A.7"; public static final String UNKNOWN_VERSION = "Unknown"; @@ -63,6 +72,7 @@ public abstract class CommonAbstractStore implements IdSequence, AutoCloseable private IdGenerator idGenerator; private boolean storeOk = true; private Throwable causeOfStoreNotOk; + private final String typeDescriptor; /** * Opens and validates the store contained in fileName @@ -85,16 +95,31 @@ public CommonAbstractStore( IdType idType, IdGeneratorFactory idGeneratorFactory, PageCache pageCache, - LogProvider logProvider ) + LogProvider logProvider, + String typeDescriptor ) { this.storageFileName = fileName; this.configuration = configuration; this.idGeneratorFactory = idGeneratorFactory; this.pageCache = pageCache; this.idType = idType; + this.typeDescriptor = typeDescriptor; this.log = logProvider.getLog( getClass() ); } + protected static long longFromIntAndMod( long base, long modifier ) + { + return modifier == 0 && base == IdGeneratorImpl.INTEGER_MINUS_ONE ? -1 : base | modifier; + } + + protected ByteBuffer intHeaderData( int value ) + { + ByteBuffer data = ByteBuffer.allocate( 4 ); + data.putInt( value ); + data.flip(); + return data; + } + void initialise( boolean createIfNotExists ) { try @@ -121,15 +146,16 @@ void initialise( boolean createIfNotExists ) } } - protected static long longFromIntAndMod( long base, long modifier ) + /** + * Returns the type and version that identifies this store. + * + * @return This store's implementation type and version identifier + */ + public String getTypeDescriptor() { - return modifier == 0 && base == IdGeneratorImpl.INTEGER_MINUS_ONE ? -1 : base | modifier; + return typeDescriptor; } - protected abstract String getTypeDescriptor(); - - protected abstract void initialiseNewStoreFile( PagedFile file ) throws IOException; - /** * This method is called by constructors. * @param createIfNotExists If true, creates and initialises the store file if it does not exist already. If false, @@ -162,6 +188,45 @@ protected void checkStorage( boolean createIfNotExists ) } } + protected void initialiseNewStoreFile( PagedFile file ) throws IOException + { + ByteBuffer headerRecord = createHeaderRecord(); + if ( headerRecord != null ) + { + try ( PageCursor pageCursor = file.io( 0, PF_SHARED_WRITE_LOCK ) ) + { + if ( pageCursor.next() ) + { + do + { + pageCursor.setOffset( 0 ); + pageCursor.putBytes( headerRecord.array() ); + } + while ( pageCursor.shouldRetry() ); + } + } + } + + File idFileName = new File( storageFileName.getPath() + ".id" ); + idGeneratorFactory.create( idFileName, 0, true ); + IdGenerator idGenerator = idGeneratorFactory.open( idFileName, 1, idType, 0 ); + initialiseNewIdGenerator( idGenerator ); + idGenerator.close(); + } + + protected void initialiseNewIdGenerator( IdGenerator idGenerator ) + { + for ( int i = 0; i < getNumberOfReservedLowIds(); i++ ) + { + idGenerator.nextId(); + } + } + + protected ByteBuffer createHeaderRecord() + { + return null; + } + /** * Should do first validation on store validating stuff like version and id * generator. This method is called by constructors. @@ -174,7 +239,7 @@ protected void loadStorage() { try { - readAndVerifyBlockSize(); + readAndVerifyHeaderRecord(); try { int filePageSize = pageCache.pageSize() - pageCache.pageSize() % getRecordSize(); @@ -203,6 +268,7 @@ protected int offsetForId( long id ) return (int) (id * getRecordSize() % storeFile.pageSize()); } + @Override public int getRecordsPerPage() { return storeFile.pageSize() / getRecordSize(); @@ -231,7 +297,10 @@ public byte[] getRawRecordData( long id ) throws IOException * Note: This method runs before the file has been mapped by the page cache, and therefore needs to * operate on the store files directly. This method is called by constructors. */ - protected abstract void readAndVerifyBlockSize() throws IOException; + protected void readAndVerifyHeaderRecord() throws IOException + { + // record size is fixed for non-dynamic stores, so nothing to do here + } private void loadIdGenerator() { @@ -259,7 +328,7 @@ private void loadIdGenerator() protected int getHeaderRecord() throws IOException { - int headerRecord = 0 ; + int headerRecord = 0; try ( PagedFile pagedFile = pageCache.map( getStorageFileName(), pageCache.pageSize() ) ) { try ( PageCursor pageCursor = pagedFile.io( 0, PF_SHARED_READ_LOCK ) ) @@ -274,15 +343,43 @@ protected int getHeaderRecord() throws IOException } } } - if ( headerRecord <= 0 ) { - throw new InvalidRecordException( "Illegal block size: " + headerRecord + " in " + getStorageFileName() ); + throw new InvalidRecordException( "Illegal block size: " + + headerRecord + " in " + getStorageFileName() ); } return headerRecord; } - protected abstract boolean isInUse( byte inUseByte ); + public boolean isInUse( long id ) + { + long pageId = pageIdForRecord( id ); + int offset = offsetForId( id ); + + try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) + { + boolean recordIsInUse = false; + if ( cursor.next() ) + { + do + { + cursor.setOffset( offset ); + recordIsInUse = isInUse( cursor ); + } + while ( cursor.shouldRetry() ); + } + return recordIsInUse; + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + protected boolean isInUse( byte inUseByte ) + { + return (inUseByte & 0x1) == Record.IN_USE.intValue(); + } /** * Should rebuild the id generator from scratch. @@ -292,7 +389,6 @@ protected int getHeaderRecord() throws IOException * map their own temporary PagedFile for the store file, and do their file IO through that, * if they need to access the data in the store file. */ - // accessible only for testing final void rebuildIdGenerator() { int blockSize = getRecordSize(); @@ -315,7 +411,7 @@ final void rebuildIdGenerator() setHighId( foundHighId ); if ( !fastRebuild ) { - try ( PageCursor cursor = storeFile.io( 0, PagedFile.PF_SHARED_WRITE_LOCK | PF_READ_AHEAD ) ) + try ( PageCursor cursor = storeFile.io( 0, PF_SHARED_WRITE_LOCK | PF_READ_AHEAD ) ) { defraggedCount = rebuildIdGeneratorSlow( cursor, getRecordsPerPage(), blockSize, foundHighId ); } @@ -366,7 +462,7 @@ private long rebuildIdGeneratorSlow( PageCursor cursor, int recordsPerPage, int break; } - if ( !isRecordInUse( cursor ) ) + if ( !isInUse( cursor ) ) { freedBatch[defragged++] = recordId; } @@ -419,11 +515,11 @@ protected boolean getStoreOk() /** * Throws cause of not being OK if {@link #getStoreOk()} returns {@code false}. */ - protected final void checkStoreOk() + protected void checkStoreOk() { if ( !storeOk ) { - throw new UnderlyingStorageException( "Store is not OK", causeOfStoreNotOk ); + throw launderedException( causeOfStoreNotOk ); } } @@ -453,12 +549,12 @@ public void freeId( long id ) } /** - * Return the highest id in use. - * If this store is not OK yet, the high id is calculated from the highest in use record on the store, - * using {@link #scanForHighId()}. + * Return the highest id in use. If this store is not OK yet, the high id is calculated from the highest + * in use record on the store, using {@link #scanForHighId()}. * - * @return The high id, highest id in use + 1 + * @return The high id, i.e. highest id in use + 1. */ + @Override public long getHighId() { return idGenerator != null ? idGenerator.getHighId() : scanForHighId(); @@ -487,13 +583,13 @@ public void setHighId( long highId ) /** * If store is not ok a call to this method will rebuild the {@link - * IdGenerator} used by this store and if successful mark it as. + * IdGenerator} used by this store and if successful mark it as OK. * * WARNING: this method must NOT be called if recovery is required, but hasn't performed. * To remove all negations from the above statement: Only call this method if store is in need of * recovery and recovery has been performed. */ - public final void makeStoreOk() + public void makeStoreOk() { if ( !storeOk ) { @@ -508,6 +604,7 @@ public final void makeStoreOk() * * @return The name of this store */ + @Override public File getStorageFileName() { return storageFileName; @@ -569,7 +666,7 @@ protected long scanForHighId() { cursor.setOffset( currentRecord * recordSize ); long recordId = (cursor.getCurrentPageId() * recordsPerPage) + currentRecord; - if ( isRecordInUse( cursor ) ) + if ( isInUse( cursor ) ) { // We've found the highest id in use return recordId + 1 /*+1 since we return the high id*/; @@ -588,9 +685,16 @@ protected long scanForHighId() } } + @Override public abstract int getRecordSize(); - protected boolean isRecordInUse( PageCursor cursor ) + @Override + public int getRecordHeaderSize() + { + return getRecordSize(); + } + + protected boolean isInUse( PageCursor cursor ) { return isInUse( cursor.getByte() ); } @@ -614,6 +718,7 @@ protected void closeIdGenerator() } } + @Override public void flush() { try @@ -643,6 +748,10 @@ protected void assertNotClosed() * Closes this store. This will cause all buffers and channels to be closed. * Requesting an operation from after this method has been invoked is * illegal and an exception will be thrown. + *

+ * This method will start by invoking the {@link #closeStorage} method + * giving the implementing store way to do anything that it needs to do + * before the pagedFile is closed. */ @Override public void close() @@ -691,17 +800,10 @@ private void closeStoreFile() throws IOException } /** @return The highest possible id in use, -1 if no id in use. */ + @Override public long getHighestPossibleIdInUse() { - if ( idGenerator != null ) - { - return idGenerator.getHighestPossibleIdInUse(); - } - else - { // If we ask for this before we've recovered we can only make a best-effort guess - // about the highest possible id in use. - return scanForHighId() - 1; - } + return idGenerator != null ? idGenerator.getHighestPossibleIdInUse() : scanForHighId() - 1; } /** @@ -724,6 +826,7 @@ public long getNumberOfIdsInUse() * @return the number of records at the beginning of the store file that are reserved for other things * than actual records. Stuff like permanent configuration data. */ + @Override public int getNumberOfReservedLowIds() { return 0; @@ -734,15 +837,15 @@ public IdType getIdType() return idType; } - public final void logVersions( Logger logger ) + public void logVersions( Logger logger ) { - logger.log( getTypeDescriptor() + " " + ALL_STORES_VERSION ); + logger.log( " " + getTypeDescriptor() + " " + ALL_STORES_VERSION ); } - public final void logIdUsage( Logger logger ) + public void logIdUsage( Logger logger ) { logger.log( String.format( " %s: used=%s high=%s", - getTypeDescriptor() + " " + ALL_STORES_VERSION, getNumberOfIdsInUse(), getHighestPossibleIdInUse() ) ); + getTypeDescriptor(), getNumberOfIdsInUse(), getHighestPossibleIdInUse() ) ); } /** @@ -755,7 +858,7 @@ public final void logIdUsage( Logger logger ) * {@link #logVersions(Logger)} * For a good samaritan to pick up later. */ - public final void visitStore( Visitor visitor ) + public void visitStore( Visitor visitor ) { visitor.visit( this ); } @@ -778,6 +881,280 @@ final void deleteIdGenerator() } } + @Override + public long getNextRecordReference( RECORD record ) + { + return Record.NULL_REFERENCE.intValue(); + } + + /** + * Acquires a {@link PageCursor} from the {@link PagedFile store file} and reads the requested record + * in the correct page and offset. + * + * @param id the record id. + * @param record the record instance to load the data into. + * @param mode how strict to be when loading, f.ex {@link RecordLoad#FORCE} will always read what's there + * and load into the record, whereas {@link RecordLoad#NORMAL} will throw {@link InvalidRecordException} + * if not in use. + */ + @Override + public RECORD getRecord( long id, RECORD record, RecordLoad mode ) + { + long pageId = pageIdForRecord( id ); + try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) + { + return getRecord( id, record, mode, cursor, pageId ); + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + protected RECORD getRecord( long id, RECORD record, RecordLoad mode, PageCursor cursor, long pageId ) + { + int offset = offsetForId( id ); + try + { + if ( cursor.next( pageId ) ) + { + // There is a page in the store that covers this record, go read it + readRecordWithRetry( cursor, id, record, mode, offset ); + } + else + { + // There was no page in the store covering this record. We mark the record with + // the correct id because often the caller depends on the id to be correct regardless + // of whether the record is in use or not. Clear the rest of the data. + record.setId( id ); + record.clear(); + mode.verify( record ); + } + return record; + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + protected void readRecordWithRetry( PageCursor cursor, long id, RECORD record, RecordLoad mode, int offset ) + throws IOException + { + do + { + // Mark the record with this id regardless of whether or not we load the contents of it. + record.setId( id ); + + // Mark this record as unused. This to simplify implementations of readRecord. + // readRecord can behave differently depending on RecordLoad argument and so it may be that + // contents of a record may be loaded even if that record is unused, where the contents + // can still be initialized data. Know that for many record stores, deleting a record means + // just setting one byte or bit in that record. + record.setInUse( false ); + cursor.setOffset( offset ); + readRecord( cursor, record, mode ); + } + while ( cursor.shouldRetry() ); + if ( !mode.verify( record ) ) + { + record.clear(); + } + } + + /** + * Reads data from {@link PageCursor} into the record. + */ + protected abstract void readRecord( PageCursor cursor, RECORD record, RecordLoad mode ); + + @Override + public void updateRecord( RECORD record ) + { + long id = record.getLongId(); + long pageId = pageIdForRecord( id ); + int offset = offsetForId( id ); + try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_WRITE_LOCK ) ) + { + if ( cursor.next() ) + { + do + { + cursor.setOffset( offset ); + writeRecord( cursor, record ); + } + while ( cursor.shouldRetry() ); + if ( !record.inUse() ) + { + freeId( id ); + } + } + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + protected abstract void writeRecord( PageCursor cursor, RECORD record ); + + /** + * Scan the given range of records both inclusive, and pass all the in-use ones to the given processor, one by one. + * + * The record passed to the NodeRecordScanner is reused instead of reallocated for every record, so it must be + * cloned if you want to save it for later. + */ + public void scanAllRecords( Visitor visitor ) throws IOException + { + long startPageId = pageIdForRecord( 0 ); + long currentPageId = startPageId; + long endPageId = pageIdForRecord( getHighestPossibleIdInUse() ); + long currentRecordId = 0; + RECORD record = newRecord(); + int recordsPerPage = storeFile.pageSize() / getRecordSize(); + + try ( PageCursor cursor = storeFile.io( startPageId, PF_SHARED_READ_LOCK | PF_READ_AHEAD ) ) + { + while ( currentPageId <= endPageId && cursor.next() ) + { + for ( int i = 0; i < recordsPerPage; i++ ) + { + if ( getRecord( currentRecordId, record, CHECK, cursor, currentPageId ).inUse() ) + { + if ( visitor.visit( record ) ) + { + return; + } + } + currentRecordId++; + } + currentPageId++; + } + } + } + + @Override + public Collection getRecords( long firstId, RecordLoad mode ) + { + // TODO we should instead be passed in a consumer of records, so we don't have to spend memory building up + // this list + List recordList = new LinkedList<>(); + long currentId = firstId; + try ( PageCursor cursor = storeFile.io( 0, PF_SHARED_READ_LOCK ) ) + { + while ( !NULL_REFERENCE.is( currentId ) && cursor.next( pageIdForRecord( currentId ) ) ) + { + RECORD record = newRecord(); + readRecordWithRetry( cursor, currentId, record, mode, offsetForId( currentId ) ); + if ( !record.inUse() ) + { + break; + } + recordList.add( record ); + currentId = getNextRecordReference( record ); + } + return recordList; + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + @Override + public RecordCursor newRecordCursor( final RECORD record ) + { + return new RecordCursor() + { + private long currentId; + private RecordLoad mode; + private PageCursor pageCursor; + + @Override + public boolean next() + { + if ( NULL_REFERENCE.is( currentId ) ) + { + return false; + } + + try + { + return getRecord( currentId, record, mode, pageCursor, pageIdForRecord( currentId ) ).inUse(); + } + finally + { + currentId = record.inUse() ? getNextRecordReference( record ) : NULL_REFERENCE.intValue(); + } + } + + @Override + public void close() + { + assert pageCursor != null; + this.pageCursor.close(); + this.pageCursor = null; + } + + @Override + public RECORD get() + { + return record; + } + + @Override + public boolean next( long id ) + { + currentId = id; + return next(); + } + + @Override + public RecordCursor init( long id, RecordLoad mode, PageCursor pageCursor ) + { + assert this.pageCursor == null; + this.currentId = id; + this.mode = mode; + this.pageCursor = pageCursor; + return this; + } + }; + } + + @SuppressWarnings( "resource" ) + @Override + public RecordCursor placeRecordCursor( final long id, final RecordCursor cursor, + final RecordLoad mode ) + { + try + { + PageCursor pageCursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ); + cursor.init( id, mode, pageCursor ); + return cursor; + } + catch ( IOException e ) + { + throw new UnderlyingStorageException( e ); + } + } + + @Override + public void ensureHeavy( RECORD record ) + { + // Do nothing by default. Some record stores have this. + } + + public static R getRecord( RecordStore store, long id, RecordLoad mode ) + { + R record = store.newRecord(); + store.getRecord( id, record, mode ); + return record; + } + + public static R getRecord( RecordStore store, long id ) + { + return getRecord( store, id, RecordLoad.NORMAL ); + } + @Override public String toString() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DelegatingRecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DelegatingRecordStore.java deleted file mode 100644 index 54a0e7e5724b4..0000000000000 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DelegatingRecordStore.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2002-2016 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.impl.store; - -import java.io.File; -import java.util.Collection; - -import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; - -public class DelegatingRecordStore implements RecordStore -{ - private final RecordStore delegate; - - public DelegatingRecordStore( RecordStore delegate ) - { - this.delegate = delegate; - } - - @Override - public String toString() - { - return delegate.toString(); - } - - @Override - public File getStorageFileName() - { - return delegate.getStorageFileName(); - } - - @Override - public long getHighId() - { - return delegate.getHighId(); - } - - @Override - public long getHighestPossibleIdInUse() - { - return delegate.getHighestPossibleIdInUse(); - } - - @Override - public long nextId() - { - return delegate.nextId(); - } - - @Override - public R getRecord( long id ) - { - return delegate.getRecord( id ); - } - - @Override - public Collection getRecords( long id ) - { - return delegate.getRecords( id ); - } - - @Override - public void updateRecord( R record ) - { - delegate.updateRecord( record ); - } - - @Override - public R forceGetRecord( long id ) - { - return delegate.forceGetRecord( id ); - } - - @Override - public void forceUpdateRecord( R record ) - { - delegate.forceUpdateRecord( record ); - } - - @Override - public void accept( Processor processor, R record ) throws FAILURE - { - delegate.accept( processor, record ); - } - - @Override - public int getRecordSize() - { - return delegate.getRecordSize(); - } - - @Override - public int getRecordsPerPage() - { - return delegate.getRecordsPerPage(); - } - - @Override - public int getRecordHeaderSize() - { - return delegate.getRecordHeaderSize(); - } - - @Override - public void close() - { - delegate.close(); - } - - @Override - public void flush() - { - delegate.flush(); - } - - @Override - public int getNumberOfReservedLowIds() - { - return delegate.getNumberOfReservedLowIds(); - } -} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicArrayStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicArrayStore.java index c8916f6017f21..785e59b6f4da5 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicArrayStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicArrayStore.java @@ -54,9 +54,10 @@ public DynamicArrayStore( IdGeneratorFactory idGeneratorFactory, PageCache pageCache, LogProvider logProvider, - int blockSize ) + int dataSizeFromConfiguration ) { - super( fileName, configuration, idType, idGeneratorFactory, pageCache, logProvider, blockSize ); + super( fileName, configuration, idType, idGeneratorFactory, pageCache, + logProvider, TYPE_DESCRIPTOR, dataSizeFromConfiguration ); } @Override @@ -66,12 +67,6 @@ public void accept( RecordStore.Processor p processor.processArray( this, record ); } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - public static void allocateFromNumbers( Collection target, Object array, Iterator recordsToUseFirst, DynamicRecordAllocator recordAllocator ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java index b7888cf7041dc..1d46e31c29bdd 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java @@ -27,6 +27,7 @@ import org.neo4j.kernel.impl.store.record.NodeRecord; import static java.lang.String.format; + import static org.neo4j.helpers.collection.IteratorUtil.first; import static org.neo4j.kernel.impl.store.AbstractDynamicStore.readFullByteArrayFromHeavyRecords; import static org.neo4j.kernel.impl.store.DynamicArrayStore.getRightArray; @@ -71,13 +72,6 @@ public long[] getIfLoaded() { return null; } - for ( DynamicRecord dynamic : node.getUsedDynamicLabelRecords() ) - { - if ( dynamic.isLight() ) - { - return null; - } - } return stripNodeId( (long[]) getRightArray( readFullByteArrayFromHeavyRecords( node.getUsedDynamicLabelRecords(), ARRAY ) ) ); } @@ -143,9 +137,9 @@ public Collection add( long labelId, NodeStore nodeStore, Dynamic public Collection remove( long labelId, NodeStore nodeStore ) { nodeStore.ensureHeavy( node, firstDynamicLabelRecordId( labelField ) ); - Collection existingRecords = node.getDynamicLabelRecords(); - long[] existingLabelIds = nodeStore.getDynamicLabelsArray( existingRecords ); + long[] existingLabelIds = nodeStore.getDynamicLabelsArray( node.getUsedDynamicLabelRecords() ); long[] newLabelIds = filter( existingLabelIds, labelId ); + Collection existingRecords = node.getDynamicLabelRecords(); if ( InlineNodeLabels.tryInlineInNodeRecord( node, newLabelIds, existingRecords ) ) { setNotInUse( existingRecords ); @@ -162,7 +156,6 @@ public Collection remove( long labelId, NodeStore nodeStore ) if ( !newRecords.contains( record ) ) { record.setInUse( false ); - record.setLength( 0 ); // so that it will not be made heavy again... } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicStringStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicStringStore.java index 64fd1d247df04..accfd80c57742 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicStringStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicStringStore.java @@ -43,9 +43,10 @@ public DynamicStringStore( IdGeneratorFactory idGeneratorFactory, PageCache pageCache, LogProvider logProvider, - int blockSize ) + int dataSizeFromConfiguration ) { - super( fileName, configuration, idType, idGeneratorFactory, pageCache, logProvider, blockSize ); + super( fileName, configuration, idType, idGeneratorFactory, pageCache, + logProvider, TYPE_DESCRIPTOR, dataSizeFromConfiguration ); } @Override @@ -54,10 +55,4 @@ public void accept( RecordStore.Processor p { processor.processString( this, record, idType ); } - - @Override - protected String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ExistingThenNewRecordAllocator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ExistingThenNewRecordAllocator.java index 2a5984c0b1441..6bc6777592bd4 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ExistingThenNewRecordAllocator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ExistingThenNewRecordAllocator.java @@ -26,15 +26,16 @@ class ExistingThenNewRecordAllocator implements DynamicRecordAllocator { - private final DynamicBlockSize blockSize; + private final int recordSize; private final IdSequence idSequence; - ExistingThenNewRecordAllocator( DynamicBlockSize blockSize, IdSequence idSequence ) + ExistingThenNewRecordAllocator( int recordSize, IdSequence idSequence ) { - this.blockSize = blockSize; + this.recordSize = recordSize; this.idSequence = idSequence; } + @Override public DynamicRecord nextUsedRecordOrNew( Iterator recordsToUseFirst ) { DynamicRecord record; @@ -58,6 +59,6 @@ record = new DynamicRecord( idSequence.nextId() ); @Override public int dataSize() { - return blockSize.getBlockSize() - AbstractDynamicStore.BLOCK_HEADER_SIZE; + return recordSize - AbstractDynamicStore.RECORD_HEADER_SIZE; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/LabelTokenStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/LabelTokenStore.java index aa38ca8810f04..b5dbf18c2f42b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/LabelTokenStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/LabelTokenStore.java @@ -47,7 +47,7 @@ public LabelTokenStore( DynamicStringStore nameStore ) { super( file, config, IdType.LABEL_TOKEN, idGeneratorFactory, pageCache, - logProvider, nameStore, new Token.Factory() ); + logProvider, nameStore, TYPE_DESCRIPTOR, new Token.Factory() ); } @Override @@ -57,15 +57,9 @@ public void accept( Processor processor, La } @Override - protected LabelTokenRecord newRecord( int id ) + public LabelTokenRecord newRecord() { - return new LabelTokenRecord( id ); - } - - @Override - protected void readRecord( LabelTokenRecord record, PageCursor cursor ) - { - record.setNameId( cursor.getInt() ); + return new LabelTokenRecord( -1 ); } @Override @@ -79,10 +73,4 @@ public int getRecordSize() { return RECORD_SIZE; } - - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/MetaDataStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/MetaDataStore.java index 8cf4ebb3e15ca..d016ac0d370f1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/MetaDataStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/MetaDataStore.java @@ -32,8 +32,10 @@ import org.neo4j.kernel.impl.store.id.IdType; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGenerator; +import org.neo4j.kernel.impl.store.record.NeoStoreActualRecord; import org.neo4j.kernel.impl.store.record.NeoStoreRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.transaction.log.LogVersionRepository; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.kernel.impl.util.ArrayQueueOutOfOrderSequence; @@ -48,7 +50,8 @@ import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -public class MetaDataStore extends AbstractStore implements TransactionIdStore, LogVersionRepository +public class MetaDataStore extends CommonAbstractStore + implements TransactionIdStore, LogVersionRepository { public static final String TYPE_DESCRIPTOR = "NeoStore"; // This value means the field has not been refreshed from the store. Normally, this should happen only once @@ -137,8 +140,7 @@ public String description() IdGeneratorFactory idGeneratorFactory, PageCache pageCache, LogProvider logProvider ) { - super( fileName, conf, IdType.NEOSTORE_BLOCK, idGeneratorFactory, pageCache, logProvider ); - + super( fileName, conf, IdType.NEOSTORE_BLOCK, idGeneratorFactory, pageCache, logProvider, TYPE_DESCRIPTOR ); this.transactionCloseWaitLogger = new CappedLogger( logProvider.getLog( MetaDataStore.class ) ); transactionCloseWaitLogger.setTimeLimit( 30, SECONDS, Clock.SYSTEM_CLOCK ); } @@ -179,7 +181,7 @@ protected void initialiseNewIdGenerator( IdGenerator idGenerator ) */ for ( int i = 0; i < META_DATA_RECORD_COUNT; i++ ) { - nextId(); + idGenerator.nextId(); } } @@ -197,12 +199,35 @@ public void setLastCommittedAndClosedTransactionId( lastCommittingTxField.set( transactionId ); lastClosedTx.set( transactionId, new long[]{logVersion, byteOffset} ); highestCommittedTransaction.set( transactionId, checksum ); - } - - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; +//======= +// long record; +// try +// { +// record = getRecord( pageCache, storageFileName, Position.STORE_VERSION ); +// } +// catch ( IOException e ) +// { +// throw new UnderlyingStorageException( e ); +// } +// +// if ( record == FIELD_NOT_PRESENT ) +// { +// // if the record cannot be read, let's assume the neo store has not been create yet +// // we'll check again when the store is gonna set to "store ok" +// return; +// } +// +// String foundVersion = versionLongToString( record ); +// if ( !ALL_STORES_VERSION.equals( foundVersion ) ) +// { +// throw new IllegalStateException( +// format( "Mismatching store version found (%s while expecting %s). The store cannot be " + +// "automatically upgraded since it isn't cleanly shutdown." +// + " Recover by starting the database using the previous Neo4j version, " + +// "followed by a clean shutdown. Then start with this version again.", +// foundVersion, ALL_STORES_VERSION ) ); +// } +//>>>>>>> Simplified store classes } @Override @@ -548,7 +573,6 @@ private void scanAllFields( int pf_flags, Visitor visito private void setRecord( Position position, long value ) { long id = position.id; - long pageId = pageIdForRecord( id ); // We need to do a little special handling of high id in neostore since it's not updated in the same // way as other stores. Other stores always gets updates via commands where records are updated and @@ -557,17 +581,17 @@ private void setRecord( Position position, long value ) // unclear from the outside which record id that refers to, so here we need to manage high id ourselves. setHighestPossibleIdInUse( id ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) - { - setRecord( cursor, position, value ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } + NeoStoreActualRecord record = new NeoStoreActualRecord(); + record.initialize( true, value ); + record.setId( id ); + updateRecord( record ); + } + + @Override + protected void writeRecord( PageCursor cursor, NeoStoreActualRecord record ) + { + cursor.putByte( Record.IN_USE.byteValue() ); + cursor.putLong( record.getValue() ); } private void setRecord( PageCursor cursor, Position position, long value ) throws IOException @@ -826,4 +850,24 @@ public boolean visit( PageCursor element ) throws IOException } } ); } + + @Override + public NeoStoreActualRecord newRecord() + { + return new NeoStoreActualRecord(); + } + + @Override + public void accept( + org.neo4j.kernel.impl.store.RecordStore.Processor processor, NeoStoreActualRecord record ) + throws FAILURE + { + throw new UnsupportedOperationException(); + } + + @Override + protected void readRecord( PageCursor cursor, NeoStoreActualRecord record, RecordLoad mode ) + { + throw new UnsupportedOperationException(); + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java index 1881b8681ebbf..45dee7d0574bd 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java @@ -20,35 +20,30 @@ package org.neo4j.kernel.impl.store; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import org.neo4j.helpers.collection.Pair; -import org.neo4j.helpers.collection.Visitor; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdType; -import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.util.Bits; import org.neo4j.logging.LogProvider; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_READ_AHEAD; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; import static org.neo4j.kernel.impl.store.AbstractDynamicStore.readFullByteArrayFromHeavyRecords; /** * Implementation of the node store. */ -public class NodeStore extends AbstractRecordStore +public class NodeStore extends CommonAbstractStore { public static Long readOwnerFromDynamicLabelsRecord( DynamicRecord record ) { @@ -66,7 +61,7 @@ public static Long readOwnerFromDynamicLabelsRecord( DynamicRecord record ) } public static abstract class Configuration - extends AbstractStore.Configuration + extends CommonAbstractStore.Configuration { } @@ -85,7 +80,7 @@ public NodeStore( LogProvider logProvider, DynamicArrayStore dynamicLabelStore ) { - super( fileName, config, IdType.NODE, idGeneratorFactory, pageCache, logProvider ); + super( fileName, config, IdType.NODE, idGeneratorFactory, pageCache, logProvider, TYPE_DESCRIPTOR ); this.dynamicLabelStore = dynamicLabelStore; } @@ -95,12 +90,6 @@ public void accept( Processor processor, No processor.processNode( this, record ); } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - @Override public int getRecordSize() { @@ -108,11 +97,6 @@ public int getRecordSize() } @Override - public int getRecordHeaderSize() - { - return getRecordSize(); - } - public void ensureHeavy( NodeRecord node ) { if ( NodeLabelsField.fieldPointsToDynamicRecordOfLabels( node.getLabelField() ) ) @@ -129,125 +113,28 @@ public void ensureHeavy( NodeRecord node, long firstDynamicLabelRecord ) } // Load any dynamic labels and populate the node record - node.setLabelField( node.getLabelField(), dynamicLabelStore.getRecords( firstDynamicLabelRecord ) ); - } - - @Override - public NodeRecord getRecord( long id ) - { - return getRecord( id, null ); - } - - public NodeRecord getRecord( long id, NodeRecord record ) - { - NodeRecord result = loadRecord( id, record ); - - if ( result == null ) - { - throw new InvalidRecordException( "NodeRecord[" + id + "] not in use" ); - } - return result; - } - - public NodeRecord loadLightNode( long id ) - { - return loadRecord( id, null ); - } - - @Override - public NodeRecord forceGetRecord( long id ) - { - NodeRecord record = loadRecord( id, null ); - if ( record == null ) - { - return new NodeRecord( - id, - false, - Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); // inUse=false by default - } - return record; - } - - public NodeRecord loadRecord( long id, NodeRecord record) - { - long pageId = pageIdForRecord( id ); - int offset = offsetForId( id ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - boolean isInUse = false; - if ( cursor.next() ) - { - do - { - cursor.setOffset( offset ); - - // [ , x] in use bit - // [ ,xxx ] higher bits for rel id - // [xxxx, ] higher bits for prop id - byte inUseByte = cursor.getByte(); - isInUse = isInUse( inUseByte ); - if ( isInUse ) - { - if ( record == null ) - { - record = new NodeRecord( id ); - } - record.setId( id ); - readIntoRecord( cursor, record, inUseByte, true ); - } - } while ( cursor.shouldRetry() ); - } - return isInUse? record : null; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } + node.setLabelField( node.getLabelField(), dynamicLabelStore.getRecords( firstDynamicLabelRecord, RecordLoad.NORMAL ) ); } @Override - public void forceUpdateRecord( NodeRecord record ) + public NodeRecord newRecord() { - writeRecord( record, true ); + return new NodeRecord( -1 ); } @Override public void updateRecord( NodeRecord record ) { - writeRecord( record, false ); - if ( !record.inUse() ) - { - freeId( record.getId() ); - } + super.updateRecord( record ); updateDynamicLabelRecords( record.getDynamicLabelRecords() ); } - private void writeRecord( NodeRecord record, boolean force ) - { - long recordId = record.getId(); - long pageId = pageIdForRecord( recordId ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - writeRecord( cursor, record, force ); - } while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - private void writeRecord( PageCursor cursor, NodeRecord record, boolean force ) + @Override + protected void writeRecord( PageCursor cursor, NodeRecord record ) { int offset = offsetForId( record.getId() ); cursor.setOffset( offset ); - if ( record.inUse() || force ) + if ( record.inUse() ) { long nextRel = record.getNextRel(); long nextProp = record.getNextProp(); @@ -280,92 +167,28 @@ private void writeRecord( PageCursor cursor, NodeRecord record, boolean force ) } } - public boolean inUse( long id ) + @Override + protected void readRecord( PageCursor cursor, NodeRecord record, RecordLoad mode ) { - long pageId = pageIdForRecord( id ); - int offset = offsetForId( id ); - - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - boolean recordIsInUse = false; - if ( cursor.next() ) - { - do - { - cursor.setOffset( offset ); - recordIsInUse = isInUse( cursor.getByte() ); - } while ( cursor.shouldRetry() ); - } - return recordIsInUse; - } - catch ( IOException e ) + byte inUseByte = cursor.getByte(); + boolean isInUse = isInUse( inUseByte ); + if ( mode.shouldLoad( isInUse ) ) { - throw new UnderlyingStorageException( e ); - } - } + long nextRel = cursor.getUnsignedInt(); + long nextProp = cursor.getUnsignedInt(); - private void readIntoRecord( PageCursor cursor, NodeRecord record, byte inUseByte, boolean inUse ) - { - long nextRel = cursor.getUnsignedInt(); - long nextProp = cursor.getUnsignedInt(); - - long relModifier = (inUseByte & 0xEL) << 31; - long propModifier = (inUseByte & 0xF0L) << 28; - - long lsbLabels = cursor.getUnsignedInt(); - long hsbLabels = cursor.getByte() & 0xFF; // so that a negative byte won't fill the "extended" bits with ones. - long labels = lsbLabels | (hsbLabels << 32); - byte extra = cursor.getByte(); - boolean dense = (extra & 0x1) > 0; - - record.setDense( dense ); - record.setNextRel( longFromIntAndMod( nextRel, relModifier ) ); - record.setNextProp( longFromIntAndMod( nextProp, propModifier ) ); - record.setInUse( inUse ); - record.setLabelField( labels, Collections.emptyList() ); - } + long relModifier = (inUseByte & 0xEL) << 31; + long propModifier = (inUseByte & 0xF0L) << 28; - /** - * Scan the given range of records both inclusive, and pass all the in-use ones to the given processor, one by one. - * - * The record passed to the NodeRecordScanner is reused instead of reallocated for every record, so it must be - * cloned if you want to save it for later. - */ - public void scanAllRecords( Visitor visitor ) throws IOException - { - long startPageId = pageIdForRecord( 0 ); - long currentPageId = startPageId; - long endPageId = pageIdForRecord( getHighestPossibleIdInUse() ); - long currentRecordId = 0; - NodeRecord record = new NodeRecord( -1 ); - int recordsPerPage = storeFile.pageSize() / getRecordSize(); - - try ( PageCursor cursor = storeFile.io( startPageId, PF_SHARED_READ_LOCK | PF_READ_AHEAD ) ) - { - while ( currentPageId <= endPageId && cursor.next() ) - { - for ( int i = 0; i < recordsPerPage; i++ ) - { - record.setId( currentRecordId ); - int offset = offsetForId( currentRecordId ); - do { - cursor.setOffset( offset ); - byte inUseByte = cursor.getByte(); - boolean isInUse = isInUse( inUseByte ); - readIntoRecord( cursor, record, inUseByte, isInUse ); - } while ( cursor.shouldRetry() ); - - if ( record.inUse() ) - { - if ( visitor.visit( record ) ) - { - return; - } - } - currentRecordId++; - } - currentPageId++; - } + long lsbLabels = cursor.getUnsignedInt(); + long hsbLabels = cursor.getByte() & 0xFF; // so that a negative byte won't fill the "extended" bits with ones. + long labels = lsbLabels | (hsbLabels << 32); + byte extra = cursor.getByte(); + boolean dense = (extra & 0x1) > 0; + + record.initialize( isInUse, + longFromIntAndMod( nextProp, propModifier ), dense, + longFromIntAndMod( nextRel, relModifier ), labels ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyKeyTokenStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyKeyTokenStore.java index 0bae688c64082..9d9d8524f38d6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyKeyTokenStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyKeyTokenStore.java @@ -48,27 +48,29 @@ public PropertyKeyTokenStore( LogProvider logProvider, DynamicStringStore nameStore ) { - super( fileName, config, IdType.PROPERTY_KEY_TOKEN, idGeneratorFactory, pageCache, - logProvider, nameStore, new Token.Factory() ); + super( fileName, config, IdType.PROPERTY_KEY_TOKEN, idGeneratorFactory, pageCache, logProvider, nameStore, + TYPE_DESCRIPTOR, new Token.Factory() ); } @Override - public void accept( RecordStore.Processor processor, PropertyKeyTokenRecord record ) throws FAILURE + public void accept( RecordStore.Processor processor, + PropertyKeyTokenRecord record ) throws FAILURE { processor.processPropertyKeyToken( this, record ); } @Override - protected PropertyKeyTokenRecord newRecord( int id ) + public PropertyKeyTokenRecord newRecord() { - return new PropertyKeyTokenRecord( id ); + return new PropertyKeyTokenRecord( -1 ); } @Override - protected void readRecord( PropertyKeyTokenRecord record, PageCursor cursor ) + protected void readRecord( PageCursor cursor, PropertyKeyTokenRecord record, boolean inUse ) { - record.setPropertyCount( cursor.getInt() ); - record.setNameId( cursor.getInt() ); + int propertyCount = cursor.getInt(); + int nameId = cursor.getInt(); + record.initialize( inUse, nameId, propertyCount ); } @Override @@ -83,10 +85,4 @@ public int getRecordSize() { return RECORD_SIZE; } - - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java index 61c7d0ae9e146..1807e10e70132 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java @@ -27,37 +27,39 @@ import java.util.List; import org.neo4j.collection.primitive.PrimitiveLongObjectMap; -import org.neo4j.cursor.GenericCursor; +import org.neo4j.cursor.Cursor; import org.neo4j.helpers.UTF8; import org.neo4j.helpers.collection.IteratorUtil; import org.neo4j.helpers.collection.Pair; import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; -import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; -import org.neo4j.kernel.impl.store.id.IdType; +import org.neo4j.io.pagecache.PagedFile; import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.api.store.PropertyRecordCursor; +import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; +import org.neo4j.kernel.impl.store.id.IdType; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.transaction.state.PropertyRecordChange; import org.neo4j.logging.LogProvider; import static org.neo4j.helpers.collection.IteratorUtil.first; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; import static org.neo4j.kernel.impl.store.DynamicArrayStore.getRightArray; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; /** * Implementation of the property store. This implementation has two dynamic * stores. One used to store keys and another for string property values. */ -public class PropertyStore extends AbstractRecordStore +public class PropertyStore extends CommonAbstractStore { - public static abstract class Configuration extends AbstractStore.Configuration + public static abstract class Configuration extends CommonAbstractStore.Configuration { } @@ -72,9 +74,9 @@ public static abstract class Configuration extends AbstractStore.Configuration + DEFAULT_PAYLOAD_SIZE /*property blocks*/; // = 41 - private final DynamicStringStore stringPropertyStore; + private final DynamicStringStore stringStore; private final PropertyKeyTokenStore propertyKeyTokenStore; - private final DynamicArrayStore arrayPropertyStore; + private final DynamicArrayStore arrayStore; private final PropertyPhysicalToLogicalConverter physicalToLogicalConverter; public PropertyStore( @@ -87,10 +89,10 @@ public PropertyStore( PropertyKeyTokenStore propertyKeyTokenStore, DynamicArrayStore arrayPropertyStore ) { - super( fileName, configuration, IdType.PROPERTY, idGeneratorFactory, pageCache, logProvider ); - this.stringPropertyStore = stringPropertyStore; + super( fileName, configuration, IdType.PROPERTY, idGeneratorFactory, pageCache, logProvider, TYPE_DESCRIPTOR ); + this.stringStore = stringPropertyStore; this.propertyKeyTokenStore = propertyKeyTokenStore; - this.arrayPropertyStore = arrayPropertyStore; + this.arrayStore = arrayPropertyStore; this.physicalToLogicalConverter = new PropertyPhysicalToLogicalConverter( this ); } @@ -103,18 +105,12 @@ public void accept( RecordStore.Processor p public DynamicStringStore getStringStore() { - return stringPropertyStore; + return stringStore; } public DynamicArrayStore getArrayStore() { - return arrayPropertyStore; - } - - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; + return arrayStore; } @Override @@ -135,39 +131,7 @@ public PropertyKeyTokenStore getPropertyKeyTokenStore() } @Override - public void updateRecord( PropertyRecord record ) - { - long pageId = pageIdForRecord( record.getId() ); - updatePropertyBlocks( record ); - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) // should always be true - { - do - { - updateRecord( record, cursor ); - } while ( cursor.shouldRetry() ); - } - else - { - throw new UnderlyingStorageException( - "Could not pin page[" + pageId + "] for updateRecord: " + record ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - - } - - @Override - public void forceUpdateRecord( PropertyRecord record ) - { - updateRecord( record ); // TODO: should we do something special for property records? - } - - private void updateRecord( PropertyRecord record, PageCursor cursor ) + protected void writeRecord( PageCursor cursor, PropertyRecord record ) { long id = record.getId(); cursor.setOffset( (int) (id * RECORD_SIZE % storeFile.pageSize()) ); @@ -205,13 +169,19 @@ private void updateRecord( PropertyRecord record, PageCursor cursor ) } else { - freeId( id ); // skip over the record header, nothing useful there cursor.setOffset( cursor.getOffset() + 9 ); cursor.putLong( 0 ); } } + @Override + public void updateRecord( PropertyRecord record ) + { + updatePropertyBlocks( record ); + super.updateRecord( record ); + } + private void updatePropertyBlocks( PropertyRecord record ) { if ( record.inUse() ) @@ -241,11 +211,11 @@ private void updateDynamicRecords( List records ) { if ( valueRecord.getType() == PropertyType.STRING.intValue() ) { - stringPropertyStore.updateRecord( valueRecord ); + stringStore.updateRecord( valueRecord ); } else if ( valueRecord.getType() == PropertyType.ARRAY.intValue() ) { - arrayPropertyStore.updateRecord( valueRecord ); + arrayStore.updateRecord( valueRecord ); } else { @@ -255,155 +225,69 @@ else if ( valueRecord.getType() == PropertyType.ARRAY.intValue() ) } } - public PropertyRecord getLightRecord( long id ) - { - return getRecord( id ); - } - - public void ensureHeavy( PropertyBlock block ) + @Override + public void ensureHeavy( PropertyRecord record ) { - if ( block.getType() == PropertyType.STRING ) - { - if ( block.isLight() ) - { - try ( GenericCursor stringRecords = stringPropertyStore.getRecordsCursor( - block.getSingleValueLong(), false ) ) - { - while ( stringRecords.next() ) - { - stringRecords.get().setType( PropertyType.STRING.intValue() ); - block.addValueRecord( stringRecords.get().clone() ); - } - } - } - for ( DynamicRecord stringRecord : block.getValueRecords() ) - { - stringPropertyStore.ensureHeavy( stringRecord ); - } - } - else if ( block.getType() == PropertyType.ARRAY ) + for ( PropertyBlock block : record ) { - if ( block.isLight() ) - { - Collection arrayRecords = arrayPropertyStore.getLightRecords( - block.getSingleValueLong() ); - for ( DynamicRecord arrayRecord : arrayRecords ) - { - arrayRecord.setType( PropertyType.ARRAY.intValue() ); - block.addValueRecord( arrayRecord ); - } - } - for ( DynamicRecord arrayRecord : block.getValueRecords() ) - { - arrayPropertyStore.ensureHeavy( arrayRecord ); - } + ensureHeavy( block ); } } - @Override - public PropertyRecord getRecord( long id ) - { - return getRecord( new PropertyRecord( id ) ); - } - - public PropertyRecord getRecord( PropertyRecord record ) + public void ensureHeavy( PropertyBlock block ) { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( record.getId() ), PF_SHARED_READ_LOCK ) ) + if ( !block.isLight() ) { - PropertyRecord tmpRecord = null; - if ( cursor.next() ) - { - do - { - tmpRecord = getRecord( cursor, record ); - } while ( cursor.shouldRetry() ); - } - - if ( tmpRecord == null || !tmpRecord.inUse() ) - { - throw new InvalidRecordException( "PropertyRecord[" + record.getId() + "] not in use" ); - } - - return record; + return; } - catch ( IOException e ) + + PropertyType type = block.getType(); + RecordStore dynamicStore = dynamicStoreForValueType( type ); + if ( dynamicStore == null ) { - throw new UnderlyingStorageException( e ); + return; } - } - public PageCursor newReadCursor( long recordId ) throws IOException - { - PageCursor cursor = storeFile.io( pageIdForRecord( recordId ), PF_SHARED_READ_LOCK ); - try + try ( Cursor dynamicRecords = dynamicStore.placeRecordCursor( block.getSingleValueLong(), + dynamicStore.newRecordCursor( dynamicStore.newRecord() ), NORMAL ) ) { - if ( !cursor.next() ) + while ( dynamicRecords.next() ) { - throw new IOException( "Record not found: " + recordId ); + dynamicRecords.get().setType( type.intValue() ); + block.addValueRecord( dynamicRecords.get().clone() ); } - cursor.setOffset( (int) (recordId * RECORD_SIZE % storeFile.pageSize()) ); - return cursor; - } - catch ( Throwable failure ) - { - cursor.close(); - throw failure; } } - public PropertyRecord getRecord( PropertyRecord record, PageCursor cursor ) + private RecordStore dynamicStoreForValueType( PropertyType type ) { - try - { - PropertyRecord tmpRecord = null; - if ( cursor.next( pageIdForRecord( record.getId() ) ) ) - { - do - { - tmpRecord = getRecord( cursor, record ); - } while ( cursor.shouldRetry() ); - } - - if ( tmpRecord == null || !tmpRecord.inUse() ) - { - throw new InvalidRecordException( "PropertyRecord[" + record.getId() + "] not in use" ); - } - - return record; - } - catch ( IOException e ) + switch ( type ) { - throw new UnderlyingStorageException( e ); + case ARRAY: return arrayStore; + case STRING: return stringStore; + default: return null; } } - @Override - public PropertyRecord forceGetRecord( long id ) + /** + * This method is too leaky and ignores the abstractions set in place by {@link RecordStore} and + * {@link AbstractBaseRecord}. The point is to get access to the raw record bytes from a {@link PageCursor} + * without creating/deserializing that data into objects. + */ + public PageCursor newReadCursor( long recordId ) throws IOException { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - PropertyRecord record = new PropertyRecord( id ); - if ( cursor.next() ) - { - do - { - record = getRecord( cursor, record ); - } while ( cursor.shouldRetry() ); - } - return record == null ? new PropertyRecord( id ) : record; - } - catch ( IOException e ) + PageCursor cursor = storeFile.io( pageIdForRecord( recordId ), PagedFile.PF_SHARED_READ_LOCK ); + if ( !cursor.next() ) { - throw new UnderlyingStorageException( e ); + cursor.close(); + throw new InvalidRecordException( "Record not found: " + recordId ); } + cursor.setOffset( offsetForId( recordId ) ); + return cursor; } - private PropertyRecord getRecordFromBuffer( long id, PageCursor cursor ) - { - return getRecordFromBuffer( cursor, new PropertyRecord( id ) ); - } - - private PropertyRecord getRecordFromBuffer( PageCursor cursor, PropertyRecord record ) + @Override + protected void readRecord( PageCursor cursor, PropertyRecord record, RecordLoad mode ) { record.clearPropertyBlocks(); @@ -420,6 +304,7 @@ private PropertyRecord getRecordFromBuffer( PageCursor cursor, PropertyRecord re record.setPrevProp( longFromIntAndMod( prevProp, prevMod ) ); record.setNextProp( longFromIntAndMod( nextProp, nextMod ) ); + record.setInUse( false ); while ( cursor.getOffset() - offsetAtBeginning < RECORD_SIZE ) { PropertyBlock newBlock = getPropertyBlock( cursor ); @@ -434,13 +319,6 @@ private PropertyRecord getRecordFromBuffer( PageCursor cursor, PropertyRecord re break; } } - return record; - } - - private PropertyRecord getRecord( PageCursor cursor, PropertyRecord record ) - { - cursor.setOffset( (int) (record.getId() * RECORD_SIZE % storeFile.pageSize()) ); - return getRecordFromBuffer( cursor, record ); } /* @@ -475,6 +353,12 @@ public Object getValue( PropertyBlock propertyBlock ) return propertyBlock.getType().getValue( propertyBlock, this ); } + @Override + public long getNextRecordReference( PropertyRecord record ) + { + return record.getNextProp(); + } + public static void allocateStringRecords( Collection target, byte[] chars, DynamicRecordAllocator allocator ) { @@ -490,7 +374,7 @@ public static void allocateArrayRecords( Collection target, Objec public void encodeValue( PropertyBlock block, int keyId, Object value ) { - encodeValue( block, keyId, value, stringPropertyStore, arrayPropertyStore ); + encodeValue( block, keyId, value, stringStore, arrayStore ); } public static void encodeValue( PropertyBlock block, int keyId, Object value, @@ -608,7 +492,7 @@ public String getStringFor( PropertyBlock propertyBlock ) public String getStringFor( Collection dynamicRecords ) { - Pair source = stringPropertyStore.readFullByteArray( dynamicRecords, PropertyType.STRING ); + Pair source = stringStore.readFullByteArray( dynamicRecords, PropertyType.STRING ); // A string doesn't have a header in the data array return decodeString( source.other() ); } @@ -621,17 +505,7 @@ public Object getArrayFor( PropertyBlock propertyBlock ) public Object getArrayFor( Iterable records ) { - return getRightArray( arrayPropertyStore.readFullByteArray( records, PropertyType.ARRAY ) ); - } - - public int getStringBlockSize() - { - return stringPropertyStore.getBlockSize(); - } - - public int getArrayBlockSize() - { - return arrayPropertyStore.getBlockSize(); + return getRightArray( arrayStore.readFullByteArray( records, PropertyType.ARRAY ) ); } @Override @@ -658,7 +532,8 @@ public Collection getPropertyRecordChain( long firstRecordId ) List toReturn = new LinkedList<>(); while ( nextProp != Record.NO_NEXT_PROPERTY.intValue() ) { - PropertyRecord propRecord = getLightRecord( nextProp ); + PropertyRecord propRecord = new PropertyRecord( nextProp ); + getRecord( nextProp, propRecord, RecordLoad.NORMAL ); toReturn.add( propRecord ); nextProp = propRecord.getNextProp(); } @@ -675,7 +550,7 @@ public Collection getPropertyRecordChain( long firstRecordId, PropertyRecord propRecord = propertyLookup.get( nextProp ); if ( propRecord == null ) { - propRecord = getLightRecord( nextProp ); + getRecord( nextProp, propRecord = newRecord(), RecordLoad.NORMAL ); } toReturn.add( propRecord ); nextProp = propRecord.getNextProp(); @@ -696,8 +571,24 @@ public void toLogicalUpdates( Collection target, * see if there are any PropertyBlocks in use in it. */ @Override - protected boolean isRecordInUse( PageCursor cursor ) + protected boolean isInUse( PageCursor cursor ) + { + int offsetAtBeginning = cursor.getOffset(); + cursor.setOffset( offsetAtBeginning + 1/*mod*/ + 4/*prev*/ + 4/*next*/ ); + while ( cursor.getOffset() - offsetAtBeginning < RECORD_SIZE ) + { + long block = cursor.getLong(); + if ( PropertyType.getPropertyType( block, true ) != null ) + { + return true; + } + } + return false; + } + + @Override + public PropertyRecord newRecord() { - return getRecordFromBuffer( 0 /*id doesn't matter here*/, cursor ).inUse(); + return new PropertyRecord( -1 ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicBlockSize.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordCursor.java similarity index 69% rename from community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicBlockSize.java rename to community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordCursor.java index 514a9c1ed5045..7a771090adee3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicBlockSize.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordCursor.java @@ -19,7 +19,14 @@ */ package org.neo4j.kernel.impl.store; -public interface DynamicBlockSize +import org.neo4j.cursor.Cursor; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; + +public interface RecordCursor extends Cursor { - int getBlockSize(); + RecordCursor init( long id, RecordLoad mode, PageCursor pageCursor ); + + boolean next( long id ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java index 6bfc01bd28e45..86deaaeadd994 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java @@ -27,6 +27,8 @@ import org.neo4j.helpers.collection.PrefetchingIterator; import org.neo4j.helpers.progress.ProgressListener; import org.neo4j.kernel.impl.store.id.IdType; +import org.neo4j.helpers.collection.IterableWrapper; +import org.neo4j.io.pagecache.PageCursor; import org.neo4j.kernel.impl.store.id.IdSequence; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -34,11 +36,28 @@ import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PropertyKeyTokenRecord; import org.neo4j.kernel.impl.store.record.PropertyRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; -public interface RecordStore extends IdSequence +/** + * A store for {@link #updateRecord(AbstractBaseRecord) updating} and + * {@link #getRecord(long, AbstractBaseRecord, RecordLoad) getting} records. + * + * There are two ways of getting records, either one-by-one using + * {@link #getRecord(long, AbstractBaseRecord, RecordLoad)}, passing in record retrieved from {@link #newRecord()}. + * This to make a conscious decision about who will create the record instance and in that process figure out + * ways to reduce number of record instances created. The other way is to use a {@link RecordCursor}, created + * by {@link #newRecordCursor(AbstractBaseRecord)} and placed at a certain record using + * {@link #placeRecordCursor(long, RecordCursor, RecordLoad)}. A {@link RecordCursor} will keep underlying + * {@link PageCursor} open until until the {@link RecordCursor} is closed and so will be efficient if multiple + * records are retrieved from it. A {@link RecordCursor} will follow {@link #getNextRecordReference(AbstractBaseRecord)} + * references to get to {@link RecordCursor#next()} record. + * + * @param type of {@link AbstractBaseRecord}. + */ +public interface RecordStore extends IdSequence { File getStorageFileName(); @@ -46,17 +65,58 @@ public interface RecordStore extends IdSequence long getHighestPossibleIdInUse(); - R getRecord( long id ); - - Collection getRecords( long id ); - - void updateRecord( R record ); - - R forceGetRecord( long id ); - - void forceUpdateRecord( R record ); - - void accept( Processor processor, R record ) throws FAILURE; + /** + * @return a new record instance for receiving data by {@link #getRecord(long, AbstractBaseRecord, RecordLoad)} + * and {@link #newRecordCursor(AbstractBaseRecord)}. + */ + RECORD newRecord(); + + /** + * Reads a record from the store into {@code target}. Depending on {@link RecordLoad} given there will + * be different behavior, although the {@code target} record will be marked with the specified + * {@code id} after participating in this method call. + *

    + *
  • {@link RecordLoad#CHECK}: As little data as possible is read to determine whether or not the record + * is in use. If not in use then no more data will be loaded into the target record and + * the the data of the record will be {@link AbstractBaseRecord#clear() cleared}.
    • + *
  • {@link RecordLoad#NORMAL}: Just like {@link RecordLoad#CHECK}, but with the difference that + * an {@link InvalidRecordException} will be thrown if the record isn't in use.
    • + *
  • {@link RecordLoad#FORCE}: The entire contents of the record will be loaded into the target record + * regardless if the record is in use or not. This leaves no guarantees about the data in the record + * after this method call, except that the id will be the specified {@code id}. + * + * @param id the id of the record to load. + * @param target record where data will be loaded into. This record will have its id set to the specified + * {@code id} as part of this method call. + * @param mode loading behaviour, read more in method description. + */ + RECORD getRecord( long id, RECORD target, RecordLoad mode ) throws InvalidRecordException; + + /** + * For stores that have other stores coupled underneath, the "top level" record will have a flag + * saying whether or not it's light. Light means that no records from the coupled store have been loaded yet. + * This method can load those records and enrich the target record with those, marking it as heavy. + */ + void ensureHeavy( RECORD record ); + + /** + * Reads records that belong together, a chain of records that as a whole forms the entirety of a data item. + * + * @param firstId record id of the first record to start loading from. + * @param mode {@link RecordLoad} mode. + * @return {@link Collection} of records in the loaded chain. + */ + Collection getRecords( long firstId, RecordLoad mode ) throws InvalidRecordException; + + RecordCursor newRecordCursor( RECORD record ); + + RecordCursor placeRecordCursor( long id, RecordCursor cursor, RecordLoad mode ); + + long getNextRecordReference( RECORD record ); + + void updateRecord( RECORD record ); + + void accept( Processor processor, RECORD record ) throws FAILURE; int getRecordSize(); @@ -76,68 +136,79 @@ class Delegator implements RecordStore { private final RecordStore actual; - public Delegator( RecordStore actual ) + @Override + public R newRecord() { - this.actual = actual; + return actual.newRecord(); } @Override - public long nextId() + public R getRecord( long id, R target, RecordLoad mode ) throws InvalidRecordException { - return actual.nextId(); + return actual.getRecord( id, target, mode ); } @Override - public File getStorageFileName() + public Collection getRecords( long firstId, RecordLoad mode ) throws InvalidRecordException { - return actual.getStorageFileName(); + return actual.getRecords( firstId, mode ); } @Override - public long getHighId() + public RecordCursor newRecordCursor( R record ) { - return actual.getHighId(); + return actual.newRecordCursor( record ); } @Override - public long getHighestPossibleIdInUse() + public RecordCursor placeRecordCursor( long id, RecordCursor cursor, RecordLoad mode ) { - return actual.getHighestPossibleIdInUse(); + return actual.placeRecordCursor( id, cursor, mode ); } @Override - public R getRecord( long id ) + public long getNextRecordReference( R record ) + { + return actual.getNextRecordReference( record ); + } + + public Delegator( RecordStore actual ) { - return actual.getRecord( id ); + this.actual = actual; } @Override - public Collection getRecords( long id ) + public long nextId() { - return actual.getRecords( id ); + return actual.nextId(); } @Override - public void updateRecord( R record ) + public File getStorageFileName() { - actual.updateRecord( record ); + return actual.getStorageFileName(); + } + + @Override + public long getHighId() + { + return actual.getHighId(); } @Override - public R forceGetRecord( long id ) + public long getHighestPossibleIdInUse() { - return actual.forceGetRecord( id ); + return actual.getHighestPossibleIdInUse(); } @Override - public void forceUpdateRecord( R record ) + public void updateRecord( R record ) { - actual.forceUpdateRecord( record ); + actual.updateRecord( record ); } @Override - public void accept( - org.neo4j.kernel.impl.store.RecordStore.Processor processor, R record ) throws FAILURE + public void accept( Processor processor, R record ) throws FAILURE { actual.accept( processor, record ); } @@ -177,6 +248,12 @@ public void flush() { actual.flush(); } + + @Override + public void ensureHeavy( R record ) + { + actual.ensureHeavy( record ); + } } @SuppressWarnings( "unchecked" ) @@ -220,6 +297,20 @@ public abstract void processLabelToken( RecordStore store, Lab public abstract void processRelationshipGroup( RecordStore store, RelationshipGroupRecord record ) throws FAILURE; + protected R getRecord( RecordStore store, long id, R into ) + { + store.getRecord( id, into, RecordLoad.FORCE ); + return into; + } + + public void applyById( RecordStore store, Iterable ids ) throws FAILURE + { + for ( R record : Scanner.scanById( store, ids ) ) + { + store.accept( this, record ); + } + } + public void applyFiltered( RecordStore store, Predicate... filters ) throws FAILURE { @@ -266,6 +357,7 @@ public static Iterable scan( final RecordStore return () -> new PrefetchingIterator() { final PrimitiveLongIterator ids = new StoreIdIterator( store, forward ); + final R record = store.newRecord(); @Override protected R fetchNextOrNull() @@ -273,12 +365,14 @@ protected R fetchNextOrNull() scan: while ( ids.hasNext() ) { - R record = store.forceGetRecord( ids.next() ); - for ( Predicate filter : filters ) + if ( store.getRecord( ids.next(), record, RecordLoad.FORCE ).inUse() ) { - if ( !filter.test( record ) ) + for ( Predicate filter : filters ) { - continue scan; + if ( !filter.test( record ) ) + { + continue scan; + } } } return record; @@ -287,5 +381,21 @@ protected R fetchNextOrNull() } }; } + + public static Iterable scanById( final RecordStore store, + Iterable ids ) + { + return new IterableWrapper( ids ) + { + private final R record = store.newRecord(); + + @Override + protected R underlyingObjectToObject( Long id ) + { + store.getRecord( id, record, RecordLoad.FORCE ); + return record; + } + }; + } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java index 1a021a658ce4b..2bd827a164fda 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java @@ -30,13 +30,11 @@ import org.neo4j.kernel.impl.store.id.IdType; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.logging.LogProvider; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; - -public class RelationshipGroupStore extends AbstractRecordStore +public class RelationshipGroupStore extends CommonAbstractStore { /* Record layout * @@ -56,149 +54,76 @@ public RelationshipGroupStore( PageCache pageCache, LogProvider logProvider ) { - super( fileName, config, IdType.RELATIONSHIP_GROUP, idGeneratorFactory, pageCache, logProvider ); - } - - @Override - protected ByteBuffer createHeaderRecord() - { - int denseNodeThreshold = configuration.get( GraphDatabaseSettings.dense_node_threshold ); - ByteBuffer headerRecord = ByteBuffer.allocate( RelationshipGroupStore.RECORD_SIZE ).putInt( denseNodeThreshold ); - headerRecord.flip(); - headerRecord.limit( headerRecord.capacity() ); - return headerRecord; + super( fileName, config, IdType.RELATIONSHIP_GROUP, idGeneratorFactory, pageCache, + logProvider, TYPE_DESCRIPTOR ); + denseNodeThreshold = configuration.get( GraphDatabaseSettings.dense_node_threshold ); } @Override - public RelationshipGroupRecord getRecord( long id ) - { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) - { - RelationshipGroupRecord record; - do - { - record = getRecord( id, cursor ); - } while ( cursor.shouldRetry() ); - - if ( record != null ) - { - return record; - } - } - throw new InvalidRecordException( "Record[" + id + "] not in use" ); - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - public RelationshipGroupRecord forceGetRecord( long id, RelationshipGroupRecord record ) + public int getNumberOfReservedLowIds() { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - readRecord( id, cursor, record ); - } - while ( cursor.shouldRetry() ); - } - else - { - record.setInUse( false ); - } - return record; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } + return 1; } @Override - public int getNumberOfReservedLowIds() + protected ByteBuffer createHeaderRecord() { - return 1; + return intHeaderData( denseNodeThreshold ); } @Override - protected void readAndVerifyBlockSize() throws IOException + protected void readAndVerifyHeaderRecord() throws IOException { denseNodeThreshold = getHeaderRecord(); } - private RelationshipGroupRecord getRecord( long id, PageCursor cursor ) + @Override + public RelationshipGroupRecord newRecord() { - RelationshipGroupRecord record = new RelationshipGroupRecord( -1, -1 ); - readRecord( id, cursor, record ); - return record; + return new RelationshipGroupRecord( -1 ); } - private void readRecord( long id, PageCursor cursor, RelationshipGroupRecord record ) + @Override + protected void readRecord( PageCursor cursor, RelationshipGroupRecord record, RecordLoad mode ) { - cursor.setOffset( offsetForId( id ) ); - // [ , x] in use // [ ,xxx ] high next id bits // [ xxx, ] high firstOut bits long inUseByte = cursor.getByte(); boolean inUse = (inUseByte&0x1) > 0; - - // [ ,xxx ] high firstIn bits - // [ xxx, ] high firstLoop bits - long highByte = cursor.getByte(); - - int type = cursor.getShort(); - long nextLowBits = cursor.getUnsignedInt(); - long nextOutLowBits = cursor.getUnsignedInt(); - long nextInLowBits = cursor.getUnsignedInt(); - long nextLoopLowBits = cursor.getUnsignedInt(); - long owningNode = cursor.getUnsignedInt() | (((long)cursor.getByte()) << 32); - - long nextMod = (inUseByte & 0xE) << 31; - long nextOutMod = (inUseByte & 0x70) << 28; - long nextInMod = (highByte & 0xE) << 31; - long nextLoopMod = (highByte & 0x70) << 28; - - record.setId( id ); - record.setType( type ); - record.setInUse( inUse ); - record.setNext( longFromIntAndMod( nextLowBits, nextMod ) ); - record.setFirstOut( longFromIntAndMod( nextOutLowBits, nextOutMod ) ); - record.setFirstIn( longFromIntAndMod( nextInLowBits, nextInMod ) ); - record.setFirstLoop( longFromIntAndMod( nextLoopLowBits, nextLoopMod ) ); - record.setOwningNode( owningNode ); - } - - @Override - public void updateRecord( RelationshipGroupRecord record ) - { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( record.getId() ), PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - updateRecord( record, cursor, false ); - } - while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) + if ( mode.shouldLoad( inUse ) ) { - throw new UnderlyingStorageException( e ); + // [ ,xxx ] high firstIn bits + // [ xxx, ] high firstLoop bits + long highByte = cursor.getByte(); + + int type = cursor.getShort(); + long nextLowBits = cursor.getUnsignedInt(); + long nextOutLowBits = cursor.getUnsignedInt(); + long nextInLowBits = cursor.getUnsignedInt(); + long nextLoopLowBits = cursor.getUnsignedInt(); + long owningNode = cursor.getUnsignedInt() | (((long)cursor.getByte()) << 32); + + long nextMod = (inUseByte & 0xE) << 31; + long nextOutMod = (inUseByte & 0x70) << 28; + long nextInMod = (highByte & 0xE) << 31; + long nextLoopMod = (highByte & 0x70) << 28; + + record.initialize( inUse, type, + longFromIntAndMod( nextOutLowBits, nextOutMod ), + longFromIntAndMod( nextInLowBits, nextInMod ), + longFromIntAndMod( nextLoopLowBits, nextLoopMod ), + owningNode, + longFromIntAndMod( nextLowBits, nextMod ) ); } } - private void updateRecord( RelationshipGroupRecord record, PageCursor cursor, boolean force ) + @Override + protected void writeRecord( PageCursor cursor, RelationshipGroupRecord record ) { long id = record.getId(); cursor.setOffset( offsetForId( id ) ); - if ( record.inUse() || force ) + if ( record.inUse() ) { long nextMod = record.getNext() == Record.NO_NEXT_RELATIONSHIP.intValue() ? 0 : (record.getNext() & 0x700000000L) >> 31; long nextOutMod = record.getFirstOut() == Record.NO_NEXT_RELATIONSHIP.intValue() ? 0 : (record.getFirstOut() & 0x700000000L) >> 28; @@ -225,48 +150,13 @@ private void updateRecord( RelationshipGroupRecord record, PageCursor cursor, bo else { cursor.putByte( Record.NOT_IN_USE.byteValue() ); - freeId( id ); - } - } - - @Override - public RelationshipGroupRecord forceGetRecord( long id ) - { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - RelationshipGroupRecord record = new RelationshipGroupRecord( id, -1 ); - if ( cursor.next() ) - { - do - { - readRecord( id, cursor, record ); - } while ( cursor.shouldRetry() ); - } - return record; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); } } @Override - public void forceUpdateRecord( RelationshipGroupRecord record ) + public long getNextRecordReference( RelationshipGroupRecord record ) { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( record.getId() ), PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) // should always be true - { - do - { - updateRecord( record, cursor, true ); - } while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } + return record.getNext(); } @Override @@ -276,24 +166,12 @@ public void accept( Processor processor, Re processor.processRelationshipGroup( this, record ); } - @Override - public int getRecordHeaderSize() - { - return getRecordSize(); - } - @Override public int getRecordSize() { return RECORD_SIZE; } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - public int getDenseNodeThreshold() { return denseNodeThreshold; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipStore.java index 4ff8d5fe48176..16ac3dd710841 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipStore.java @@ -20,25 +20,20 @@ package org.neo4j.kernel.impl.store; import java.io.File; -import java.io.IOException; - import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdType; -import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.record.Record; import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.logging.LogProvider; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; - /** * Implementation of the relationship store. */ -public class RelationshipStore extends AbstractRecordStore +public class RelationshipStore extends CommonAbstractStore { public static final String TYPE_DESCRIPTOR = "RelationshipStore"; @@ -55,7 +50,8 @@ public RelationshipStore( PageCache pageCache, LogProvider logProvider ) { - super( fileName, configuration, IdType.RELATIONSHIP, idGeneratorFactory, pageCache, logProvider ); + super( fileName, configuration, IdType.RELATIONSHIP, idGeneratorFactory, + pageCache, logProvider, TYPE_DESCRIPTOR ); } @Override @@ -64,12 +60,6 @@ public void accept( Processor processor, Re processor.processRelationship( this, record ); } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - @Override public int getRecordSize() { @@ -77,136 +67,15 @@ public int getRecordSize() } @Override - public int getRecordHeaderSize() - { - return getRecordSize(); - } - - @Override - public RelationshipRecord getRecord( long id ) - { - return getRecord( new RelationshipRecord( id ) ); - } - - public RelationshipRecord getRecord( RelationshipRecord record ) - { - return fillRecord( record.getId(), record, RecordLoad.NORMAL ) ? record : null; - } - - @Override - public RelationshipRecord forceGetRecord( long id ) - { - RelationshipRecord record = new RelationshipRecord( -1 ); - return fillRecord( id, record, RecordLoad.FORCE ) ? record : null; - } - - public RelationshipRecord getLightRel( long id ) - { - RelationshipRecord record = new RelationshipRecord( id ); - return fillRecord( id, record, RecordLoad.CHECK ) ? record : null; - } - - /** - * @return {@code true} if record successfully loaded and in use. - * If not in use the return value depends on the {@code loadMode}: - *
      - *
    1. NORMAL: throws {@link InvalidRecordException}
      2. - *
    2. CHECK: returns {@code false}
      3. - *
    3. FORCE: return {@code true}
      4. - *
    - */ - public boolean fillRecord( long id, RelationshipRecord target, RecordLoad loadMode ) - { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - boolean success = false; - if ( cursor.next() ) - { - do - { - success = readRecord( id, cursor, target ); - } while ( cursor.shouldRetry() ); - } - - if ( !success ) - { - if ( loadMode == RecordLoad.NORMAL ) - { - throw new InvalidRecordException( "RelationshipRecord[" + id + "] not in use" ); - } - else if ( loadMode == RecordLoad.CHECK ) - { - return false; - } - } - return true; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - public boolean inUse( long id ) - { - long pageId = pageIdForRecord( id ); - int offset = offsetForId( id ); - - try ( PageCursor cursor = storeFile.io( pageId, PF_SHARED_READ_LOCK ) ) - { - boolean recordIsInUse = false; - if ( cursor.next() ) - { - do - { - cursor.setOffset( offset ); - recordIsInUse = isInUse( cursor.getByte() ); - } while ( cursor.shouldRetry() ); - } - return recordIsInUse; - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - @Override - public void forceUpdateRecord( RelationshipRecord record ) + public RelationshipRecord newRecord() { - updateRecord( record, true ); + return new RelationshipRecord( -1 ); } @Override - public void updateRecord( RelationshipRecord record ) - { - updateRecord( record, false ); - } - - private void updateRecord( RelationshipRecord record, boolean force ) + protected void writeRecord( PageCursor cursor, RelationshipRecord record ) { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( record.getId() ), PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) // should always be true - { - do - { - updateRecord( record, cursor, force ); - } while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - private void updateRecord( RelationshipRecord record, - PageCursor cursor, boolean force ) - { - long id = record.getId(); - cursor.setOffset( offsetForId( id ) ); - if ( record.inUse() || force ) + if ( record.inUse() ) { long firstNode = record.getFirstNode(); short firstNodeMod = (short)((firstNode & 0x700000000L) >> 31); @@ -262,73 +131,62 @@ private void updateRecord( RelationshipRecord record, else { cursor.putByte( Record.NOT_IN_USE.byteValue() ); - freeId( id ); } } - private boolean readRecord( long id, PageCursor cursor, - RelationshipRecord record ) + @Override + protected void readRecord( PageCursor cursor, RelationshipRecord record, RecordLoad mode ) { - cursor.setOffset( offsetForId( id ) ); - // [ , x] in use flag // [ ,xxx ] first node high order bits // [xxxx, ] next prop high order bits long inUseByte = cursor.getByte(); - boolean inUse = (inUseByte & 0x1) == Record.IN_USE.intValue(); + if ( mode.shouldLoad( inUse ) ) + { + long firstNode = cursor.getUnsignedInt(); + long firstNodeMod = (inUseByte & 0xEL) << 31; - long firstNode = cursor.getUnsignedInt(); - long firstNodeMod = (inUseByte & 0xEL) << 31; - - long secondNode = cursor.getUnsignedInt(); - - // [ xxx, ][ , ][ , ][ , ] second node high order bits, 0x70000000 - // [ ,xxx ][ , ][ , ][ , ] first prev rel high order bits, 0xE000000 - // [ , x][xx , ][ , ][ , ] first next rel high order bits, 0x1C00000 - // [ , ][ xx,x ][ , ][ , ] second prev rel high order bits, 0x380000 - // [ , ][ , xxx][ , ][ , ] second next rel high order bits, 0x70000 - // [ , ][ , ][xxxx,xxxx][xxxx,xxxx] type - long typeInt = cursor.getInt(); - long secondNodeMod = (typeInt & 0x70000000L) << 4; - int type = (int)(typeInt & 0xFFFF); - - record.setId( id ); - record.setFirstNode( longFromIntAndMod( firstNode, firstNodeMod ) ); - record.setSecondNode( longFromIntAndMod( secondNode, secondNodeMod ) ); - record.setType( type ); - record.setInUse( inUse ); - - long firstPrevRel = cursor.getUnsignedInt(); - long firstPrevRelMod = (typeInt & 0xE000000L) << 7; - record.setFirstPrevRel( longFromIntAndMod( firstPrevRel, firstPrevRelMod ) ); - - long firstNextRel = cursor.getUnsignedInt(); - long firstNextRelMod = (typeInt & 0x1C00000L) << 10; - record.setFirstNextRel( longFromIntAndMod( firstNextRel, firstNextRelMod ) ); - - long secondPrevRel = cursor.getUnsignedInt(); - long secondPrevRelMod = (typeInt & 0x380000L) << 13; - record.setSecondPrevRel( longFromIntAndMod( secondPrevRel, secondPrevRelMod ) ); - - long secondNextRel = cursor.getUnsignedInt(); - long secondNextRelMod = (typeInt & 0x70000L) << 16; - record.setSecondNextRel( longFromIntAndMod( secondNextRel, secondNextRelMod ) ); - - long nextProp = cursor.getUnsignedInt(); - long nextPropMod = (inUseByte & 0xF0L) << 28; - - byte extraByte = cursor.getByte(); - - record.setFirstInFirstChain( (extraByte & 0x1) != 0 ); - record.setFirstInSecondChain( (extraByte & 0x2) != 0 ); + long secondNode = cursor.getUnsignedInt(); - record.setNextProp( longFromIntAndMod( nextProp, nextPropMod ) ); - return inUse; - } - - public boolean fillChainRecord( long id, RelationshipRecord record ) - { - return fillRecord( id, record, RecordLoad.CHECK ); + // [ xxx, ][ , ][ , ][ , ] second node high order bits, 0x70000000 + // [ ,xxx ][ , ][ , ][ , ] first prev rel high order bits, 0xE000000 + // [ , x][xx , ][ , ][ , ] first next rel high order bits, 0x1C00000 + // [ , ][ xx,x ][ , ][ , ] second prev rel high order bits, 0x380000 + // [ , ][ , xxx][ , ][ , ] second next rel high order bits, 0x70000 + // [ , ][ , ][xxxx,xxxx][xxxx,xxxx] type + long typeInt = cursor.getInt(); + long secondNodeMod = (typeInt & 0x70000000L) << 4; + int type = (int)(typeInt & 0xFFFF); + + long firstPrevRel = cursor.getUnsignedInt(); + long firstPrevRelMod = (typeInt & 0xE000000L) << 7; + + long firstNextRel = cursor.getUnsignedInt(); + long firstNextRelMod = (typeInt & 0x1C00000L) << 10; + + long secondPrevRel = cursor.getUnsignedInt(); + long secondPrevRelMod = (typeInt & 0x380000L) << 13; + + long secondNextRel = cursor.getUnsignedInt(); + long secondNextRelMod = (typeInt & 0x70000L) << 16; + + long nextProp = cursor.getUnsignedInt(); + long nextPropMod = (inUseByte & 0xF0L) << 28; + + byte extraByte = cursor.getByte(); + + record.initialize( inUse, + longFromIntAndMod( nextProp, nextPropMod ), + longFromIntAndMod( firstNode, firstNodeMod ), + longFromIntAndMod( secondNode, secondNodeMod ), + type, + longFromIntAndMod( firstPrevRel, firstPrevRelMod ), + longFromIntAndMod( firstNextRel, firstNextRelMod ), + longFromIntAndMod( secondPrevRel, secondPrevRelMod ), + longFromIntAndMod( secondNextRel, secondNextRelMod ), + (extraByte & 0x1) != 0, + (extraByte & 0x2) != 0 ); + } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipTypeTokenStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipTypeTokenStore.java index 86df1f3ffe6a7..9eac7aeb5c9a3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipTypeTokenStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipTypeTokenStore.java @@ -48,8 +48,8 @@ public RelationshipTypeTokenStore( LogProvider logProvider, DynamicStringStore nameStore ) { - super( fileName, config, IdType.RELATIONSHIP_TYPE_TOKEN, idGeneratorFactory, pageCache, logProvider, nameStore, - new RelationshipTypeToken.Factory() ); + super( fileName, config, IdType.RELATIONSHIP_TYPE_TOKEN, idGeneratorFactory, pageCache, + logProvider, nameStore, TYPE_DESCRIPTOR, new RelationshipTypeToken.Factory() ); } @Override @@ -60,9 +60,9 @@ public void accept( Processor processor, Re } @Override - protected RelationshipTypeTokenRecord newRecord( int id ) + public RelationshipTypeTokenRecord newRecord() { - return new RelationshipTypeTokenRecord( id ); + return new RelationshipTypeTokenRecord( -1 ); } @Override @@ -71,12 +71,6 @@ public int getRecordSize() return RECORD_SIZE; } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - @Override protected boolean isRecordReserved( PageCursor cursor ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStorage.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStorage.java index 27f1c15ddaca7..e3a14a4658b1b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStorage.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStorage.java @@ -36,6 +36,7 @@ import org.neo4j.kernel.impl.store.record.IndexRule; import org.neo4j.kernel.impl.store.record.NodePropertyConstraintRule; import org.neo4j.kernel.impl.store.record.NodePropertyExistenceConstraintRule; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipPropertyConstraintRule; import org.neo4j.kernel.impl.store.record.RelationshipPropertyExistenceConstraintRule; import org.neo4j.kernel.impl.store.record.UniquePropertyConstraintRule; @@ -205,6 +206,7 @@ public Iterator loadAllSchemaRules() private final long highestId = schemaStore.getHighestPossibleIdInUse(); private long currentId = 1; /*record 0 contains the block size*/ private final byte[] scratchData = newRecordBuffer(); + private final DynamicRecord record = schemaStore.newRecord(); @Override protected SchemaRule fetchNextOrNull() @@ -212,7 +214,7 @@ protected SchemaRule fetchNextOrNull() while ( currentId <= highestId ) { long id = currentId++; - DynamicRecord record = schemaStore.forceGetRecord( id ); + schemaStore.getRecord( id, record, RecordLoad.FORCE ); if ( record.inUse() && record.isStartRecord() ) { try @@ -247,7 +249,7 @@ private SchemaRule getSchemaRule( long id, byte[] buffer ) throws MalformedSchem Collection records; try { - records = schemaStore.getRecords( id ); + records = schemaStore.getRecords( id, RecordLoad.NORMAL ); } catch ( Exception e ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStore.java index 7a8404b127209..781801f9a648c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/SchemaStore.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.List; import org.neo4j.helpers.collection.IteratorUtil; import org.neo4j.io.pagecache.PageCache; @@ -37,6 +38,8 @@ import org.neo4j.logging.LogProvider; import org.neo4j.storageengine.api.schema.SchemaRule; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; + public class SchemaStore extends AbstractDynamicStore implements Iterable { // store version, each store ends with this string (byte encoded) @@ -52,7 +55,7 @@ public SchemaStore( PageCache pageCache, LogProvider logProvider ) { - super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider, BLOCK_SIZE ); + super( fileName, conf, idType, idGeneratorFactory, pageCache, logProvider, TYPE_DESCRIPTOR, BLOCK_SIZE ); } @Override @@ -61,19 +64,13 @@ public void accept( Processor processor, Dy processor.processSchema( this, record ); } - @Override - public String getTypeDescriptor() - { - return TYPE_DESCRIPTOR; - } - - public Collection allocateFrom( SchemaRule rule ) + public List allocateFrom( SchemaRule rule ) { RecordSerializer serializer = new RecordSerializer(); serializer = serializer.append( (AbstractSchemaRule)rule ); - Collection records = new ArrayList<>(); + List records = new ArrayList<>(); allocateRecordsFromBytes( records, serializer.serialize(), - IteratorUtil.iterator( forceGetRecord( rule.getId() ) ), this ); + IteratorUtil.iterator( getRecord( rule.getId(), newRecord(), FORCE ) ), this ); return records; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java index 8471ee1afa08b..89993d5aa5de1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/StoreAccess.java @@ -215,10 +215,12 @@ protected RecordStore[] allStores() }; } - private static RecordStore wrapNodeDynamicLabelStore( RecordStore store ) { - return new DelegatingRecordStore( store ) { + private static RecordStore wrapNodeDynamicLabelStore( RecordStore store ) + { + return new RecordStore.Delegator( store ) + { @Override - public void accept( Processor processor, DynamicRecord record) + public void accept( Processor processor, DynamicRecord record ) throws FAILURE { processor.processLabelArrayWithOwner( this, record ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/TokenStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/TokenStore.java index cdb633d68ce14..dbac468fa1f18 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/TokenStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/TokenStore.java @@ -20,7 +20,6 @@ package org.neo4j.kernel.impl.store; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; @@ -29,21 +28,21 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdType; -import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.TokenRecord; import org.neo4j.logging.LogProvider; import org.neo4j.storageengine.api.Token; import org.neo4j.storageengine.api.TokenFactory; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_WRITE_LOCK; -import static org.neo4j.io.pagecache.PagedFile.PF_SHARED_READ_LOCK; import static org.neo4j.kernel.impl.store.PropertyStore.decodeString; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; -public abstract class TokenStore extends AbstractRecordStore +public abstract class TokenStore extends CommonAbstractStore { public static final int NAME_STORE_BLOCK_SIZE = 30; @@ -58,9 +57,10 @@ public TokenStore( PageCache pageCache, LogProvider logProvider, DynamicStringStore nameStore, + String typeDescriptor, TokenFactory tokenFactory ) { - super( file, configuration, idType, idGeneratorFactory, pageCache, logProvider); + super( file, configuration, idType, idGeneratorFactory, pageCache, logProvider, typeDescriptor ); this.nameStore = nameStore; this.tokenFactory = tokenFactory; } @@ -70,12 +70,6 @@ public DynamicStringStore getNameStore() return nameStore; } - @Override - public int getRecordHeaderSize() - { - return getRecordSize(); - } - @Override protected boolean doFastIdGeneratorRebuild() { @@ -87,17 +81,14 @@ public List getTokens( int maxCount ) LinkedList records = new LinkedList<>(); long maxIdInUse = getHighestPossibleIdInUse(); int found = 0; + RECORD record = newRecord(); for ( int i = 0; i <= maxIdInUse && found < maxCount; i++ ) { - RECORD record; - try - { - record = getRecord( i ); - } - catch ( InvalidRecordException t ) + if ( !getRecord( i, record, RecordLoad.CHECK ).inUse() ) { continue; } + found++; if ( record != null && record.inUse() && record.getNameId() != Record.RESERVED.intValue() ) { @@ -110,84 +101,10 @@ record = getRecord( i ); public TOKEN getToken( int id ) { - RECORD record = getRecord( id ); + RECORD record = getRecord( id, newRecord(), NORMAL ); return tokenFactory.newToken( getStringFor( record ), record.getId() ); } - public RECORD getRecord( int id ) - { - RECORD record = newRecord( id ); - byte inUseByte = Record.NOT_IN_USE.byteValue(); - - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - inUseByte = getRecord( id, record, cursor ); - } while ( cursor.shouldRetry() ); - - } - - if ( inUseByte != Record.IN_USE.byteValue() ) - { - throw new InvalidRecordException( getClass().getSimpleName() + " Record[" + id + "] not in use" ); - } - - checkInUseByteValidity( id, inUseByte ); - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - - record.addNameRecords( nameStore.getLightRecords( record.getNameId() ) ); - return record; - } - - @Override - public RECORD getRecord( long id ) - { - return getRecord( (int) id ); - } - - @Override - public RECORD forceGetRecord( long id ) - { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( id ), PF_SHARED_READ_LOCK ) ) - { - if ( cursor.next() ) - { - RECORD record = newRecord( (int) id ); - do - { - getRecord( (int) id, record, cursor ); - } - while ( cursor.shouldRetry() ); - - record.setIsLight( true ); - return record; - } - else - { - return newRecord( (int) id ); - } - } - catch ( IOException e ) - { - return newRecord( (int) id ); - } - } - - private void checkInUseByteValidity( long id, byte inUseByte ) - { - if ( inUseByte != Record.IN_USE.byteValue() && inUseByte != Record.NOT_IN_USE.byteValue() ) - { - throw new InvalidRecordException( getClass().getSimpleName() + " Record[" + id + "] unknown in use flag[" + inUseByte + "]" ); - } - } - public Collection allocateNameRecords( byte[] chars ) { Collection records = new ArrayList<>(); @@ -198,7 +115,7 @@ public Collection allocateNameRecords( byte[] chars ) @Override public void updateRecord( RECORD record ) { - forceUpdateRecord( record ); + super.updateRecord( record ); if ( !record.isLight() ) { for ( DynamicRecord keyRecord : record.getNameRecords() ) @@ -209,49 +126,24 @@ public void updateRecord( RECORD record ) } @Override - public void forceUpdateRecord( RECORD record ) + protected void readRecord( PageCursor cursor, RECORD record, RecordLoad mode ) { - try ( PageCursor cursor = storeFile.io( pageIdForRecord( record.getId() ), PF_SHARED_WRITE_LOCK ) ) - { - if ( cursor.next() ) - { - do - { - updateRecord( record, cursor ); - } while ( cursor.shouldRetry() ); - } - } - catch ( IOException e ) - { - throw new UnderlyingStorageException( e ); - } - } - - protected abstract RECORD newRecord( int id ); - - protected byte getRecord( int id, RECORD record, PageCursor cursor ) - { - cursor.setOffset( offsetForId( id ) ); byte inUseByte = cursor.getByte(); boolean inUse = (inUseByte == Record.IN_USE.byteValue()); - - record.setInUse( inUse ); - if ( inUse ) + if ( mode.shouldLoad( inUse ) ) { - readRecord( record, cursor ); + readRecord( cursor, record, inUse ); } - return inUseByte; } - protected void readRecord( RECORD record, PageCursor cursor ) + protected void readRecord( PageCursor cursor, RECORD record, boolean inUse ) { - record.setNameId( cursor.getInt() ); + record.initialize( inUse, cursor.getInt() ); } - protected void updateRecord( RECORD record, PageCursor cursor ) + @Override + protected void writeRecord( PageCursor cursor, RECORD record ) { - int id = record.getId(); - cursor.setOffset( offsetForId( id ) ); if ( record.inUse() ) { cursor.putByte( Record.IN_USE.byteValue() ); @@ -260,7 +152,6 @@ protected void updateRecord( RECORD record, PageCursor cursor ) else { cursor.putByte( Record.NOT_IN_USE.byteValue() ); - freeId( id ); } } @@ -269,19 +160,20 @@ protected void writeRecord( RECORD record, PageCursor cursor ) cursor.putInt( record.getNameId() ); } + @Override public void ensureHeavy( RECORD record ) { - if (!record.isLight()) + if ( !record.isLight() ) { return; } - record.setIsLight( false ); - record.addNameRecords( nameStore.getRecords( record.getNameId() ) ); + record.addNameRecords( nameStore.getRecords( record.getNameId(), NORMAL ) ); } public String getStringFor( RECORD nameRecord ) { + ensureHeavy( nameRecord ); int recordToFind = nameRecord.getNameId(); Iterator records = nameRecord.getNameRecords().iterator(); Collection relevantRecords = new ArrayList<>(); @@ -291,7 +183,7 @@ public String getStringFor( RECORD nameRecord ) if ( record.inUse() && record.getId() == recordToFind ) { recordToFind = (int) record.getNextBlock(); -// // TODO: optimize here, high chance next is right one + // TODO: optimize here, high chance next is right one relevantRecords.add( record ); records = nameRecord.getNameRecords().iterator(); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java index b9595849de1ba..1374900ae2777 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/counts/CountsTracker.java @@ -50,6 +50,7 @@ import org.neo4j.register.Register; import static java.lang.String.format; + import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.indexSampleKey; import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.indexStatisticsKey; import static org.neo4j.kernel.impl.store.counts.keys.CountsKeyFactory.nodeKey; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/Abstract64BitRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/Abstract64BitRecord.java index e5b80686a2de5..2d00192fa1e2d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/Abstract64BitRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/Abstract64BitRecord.java @@ -23,14 +23,17 @@ public abstract class Abstract64BitRecord extends AbstractBaseRecord { private long id; - protected Abstract64BitRecord() + public Abstract64BitRecord( long id ) { - + this.id = id; + clear(); } - protected Abstract64BitRecord( long id ) + @Override + protected Abstract64BitRecord initialize( boolean inUse ) { - this.id = id; + super.initialize( inUse ); + return this; } public long getId() @@ -38,14 +41,15 @@ public long getId() return id; } + @Override public void setId( long id ) { this.id = id; } - + @Override public long getLongId() { - return id; + return getId(); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractBaseRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractBaseRecord.java index f3062d9403af0..0c4b848c7c878 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractBaseRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractBaseRecord.java @@ -23,13 +23,39 @@ import org.neo4j.helpers.CloneableInPublic; +/** + * {@link AbstractBaseRecord records} are intended to be reusable. Created with a zero-arg constructor + * and initialized with the public {@code initialize} method exposed by the specific record implementations, + * or {@link #clear() cleared} if reading a record that isn't in use. + */ public abstract class AbstractBaseRecord implements CloneableInPublic { - private boolean inUse = false; - private boolean created = false; + private boolean inUse; + private boolean created; + + protected AbstractBaseRecord initialize( boolean inUse ) + { + this.inUse = inUse; + this.created = false; + return this; + } + + /** + * Clears this record to its initial state. Initializing this record with an {@code initialize-method} + * doesn't require clear the record first, either initialize or clear suffices. + * Subclasses, most specific subclasses only, implements this method by calling initialize with + * zero-like arguments. + */ + public void clear() + { + inUse = false; + created = false; + } public abstract long getLongId(); + public abstract void setId( long id ); + public final boolean inUse() { return inUse; @@ -53,11 +79,8 @@ public final boolean isCreated() @Override public int hashCode() { - final int prime = 31; - int result = 1; long id = getLongId(); - result = prime * result + (int) (id ^ (id >>> 32)); - return result; + return (int) (( id >>> 32 ) ^ id ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractRecord.java index 7e94a1efb341a..d52a3ccc28886 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/AbstractRecord.java @@ -19,13 +19,16 @@ */ package org.neo4j.kernel.impl.store.record; +import org.neo4j.unsafe.impl.batchimport.Utils; + public abstract class AbstractRecord extends AbstractBaseRecord { - private final int id; + private int id; public AbstractRecord( int id ) { this.id = id; + clear(); } public int getId() @@ -33,6 +36,12 @@ public int getId() return id; } + @Override + public void setId( long id ) + { + this.id = Utils.safeCastLongToInt( id ); + } + @Override public long getLongId() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/DynamicRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/DynamicRecord.java index d3c3bcc7cc072..e47d531e32039 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/DynamicRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/DynamicRecord.java @@ -22,17 +22,18 @@ import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.PropertyType; - public class DynamicRecord extends Abstract64BitRecord { + private static final byte[] NO_DATA = new byte[0]; private static final int MAX_BYTES_IN_TO_STRING = 8, MAX_CHARS_IN_TO_STRING = 16; - private byte[] data = null; + private byte[] data; private int length; - private long nextBlock = Record.NO_NEXT_BLOCK.intValue(); + private long nextBlock; private int type; - private boolean startRecord = true; + private boolean startRecord; + @Deprecated public static DynamicRecord dynamicRecord( long id, boolean inUse ) { DynamicRecord record = new DynamicRecord( id ); @@ -40,6 +41,7 @@ public static DynamicRecord dynamicRecord( long id, boolean inUse ) return record; } + @Deprecated public static DynamicRecord dynamicRecord( long id, boolean inUse, boolean isStartRecord, long nextBlock, int type, byte [] data ) { @@ -56,12 +58,30 @@ public DynamicRecord( long id ) { super( id ); } - + + public DynamicRecord initialize( boolean inUse, boolean isStartRecord, long nextBlock, + int type, int length ) + { + super.initialize( inUse ); + this.startRecord = isStartRecord; + this.nextBlock = nextBlock; + this.type = type; + this.data = NO_DATA; + this.length = length; + return this; + } + + @Override + public void clear() + { + initialize( false, true, Record.NO_NEXT_BLOCK.intValue(), -1, 0 ); + } + public void setStartRecord( boolean startRecord ) { this.startRecord = startRecord; } - + public boolean isStartRecord() { return startRecord; @@ -77,26 +97,11 @@ public void setType( int type ) this.type = type; } - public boolean isLight() - { - return data == null; - } - public void setLength( int length ) { this.length = length; } - @Override - public void setInUse( boolean inUse ) - { - super.setInUse( inUse ); - if ( !inUse ) - { - data = null; - } - } - public void setInUse( boolean inUse, int type ) { this.type = type; @@ -136,72 +141,47 @@ public String toString() buf.append( "DynamicRecord[" ) .append( getId() ) .append( ",used=" ).append(inUse() ).append( "," ) - .append( "light=" ).append( isLight() ) .append("(" ).append( length ).append( "),type=" ); PropertyType type = PropertyType.getPropertyType( this.type << 24, true ); if ( type == null ) buf.append( this.type ); else buf.append( type.name() ); buf.append( ",data=" ); - if ( data != null ) + if ( type == PropertyType.STRING && data.length <= MAX_CHARS_IN_TO_STRING ) + { + buf.append( '"' ); + buf.append( PropertyStore.decodeString( data ) ); + buf.append( "\"," ); + } + else { - if ( type == PropertyType.STRING && data.length <= MAX_CHARS_IN_TO_STRING ) + buf.append( "byte[" ); + if ( data.length <= MAX_BYTES_IN_TO_STRING ) { - buf.append( '"' ); - buf.append( PropertyStore.decodeString( data ) ); - buf.append( "\"," ); + for ( int i = 0; i < data.length; i++ ) + { + if (i != 0) buf.append( ',' ); + buf.append( data[i] ); + } } else { - buf.append( "byte[" ); - if ( data.length <= MAX_BYTES_IN_TO_STRING ) - { - for ( int i = 0; i < data.length; i++ ) - { - if (i != 0) buf.append( ',' ); - buf.append( data[i] ); - } - } - else - { - buf.append( "size=" ).append( data.length ); - } - buf.append( "]," ); + buf.append( "size=" ).append( data.length ); } - } - else - { - buf.append( "null," ); + buf.append( "]," ); } buf.append( "start=" ).append( startRecord ); buf.append( ",next=" ).append( nextBlock ).append( "]" ); return buf.toString(); } - + @Override public DynamicRecord clone() { - DynamicRecord result = new DynamicRecord( getLongId() ); + DynamicRecord clone = new DynamicRecord( getLongId() ).initialize( inUse(), + startRecord, nextBlock, type, length ); if ( data != null ) - result.data = data.clone(); - result.setInUse( inUse() ); - result.length = length; - result.nextBlock = nextBlock; - result.type = type; - result.startRecord = startRecord; - return result; - } - - @Override - public boolean equals( Object obj ) - { - if ( !( obj instanceof DynamicRecord ) ) - return false; - return ((DynamicRecord) obj).getId() == getId(); - } - - @Override - public int hashCode() - { - long id = getId(); - return (int) (( id >>> 32 ) ^ id ); + { + clone.setData( data.clone() ); + } + return clone; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/LabelTokenRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/LabelTokenRecord.java index 473dae39cd83f..6db7698536cd3 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/LabelTokenRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/LabelTokenRecord.java @@ -19,7 +19,6 @@ */ package org.neo4j.kernel.impl.store.record; - public class LabelTokenRecord extends TokenRecord { public LabelTokenRecord( int id ) @@ -27,6 +26,13 @@ public LabelTokenRecord( int id ) super( id ); } + @Override + public LabelTokenRecord initialize( boolean inUse, int nameId ) + { + super.initialize( inUse, nameId ); + return this; + } + @Override protected String simpleName() { @@ -42,7 +48,6 @@ public LabelTokenRecord clone() { labelTokenRecord.setCreated(); } - labelTokenRecord.setIsLight( isLight() ); labelTokenRecord.setNameId( getNameId() ); labelTokenRecord.addNameRecords( getNameRecords() ); return labelTokenRecord; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreActualRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreActualRecord.java new file mode 100644 index 0000000000000..e1fe80282e862 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreActualRecord.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.store.record; + +public class NeoStoreActualRecord extends Abstract64BitRecord +{ + private long value; + + public NeoStoreActualRecord() + { + super( -1 ); + } + + public NeoStoreActualRecord initialize( boolean inUse, long value ) + { + super.initialize( inUse ); + this.value = value; + return this; + } + + @Override + public void clear() + { + initialize( false, -1 ); + } + + public long getValue() + { + return value; + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreRecord.java index f89afc274a8fc..8b9f3a693893d 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NeoStoreRecord.java @@ -19,15 +19,27 @@ */ package org.neo4j.kernel.impl.store.record; - public class NeoStoreRecord extends PrimitiveRecord { public NeoStoreRecord() { - super( -1, Record.NO_NEXT_PROPERTY.intValue() ); + super( -1 ); setInUse( true ); } + @Override + public NeoStoreRecord initialize( boolean inUse, long nextProp ) + { + super.initialize( inUse, nextProp ); + return this; + } + + @Override + public void clear() + { + initialize( false, -1 ); + } + @Override public String toString() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NodeRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NodeRecord.java index 977bb2ea0cd42..c0c36954b1a9a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NodeRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/NodeRecord.java @@ -30,22 +30,35 @@ public class NodeRecord extends PrimitiveRecord { - private long nextRel = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long labels = Record.NO_LABELS_FIELD.intValue(); - private Collection dynamicLabelRecords = emptyList(); - private boolean isLight = true; + private long nextRel; + private long labels; + private Collection dynamicLabelRecords; + private boolean isLight; private boolean dense; public NodeRecord( long id ) { - super( id, Record.NO_NEXT_PROPERTY.intValue() ); + super( id ); } + public NodeRecord initialize( boolean inUse, long nextProp, boolean dense, long nextRel, long labels ) + { + super.initialize( inUse, nextProp ); + this.nextRel = nextRel; + this.dense = dense; + this.labels = labels; + this.dynamicLabelRecords = emptyList(); + this.isLight = true; + return this; + } + + @Deprecated public NodeRecord( long id, boolean dense, long nextRel, long nextProp ) { this( id, false, dense, nextRel, nextProp, 0 ); } + @Deprecated public NodeRecord( long id, boolean inUse, boolean dense, long nextRel, long nextProp, long labels ) { super( id, nextProp ); @@ -55,12 +68,20 @@ public NodeRecord( long id, boolean inUse, boolean dense, long nextRel, long nex setInUse( inUse ); } + @Deprecated public NodeRecord( long id, boolean dense, long nextRel, long nextProp, boolean inUse ) { this(id, dense, nextRel, nextProp); setInUse( inUse ); } + @Override + public void clear() + { + initialize( false, Record.NO_NEXT_PROPERTY.intValue(), false, + Record.NO_NEXT_RELATIONSHIP.intValue(), Record.NO_LABELS_FIELD.intValue() ); + } + public long getNextRel() { return nextRel; @@ -149,12 +170,10 @@ public void setIdTo( PropertyRecord property ) @Override public NodeRecord clone() { - NodeRecord clone = new NodeRecord( getId(), dense, nextRel, getNextProp() ); - clone.labels = labels; + NodeRecord clone = new NodeRecord( getId() ).initialize( inUse(), nextProp, dense, nextRel, labels ); clone.isLight = isLight; - clone.setInUse( inUse() ); - if( dynamicLabelRecords.size() > 0 ) + if ( dynamicLabelRecords.size() > 0 ) { List clonedLabelRecords = new ArrayList<>(dynamicLabelRecords.size()); for ( DynamicRecord labelRecord : dynamicLabelRecords ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PrimitiveRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PrimitiveRecord.java index a46731ffde8ea..4b5306d5a297e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PrimitiveRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PrimitiveRecord.java @@ -21,18 +21,34 @@ public abstract class PrimitiveRecord extends Abstract64BitRecord { - private long nextProp; + protected long nextProp; - public PrimitiveRecord() + PrimitiveRecord( long id ) { + super( id ); } - public PrimitiveRecord( long id, long nextProp ) + @Deprecated + PrimitiveRecord( long id, long nextProp ) { super( id ); this.nextProp = nextProp; } + @Override + public void clear() + { + super.clear(); + nextProp = Record.NO_NEXT_PROPERTY.intValue(); + } + + protected PrimitiveRecord initialize( boolean inUse, long nextProp ) + { + super.initialize( inUse ); + this.nextProp = nextProp; + return this; + } + public long getNextProp() { return nextProp; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyKeyTokenRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyKeyTokenRecord.java index 2d035e282700f..7e02cc76ec645 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyKeyTokenRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyKeyTokenRecord.java @@ -19,16 +19,35 @@ */ package org.neo4j.kernel.impl.store.record; - public class PropertyKeyTokenRecord extends TokenRecord { - private int propCount = 0; + private int propCount; public PropertyKeyTokenRecord( int id ) { super( id ); } + public PropertyKeyTokenRecord initialize( boolean inUse, int nameId, int propertyCount ) + { + super.initialize( inUse, nameId ); + this.propCount = propertyCount; + return this; + } + + @Override + public void setId( long id ) + { + super.setId( id ); + } + + @Override + public void clear() + { + super.clear(); + propCount = 0; + } + @Override protected String simpleName() { @@ -61,7 +80,6 @@ public PropertyKeyTokenRecord clone() { propertyKeyTokenRecord.setCreated(); } - propertyKeyTokenRecord.setIsLight( isLight() ); propertyKeyTokenRecord.setNameId( getNameId() ); propertyKeyTokenRecord.addNameRecords( getNameRecords() ); propertyKeyTokenRecord.setPropertyCount( getPropertyCount() ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyRecord.java index a0773c386946a..1b5dd7e748cc1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/PropertyRecord.java @@ -25,8 +25,11 @@ import java.util.List; import java.util.NoSuchElementException; -import org.neo4j.kernel.impl.store.PropertyType; import org.neo4j.kernel.impl.api.store.PropertyBlockCursor; +import org.neo4j.kernel.impl.store.PropertyType; + +import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_PROPERTY; +import static org.neo4j.kernel.impl.store.record.Record.NO_PREVIOUS_PROPERTY; /** * PropertyRecord is a container for PropertyBlocks. PropertyRecords form @@ -40,12 +43,12 @@ public class PropertyRecord extends Abstract64BitRecord implements Iterable deletedRecords; @@ -61,10 +64,33 @@ public PropertyRecord( long id ) public PropertyRecord( long id, PrimitiveRecord primitive ) { super( id ); - setCreated(); primitive.setIdTo( this ); } + public PropertyRecord initialize( boolean inUse, PrimitiveRecord primitive, long prevProp, long nextProp ) + { + super.initialize( inUse ); + setCreated(); // TODO why? + primitive.setIdTo( this ); + this.prevProp = prevProp; + this.nextProp = nextProp; + this.deletedRecords = null; + this.blockRecordsCursor = 0; + return this; + } + + @Override + public void clear() + { + super.initialize( false ); + this.entityId = -1; + this.entityType = 0; + this.prevProp = NO_PREVIOUS_PROPERTY.intValue(); + this.nextProp = NO_NEXT_PROPERTY.intValue(); + this.deletedRecords = null; + this.blockRecordsCursor = 0; + } + public void setNodeId( long nodeId ) { entityType = TYPE_NODE; @@ -283,8 +309,7 @@ public void setPrevProp( long prev ) @Override public PropertyRecord clone() { - PropertyRecord result = new PropertyRecord( getLongId() ); - result.setInUse( inUse() ); + PropertyRecord result = (PropertyRecord) new PropertyRecord( getId() ).initialize( inUse() ); result.nextProp = nextProp; result.prevProp = prevProp; result.entityId = entityId; @@ -304,10 +329,9 @@ public PropertyRecord clone() return result; } - public PropertyBlockCursor getPropertyBlockCursor(PropertyBlockCursor cursor) + public PropertyBlockCursor getPropertyBlockCursor( PropertyBlockCursor cursor ) { - cursor.init(blockRecords, blockRecordsCursor); - + cursor.init( blockRecords, blockRecordsCursor ); return cursor; } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RecordLoad.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RecordLoad.java index 4f838934451e8..b8f44a327c381 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RecordLoad.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RecordLoad.java @@ -19,7 +19,97 @@ */ package org.neo4j.kernel.impl.store.record; +import org.neo4j.io.pagecache.PageCursor; +import org.neo4j.kernel.impl.store.InvalidRecordException; + +/** + * Specifies what happens when loading records, based on inUse status. + * + * Roughly this is what happens for the different modes: + *
      + *
    • {@link RecordLoad#CHECK}: Load at least data to determine whether or not it's in use. + * If in use then record is loaded into target and returns {@code true}, + * otherwise return {@code false}.
      • + *
    • {@link RecordLoad#NORMAL}: Load at least data to determine whether or not it's in use. + * if in use then record is loaded into target returns {@code true}, + * otherwise throws {@link InvalidRecordException}.
      • + *
    • {@link RecordLoad#FORCE}: Loads record data into target regardless of whether or not record in use. + * Returns whether or not record is in use. + * + */ public enum RecordLoad { - NORMAL, CHECK, FORCE + NORMAL + { + @Override + public boolean shouldLoad( boolean inUse ) + { + return inUse; + } + + @Override + public boolean verify( AbstractBaseRecord record ) + { + if ( !record.inUse() ) + { + throw new InvalidRecordException( record + " not in use" ); + } + return true; + } + }, + CHECK + { + @Override + public boolean shouldLoad( boolean inUse ) + { + return inUse; + } + + @Override + public boolean verify( AbstractBaseRecord record ) + { + return record.inUse(); + } + + @Override + public void report( String message ) + { + } + }, + FORCE + { + @Override + public boolean shouldLoad( boolean inUse ) + { + // Always return true so that record data will always be loaded, even if not in use. + return true; + } + + @Override + public boolean verify( AbstractBaseRecord record ) + { + return true; + } + + @Override + public void report( String message ) + { + // Don't report + } + }; + + /** + * Checks whether or not a record should be fully loaded from {@link PageCursor}, based on inUse status. + */ + public abstract boolean shouldLoad( boolean inUse ); + + /** + * Verifies that a record's in use status is in line with the mode, might throw {@link InvalidRecordException}. + */ + public abstract boolean verify( AbstractBaseRecord record ); + + public void report( String message ) + { + throw new InvalidRecordException( message ); + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipGroupRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipGroupRecord.java index dba67272ae5e3..e5fc370d954ad 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipGroupRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipGroupRecord.java @@ -19,33 +19,37 @@ */ package org.neo4j.kernel.impl.store.record; +import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP; import java.util.Objects; public class RelationshipGroupRecord extends Abstract64BitRecord { private int type; - private long next = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long firstOut = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long firstIn = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long firstLoop = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long owningNode = Record.NO_NEXT_RELATIONSHIP.intValue(); + private long next; + private long firstOut; + private long firstIn; + private long firstLoop; + private long owningNode; // Not stored, just kept in memory temporarily when loading the group chain - private long prev = Record.NO_NEXT_RELATIONSHIP.intValue(); + private long prev; + @Deprecated public RelationshipGroupRecord( long id, int type ) { super( id ); this.type = type; } + @Deprecated public RelationshipGroupRecord( long id, int type, long firstOut, long firstIn, long firstLoop, long owningNode, boolean inUse ) { this( id, type, firstOut, firstIn, firstLoop, owningNode, Record.NO_NEXT_RELATIONSHIP.intValue(), inUse ); } + @Deprecated public RelationshipGroupRecord( long id, int type, long firstOut, long firstIn, long firstLoop, long owningNode, long next, boolean inUse ) { @@ -59,6 +63,33 @@ public RelationshipGroupRecord( long id, int type, long firstOut, long firstIn, this.next = next; } + public RelationshipGroupRecord( long id ) + { + super( id ); + } + + public RelationshipGroupRecord initialize( boolean inUse, int type, + long firstOut, long firstIn, long firstLoop, long owningNode, long next ) + { + super.initialize( inUse ); + this.type = type; + this.firstOut = firstOut; + this.firstIn = firstIn; + this.firstLoop = firstLoop; + this.owningNode = owningNode; + this.next = next; + this.prev = NO_NEXT_RELATIONSHIP.intValue(); + return this; + } + + @Override + public void clear() + { + initialize( false, -1, NO_NEXT_RELATIONSHIP.intValue(), NO_NEXT_RELATIONSHIP.intValue(), + NO_NEXT_RELATIONSHIP.intValue(), NO_NEXT_RELATIONSHIP.intValue(), NO_NEXT_RELATIONSHIP.intValue() ); + prev = NO_NEXT_RELATIONSHIP.intValue(); + } + public int getType() { return type; @@ -160,7 +191,8 @@ public String toString() @Override public RelationshipGroupRecord clone() { - return new RelationshipGroupRecord( getId(), type, firstOut, firstIn, firstLoop, owningNode, next, inUse() ); + return new RelationshipGroupRecord( getId() ).initialize( inUse(), type, firstOut, firstIn, firstLoop, + owningNode, next ); } @Override @@ -178,8 +210,8 @@ public boolean equals( Object o ) firstOut == that.firstOut && firstIn == that.firstIn && firstLoop == that.firstLoop && - owningNode == that.owningNode && - prev == that.prev; + owningNode == that.owningNode; + // don't compare prev since it's not persisted } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipRecord.java index 7d579532fa556..be19a7385a41f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipRecord.java @@ -20,24 +20,22 @@ package org.neo4j.kernel.impl.store.record; import java.util.Objects; +import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_PROPERTY; +import static org.neo4j.kernel.impl.store.record.Record.NO_NEXT_RELATIONSHIP; public class RelationshipRecord extends PrimitiveRecord { private long firstNode; private long secondNode; private int type; - private long firstPrevRel = 1; - private long firstNextRel = Record.NO_NEXT_RELATIONSHIP.intValue(); - private long secondPrevRel = 1; - private long secondNextRel = Record.NO_NEXT_RELATIONSHIP.intValue(); - private boolean firstInFirstChain = true; - private boolean firstInSecondChain = true; - - public RelationshipRecord( long id ) - { - super( id, Record.NO_NEXT_PROPERTY.intValue() ); - } - + private long firstPrevRel; + private long firstNextRel; + private long secondPrevRel; + private long secondNextRel; + private boolean firstInFirstChain; + private boolean firstInSecondChain; + + @Deprecated public RelationshipRecord( long id, long firstNode, long secondNode, int type ) { this( id ); @@ -46,6 +44,7 @@ public RelationshipRecord( long id, long firstNode, long secondNode, int type ) this.type = type; } + @Deprecated public RelationshipRecord( long id, boolean inUse, long firstNode, long secondNode, int type, long firstPrevRel, long firstNextRel, long secondPrevRel, long secondNextRel, boolean firstInFirstChain, boolean firstInSecondChain ) @@ -61,6 +60,36 @@ public RelationshipRecord( long id, boolean inUse, long firstNode, long secondNo } + public RelationshipRecord( long id ) + { + super( id ); + } + + public RelationshipRecord initialize( boolean inUse, long nextProp, long firstNode, long secondNode, + int type, long firstPrevRel, long firstNextRel, long secondPrevRel, long secondNextRel, + boolean firstInFirstChain, boolean firstInSecondChain ) + { + super.initialize( inUse, nextProp ); + this.firstNode = firstNode; + this.secondNode = secondNode; + this.type = type; + this.firstPrevRel = firstPrevRel; + this.firstNextRel = firstNextRel; + this.secondPrevRel = secondPrevRel; + this.secondNextRel = secondNextRel; + this.firstInFirstChain = firstInFirstChain; + this.firstInSecondChain = firstInSecondChain; + return this; + } + + @Override + public void clear() + { + initialize( false, NO_NEXT_PROPERTY.intValue(), -1, -1, -1, + 1, NO_NEXT_RELATIONSHIP.intValue(), + 1, NO_NEXT_RELATIONSHIP.intValue(), true, true ); + } + public void setLinks( long firstNode, long secondNode, int type ) { this.firstNode = firstNode; @@ -180,7 +209,7 @@ public String toString() @Override public RelationshipRecord clone() { - return new RelationshipRecord( getId(), inUse(), + return new RelationshipRecord( getId() ).initialize( inUse(), nextProp, firstNode, secondNode, type, firstPrevRel, firstNextRel, secondPrevRel, secondNextRel, firstInFirstChain, firstInSecondChain ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipTypeTokenRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipTypeTokenRecord.java index 8cd79dfa180f6..17cc5c19bd1d6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipTypeTokenRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/RelationshipTypeTokenRecord.java @@ -26,6 +26,13 @@ public RelationshipTypeTokenRecord( int id ) super( id ); } + @Override + public RelationshipTypeTokenRecord initialize( boolean inUse, int nameId ) + { + super.initialize( inUse, nameId ); + return this; + } + @Override protected String simpleName() { @@ -41,7 +48,6 @@ public RelationshipTypeTokenRecord clone() { relationshipTypeTokenRecord.setCreated(); } - relationshipTypeTokenRecord.setIsLight( isLight() ); relationshipTypeTokenRecord.setNameId( getNameId() ); relationshipTypeTokenRecord.addNameRecords( getNameRecords() ); return relationshipTypeTokenRecord; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/SchemaRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/SchemaRecord.java new file mode 100644 index 0000000000000..249044e1dac4a --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/SchemaRecord.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.store.record; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + +import static org.neo4j.kernel.impl.store.record.Record.NULL_REFERENCE; + +public class SchemaRecord extends Abstract64BitRecord implements Iterable +{ + private Collection records; + + public SchemaRecord( Collection records ) + { + super( -1 ); + initialize( records ); + } + + public SchemaRecord initialize( Collection records ) + { + Iterator iterator = records.iterator(); + long id = iterator.hasNext() ? iterator.next().getId() : NULL_REFERENCE.intValue(); + super.initialize( true ); + setId( id ); + setInUse( true ); + this.records = records; + return this; + } + + public void setDynamicRecords( Collection records ) + { + this.records.clear(); + this.records.addAll( records ); + } + + @Override + public void clear() + { + super.initialize( false ); + this.records = null; + } + + @Override + public Iterator iterator() + { + return records.iterator(); + } + + public int size() + { + return records.size(); + } + + @Override + public SchemaRecord clone() + { + List list = new ArrayList<>( records.size() ); + for ( DynamicRecord record : records ) + { + list.add( record.clone() ); + } + return new SchemaRecord( list ); + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/TokenRecord.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/TokenRecord.java index c7aa488f45e59..81e91b9b0f11a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/TokenRecord.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/record/TokenRecord.java @@ -25,23 +25,31 @@ public abstract class TokenRecord extends AbstractRecord { - private int nameId = Record.NO_NEXT_BLOCK.intValue(); - private final List nameRecords = new ArrayList(); - private boolean isLight; + private int nameId; + private List nameRecords; - TokenRecord( int id ) + public TokenRecord( int id ) { super( id ); } - public void setIsLight( boolean status ) + public TokenRecord initialize( boolean inUse, int nameId ) { - isLight = status; + super.initialize( inUse ); + this.nameId = nameId; + this.nameRecords = new ArrayList<>(); + return this; + } + + @Override + public void clear() + { + initialize( false, Record.NO_NEXT_BLOCK.intValue() ); } public boolean isLight() { - return isLight; + return nameRecords == null || nameRecords.isEmpty(); } public int getNameId() @@ -79,7 +87,7 @@ public String toString() buf.append( getId() ).append( "," ).append( inUse() ? "in" : "no" ).append( " use" ); buf.append( ",nameId=" ).append( nameId ); additionalToString( buf ); - if ( !isLight ) + if ( !isLight() ) { for ( DynamicRecord dyn : nameRecords ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreFile.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreFile.java index 18c49ae3941d6..54bfcd03d4bde 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreFile.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreFile.java @@ -50,7 +50,6 @@ import static org.neo4j.helpers.collection.Iterables.iterable; - public enum StoreFile { // all store files in Neo4j diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyRelationshipStoreReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyRelationshipStoreReader.java index 92840bc41892a..04b7c0af21c73 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyRelationshipStoreReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/LegacyRelationshipStoreReader.java @@ -106,16 +106,10 @@ public long id() public RelationshipRecord createRecord() { - if( record == null) + if ( record == null) { - record = new RelationshipRecord( recordId, firstNode, secondNode, type ); - record.setInUse( inUse ); - record.setFirstPrevRel( firstPrevRel ); - record.setFirstNextRel( firstNextRel ); - record.setSecondPrevRel( secondPrevRel ); - record.setSecondNextRel( secondNextRel ); - record.setNextProp( nextProp ); - + record = new RelationshipRecord( recordId ).initialize( inUse, nextProp, firstNode, secondNode, + type, firstPrevRel, firstNextRel, secondPrevRel, secondNextRel, true, true ); } return record; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v19/Legacy19NodeStoreReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v19/Legacy19NodeStoreReader.java index cf97a5953ab6b..76675216f9236 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v19/Legacy19NodeStoreReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v19/Legacy19NodeStoreReader.java @@ -89,9 +89,11 @@ protected NodeRecord fetchNextOrNull() long relModifier = (inUseByte & 0xEL) << 31; long nextProp = getUnsignedInt( buffer ); long propModifier = (inUseByte & 0xF0L) << 28; - nodeRecord = new NodeRecord( id, false, longFromIntAndMod( nextRel, relModifier ), - longFromIntAndMod( nextProp, propModifier ) ); - nodeRecord.setInUse( inUse ); + nodeRecord = new NodeRecord( id ).initialize( true, + longFromIntAndMod( nextProp, propModifier ), + false, + longFromIntAndMod( nextRel, relModifier ), + Record.NO_LABELS_FIELD.intValue() ); } else { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v20/Legacy20NodeStoreReader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v20/Legacy20NodeStoreReader.java index 2e7a5cac45488..f79b4269d154a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v20/Legacy20NodeStoreReader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v20/Legacy20NodeStoreReader.java @@ -141,14 +141,16 @@ private NodeRecord readRecord( ByteBuffer buffer, long id ) long lsbLabels = Legacy20Store.getUnsignedInt( buffer ); long hsbLabels = buffer.get() & 0xFF; // so that a negative byte won't fill the "extended" bits with ones. long labels = lsbLabels | (hsbLabels << 32); - nodeRecord = new NodeRecord( id, false, Legacy20Store.longFromIntAndMod( nextRel, relModifier ), - Legacy20Store.longFromIntAndMod( nextProp, propModifier ) ); + nodeRecord = new NodeRecord( id ).initialize( true, + Legacy20Store.longFromIntAndMod( nextProp, propModifier ), + false, + Legacy20Store.longFromIntAndMod( nextRel, relModifier ), + Record.NO_LABELS_FIELD.intValue() ); nodeRecord.setLabelField( labels, Collections.emptyList() ); // no need to load 'em heavy } else { - nodeRecord = new NodeRecord( id, false, - Record.NO_NEXT_RELATIONSHIP.intValue(), Record.NO_NEXT_PROPERTY.intValue() ); + nodeRecord = new NodeRecord( id ); } nodeRecord.setInUse( inUse ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/DuplicatePropertyRemover.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/DuplicatePropertyRemover.java index 0044e29fbdf6a..876912b17e5fa 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/DuplicatePropertyRemover.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/DuplicatePropertyRemover.java @@ -25,15 +25,19 @@ import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + class DuplicatePropertyRemover { private final NodeStore nodeStore; private final PropertyStore propertyStore; + private final PropertyRecord otherPropertyRecord; DuplicatePropertyRemover( NodeStore nodeStore, PropertyStore propertyStore ) { this.nodeStore = nodeStore; this.propertyStore = propertyStore; + this.otherPropertyRecord = propertyStore.newRecord(); } public void fixUpPropertyLinksAroundUnusedRecord( NodeRecord nodeRecord, PropertyRecord duplicateRecord ) @@ -49,15 +53,15 @@ public void fixUpPropertyLinksAroundUnusedRecord( NodeRecord nodeRecord, Propert long nextRecordId = duplicateRecord.getNextProp(); if ( previousRecordId != Record.NO_PREVIOUS_PROPERTY.intValue() ) { - PropertyRecord property = propertyStore.getRecord( previousRecordId ); - property.setNextProp( nextRecordId ); - propertyStore.updateRecord( property ); + propertyStore.getRecord( previousRecordId, otherPropertyRecord, NORMAL ); + otherPropertyRecord.setNextProp( nextRecordId ); + propertyStore.updateRecord( otherPropertyRecord ); } if ( nextRecordId != Record.NO_NEXT_PROPERTY.intValue() ) { - PropertyRecord property = propertyStore.getRecord( nextRecordId ); - property.setPrevProp( previousRecordId ); - propertyStore.updateRecord( property ); + propertyStore.getRecord( nextRecordId, otherPropertyRecord, NORMAL ); + otherPropertyRecord.setPrevProp( previousRecordId ); + propertyStore.updateRecord( otherPropertyRecord ); } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/IndexConsultedPropertyBlockSweeper.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/IndexConsultedPropertyBlockSweeper.java index 5bc6521ce3b85..81e9aec895212 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/IndexConsultedPropertyBlockSweeper.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/IndexConsultedPropertyBlockSweeper.java @@ -28,6 +28,8 @@ import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + class IndexConsultedPropertyBlockSweeper implements PrimitiveLongVisitor { private final int propertyKeyId; @@ -35,6 +37,7 @@ class IndexConsultedPropertyBlockSweeper implements PrimitiveLongVisitor it = record.iterator(); + Iterator it = propertyRecord.iterator(); while ( it.hasNext() ) { PropertyBlock block = it.next(); @@ -76,11 +80,11 @@ public boolean visited( long propRecordId ) throws IOException } if ( changed ) { - if ( record.numberOfProperties() == 0 ) + if ( propertyRecord.numberOfProperties() == 0 ) { - propertyRemover.fixUpPropertyLinksAroundUnusedRecord( nodeRecord, record ); + propertyRemover.fixUpPropertyLinksAroundUnusedRecord( nodeRecord, propertyRecord ); } - propertyStore.updateRecord( record ); + propertyStore.updateRecord( propertyRecord ); } return false; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/NonIndexedConflictResolver.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/NonIndexedConflictResolver.java index 798d45b88d4e6..3687f93d94479 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/NonIndexedConflictResolver.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/NonIndexedConflictResolver.java @@ -36,6 +36,8 @@ import org.neo4j.storageengine.api.Token; import org.neo4j.unsafe.batchinsert.DirectRecordAccess; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + class NonIndexedConflictResolver implements PrimitiveLongObjectVisitor, IOException> { private final PropertyKeyTokenStore keyTokenStore; @@ -103,6 +105,7 @@ private class DuplicateNameAssigner implements PrimitiveLongVisitor private final DuplicateCluster duplicateCluster; private final String oldName; private int index; + private final PropertyRecord record = store.newRecord(); public DuplicateNameAssigner( DuplicateCluster duplicateCluster, String oldName ) { @@ -113,7 +116,7 @@ public DuplicateNameAssigner( DuplicateCluster duplicateCluster, String oldName @Override public boolean visited( long propertyRecordId ) throws IOException { - PropertyRecord record = store.getRecord( propertyRecordId ); + store.getRecord( propertyRecordId, record, NORMAL ); for ( PropertyBlock block : record ) { if ( block.getKeyIndexId() == duplicateCluster.propertyKeyId ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java index fefd1e8200d8c..973ca5c4539a6 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/legacystore/v21/propertydeduplication/PropertyDeduplicator.java @@ -44,6 +44,9 @@ import org.neo4j.kernel.impl.store.record.Record; import org.neo4j.logging.NullLogProvider; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + public class PropertyDeduplicator { private final FileSystemAbstraction fileSystem; @@ -85,12 +88,13 @@ private PrimitiveLongObjectMap> collectConflictingPropert final PrimitiveLongObjectMap> duplicateClusters = Primitive.longObjectMap(); long highId = store.getHighId(); + PropertyRecord head = store.newRecord(), tail = store.newRecord(); for ( long headRecordId = 0; headRecordId < highId; ++headRecordId ) { - PropertyRecord record = store.forceGetRecord( headRecordId ); + store.getRecord( headRecordId, head, FORCE ); // Skip property propertyRecordIds that are not in use. // Skip property propertyRecordIds that are not at the start of a chain. - if ( !record.inUse() || record.getPrevProp() != Record.NO_NEXT_PROPERTY.intValue() ) + if ( !head.inUse() || head.getPrevProp() != Record.NO_NEXT_PROPERTY.intValue() ) { continue; } @@ -98,12 +102,12 @@ private PrimitiveLongObjectMap> collectConflictingPropert long propertyId = headRecordId; while ( propertyId != Record.NO_NEXT_PROPERTY.intValue() ) { - record = store.getRecord( propertyId ); + store.getRecord( propertyId, tail, NORMAL ); - Iterable propertyBlocks = record; + Iterable propertyBlocks = tail; scanForDuplicates( propertyId, propertyBlocks ); - propertyId = record.getNextProp(); + propertyId = tail.getNextProp(); } final long localHeadRecordId = headRecordId; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/Command.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/Command.java index a6d6b800dffe7..5b5eb5cd3170a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/Command.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/Command.java @@ -36,6 +36,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRecord; import org.neo4j.kernel.impl.store.record.TokenRecord; import org.neo4j.kernel.impl.transaction.state.PropertyRecordChange; import org.neo4j.storageengine.api.StorageCommand; @@ -43,7 +44,7 @@ import org.neo4j.storageengine.api.schema.SchemaRule; import static java.lang.String.format; -import static java.util.Collections.unmodifiableCollection; + import static org.neo4j.helpers.collection.IteratorUtil.first; import static org.neo4j.kernel.impl.util.Bits.bitFlag; import static org.neo4j.kernel.impl.util.Bits.bitFlags; @@ -134,7 +135,12 @@ protected String beforeAndAfterToString( AbstractBaseRecord before, AbstractBase void writeDynamicRecords( WritableChannel channel, Collection records ) throws IOException { - channel.putInt( records.size() ); // 4 + writeDynamicRecords( channel, records, records.size() ); + } + + void writeDynamicRecords( WritableChannel channel, Iterable records, int size ) throws IOException + { + channel.putInt( size ); // 4 for ( DynamicRecord record : records ) { writeDynamicRecord( channel, record ); @@ -579,12 +585,18 @@ private void writeLabelTokenRecord( WritableChannel channel, LabelTokenRecord re public static class SchemaRuleCommand extends Command { - private final Collection recordsBefore; - private final Collection recordsAfter; + private final SchemaRecord recordsBefore; + private final SchemaRecord recordsAfter; private final SchemaRule schemaRule; public SchemaRuleCommand( Collection recordsBefore, Collection recordsAfter, SchemaRule schemaRule ) + { + this( new SchemaRecord( recordsBefore ), new SchemaRecord( recordsAfter ), schemaRule ); + } + + public SchemaRuleCommand( SchemaRecord recordsBefore, SchemaRecord recordsAfter, + SchemaRule schemaRule ) { setup( first( recordsAfter ).getId(), Mode.fromRecordState( first( recordsAfter ) ) ); this.recordsBefore = recordsBefore; @@ -608,9 +620,9 @@ public boolean handle( CommandVisitor handler ) throws IOException return handler.visitSchemaRuleCommand( this ); } - public Collection getRecordsAfter() + public SchemaRecord getRecordsAfter() { - return unmodifiableCollection( recordsAfter ); + return recordsAfter; } public SchemaRule getSchemaRule() @@ -618,7 +630,7 @@ public SchemaRule getSchemaRule() return schemaRule; } - public Collection getRecordsBefore() + public SchemaRecord getRecordsBefore() { return recordsBefore; } @@ -627,8 +639,8 @@ public Collection getRecordsBefore() public void serialize( WritableChannel channel ) throws IOException { channel.put( NeoCommandType.SCHEMA_RULE_COMMAND ); - writeDynamicRecords( channel, recordsBefore ); - writeDynamicRecords( channel, recordsAfter ); + writeDynamicRecords( channel, recordsBefore, recordsBefore.size() ); + writeDynamicRecords( channel, recordsAfter, recordsAfter.size() ); channel.put( first( recordsAfter ).isCreated() ? (byte) 1 : 0 ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV1_9.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV1_9.java index 53f17bbc63547..f4cbb31316c3e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV1_9.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV1_9.java @@ -20,6 +20,7 @@ package org.neo4j.kernel.impl.transaction.command; import java.io.IOException; + import org.neo4j.kernel.impl.store.PropertyType; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NeoStoreRecord; @@ -32,6 +33,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; import org.neo4j.kernel.impl.transaction.command.CommandReading.DynamicRecordAdder; import org.neo4j.storageengine.api.ReadableChannel; + import static org.neo4j.kernel.impl.transaction.command.CommandReading.PROPERTY_BLOCK_DYNAMIC_RECORD_ADDER; import static org.neo4j.kernel.impl.transaction.command.CommandReading.PROPERTY_DELETED_DYNAMIC_RECORD_ADDER; import static org.neo4j.kernel.impl.transaction.command.CommandReading.PROPERTY_INDEX_DYNAMIC_RECORD_ADDER; @@ -108,8 +110,7 @@ record = new RelationshipRecord( id, channel.getLong(), channel.getLong(), chann } else { - record = new RelationshipRecord( id, -1, -1, -1 ); - record.setInUse( false ); + record = new RelationshipRecord( id ); } return new Command.RelationshipCommand( null, record ); } @@ -209,8 +210,7 @@ record = new NodeRecord( id, false, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } record.setInUse( inUse ); return record; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_0.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_0.java index 2347dbb3e6cf1..fd55361e5954a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_0.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_0.java @@ -134,8 +134,7 @@ record = new RelationshipRecord( id, channel.getLong(), channel.getLong(), chann } else { - record = new RelationshipRecord( id, -1, -1, -1 ); - record.setInUse( false ); + record = new RelationshipRecord( id ); } return new Command.RelationshipCommand( null, record ); } @@ -298,8 +297,7 @@ record = new NodeRecord( id, false, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } record.setInUse( inUse ); return record; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_1.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_1.java index efb43855019c2..ed17cf5618241 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_1.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_1.java @@ -39,8 +39,6 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; -import org.neo4j.kernel.impl.transaction.command.Command.NodeCountsCommand; -import org.neo4j.kernel.impl.transaction.command.Command.RelationshipCountsCommand; import org.neo4j.kernel.impl.transaction.command.CommandReading.DynamicRecordAdder; import org.neo4j.storageengine.api.ReadableChannel; import org.neo4j.storageengine.api.schema.SchemaRule; @@ -132,7 +130,7 @@ record = new RelationshipRecord( id, channel.getLong(), channel.getLong(), chann } else { - record = new RelationshipRecord( id, -1, -1, -1 ); + record = new RelationshipRecord( id ); record.setInUse( false ); } if ( bitFlag( flags, Record.CREATED_IN_TX ) ) @@ -321,8 +319,7 @@ record = new NodeRecord( id, dense, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } record.setInUse( inUse ); return record; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2.java index 0602087a4a485..4dc6322f809cf 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2.java @@ -368,8 +368,7 @@ record = new NodeRecord( id, dense, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } readDynamicRecords( channel, dynamicLabelRecords, COLLECTION_DYNAMIC_RECORD_ADDER ); record.setLabelField( labelField, dynamicLabelRecords ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2_4.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2_4.java index caffcd8bd3270..d5d3c59199369 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2_4.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV2_2_4.java @@ -368,8 +368,7 @@ record = new NodeRecord( id, dense, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } readDynamicRecords( channel, dynamicLabelRecords, COLLECTION_DYNAMIC_RECORD_ADDER ); record.setLabelField( labelField, dynamicLabelRecords ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV3_0.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV3_0.java index a3197a91c8072..faf5ce4819f0f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV3_0.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/PhysicalLogCommandReaderV3_0.java @@ -419,8 +419,7 @@ record = new NodeRecord( id, dense, channel.getLong(), channel.getLong() ); } else { - record = new NodeRecord( id, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ); + record = new NodeRecord( id ); } readDynamicRecords( channel, dynamicLabelRecords, COLLECTION_DYNAMIC_RECORD_ADDER ); record.setLabelField( labelField, dynamicLabelRecords ); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java index c0167d80cc304..c2b01afe1aa0c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java @@ -19,9 +19,6 @@ */ package org.neo4j.kernel.impl.transaction.state; -import java.util.ArrayList; -import java.util.Collection; - import org.neo4j.kernel.impl.core.RelationshipTypeToken; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; @@ -38,21 +35,24 @@ import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyKeyTokenRecord; import org.neo4j.kernel.impl.store.record.PropertyRecord; -import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRecord; import org.neo4j.kernel.impl.transaction.state.RecordAccess.Loader; import org.neo4j.storageengine.api.Token; import org.neo4j.storageengine.api.schema.SchemaRule; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + public class Loaders { private final Loader nodeLoader; private final Loader propertyLoader; private final Loader relationshipLoader; private final Loader relationshipGroupLoader; - private final Loader,SchemaRule> schemaRuleLoader; + private final Loader schemaRuleLoader; private final Loader propertyKeyTokenLoader; private final Loader labelTokenLoader; private final Loader relationshipTypeTokenLoader; @@ -89,7 +89,7 @@ public Loader relationshipGroupLoader() return relationshipGroupLoader; } - public Loader,SchemaRule> schemaRuleLoader() + public Loader schemaRuleLoader() { return schemaRuleLoader; } @@ -116,14 +116,13 @@ public static Loader nodeLoader( final NodeStore store ) @Override public NodeRecord newUnused( Long key, Void additionalData ) { - return andMarkAsCreated( new NodeRecord( key, false, Record.NO_NEXT_RELATIONSHIP.intValue(), - Record.NO_NEXT_PROPERTY.intValue() ) ); + return andMarkAsCreated( new NodeRecord( key ) ); } @Override public NodeRecord load( Long key, Void additionalData ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override @@ -163,7 +162,7 @@ private void setOwner( PropertyRecord record, PrimitiveRecord owner ) @Override public PropertyRecord load( Long key, PrimitiveRecord additionalData ) { - PropertyRecord record = store.getRecord( key.longValue() ); + PropertyRecord record = store.getRecord( key, store.newRecord(), NORMAL ); setOwner( record, additionalData ); return record; } @@ -198,7 +197,7 @@ public RelationshipRecord newUnused( Long key, Void additionalData ) @Override public RelationshipRecord load( Long key, Void additionalData ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override @@ -222,13 +221,15 @@ public static Loader relationshipGroupLoad @Override public RelationshipGroupRecord newUnused( Long key, Integer type ) { - return andMarkAsCreated( new RelationshipGroupRecord( key, type ) ); + RelationshipGroupRecord record = new RelationshipGroupRecord( key ); + record.setType( type ); + return andMarkAsCreated( record ); } @Override public RelationshipGroupRecord load( Long key, Integer type ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override @@ -244,41 +245,36 @@ public RelationshipGroupRecord clone( RelationshipGroupRecord record ) }; } - public static Loader,SchemaRule> schemaRuleLoader( final SchemaStore store ) + public static Loader schemaRuleLoader( final SchemaStore store ) { - return new Loader, SchemaRule>() + return new Loader() { @Override - public Collection newUnused(Long key, SchemaRule additionalData ) + public SchemaRecord newUnused( Long key, SchemaRule additionalData ) { // Don't blindly mark as created here since some records may be reused. - return store.allocateFrom( additionalData ); + return new SchemaRecord( store.allocateFrom( additionalData ) ); } @Override - public Collection load(Long key, SchemaRule additionalData ) + public SchemaRecord load( Long key, SchemaRule additionalData ) { - return store.getRecords( key ); + return new SchemaRecord( store.getRecords( key, RecordLoad.NORMAL ) ); } @Override - public void ensureHeavy(Collection dynamicRecords ) + public void ensureHeavy( SchemaRecord records ) { - for ( DynamicRecord record : dynamicRecords) + for ( DynamicRecord record : records) { store.ensureHeavy(record); } } @Override - public Collection clone( Collection dynamicRecords ) + public SchemaRecord clone( SchemaRecord records ) { - Collection list = new ArrayList<>( dynamicRecords.size() ); - for ( DynamicRecord record : dynamicRecords ) - { - list.add( record.clone() ); - } - return list; + return records.clone(); } }; } @@ -297,7 +293,7 @@ public PropertyKeyTokenRecord newUnused( Integer key, Void additionalData ) @Override public PropertyKeyTokenRecord load( Integer key, Void additionalData ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override @@ -328,7 +324,7 @@ public LabelTokenRecord newUnused( Integer key, Void additionalData ) @Override public LabelTokenRecord load( Integer key, Void additionalData ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override @@ -359,7 +355,7 @@ public RelationshipTypeTokenRecord newUnused( Integer key, Void additionalData ) @Override public RelationshipTypeTokenRecord load( Integer key, Void additionalData ) { - return store.getRecord( key ); + return store.getRecord( key, store.newRecord(), NORMAL ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreIndexStoreView.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreIndexStoreView.java index ff61282680419..da34c160b43c2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreIndexStoreView.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/NeoStoreIndexStoreView.java @@ -54,6 +54,7 @@ import static org.neo4j.kernel.api.index.NodePropertyUpdate.add; import static org.neo4j.kernel.api.labelscan.NodeLabelUpdate.labelChanges; import static org.neo4j.kernel.impl.store.NodeLabelsField.parseLabelsField; +import static org.neo4j.kernel.impl.store.record.RecordLoad.FORCE; public class NeoStoreIndexStoreView implements IndexStoreView { @@ -152,8 +153,8 @@ protected void process( NodeRecord node ) throws FAILURE @Override public void nodeAsUpdates( long nodeId, Collection target ) { - NodeRecord node = nodeStore.loadRecord( nodeId, new NodeRecord( nodeId ) ); - if ( node == null || !node.inUse() ) + NodeRecord node = nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ); + if ( !node.inUse() ) { return; } @@ -180,7 +181,7 @@ public void nodeAsUpdates( long nodeId, Collection target ) @Override public Property getProperty( long nodeId, int propertyKeyId ) throws EntityNotFoundException { - NodeRecord node = nodeStore.forceGetRecord( nodeId ); + NodeRecord node = nodeStore.getRecord( nodeId, nodeStore.newRecord(), FORCE ); if ( !node.inUse() ) { throw new EntityNotFoundException( EntityType.NODE, nodeId ); @@ -271,6 +272,7 @@ protected PropertyBlock fetchNextOrNull() abstract static class NodeStoreScan implements StoreScan { private volatile boolean continueScanning; + private final NodeRecord record; protected final NodeStore nodeStore; protected final LockService locks; @@ -283,6 +285,7 @@ abstract static class NodeStoreScan implements StoreS public NodeStoreScan( NodeStore nodeStore, LockService locks, long totalCount ) { this.nodeStore = nodeStore; + this.record = nodeStore.newRecord(); this.locks = locks; this.totalCount = totalCount; } @@ -292,18 +295,16 @@ public void run() throws FAILURE { PrimitiveLongIterator nodeIds = new StoreIdIterator( nodeStore ); continueScanning = true; - NodeRecord record = new NodeRecord( -1 ); while ( continueScanning && nodeIds.hasNext() ) { long id = nodeIds.next(); try ( Lock ignored = locks.acquireNodeLock( id, LockService.LockType.READ_LOCK ) ) { - NodeRecord loaded = nodeStore.loadRecord( id, record ); - if ( loaded != null ) + count++; + if ( nodeStore.getRecord( id, record, FORCE ).inUse() ) { - process( loaded ); + process( record ); } - count++; } } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java index 7bdd54f45730e..203cdc63e522f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java @@ -41,6 +41,7 @@ import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PropertyRecord; +import org.neo4j.kernel.impl.store.record.RecordLoad; import org.neo4j.kernel.impl.transaction.command.Command.Mode; import org.neo4j.kernel.impl.transaction.command.Command.NodeCommand; import org.neo4j.kernel.impl.transaction.command.Command.PropertyCommand; @@ -65,6 +66,7 @@ public class OnlineIndexUpdates implements IndexUpdates private final PropertyStore propertyStore; private final PropertyLoader propertyLoader; private final Collection updates = new ArrayList<>(); + private NodeRecord nodeRecord; public OnlineIndexUpdates( NodeStore nodeStore, PropertyStore propertyStore, @@ -154,7 +156,7 @@ private void gatherUpdatesFromPropertyCommandsForNode( long nodeId, * if this happens and we're in recovery mode that the node in question will be deleted * in an upcoming transaction, so just skip this update. */ - NodeRecord nodeRecord = nodeStore.getRecord( nodeId ); + NodeRecord nodeRecord = loadNode( nodeId ); nodeLabelsBefore = nodeLabelsAfter = parseLabelsField( nodeRecord ).get( nodeStore ); } @@ -179,6 +181,16 @@ public boolean visited( long key, NodeCommand nodeCommand ) } ); } + private NodeRecord loadNode( long nodeId ) + { + if ( nodeRecord == null ) + { + nodeRecord = nodeStore.newRecord(); + } + nodeStore.getRecord( nodeId, nodeRecord, RecordLoad.NORMAL ); + return nodeRecord; + } + private void gatherUpdatesFromNodeCommand( NodeCommand nodeCommand, PrimitiveLongObjectMap nodeCommands, PrimitiveLongObjectMap> propertyCommands, @@ -225,7 +237,7 @@ private Iterator nodeFullyLoadProperties( long nodeId, PrimitiveLongObjectMap> propertyCommands ) { NodeCommand nodeCommand = nodeCommands.get( nodeId ); - NodeRecord nodeRecord = (nodeCommand == null) ? nodeStore.getRecord( nodeId ) : nodeCommand.getAfter(); + NodeRecord nodeRecord = (nodeCommand == null) ? loadNode( nodeId ) : nodeCommand.getAfter(); IteratingPropertyReceiver receiver = new IteratingPropertyReceiver(); PrimitiveLongObjectMap propertiesById = diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/PropertyLoader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/PropertyLoader.java index d0bdc924a2383..873e33d8880a8 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/PropertyLoader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/PropertyLoader.java @@ -33,6 +33,8 @@ import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.transaction.state.TransactionRecordState.PropertyReceiver; +import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL; + public class PropertyLoader { private final NodeStore nodeStore; @@ -50,7 +52,7 @@ public PropertyLoader( NeoStores neoStores ) public RECEIVER nodeLoadProperties( long nodeId, RECEIVER receiver ) { - NodeRecord nodeRecord = nodeStore.getRecord( nodeId ); + NodeRecord nodeRecord = nodeStore.getRecord( nodeId, nodeStore.newRecord(), NORMAL ); loadProperties( nodeRecord.getNextProp(), receiver ); return receiver; } @@ -63,7 +65,7 @@ public RECEIVER nodeLoadProperties( NodeReco public RECEIVER relLoadProperties( long relId, RECEIVER receiver ) { - RelationshipRecord relRecord = relationshipStore.getRecord( relId ); + RelationshipRecord relRecord = relationshipStore.getRecord( relId, relationshipStore.newRecord(), NORMAL ); return loadProperties( relRecord.getNextProp(), receiver ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordAccessSet.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordAccessSet.java index 1ec3f6e5b05d8..e583af730b7e9 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordAccessSet.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordAccessSet.java @@ -19,9 +19,6 @@ */ package org.neo4j.kernel.impl.transaction.state; -import java.util.Collection; - -import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.LabelTokenRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PrimitiveRecord; @@ -30,6 +27,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRecord; import org.neo4j.storageengine.api.schema.SchemaRule; public interface RecordAccessSet @@ -42,7 +40,7 @@ public interface RecordAccessSet RecordAccess getRelGroupRecords(); - RecordAccess, SchemaRule> getSchemaRuleChanges(); + RecordAccess getSchemaRuleChanges(); RecordAccess getPropertyKeyTokenChanges(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordChangeSet.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordChangeSet.java index c613e3afc420e..a2b1dda4f6eba 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordChangeSet.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RecordChangeSet.java @@ -19,9 +19,6 @@ */ package org.neo4j.kernel.impl.transaction.state; -import java.util.Collection; - -import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.LabelTokenRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PrimitiveRecord; @@ -30,6 +27,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRecord; import org.neo4j.kernel.impl.transaction.state.RecordAccess.Loader; import org.neo4j.kernel.impl.util.statistics.IntCounter; import org.neo4j.storageengine.api.schema.SchemaRule; @@ -40,7 +38,7 @@ public class RecordChangeSet implements RecordAccessSet private final RecordAccess propertyRecords; private final RecordAccess relRecords; private final RecordAccess relGroupRecords; - private final RecordAccess, SchemaRule> schemaRuleChanges; + private final RecordAccess schemaRuleChanges; private final RecordAccess propertyKeyTokenChanges; private final RecordAccess labelTokenChanges; private final RecordAccess relationshipTypeTokenChanges; @@ -63,7 +61,7 @@ public RecordChangeSet( Loader propertyLoader, Loader relationshipLoader, Loader relationshipGroupLoader, - Loader,SchemaRule> schemaRuleLoader, + Loader schemaRuleLoader, Loader propertyKeyTokenLoader, Loader labelTokenLoader, Loader relationshipTypeTokenLoader ) @@ -103,7 +101,7 @@ public RecordAccess getRelGroupRecords() } @Override - public RecordAccess, SchemaRule> getSchemaRuleChanges() + public RecordAccess getSchemaRuleChanges() { return schemaRuleChanges; } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordState.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordState.java index b2c3f528353a1..5ba76816bd315 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordState.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordState.java @@ -43,6 +43,7 @@ import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; +import org.neo4j.kernel.impl.store.record.SchemaRecord; import org.neo4j.kernel.impl.transaction.command.Command; import org.neo4j.kernel.impl.transaction.command.Command.Mode; import org.neo4j.kernel.impl.transaction.state.RecordAccess.RecordProxy; @@ -213,8 +214,7 @@ public void extractCommands( Collection commands ) throws Transa commands.add( new Command.NeoStoreCommand( change.getBefore(), change.forReadingData() ) ); } } - for ( RecordProxy, SchemaRule> change : - recordChangeSet.getSchemaRuleChanges().changes() ) + for ( RecordProxy change : recordChangeSet.getSchemaRuleChanges().changes() ) { integrityValidator.validateSchemaRule( change.getAdditionalData() ); commands.add( new Command.SchemaRuleCommand( @@ -552,9 +552,9 @@ public void createSchemaRule( SchemaRule schemaRule ) public void dropSchemaRule( SchemaRule rule ) { - RecordProxy, SchemaRule> change = + RecordProxy change = recordChangeSet.getSchemaRuleChanges().getOrLoad( rule.getId(), rule ); - Collection records = change.forChangingData(); + SchemaRecord records = change.forChangingData(); for ( DynamicRecord record : records ) { record.setInUse( false ); @@ -575,14 +575,13 @@ public void removeLabelFromNode( int labelId, long nodeId ) public void setConstraintIndexOwner( IndexRule indexRule, long constraintId ) { - RecordProxy, SchemaRule> change = + RecordProxy change = recordChangeSet.getSchemaRuleChanges().getOrLoad( indexRule.getId(), indexRule ); - Collection records = change.forChangingData(); + SchemaRecord records = change.forChangingData(); indexRule = indexRule.withOwningConstraint( constraintId ); - records.clear(); - records.addAll( schemaStore.allocateFrom( indexRule ) ); + records.setDynamicRecords( schemaStore.allocateFrom( indexRule ) ); } public interface PropertyReceiver

      diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java index 145bcb9d91a37..6eb15b23dd6ec 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java @@ -179,6 +179,7 @@ public class BatchInserterImpl implements BatchInserter private final BatchInserterImpl.BatchSchemaActions actions; private final StoreLocker storeLocker; private boolean labelsTouched; + private boolean isShutdown; private final LongFunction