diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java index 4462af36ad54..33061303a4cc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractNodeCursor.java @@ -33,6 +33,7 @@ import org.neo4j.kernel.impl.store.InvalidRecordException; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.record.NodeRecord; @@ -59,7 +60,7 @@ public abstract class StoreAbstractNodeCursor extends NodeItemHelper implements { protected final NodeRecord nodeRecord; protected NodeStore nodeStore; - protected RelationshipGroupStore relationshipGroupStore; + protected RecordStore relationshipGroupStore; protected RelationshipStore relationshipStore; protected final LockService lockService; protected StoreStatement storeStatement; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java index 3c957c39d47d..624e7f8795be 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/store/StoreAbstractRelationshipCursor.java @@ -24,9 +24,10 @@ import org.neo4j.kernel.impl.locking.Lock; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.NeoStores; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.util.InstanceCache; import org.neo4j.storageengine.api.PropertyItem; @@ -43,7 +44,7 @@ public abstract class StoreAbstractRelationshipCursor extends EntityItemHelper { protected final RelationshipRecord relationshipRecord; protected final RelationshipStore relationshipStore; - protected final RelationshipGroupStore relationshipGroupStore; + protected final RecordStore relationshipGroupStore; private final LockService lockService; protected StoreStatement storeStatement; diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java index 3fcfb806a04e..80ad0dd5c186 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storageengine/impl/recordstorage/RecordStorageEngine.java @@ -52,6 +52,7 @@ import org.neo4j.kernel.impl.api.TransactionApplierFacade; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.api.index.IndexingServiceFactory; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig; import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; import org.neo4j.kernel.impl.api.store.CacheLayer; @@ -150,6 +151,7 @@ public class RecordStorageEngine implements StorageEngine, Lifecycle private final WorkSync indexUpdatesSync; private final NeoStoreIndexStoreView indexStoreView; private final LegacyIndexProviderLookup legacyIndexProviderLookup; + private final PropertyPhysicalToLogicalConverter indexUpdatesConverter; // Immutable state for creating/applying commands private final Loaders loaders; @@ -199,6 +201,7 @@ public RecordStorageEngine( try { + indexUpdatesConverter = new PropertyPhysicalToLogicalConverter( neoStores.getPropertyStore() ); schemaCache = new SchemaCache( constraintSemantics, Collections.emptyList() ); schemaStorage = new SchemaStorage( neoStores.getSchemaStore() ); @@ -358,7 +361,8 @@ private BatchTransactionApplierFacade applier( TransactionApplicationMode mode ) // Schema index application appliers.add( new IndexBatchTransactionApplier( indexingService, labelScanStoreSync, indexUpdatesSync, - neoStores.getNodeStore(), neoStores.getPropertyStore(), new PropertyLoader( neoStores ), mode ) ); + neoStores.getNodeStore(), neoStores.getPropertyStore(), new PropertyLoader( neoStores ), + indexUpdatesConverter, mode ) ); // Legacy index application appliers.add( diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java index c9101fa91838..644dad7e2dd5 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/CommonAbstractStore.java @@ -1178,6 +1178,12 @@ public String toString() return getClass().getSimpleName(); } + @Override + public int getStoreHeaderInt() + { + throw new UnsupportedOperationException( "No header" ); + } + public static abstract class Configuration { public static final Setting rebuild_idgenerators_fast = diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ComposableRecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ComposableRecordStore.java index 00d1be09e908..602072796446 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ComposableRecordStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/ComposableRecordStore.java @@ -111,6 +111,12 @@ protected boolean isInUse( PageCursor cursor ) return recordFormat.isInUse( cursor ); } + @Override + public int getStoreHeaderInt() + { + return ((IntStoreHeader) storeHeader).value(); + } + @Override public void accept( org.neo4j.kernel.impl.store.RecordStore.Processor processor, RECORD record ) throws FAILURE diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java index 1d46e31c29bd..5e3da8b7bdc2 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DynamicNodeLabels.java @@ -19,10 +19,12 @@ */ package org.neo4j.kernel.impl.store; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import org.neo4j.helpers.collection.Pair; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; @@ -36,7 +38,6 @@ import static org.neo4j.kernel.impl.store.NodeLabelsField.fieldPointsToDynamicRecordOfLabels; import static org.neo4j.kernel.impl.store.NodeLabelsField.firstDynamicLabelRecordId; import static org.neo4j.kernel.impl.store.NodeLabelsField.parseLabelsBody; -import static org.neo4j.kernel.impl.store.NodeStore.getDynamicLabelsArrayFromHeavyRecords; import static org.neo4j.kernel.impl.store.PropertyType.ARRAY; public class DynamicNodeLabels implements NodeLabels @@ -62,7 +63,7 @@ public static long[] get( NodeRecord node, NodeStore nodeStore ) { nodeStore.ensureHeavy( node, firstDynamicLabelRecordId( node.getLabelField() ) ); } - return nodeStore.getDynamicLabelsArray( node.getUsedDynamicLabelRecords() ); + return getDynamicLabelsArray( node.getUsedDynamicLabelRecords(), nodeStore.getDynamicLabelStore() ); } @Override @@ -104,7 +105,7 @@ public static Collection putSorted( NodeRecord node, long[] label { Iterator recycledRecords = changedDynamicRecords.iterator(); Collection allocatedRecords = - NodeStore.allocateRecordsForDynamicLabels( node.getId(), labelIds, + allocateRecordsForDynamicLabels( node.getId(), labelIds, recycledRecords, allocator ); // Set the rest of the previously set dynamic records as !inUse while ( recycledRecords.hasNext() ) @@ -125,10 +126,10 @@ public Collection add( long labelId, NodeStore nodeStore, Dynamic { nodeStore.ensureHeavy( node, firstDynamicLabelRecordId( labelField ) ); Collection existingRecords = node.getDynamicLabelRecords(); - long[] existingLabelIds = nodeStore.getDynamicLabelsArray( existingRecords ); + long[] existingLabelIds = getDynamicLabelsArray( existingRecords, nodeStore.getDynamicLabelStore() ); long[] newLabelIds = LabelIdArray.concatAndSort( existingLabelIds, labelId ); Collection changedDynamicRecords = - NodeStore.allocateRecordsForDynamicLabels( node.getId(), newLabelIds, existingRecords.iterator(), allocator ); + allocateRecordsForDynamicLabels( node.getId(), newLabelIds, existingRecords.iterator(), allocator ); node.setLabelField( dynamicPointer( changedDynamicRecords ), changedDynamicRecords ); return changedDynamicRecords; } @@ -137,7 +138,8 @@ public Collection add( long labelId, NodeStore nodeStore, Dynamic public Collection remove( long labelId, NodeStore nodeStore ) { nodeStore.ensureHeavy( node, firstDynamicLabelRecordId( labelField ) ); - long[] existingLabelIds = nodeStore.getDynamicLabelsArray( node.getUsedDynamicLabelRecords() ); + long[] existingLabelIds = getDynamicLabelsArray( node.getUsedDynamicLabelRecords(), + nodeStore.getDynamicLabelStore() ); long[] newLabelIds = filter( existingLabelIds, labelId ); Collection existingRecords = node.getDynamicLabelRecords(); if ( InlineNodeLabels.tryInlineInNodeRecord( node, newLabelIds, existingRecords ) ) @@ -146,8 +148,8 @@ public Collection remove( long labelId, NodeStore nodeStore ) } else { - Collection newRecords = - nodeStore.allocateRecordsForDynamicLabels( node.getId(), newLabelIds, existingRecords.iterator() ); + Collection newRecords = allocateRecordsForDynamicLabels( node.getId(), + newLabelIds, existingRecords.iterator(), nodeStore.getDynamicLabelStore() ); node.setLabelField( dynamicPointer( newRecords ), existingRecords ); if ( !newRecords.equals( existingRecords ) ) { // One less dynamic record, mark that one as not in use @@ -197,4 +199,42 @@ public String toString() return format( "Dynamic(id:%d,[%s])", firstDynamicLabelRecordId( node.getLabelField() ), Arrays.toString( getDynamicLabelsArrayFromHeavyRecords( node.getUsedDynamicLabelRecords() ) ) ); } + + public static Collection allocateRecordsForDynamicLabels( long nodeId, long[] labels, + Iterator useFirst, AbstractDynamicStore dynamicLabelStore ) + { + return allocateRecordsForDynamicLabels( nodeId, labels, useFirst, (DynamicRecordAllocator)dynamicLabelStore ); + } + + public static Collection allocateRecordsForDynamicLabels( long nodeId, long[] labels, + Iterator useFirst, DynamicRecordAllocator allocator ) + { + long[] storedLongs = LabelIdArray.prependNodeId( nodeId, labels ); + Collection records = new ArrayList<>(); + DynamicArrayStore.allocateRecords( records, storedLongs, useFirst, allocator ); + return records; + } + + public static long[] getDynamicLabelsArray( Iterable records, + AbstractDynamicStore dynamicLabelStore ) + { + long[] storedLongs = (long[]) + DynamicArrayStore.getRightArray( dynamicLabelStore.readFullByteArray( records, PropertyType.ARRAY ) ); + return LabelIdArray.stripNodeId( storedLongs ); + } + + public static long[] getDynamicLabelsArrayFromHeavyRecords( Iterable records ) + { + long[] storedLongs = (long[]) + DynamicArrayStore.getRightArray( readFullByteArrayFromHeavyRecords( records, PropertyType.ARRAY ) ); + return LabelIdArray.stripNodeId( storedLongs ); + } + + public static Pair getDynamicLabelsArrayAndOwner( Iterable records, + AbstractDynamicStore dynamicLabelStore ) + { + long[] storedLongs = (long[]) + DynamicArrayStore.getRightArray( dynamicLabelStore.readFullByteArray( records, PropertyType.ARRAY ) ); + return Pair.of(storedLongs[0], LabelIdArray.stripNodeId( storedLongs )); + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java index 7d23d0d43162..3d3018ab98b1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NeoStores.java @@ -42,6 +42,7 @@ import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; import org.neo4j.kernel.impl.store.id.IdType; import org.neo4j.kernel.impl.store.kvstore.DataInitializer; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.info.DiagnosticsManager; import org.neo4j.logging.Log; import org.neo4j.logging.LogProvider; @@ -326,10 +327,7 @@ private DynamicStringStore getPropertyKeyTokenNamesStore() return (DynamicStringStore) getStore( StoreType.PROPERTY_KEY_TOKEN_NAME ); } - /** - * @return the {@link RelationshipGroupStore} - */ - public RelationshipGroupStore getRelationshipGroupStore() + public RecordStore getRelationshipGroupStore() { return (RelationshipGroupStore) getStore( StoreType.RELATIONSHIP_GROUP ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java index b23389729415..cde8bc8cdfb5 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/NodeStore.java @@ -20,12 +20,8 @@ package org.neo4j.kernel.impl.store; import java.io.File; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Iterator; -import org.neo4j.helpers.collection.Pair; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.impl.store.format.RecordFormat; @@ -37,7 +33,6 @@ import org.neo4j.kernel.impl.util.Bits; import org.neo4j.logging.LogProvider; -import static org.neo4j.kernel.impl.store.AbstractDynamicStore.readFullByteArrayFromHeavyRecords; import static org.neo4j.kernel.impl.store.NoStoreHeaderFormat.NO_STORE_HEADER_FORMAT; /** @@ -121,42 +116,6 @@ public DynamicArrayStore getDynamicLabelStore() return dynamicLabelStore; } - public Collection allocateRecordsForDynamicLabels( long nodeId, long[] labels, - Iterator useFirst ) - { - return allocateRecordsForDynamicLabels( nodeId, labels, useFirst, dynamicLabelStore ); - } - - public static Collection allocateRecordsForDynamicLabels( long nodeId, long[] labels, - Iterator useFirst, DynamicRecordAllocator allocator ) - { - long[] storedLongs = LabelIdArray.prependNodeId( nodeId, labels ); - Collection records = new ArrayList<>(); - DynamicArrayStore.allocateRecords( records, storedLongs, useFirst, allocator ); - return records; - } - - public long[] getDynamicLabelsArray( Iterable records ) - { - long[] storedLongs = (long[]) - DynamicArrayStore.getRightArray( dynamicLabelStore.readFullByteArray( records, PropertyType.ARRAY ) ); - return LabelIdArray.stripNodeId( storedLongs ); - } - - public static long[] getDynamicLabelsArrayFromHeavyRecords( Iterable records ) - { - long[] storedLongs = (long[]) - DynamicArrayStore.getRightArray( readFullByteArrayFromHeavyRecords( records, PropertyType.ARRAY ) ); - return LabelIdArray.stripNodeId( storedLongs ); - } - - public Pair getDynamicLabelsArrayAndOwner( Iterable records ) - { - long[] storedLongs = (long[]) - DynamicArrayStore.getRightArray( dynamicLabelStore.readFullByteArray( records, PropertyType.ARRAY ) ); - return Pair.of(storedLongs[0], LabelIdArray.stripNodeId( storedLongs )); - } - public void updateDynamicLabelRecords( Iterable dynamicLabelRecords ) { for ( DynamicRecord record : dynamicLabelRecords ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java index 793cb09bf974..59dfa0396179 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/PropertyStore.java @@ -34,9 +34,7 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCursor; import org.neo4j.io.pagecache.PagedFile; -import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.api.store.PropertyRecordCursor; import org.neo4j.kernel.impl.store.format.RecordFormat; import org.neo4j.kernel.impl.store.id.IdGeneratorFactory; @@ -47,7 +45,6 @@ import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; import org.neo4j.kernel.impl.store.record.RecordLoad; -import org.neo4j.kernel.impl.transaction.state.PropertyRecordChange; import org.neo4j.logging.LogProvider; import static org.neo4j.helpers.collection.IteratorUtil.first; @@ -70,7 +67,6 @@ public static abstract class Configuration extends CommonAbstractStore.Configura private final DynamicStringStore stringStore; private final PropertyKeyTokenStore propertyKeyTokenStore; private final DynamicArrayStore arrayStore; - private final PropertyPhysicalToLogicalConverter physicalToLogicalConverter; public PropertyStore( File fileName, @@ -88,7 +84,6 @@ public PropertyStore( this.stringStore = stringPropertyStore; this.propertyKeyTokenStore = propertyKeyTokenStore; this.arrayStore = arrayPropertyStore; - this.physicalToLogicalConverter = new PropertyPhysicalToLogicalConverter( this ); } @Override @@ -428,35 +423,6 @@ public Collection getPropertyRecordChain( long firstRecordId, return toReturn; } - public void toLogicalUpdates( Collection target, - Iterable changes, - long[] nodeLabelsBefore, - long[] nodeLabelsAfter ) - { - physicalToLogicalConverter.apply( target, changes, nodeLabelsBefore, nodeLabelsAfter ); - } - - /** - * For property records there's no "inUse" byte and we need to read the whole record to - * see if there are any PropertyBlocks in use in it. - */ - @Override - protected boolean isInUse( PageCursor cursor ) - { - int offsetAtBeginning = cursor.getOffset(); - cursor.setOffset( offsetAtBeginning + 1/*mod*/ + 4/*prev*/ + 4/*next*/ ); - int recordSize = getRecordSize(); - while ( cursor.getOffset() - offsetAtBeginning < recordSize ) - { - long block = cursor.getLong(); - if ( PropertyType.getPropertyType( block, true ) != null ) - { - return true; - } - } - return false; - } - @Override public PropertyRecord newRecord() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java index ab9ebe954554..eb740a26407b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RecordStore.java @@ -75,6 +75,16 @@ public interface RecordStore extends IdSequen */ long getHighestPossibleIdInUse(); + /** + * Sets highest id in use for this store. This is for when records are applied to this store where + * the ids have been generated through some other means. Having an up to date highest possible id + * makes sure that closing this store truncates at the right place and that "all record scans" can + * see all records. + * + * @param highestIdInUse highest id that is now in use in this store. + */ + void setHighestPossibleIdInUse( long highestIdInUse ); + /** * @return a new record instance for receiving data by {@link #getRecord(long, AbstractBaseRecord, RecordLoad)} * and {@link #newRecordCursor(AbstractBaseRecord)}. @@ -213,12 +223,26 @@ public interface RecordStore extends IdSequen */ int getNumberOfReservedLowIds(); + /** + * Returns store header (see {@link #getNumberOfReservedLowIds()}) as {@code int}. Exposed like this + * for convenience since all known store headers are ints. + * + * @return store header as an int value, e.g the first 4 bytes of the first (reserved) record in this store. + */ + int getStoreHeaderInt(); + Predicate IN_USE = AbstractBaseRecord::inUse; class Delegator implements RecordStore { private final RecordStore actual; + @Override + public void setHighestPossibleIdInUse( long highestIdInUse ) + { + actual.setHighestPossibleIdInUse( highestIdInUse ); + } + @Override public R newRecord() { @@ -314,6 +338,12 @@ public int getRecordsPerPage() return actual.getRecordsPerPage(); } + @Override + public int getStoreHeaderInt() + { + return actual.getStoreHeaderInt(); + } + @Override public void close() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java index 7e73e12b3702..3cf7c1c50345 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/RelationshipGroupStore.java @@ -53,9 +53,4 @@ public void accept( Processor processor, Re { processor.processRelationshipGroup( this, record ); } - - public int getDenseNodeThreshold() - { - return storeHeader.value(); - } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/current/PropertyRecordFormat.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/current/PropertyRecordFormat.java index 1241755f6fb0..1c5f88337b09 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/current/PropertyRecordFormat.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/format/current/PropertyRecordFormat.java @@ -159,4 +159,24 @@ public long getNextRecordReference( PropertyRecord record ) { return record.getNextProp(); } + + /** + * For property records there's no "inUse" byte and we need to read the whole record to + * see if there are any PropertyBlocks in use in it. + */ + @Override + public boolean isInUse( PageCursor cursor ) + { + cursor.setOffset( cursor.getOffset() /*skip...*/ + 1/*mod*/ + 4/*prev*/ + 4/*next*/ ); + int blocks = PropertyType.getPayloadSizeLongs(); + for ( int i = 0; i < blocks; i++ ) + { + long block = cursor.getLong(); + if ( PropertyType.getPropertyType( block, true ) != null ) + { + return true; + } + } + return false; + } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplier.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplier.java index 36e6ea583b0e..2c70d7189e43 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplier.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/HighIdTransactionApplier.java @@ -24,12 +24,11 @@ import java.util.HashMap; import java.util.Map; -import org.neo4j.kernel.impl.api.CommandVisitor; import org.neo4j.kernel.impl.api.TransactionApplier; -import org.neo4j.kernel.impl.store.CommonAbstractStore; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.SchemaStore; import org.neo4j.kernel.impl.store.TokenStore; import org.neo4j.kernel.impl.store.record.Abstract64BitRecord; @@ -50,7 +49,7 @@ public class HighIdTransactionApplier extends TransactionApplier.Adapter { private final NeoStores neoStores; - private final Map highIds = new HashMap<>(); + private final Map,HighId> highIds = new HashMap<>(); public HighIdTransactionApplier( NeoStores neoStores ) { @@ -140,13 +139,13 @@ public void close() throws Exception { // Notifies the stores about the recovered ids and will bump those high ids atomically if // they surpass the current high ids - for ( Map.Entry highId : highIds.entrySet() ) + for ( Map.Entry,HighId> highId : highIds.entrySet() ) { highId.getKey().setHighestPossibleIdInUse( highId.getValue().id ); } } - private void track( CommonAbstractStore store, long id ) + private void track( RecordStore store, long id ) { HighId highId = highIds.get( store ); if ( highId == null ) @@ -159,12 +158,12 @@ private void track( CommonAbstractStore store, long id ) } } - private void track( CommonAbstractStore store, Command command ) + private void track( RecordStore store, Command command ) { track( store, command.getKey() ); } - private void track( CommonAbstractStore store, Collection records ) + private void track( RecordStore store, Collection records ) { for ( Abstract64BitRecord record : records ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplier.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplier.java index 3ebec91c7c65..aae2502a9b9a 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplier.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplier.java @@ -34,6 +34,7 @@ import org.neo4j.kernel.impl.api.TransactionApplier; import org.neo4j.kernel.impl.api.index.IndexingService; import org.neo4j.kernel.impl.api.index.NodePropertyCommandsExtractor; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.store.NodeLabels; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; @@ -59,6 +60,7 @@ public class IndexBatchTransactionApplier extends BatchTransactionApplier.Adapte private final WorkSync,LabelUpdateWork> labelScanStoreSync; private final WorkSync indexUpdatesSync; private final SingleTransactionApplier transactionApplier; + private final PropertyPhysicalToLogicalConverter indexUpdateConverter; private List labelUpdates; private IndexUpdates indexUpdates; @@ -67,11 +69,13 @@ public IndexBatchTransactionApplier( IndexingService indexingService, WorkSync,LabelUpdateWork> labelScanStoreSync, WorkSync indexUpdatesSync, NodeStore nodeStore, PropertyStore propertyStore, PropertyLoader propertyLoader, + PropertyPhysicalToLogicalConverter indexUpdateConverter, TransactionApplicationMode mode ) { this.indexingService = indexingService; this.labelScanStoreSync = labelScanStoreSync; this.indexUpdatesSync = indexUpdatesSync; + this.indexUpdateConverter = indexUpdateConverter; this.transactionApplier = new SingleTransactionApplier( nodeStore, propertyStore, propertyLoader, mode ); } @@ -159,8 +163,8 @@ private IndexUpdates indexUpdates() private IndexUpdates createIndexUpdates() { - return mode == TransactionApplicationMode.RECOVERY ? - new RecoveryIndexUpdates() : new OnlineIndexUpdates( nodeStore, propertyStore, propertyLoader ); + return mode == TransactionApplicationMode.RECOVERY ? new RecoveryIndexUpdates() : + new OnlineIndexUpdates( nodeStore, propertyStore, propertyLoader, indexUpdateConverter ); } @Override diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java index c2b01afe1aa0..36d1ea566165 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/Loaders.java @@ -23,6 +23,7 @@ import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.SchemaStore; @@ -214,7 +215,7 @@ public RelationshipRecord clone(RelationshipRecord relationshipRecord) } public static Loader relationshipGroupLoader( - final RelationshipGroupStore store ) + final RecordStore store ) { return new Loader() { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java index 203cdc63e522..23fe60abbf1c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/OnlineIndexUpdates.java @@ -35,6 +35,7 @@ import org.neo4j.helpers.collection.Pair; import org.neo4j.kernel.api.index.NodePropertyUpdate; import org.neo4j.kernel.api.properties.DefinedProperty; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.api.index.UpdateMode; import org.neo4j.kernel.impl.core.IteratingPropertyReceiver; import org.neo4j.kernel.impl.store.NodeStore; @@ -65,16 +66,19 @@ public class OnlineIndexUpdates implements IndexUpdates private final NodeStore nodeStore; private final PropertyStore propertyStore; private final PropertyLoader propertyLoader; + private final PropertyPhysicalToLogicalConverter converter; private final Collection updates = new ArrayList<>(); private NodeRecord nodeRecord; public OnlineIndexUpdates( NodeStore nodeStore, PropertyStore propertyStore, - PropertyLoader propertyLoader ) + PropertyLoader propertyLoader, + PropertyPhysicalToLogicalConverter converter ) { this.nodeStore = nodeStore; this.propertyStore = propertyStore; this.propertyLoader = propertyLoader; + this.converter = converter; } @Override @@ -160,7 +164,7 @@ private void gatherUpdatesFromPropertyCommandsForNode( long nodeId, nodeLabelsBefore = nodeLabelsAfter = parseLabelsField( nodeRecord ).get( nodeStore ); } - propertyStore.toLogicalUpdates( updates, + converter.apply( updates, Iterables.cast( propertyCommandsForNode ), nodeLabelsBefore, nodeLabelsAfter ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetter.java index 4a59e3f0e41c..fad477964e5e 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetter.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetter.java @@ -19,6 +19,7 @@ */ package org.neo4j.kernel.impl.transaction.state; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.Record; @@ -27,9 +28,9 @@ public class RelationshipGroupGetter { - private final RelationshipGroupStore store; + private final RecordStore store; - public RelationshipGroupGetter( RelationshipGroupStore store ) + public RelationshipGroupGetter( RecordStore store ) { this.store = store; } diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java index 6067b0a0e087..a61cad39021c 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/BatchInserterImpl.java @@ -105,7 +105,7 @@ import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyKeyTokenStore; import org.neo4j.kernel.impl.store.PropertyStore; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.RelationshipTypeTokenStore; import org.neo4j.kernel.impl.store.SchemaStore; @@ -127,6 +127,7 @@ import org.neo4j.kernel.impl.store.record.PropertyKeyTokenRecord; import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipPropertyExistenceConstraintRule; import org.neo4j.kernel.impl.store.record.RelationshipRecord; import org.neo4j.kernel.impl.store.record.RelationshipTypeTokenRecord; @@ -204,7 +205,7 @@ public Label apply( long from ) private final RelationshipTypeTokenStore relationshipTypeTokenStore; private final PropertyKeyTokenStore propertyKeyTokenStore; private final PropertyStore propertyStore; - private final RelationshipGroupStore relationshipGroupStore; + private final RecordStore relationshipGroupStore; private final SchemaStore schemaStore; private final NeoStoreIndexStoreView indexStoreView; @@ -316,7 +317,7 @@ public Label apply( long from ) // Record access recordAccess = new DirectRecordAccessSet( neoStores ); relationshipCreator = new RelationshipCreator( - new RelationshipGroupGetter( relationshipGroupStore ), relationshipGroupStore.getDenseNodeThreshold() ); + new RelationshipGroupGetter( relationshipGroupStore ), relationshipGroupStore.getStoreHeaderInt() ); propertyTraverser = new PropertyTraverser(); propertyCreator = new PropertyCreator( propertyStore, propertyTraverser ); propertyDeletor = new PropertyDeleter( propertyStore, propertyTraverser ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccess.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccess.java index 887775fa795e..f78312922fc1 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccess.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccess.java @@ -27,7 +27,7 @@ import java.util.Map; import org.neo4j.helpers.collection.IterableWrapper; -import org.neo4j.kernel.impl.store.CommonAbstractStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import org.neo4j.kernel.impl.transaction.state.RecordAccess; import org.neo4j.kernel.impl.util.statistics.IntCounter; @@ -39,13 +39,13 @@ public class DirectRecordAccess,RECORD extends AbstractBaseRecord,ADDITIONAL> implements RecordAccess { - private final CommonAbstractStore store; + private final RecordStore store; private final Loader loader; private final Map batch = new HashMap<>(); private final IntCounter changeCounter = new IntCounter(); - public DirectRecordAccess( CommonAbstractStore store, Loader loader ) + public DirectRecordAccess( RecordStore store, Loader loader ) { this.store = store; this.loader = loader; @@ -114,9 +114,9 @@ private DirectRecordProxy proxy( final KEY key, final RECORD record, final ADDIT private class DirectRecordProxy implements RecordProxy { - private KEY key; - private RECORD record; - private ADDITIONAL additionalData; + private final KEY key; + private final RECORD record; + private final ADDITIONAL additionalData; private boolean changed = false; public DirectRecordProxy( KEY key, RECORD record, ADDITIONAL additionalData, boolean created ) diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccessSet.java b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccessSet.java index b42da61c5feb..2d25f889f4b9 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccessSet.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/batchinsert/DirectRecordAccessSet.java @@ -24,6 +24,7 @@ import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyKeyTokenStore; import org.neo4j.kernel.impl.store.PropertyStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.RelationshipTypeTokenStore; @@ -58,7 +59,7 @@ public DirectRecordAccessSet( NeoStores neoStores ) NodeStore nodeStore = neoStores.getNodeStore(); PropertyStore propertyStore = neoStores.getPropertyStore(); RelationshipStore relationshipStore = neoStores.getRelationshipStore(); - RelationshipGroupStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); + RecordStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); PropertyKeyTokenStore propertyKeyTokenStore = neoStores.getPropertyKeyTokenStore(); RelationshipTypeTokenStore relationshipTypeTokenStore = neoStores.getRelationshipTypeTokenStore(); LabelTokenStore labelTokenStore = neoStores.getLabelTokenStore(); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java index bee7fa76a994..d67a52e16848 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipProcessor.java @@ -19,7 +19,7 @@ */ package org.neo4j.unsafe.impl.batchimport; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache; @@ -35,12 +35,13 @@ */ public class NodeFirstRelationshipProcessor implements RecordProcessor, GroupVisitor { - private final RelationshipGroupStore relGroupStore; + private final RecordStore relGroupStore; private final NodeRelationshipCache cache; private long nextGroupId = -1; - public NodeFirstRelationshipProcessor( RelationshipGroupStore relGroupStore, NodeRelationshipCache cache ) + public NodeFirstRelationshipProcessor( RecordStore relGroupStore, + NodeRelationshipCache cache ) { this.relGroupStore = relGroupStore; this.cache = cache; diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java index e12d3ef44b8c..b23e336e2633 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeFirstRelationshipStage.java @@ -21,8 +21,9 @@ import org.neo4j.kernel.api.labelscan.LabelScanStore; import org.neo4j.kernel.impl.store.NodeStore; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.NodeRecord; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.unsafe.impl.batchimport.cache.NodeRelationshipCache; import org.neo4j.unsafe.impl.batchimport.input.Collector; import org.neo4j.unsafe.impl.batchimport.staging.Stage; @@ -33,7 +34,7 @@ public class NodeFirstRelationshipStage extends Stage { public NodeFirstRelationshipStage( Configuration config, NodeStore nodeStore, - RelationshipGroupStore relationshipGroupStore, NodeRelationshipCache cache, final Collector collector, + RecordStore relationshipGroupStore, NodeRelationshipCache cache, final Collector collector, LabelScanStore labelScanStore ) { super( "Node --> Relationship", config ); diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java index 0becbc542fc5..fc871061f0fa 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/store/BatchingNeoStores.java @@ -44,11 +44,12 @@ import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.StoreFactory; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.counts.CountsTracker; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.util.Dependencies; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.logging.LogProvider; @@ -62,6 +63,7 @@ import org.neo4j.unsafe.impl.batchimport.store.io.IoTracer; import static java.lang.String.valueOf; + import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; import static org.neo4j.helpers.collection.MapUtil.stringMap; @@ -227,7 +229,7 @@ public RelationshipStore getRelationshipStore() return neoStores.getRelationshipStore(); } - public RelationshipGroupStore getRelationshipGroupStore() + public RecordStore getRelationshipGroupStore() { return neoStores.getRelationshipGroupStore(); } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java index f73058b43790..ca7b83737e11 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/store/StoreNodeRelationshipCursorTest.java @@ -25,6 +25,7 @@ import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; import org.neo4j.kernel.impl.store.record.RelationshipRecord; @@ -51,7 +52,7 @@ public void shouldHandleDenseNodeWithNoRelationships() throws Exception NeoStores stores = mock( NeoStores.class ); NodeStore nodeStore = mock( NodeStore.class ); when( stores.getNodeStore() ).thenReturn( nodeStore ); - RelationshipGroupStore relationshipGroupStore = mock( RelationshipGroupStore.class ); + RecordStore relationshipGroupStore = mock( RelationshipGroupStore.class ); when( stores.getRelationshipGroupStore() ).thenReturn( relationshipGroupStore ); @SuppressWarnings( "unchecked" ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresMocking.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresMocking.java index ecf39e431d63..778c006d21aa 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresMocking.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NeoStoresMocking.java @@ -19,6 +19,8 @@ */ package org.neo4j.kernel.impl.store; +import org.neo4j.kernel.impl.store.record.RelationshipGroupRecord; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -35,7 +37,7 @@ public static NeoStores mockNeoStores() when( propertyStore.getStringStore() ).thenReturn( stringStore ); DynamicArrayStore arrayStore = mock( DynamicArrayStore.class ); when( propertyStore.getArrayStore() ).thenReturn( arrayStore ); - RelationshipGroupStore relationshipGroupStore = mock( RelationshipGroupStore.class ); + RecordStore relationshipGroupStore = mock( RelationshipGroupStore.class ); when( mockStore.getRelationshipGroupStore() ).thenReturn( relationshipGroupStore ); return mockStore; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeRecordTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeRecordTest.java index 0186b57b8e5d..66d2b64154a3 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeRecordTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/NodeRecordTest.java @@ -38,8 +38,8 @@ import static java.util.Arrays.asList; import static org.neo4j.helpers.collection.Iterables.toList; +import static org.neo4j.kernel.impl.store.DynamicNodeLabels.allocateRecordsForDynamicLabels; import static org.neo4j.kernel.impl.store.DynamicNodeLabels.dynamicPointer; -import static org.neo4j.kernel.impl.store.NodeStore.allocateRecordsForDynamicLabels; import static org.neo4j.kernel.impl.store.record.DynamicRecord.dynamicRecord; public class NodeRecordTest diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java index 3005955dddb7..acefd3fdab6f 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/RelationshipGroupStoreTest.java @@ -141,18 +141,18 @@ private void createAndVerify( Integer customThreshold ) int expectedThreshold = customThreshold != null ? customThreshold : defaultThreshold; StoreFactory factory = factory( customThreshold ); NeoStores neoStores = factory.openAllNeoStores( true ); - assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getDenseNodeThreshold() ); + assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getStoreHeaderInt() ); neoStores.close(); // Next time we open it it should be the same neoStores = factory.openAllNeoStores(); - assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getDenseNodeThreshold() ); + assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getStoreHeaderInt() ); neoStores.close(); // Even if we open with a different config setting it should just ignore it factory = factory( 999999 ); neoStores = factory.openAllNeoStores(); - assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getDenseNodeThreshold() ); + assertEquals( expectedThreshold, neoStores.getRelationshipGroupStore().getStoreHeaderInt() ); neoStores.close(); } @@ -226,7 +226,7 @@ public void verifyRecordsForDenseNodeWithOneRelType() throws Exception NodeStore nodeStore = neoStores.getNodeStore(); NodeRecord nodeRecord = getRecord( nodeStore, node.getId() ); long group = nodeRecord.getNextRel(); - RelationshipGroupStore groupStore = neoStores.getRelationshipGroupStore(); + RecordStore groupStore = neoStores.getRelationshipGroupStore(); RelationshipGroupRecord groupRecord = getRecord( groupStore, group ); assertEquals( -1, groupRecord.getNext() ); assertEquals( -1, groupRecord.getPrev() ); @@ -261,7 +261,7 @@ public void verifyRecordsForDenseNodeWithTwoRelTypes() throws Exception NodeRecord nodeRecord = getRecord( nodeStore, node.getId() ); long group = nodeRecord.getNextRel(); - RelationshipGroupStore groupStore = neoStores.getRelationshipGroupStore(); + RecordStore groupStore = neoStores.getRelationshipGroupStore(); RelationshipGroupRecord groupRecord = getRecord( groupStore, group ); assertFalse( groupRecord.getNext() == -1 ); assertRelationshipChain( neoStores.getRelationshipStore(), node, groupRecord.getFirstOut(), rel1.getId(), rel2.getId(), rel3.getId() ); @@ -294,7 +294,7 @@ public void verifyGroupIsDeletedWhenNeeded() throws Exception NodeRecord nodeRecord = getRecord( nodeStore, node.getId() ); long group = nodeRecord.getNextRel(); - RelationshipGroupStore groupStore = neoStores.getRelationshipGroupStore(); + RecordStore groupStore = neoStores.getRelationshipGroupStore(); RelationshipGroupRecord groupRecord = getRecord( groupStore, group ); assertFalse( groupRecord.getNext() == -1 ); RelationshipGroupRecord otherGroupRecord = groupStore.getRecord( groupRecord.getNext(), groupStore.newRecord(), @@ -314,7 +314,7 @@ public void checkingIfRecordIsInUseMustHappenAfterConsistentRead() try ( NeoStores neoStores = factory.openAllNeoStores( true ) ) { - RelationshipGroupStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); + RecordStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); RelationshipGroupRecord record = new RelationshipGroupRecord( 1 ).initialize( true, 2, 3, 4, 5, 6, Record.NO_NEXT_RELATIONSHIP.intValue() ); relationshipGroupStore.updateRecord( record ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplierTest.java index 933c793fa572..61588593eb88 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/IndexBatchTransactionApplierTest.java @@ -30,6 +30,7 @@ import org.neo4j.kernel.impl.api.TransactionApplier; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.IndexingService; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.store.NodeLabelsField; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; @@ -58,9 +59,11 @@ public void shouldProvideLabelScanStoreUpdatesSortedByNodeId() throws Exception spy( new WorkSync<>( singletonProvider( writer ) ) ); WorkSync indexUpdatesSync = new WorkSync<>( indexing ); TransactionToApply tx = mock( TransactionToApply.class ); + PropertyStore propertyStore = mock( PropertyStore.class ); try ( IndexBatchTransactionApplier applier = new IndexBatchTransactionApplier( indexing, labelScanSync, - indexUpdatesSync, mock( NodeStore.class ), mock( PropertyStore.class ), - mock( PropertyLoader.class ), TransactionApplicationMode.INTERNAL ) ) + indexUpdatesSync, mock( NodeStore.class ), propertyStore, + mock( PropertyLoader.class ), new PropertyPhysicalToLogicalConverter( propertyStore ), + TransactionApplicationMode.INTERNAL ) ) { try ( TransactionApplier txApplier = applier.startTx( tx ) ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoStoreTransactionApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoStoreTransactionApplierTest.java index ce4e34d53191..53b3eb45dc5a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoStoreTransactionApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoStoreTransactionApplierTest.java @@ -39,6 +39,7 @@ import org.neo4j.kernel.impl.api.BatchTransactionApplierFacade; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.IndexingService; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.core.CacheAccessBackDoor; import org.neo4j.kernel.impl.core.RelationshipTypeToken; import org.neo4j.kernel.impl.locking.LockService; @@ -49,6 +50,7 @@ import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyKeyTokenStore; import org.neo4j.kernel.impl.store.PropertyStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.RelationshipGroupStore; import org.neo4j.kernel.impl.store.RelationshipStore; import org.neo4j.kernel.impl.store.RelationshipTypeTokenStore; @@ -96,7 +98,7 @@ public class NeoStoreTransactionApplierTest private final NodeStore nodeStore = mock( NodeStore.class ); private final RelationshipStore relationshipStore = mock( RelationshipStore.class ); private final PropertyStore propertyStore = mock( PropertyStore.class ); - private final RelationshipGroupStore relationshipGroupStore = mock( RelationshipGroupStore.class ); + private final RecordStore relationshipGroupStore = mock( RelationshipGroupStore.class ); private final RelationshipTypeTokenStore relationshipTypeTokenStore = mock( RelationshipTypeTokenStore.class ); private final LabelTokenStore labelTokenStore = mock( LabelTokenStore.class ); private final PropertyKeyTokenStore propertyKeyTokenStore = mock( PropertyKeyTokenStore.class ); @@ -929,6 +931,7 @@ private BatchTransactionApplier newApplierFacade( BatchTransactionApplier... app private BatchTransactionApplier newIndexApplier() { return new IndexBatchTransactionApplier( indexingService, labelScanStoreSynchronizer, - indexUpdatesSync, nodeStore, propertyStore, new PropertyLoader( neoStores ), INTERNAL ); + indexUpdatesSync, nodeStore, propertyStore, new PropertyLoader( neoStores ), + new PropertyPhysicalToLogicalConverter( propertyStore ), INTERNAL ); } } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoTransactionIndexApplierTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoTransactionIndexApplierTest.java index b61431c162fc..32de876bb934 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoTransactionIndexApplierTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/command/NeoTransactionIndexApplierTest.java @@ -32,6 +32,7 @@ import org.neo4j.kernel.impl.api.TransactionApplier; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.IndexingService; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.store.NodeStore; import org.neo4j.kernel.impl.store.PropertyStore; import org.neo4j.kernel.impl.store.record.DynamicRecord; @@ -92,9 +93,11 @@ public void shouldUpdateLabelStoreScanOnNodeCommands() throws Exception private IndexBatchTransactionApplier newIndexTransactionApplier() { + PropertyStore propertyStore = mock( PropertyStore.class ); return new IndexBatchTransactionApplier( indexingService, - labelScanStoreSynchronizer, indexUpdatesSync, mock( NodeStore.class ), mock( PropertyStore.class ), mock( - PropertyLoader.class ), TransactionApplicationMode.INTERNAL ); + labelScanStoreSynchronizer, indexUpdatesSync, mock( NodeStore.class ), propertyStore, mock( + PropertyLoader.class ), new PropertyPhysicalToLogicalConverter( propertyStore ), + TransactionApplicationMode.INTERNAL ); } @Test diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java index cf9b368de190..7bdb0a3f6500 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/NodeLabelsFieldTest.java @@ -39,6 +39,7 @@ import org.neo4j.helpers.collection.IteratorUtil; import org.neo4j.helpers.collection.Pair; import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.store.DynamicNodeLabels; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeLabels; import org.neo4j.kernel.impl.store.NodeLabelsField; @@ -177,7 +178,7 @@ public void shouldSpillOverToDynamicRecordIfExceedsInlinedSpace() throws Excepti assertEquals( 1, count( changedDynamicRecords ) ); assertEquals( dynamicLabelsLongRepresentation( changedDynamicRecords ), node.getLabelField() ); assertTrue( Arrays.equals( new long[] {labelId1, labelId2, labelId3}, - nodeStore.getDynamicLabelsArray( changedDynamicRecords ) ) ); + DynamicNodeLabels.getDynamicLabelsArray( changedDynamicRecords, nodeStore.getDynamicLabelStore() ) ) ); } @Test @@ -208,7 +209,8 @@ public void oneDynamicRecordShouldStoreItsOwner() throws Exception Collection initialRecords = node.getDynamicLabelRecords(); // WHEN - Pair pair = nodeStore.getDynamicLabelsArrayAndOwner( initialRecords ); + Pair pair = DynamicNodeLabels.getDynamicLabelsArrayAndOwner( initialRecords, + nodeStore.getDynamicLabelStore() ); // THEN assertEquals( nodeId, pair.first() ); @@ -248,7 +250,8 @@ public void twoDynamicRecordsShouldShrinkToOneWhenRemovingWithoutChangingItsOwne new ArrayList() ); // WHEN - Pair changedPair = nodeStore.getDynamicLabelsArrayAndOwner( changedDynamicRecords ); + Pair changedPair = DynamicNodeLabels.getDynamicLabelsArrayAndOwner( changedDynamicRecords, + nodeStore.getDynamicLabelStore() ); // THEN assertEquals( nodeId, changedPair.first() ); @@ -493,8 +496,8 @@ private NodeRecord nodeRecordWithDynamicLabels( long nodeId, NodeStore nodeStore private Collection allocateAndApply( NodeStore nodeStore, long nodeId, long[] longs ) { - Collection records = nodeStore.allocateRecordsForDynamicLabels( nodeId, longs, - IteratorUtil.emptyIterator() ); + Collection records = DynamicNodeLabels.allocateRecordsForDynamicLabels( nodeId, longs, + IteratorUtil.emptyIterator(), nodeStore.getDynamicLabelStore() ); nodeStore.updateDynamicLabelRecords( records ); return records; } diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java index 9058bdf2358f..2f4652069118 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/RelationshipGroupGetterTest.java @@ -26,7 +26,7 @@ import java.io.File; import org.neo4j.kernel.impl.store.NeoStores; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.StoreFactory; import org.neo4j.kernel.impl.store.StoreType; import org.neo4j.kernel.impl.store.record.NodeRecord; @@ -63,7 +63,7 @@ public void shouldAbortLoadingGroupChainIfComeTooFar() throws Exception NullLogProvider.getInstance() ); try ( NeoStores stores = storeFactory.openNeoStores( true, StoreType.RELATIONSHIP_GROUP ) ) { - RelationshipGroupStore store = spy( stores.getRelationshipGroupStore() ); + RecordStore store = spy( stores.getRelationshipGroupStore() ); RelationshipGroupRecord group_2 = group( 0, 2 ); RelationshipGroupRecord group_4 = group( 1, 4 ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/SchemaRuleCommandTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/SchemaRuleCommandTest.java index 5cadda29b505..7e94f8f53c45 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/SchemaRuleCommandTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/SchemaRuleCommandTest.java @@ -29,6 +29,7 @@ import org.neo4j.kernel.impl.api.BatchTransactionApplier; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.IndexingService; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.core.CacheAccessBackDoor; import org.neo4j.kernel.impl.locking.LockService; import org.neo4j.kernel.impl.store.MetaDataStore; @@ -209,9 +210,11 @@ public void shouldRecreateSchemaRuleWhenDeleteCommandReadFromDisk() throws Excep private final WorkSync,LabelUpdateWork> labelScanStoreSynchronizer = new WorkSync<>( labelScanStore ); private final WorkSync indexUpdatesSync = new WorkSync<>( indexes ); + private final PropertyStore propertyStore = mock( PropertyStore.class ); private final IndexBatchTransactionApplier indexApplier = new IndexBatchTransactionApplier( indexes, - labelScanStoreSynchronizer, indexUpdatesSync, mock( NodeStore.class ), mock( PropertyStore.class ), - mock( PropertyLoader.class ), TransactionApplicationMode.INTERNAL ); + labelScanStoreSynchronizer, indexUpdatesSync, mock( NodeStore.class ), propertyStore, + mock( PropertyLoader.class ), new PropertyPhysicalToLogicalConverter( propertyStore ), + TransactionApplicationMode.INTERNAL ); private final PhysicalLogCommandReaderV2_2 reader = new PhysicalLogCommandReaderV2_2(); private final IndexRule rule = IndexRule.indexRule( id, labelId, propertyKey, PROVIDER_DESCRIPTOR ); diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java index 4cc73f35c76a..4c6f1401e80f 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/transaction/state/TransactionRecordStateTest.java @@ -43,6 +43,7 @@ import org.neo4j.kernel.impl.api.CommandVisitor; import org.neo4j.kernel.impl.api.TransactionToApply; import org.neo4j.kernel.impl.api.index.NodePropertyCommandsExtractor; +import org.neo4j.kernel.impl.api.index.PropertyPhysicalToLogicalConverter; import org.neo4j.kernel.impl.core.CacheAccessBackDoor; import org.neo4j.kernel.impl.locking.Lock; import org.neo4j.kernel.impl.locking.LockService; @@ -51,7 +52,7 @@ import org.neo4j.kernel.impl.store.DynamicArrayStore; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; -import org.neo4j.kernel.impl.store.RelationshipGroupStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.record.DynamicRecord; import org.neo4j.kernel.impl.store.record.NodeRecord; import org.neo4j.kernel.impl.store.record.PropertyBlock; @@ -1107,7 +1108,7 @@ private void assertRelationshipGroupsInOrder( NeoStores neoStores, long nodeId, List seen = new ArrayList<>(); while ( groupId != Record.NO_NEXT_RELATIONSHIP.intValue() ) { - RelationshipGroupStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); + RecordStore relationshipGroupStore = neoStores.getRelationshipGroupStore(); RelationshipGroupRecord group = relationshipGroupStore.getRecord( groupId, relationshipGroupStore.newRecord(), NORMAL ); seen.add( group ); @@ -1130,7 +1131,8 @@ private Iterable indexUpdatesOf( NeoStores neoStores, Transa transaction.accept( extractor ); OnlineIndexUpdates lazyIndexUpdates = new OnlineIndexUpdates( neoStores.getNodeStore(), - neoStores.getPropertyStore(), new PropertyLoader( neoStores ) ); + neoStores.getPropertyStore(), new PropertyLoader( neoStores ), + new PropertyPhysicalToLogicalConverter( neoStores.getPropertyStore() ) ); lazyIndexUpdates.feed( extractor.propertyCommandsByNodeIds(), extractor.nodeCommandsById() ); return lazyIndexUpdates; } @@ -1232,7 +1234,7 @@ private TransactionRecordState newTransactionRecordState( NeoStores neoStores ) return new TransactionRecordState( neoStores, integrityValidator, recordChangeSet, 0, new NoOpClient(), new RelationshipCreator( relationshipGroupGetter, - neoStores.getRelationshipGroupStore().getDenseNodeThreshold() ), + neoStores.getRelationshipGroupStore().getStoreHeaderInt() ), new RelationshipDeleter( relationshipGroupGetter, propertyDeleter ), new PropertyCreator( neoStores.getPropertyStore(), propertyTraverser ), propertyDeleter ); diff --git a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java index 612df5528768..09a6bc1da78b 100644 --- a/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java +++ b/tools/src/main/java/org/neo4j/tools/dump/DumpStore.java @@ -31,6 +31,7 @@ import org.neo4j.kernel.impl.store.CommonAbstractStore; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.NodeStore; +import org.neo4j.kernel.impl.store.RecordStore; import org.neo4j.kernel.impl.store.SchemaStorage; import org.neo4j.kernel.impl.store.SchemaStore; import org.neo4j.kernel.impl.store.StoreFactory; @@ -54,7 +55,7 @@ * @param type of record to dump * @param type of store to dump */ -public class DumpStore> +public class DumpStore> { public static void main( String... args ) throws Exception @@ -151,7 +152,7 @@ private static LogProvider logProvider() return Boolean.getBoolean( "logger" ) ? FormattedLogProvider.toOutputStream( System.out ) : NullLogProvider.getInstance(); } - private static > void dump( + private static > void dump( long[] ids, S store ) throws Exception { new DumpStore( System.out ).dump( store, ids ); @@ -302,7 +303,8 @@ private boolean dumpRecord( STORE store, int size, long id ) throws Exception else { out.print( record ); - byte[] rawRecord = store.getRawRecordData( id ); + // TODO Hmm, please don't do this + byte[] rawRecord = ((CommonAbstractStore)store).getRawRecordData( id ); dumpHex( record, ByteBuffer.wrap( rawRecord ), id, size ); } return record.inUse();