Merge pull request #6561 from tinwelint/3.0-generic-store-migration

Store migration for 3.0 enterprise format

community/collections/src/main/java/org/neo4j/helpers/collection/IteratorUtil.java

@@ -920,10 +920,4 @@ public static <T> ResourceIterable<T> resourceIterable( final Iterable<T> iterab
     {
         return () -> resourceIterator( iterable.iterator(), Resource.EMPTY );
     }
-
-    @SafeVarargs
-    public static <T> T[] array( T... items )
-    {
-        return items;
-    }
 }

community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java

@@ -34,6 +34,7 @@
 import org.neo4j.graphdb.DependencyResolver;
 import org.neo4j.graphdb.GraphDatabaseService;
 import org.neo4j.graphdb.factory.GraphDatabaseBuilder;
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.index.lucene.LuceneLabelScanStoreBuilder;
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.io.fs.FileSystemAbstraction;
@@ -432,7 +433,12 @@ protected void applyTransaction( Transaction transaction ) throws TransactionFai
     private void generateInitialData()
     {
         GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( directory );
-        GraphDatabaseAPI graphDb = (GraphDatabaseAPI) builder.newGraphDatabase();
+        GraphDatabaseAPI graphDb = (GraphDatabaseAPI) builder
+                // Some tests using this fixture were written when the label_block_size was 60 and so hardcoded
+                // tests and records around that. Those tests could change, but the simpler option is to just
+                // keep the block size to 60 and let them be.
+                .setConfig( GraphDatabaseSettings.label_block_size, "60" )
+                .newGraphDatabase();
         try
         {
             generateInitialData( graphDb );

community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/NodeInUseWithCorrectLabelsCheckTest.java

@@ -152,7 +152,6 @@ private NodeRecord withDynamicLabels( RecordAccessStub recordAccess, NodeRecord
         return nodeRecord;
     }
 
-    @SuppressWarnings("unchecked")
     private Engine engineFor( ConsistencyReport.LabelScanConsistencyReport report )
     {
         Engine engine = mock( Engine.class );
@@ -162,7 +161,7 @@ private Engine engineFor( ConsistencyReport.LabelScanConsistencyReport report )
 
     private NodeInUseWithCorrectLabelsCheck<LabelScanDocument, ConsistencyReport.LabelScanConsistencyReport> checker( long[] expectedLabels )
     {
-        return new NodeInUseWithCorrectLabelsCheck<LabelScanDocument, ConsistencyReport.LabelScanConsistencyReport>( expectedLabels );
+        return new NodeInUseWithCorrectLabelsCheck<>( expectedLabels );
     }
 
     interface Engine extends CheckerEngine<LabelScanDocument, ConsistencyReport.LabelScanConsistencyReport>

community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java

@@ -68,7 +68,6 @@
 import org.neo4j.unsafe.impl.batchimport.input.Inputs;
 import org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator;
 import org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor;
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
@@ -222,7 +221,7 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete
                 result.isSuccessful() );
     }
 
-    private static abstract class InputIdGenerator
+    public static abstract class InputIdGenerator
     {
         abstract void reset();
 

community/import-tool/src/main/java/org/neo4j/tooling/ImportTool.java

@@ -47,6 +47,7 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.logging.StoreLogService;
+import org.neo4j.kernel.impl.storemigration.ExistingTargetStrategy;
 import org.neo4j.kernel.impl.storemigration.FileOperation;
 import org.neo4j.kernel.impl.storemigration.StoreFile;
 import org.neo4j.kernel.impl.storemigration.StoreFileType;
@@ -415,7 +416,7 @@ public static void main( String[] incomingArguments, boolean defaultSettingsSuit
                 {
                     StoreFile.fileOperation( FileOperation.DELETE, fs, storeDir, null,
                             Iterables.<StoreFile,StoreFile>iterable( StoreFile.values() ),
-                            false, false, StoreFileType.values() );
+                            false, ExistingTargetStrategy.FAIL, StoreFileType.values() );
                 }
                 catch ( IOException e )
                 {

community/import-tool/src/test/java/org/neo4j/tooling/ImportToolTest.java

@@ -55,7 +55,7 @@
 import org.neo4j.kernel.internal.Version;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
 import org.neo4j.kernel.impl.store.NeoStores;
-import org.neo4j.kernel.impl.store.format.lowlimit.LowLimit;
+import org.neo4j.kernel.impl.store.format.lowlimit.LowLimitV3_0;
 import org.neo4j.kernel.impl.util.Validator;
 import org.neo4j.kernel.impl.util.Validators;
 import org.neo4j.test.EmbeddedDatabaseRule;
@@ -1547,7 +1547,7 @@ public void shouldRespectDbConfig() throws Exception
         // THEN
         NeoStores stores = dbRule.getGraphDatabaseAPI().getDependencyResolver()
                 .resolveDependency( RecordStorageEngine.class ).testAccessNeoStores();
-        int headerSize = LowLimit.RECORD_FORMATS.dynamic().getRecordHeaderSize();
+        int headerSize = LowLimitV3_0.RECORD_FORMATS.dynamic().getRecordHeaderSize();
         assertEquals( arrayBlockSize + headerSize, stores.getPropertyStore().getArrayStore().getRecordSize() );
         assertEquals( stringBlockSize + headerSize, stores.getPropertyStore().getStringStore().getRecordSize() );
     }

community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java

@@ -33,8 +33,11 @@
 import org.neo4j.kernel.configuration.Migrator;
 import org.neo4j.kernel.configuration.Title;
 import org.neo4j.kernel.impl.cache.MonitorGc;
+import org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector;
 import org.neo4j.logging.Level;
 
+import static java.lang.String.valueOf;
+
 import static org.neo4j.kernel.configuration.Settings.ANY;
 import static org.neo4j.kernel.configuration.Settings.BOOLEAN;
 import static org.neo4j.kernel.configuration.Settings.BYTES;
@@ -355,36 +358,40 @@ private static String defaultPageCacheMemory()
             setting( "dbms.pagecache.swapper", STRING, (String) null );
 
     @Description("How many relationships to read at a time during iteration")
-    public static final Setting<Integer> relationship_grab_size = setting("relationship_grab_size", INTEGER, "100", min( 1 ));
+    public static final Setting<Integer> relationship_grab_size = setting("relationship_grab_size", INTEGER,
+            "100", min( 1 ));
 
     @Description("Specifies the block size for storing strings. This parameter is only honored when the store is " +
             "created, otherwise it is ignored. " +
-            "Note that each character in a string occupies two bytes, meaning that a block size of 120 (the default " +
-            "size) will hold a 60 character " +
-            "long string before overflowing into a second block. Also note that each block carries an overhead of 8 " +
-            "bytes. " +
-            "This means that if the block size is 120, the size of the stored records will be 128 bytes.")
+            "Note that each character in a string occupies two bytes, meaning that e.g a block size of 120 will hold " +
+            "a 60 character long string before overflowing into a second block. " +
+            "Also note that each block carries a ~10B of overhead so record size on disk will be slightly larger " +
+            "than the configured block size" )
     @Internal
-    public static final Setting<Integer> string_block_size = setting("string_block_size", INTEGER, "120",min(1));
+    public static final Setting<Integer> string_block_size = setting("string_block_size", INTEGER,
+            valueOf( dynamicRecordDataSizeForAligningWith( 128 ) ), min( dynamicRecordDataSizeForAligningWith( 16 ) ) );
 
     @Description("Specifies the block size for storing arrays. This parameter is only honored when the store is " +
             "created, otherwise it is ignored. " +
-            "The default block size is 120 bytes, and the overhead of each block is the same as for string blocks, " +
-            "i.e., 8 bytes.")
+            "Also note that each block carries a ~10B of overhead so record size on disk will be slightly larger " +
+            "than the configured block size" )
     @Internal
-    public static final Setting<Integer> array_block_size = setting("array_block_size", INTEGER, "120",min(1));
+    public static final Setting<Integer> array_block_size = setting("array_block_size", INTEGER,
+            valueOf( dynamicRecordDataSizeForAligningWith( 128 ) ), min( dynamicRecordDataSizeForAligningWith( 16 ) ) );
 
     @Description("Specifies the block size for storing labels exceeding in-lined space in node record. " +
     		"This parameter is only honored when the store is created, otherwise it is ignored. " +
-            "The default block size is 60 bytes, and the overhead of each block is the same as for string blocks, " +
-            "i.e., 8 bytes.")
+            "Also note that each block carries a ~10B of overhead so record size on disk will be slightly larger " +
+            "than the configured block size" )
     @Internal
-    public static final Setting<Integer> label_block_size = setting("label_block_size", INTEGER, "60",min(1));
+    public static final Setting<Integer> label_block_size = setting("label_block_size", INTEGER,
+            valueOf( dynamicRecordDataSizeForAligningWith( 64 ) ), min( dynamicRecordDataSizeForAligningWith( 16 ) ) );
 
     @Description("An identifier that uniquely identifies this graph database instance within this JVM. " +
             "Defaults to an auto-generated number depending on how many instance are started in this JVM.")
     @Internal
-    public static final Setting<String> forced_kernel_id = setting("forced_kernel_id", STRING, NO_DEFAULT, illegalValueMessage("has to be a valid kernel identifier", matches("[a-zA-Z0-9]*")));
+    public static final Setting<String> forced_kernel_id = setting("forced_kernel_id", STRING, NO_DEFAULT,
+            illegalValueMessage("has to be a valid kernel identifier", matches("[a-zA-Z0-9]*")));
 
     @Internal
     public static final Setting<Boolean> execution_guard_enabled = setting("execution_guard_enabled", BOOLEAN, FALSE );
@@ -433,4 +440,17 @@ private static String defaultPageCacheMemory()
     @Internal
     public static final Setting<File> auth_store = setting("dbms.security.auth_store.location", PATH, "data/dbms/auth");
 
+    /**
+     * Uses the default selected record format to figure out the correct dynamic record data size to create
+     * a store with.
+     *
+     * @param recordSize the desired record size, which optimally is a power-of-two, e.g. 64 or 128.
+     * The dynamic record header size from the selected record format will then be subtracted from this value
+     * to get to the data size, which is the value that the configuration will end up having.
+     * @return the dynamic record data size based on the desired record size and with the header size in mind.
+     */
+    private static int dynamicRecordDataSizeForAligningWith( int recordSize )
+    {
+        return recordSize - InternalRecordFormatSelector.select().dynamic().getRecordHeaderSize();
+    }
 }

community/kernel/src/main/java/org/neo4j/helpers/ArrayUtil.java

@@ -22,7 +22,6 @@
 import java.lang.reflect.Array;
 import java.util.Arrays;
 import java.util.function.Function;
-
 import static java.util.Arrays.copyOf;
 
 /**

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -85,6 +85,7 @@
 import org.neo4j.kernel.impl.store.StoreId;
 import org.neo4j.kernel.impl.store.UnderlyingStorageException;
 import org.neo4j.kernel.impl.store.format.InternalRecordFormatSelector;
+import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.storemigration.DatabaseMigrator;
 import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor;
@@ -420,7 +421,8 @@ public void start() throws IOException
         life.add( new Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
 
         // Upgrade the store before we begin
-        upgradeStore();
+        RecordFormats format = InternalRecordFormatSelector.select( config, logService );
+        upgradeStore( format );
 
         // Build all modules and their services
         StorageEngine storageEngine = null;
@@ -433,7 +435,7 @@ public void start() throws IOException
 
             storageEngine = buildStorageEngine(
                     propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup,
-                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering );
+                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering, format );
 
             // We pretend that the storage engine abstract hides all details within it. Whereas that's mostly
             // true it's not entirely true for the time being. As long as we need this call below, which
@@ -529,7 +531,7 @@ public void start() throws IOException
         databaseHealth.healed();
     }
 
-    private void upgradeStore()
+    private void upgradeStore( RecordFormats format )
     {
         LabelScanStoreProvider labelScanStoreProvider =
                 dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
@@ -545,14 +547,16 @@ private void upgradeStore()
                 schemaIndexProvider,
                 labelScanStoreProvider,
                 indexProviders,
-                pageCache ).migrate( storeDir );
+                pageCache,
+                format ).migrate( storeDir );
     }
 
     private StorageEngine buildStorageEngine(
             PropertyKeyTokenHolder propertyKeyTokenHolder, LabelTokenHolder labelTokens,
             RelationshipTypeTokenHolder relationshipTypeTokens,
             LegacyIndexProviderLookup legacyIndexProviderLookup, IndexConfigStore indexConfigStore,
-            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering )
+            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering,
+            RecordFormats format )
     {
         LabelScanStoreProvider labelScanStore = dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
                 HighestSelectionStrategy.getInstance());
@@ -561,7 +565,7 @@ private StorageEngine buildStorageEngine(
                         labelTokens, relationshipTypeTokens, schemaStateChangeCallback, constraintSemantics, scheduler,
                         tokenNameLookup, lockService, schemaIndexProvider, indexingServiceMonitor, databaseHealth,
                         labelScanStore, legacyIndexProviderLookup, indexConfigStore, legacyIndexTransactionOrdering,
-                        InternalRecordFormatSelector.select(config, logService) ) );
+                        format ) );
     }
 
     private TransactionLogModule buildTransactionLogs(

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java

@@ -80,7 +80,6 @@
 import static org.neo4j.kernel.api.proc.Neo4jTypes.NTNode;
 import static org.neo4j.kernel.api.proc.Neo4jTypes.NTPath;
 import static org.neo4j.kernel.api.proc.Neo4jTypes.NTRelationship;
-import static org.neo4j.kernel.impl.factory.GraphDatabaseFacadeFactory.Configuration.execution_guard_enabled;
 
 /**
  * Datasource module for {@link GraphDatabaseFacadeFactory}. This implements all the
@@ -147,9 +146,9 @@ public DataSourceModule( final GraphDatabaseFacadeFactory.Dependencies dependenc
 
         SchemaWriteGuard schemaWriteGuard = deps.satisfyDependency( editionModule.schemaWriteGuard );
 
-        Guard guard = config.get( execution_guard_enabled ) ?
-                      deps.satisfyDependency( new Guard( logging.getInternalLog( Guard.class ) ) ) :
-                      null;
+        Boolean isGuardEnabled = config.get( GraphDatabaseSettings.execution_guard_enabled );
+        Guard guard =
+                isGuardEnabled ? deps.satisfyDependency( new Guard( logging.getInternalLog( Guard.class ) ) ) : null;
 
         kernelEventHandlers = new KernelEventHandlers( logging.getInternalLog( KernelEventHandlers.class ) );
 

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacadeFactory.java

@@ -23,7 +23,6 @@
 import java.util.Map;
 
 import org.neo4j.graphdb.config.Setting;
-import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.graphdb.security.URLAccessRule;
 import org.neo4j.helpers.Exceptions;
 import org.neo4j.kernel.AvailabilityGuard;
@@ -79,8 +78,6 @@ public interface Dependencies
 
     public static class Configuration
     {
-        public static final Setting<Boolean> read_only = GraphDatabaseSettings.read_only;
-        public static final Setting<Boolean> execution_guard_enabled = GraphDatabaseSettings.execution_guard_enabled;
         public static final Setting<Boolean> ephemeral = setting( "ephemeral", Settings.BOOLEAN, Settings.FALSE );
         public static final Setting<String> ephemeral_keep_logical_logs = setting( "keep_logical_logs", STRING, "1 " +
                 "files", illegalValueMessage( "must be `true`/`false` or of format '<number><optional unit> <type>' " +

community/kernel/src/main/java/org/neo4j/kernel/impl/store/AbstractDynamicStore.java

@@ -55,6 +55,12 @@
  * <p>
  * Note, the first block of a dynamic store is reserved and contains information
  * about the store.
+ * <p>
+ * About configuring block size: Record size is the whole record size including the header (next pointer
+ * and what not). The term block size is equivalent to data size, which is the size of the record - header size.
+ * User configures block size and the block size is what is passed into the constructor to the store.
+ * The record size is what's stored in the header (first record). {@link #getRecordDataSize()} returns
+ * the size which was configured at the store creation, {@link #getRecordSize()} returns what the store header says.
  */
 public abstract class AbstractDynamicStore extends ComposableRecordStore<DynamicRecord,IntStoreHeader>
         implements DynamicRecordAllocator