Automatic detection of record format

This commit removes the need to always explicitly configure record format, if
it is something else than default. This means that previously, tools like
consistency checked and backup always needed an explicit config parameter
`GraphDatabaseSettings.record_format` when applied to a high_limit store.

Detection of format happens inside `NeoStoreDataSource` and `StoreFactory`.
Later is often directly used by tools like consistency checker.

community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java

@@ -51,12 +51,10 @@
 import org.neo4j.kernel.configuration.Settings;
 import org.neo4j.kernel.impl.api.index.IndexStoreView;
 import org.neo4j.kernel.impl.locking.LockService;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
 import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.state.NeoStoreIndexStoreView;
 import org.neo4j.logging.DuplicatingLog;
@@ -125,9 +123,7 @@ public Result runFullConsistencyCheck( final File storeDir, Config tuningConfigu
         Config consistencyCheckerConfig = tuningConfiguration.with(
                 MapUtil.stringMap( GraphDatabaseSettings.read_only.name(), Settings.TRUE ) );
         StoreFactory factory = new StoreFactory( storeDir, consistencyCheckerConfig,
-                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem,
-                RecordFormatSelector.autoSelectFormat( consistencyCheckerConfig, NullLogService.getInstance() ),
-                logProvider );
+                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider );
 
         ConsistencySummaryStatistics summary;
         // With the added neo4j_home config the logs directory will end up in db location

community/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java

@@ -46,20 +46,16 @@
 import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
 import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
-import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess;
 import org.neo4j.kernel.impl.api.TransactionToApply;
 import org.neo4j.kernel.impl.api.index.IndexStoreView;
 import org.neo4j.kernel.impl.locking.LockService;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.NodeLabelsField;
 import org.neo4j.kernel.impl.store.NodeStore;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
-import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.kernel.impl.store.record.DynamicRecord;
 import org.neo4j.kernel.impl.store.record.NeoStoreRecord;
 import org.neo4j.kernel.impl.store.record.NodeRecord;
@@ -73,6 +69,7 @@
 import org.neo4j.kernel.impl.transaction.tracing.CommitEvent;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.logging.FormattedLogProvider;
+import org.neo4j.logging.LogProvider;
 import org.neo4j.logging.NullLog;
 import org.neo4j.logging.NullLogProvider;
 import org.neo4j.storageengine.api.StorageEngine;
@@ -84,7 +81,6 @@
 
 import static java.lang.System.currentTimeMillis;
 import static org.neo4j.consistency.ConsistencyCheckService.defaultConsistencyCheckThreadsNumber;
-import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.fullStoreLabelUpdateStream;
 
 public abstract class GraphStoreFixture extends PageCacheRule implements TestRule
@@ -105,6 +101,10 @@ public abstract class GraphStoreFixture extends PageCacheRule implements TestRul
     private long arrayPropId;
     private int relTypeId;
     private int propKeyId;
+
+    /**
+     * Record format used to generate initial database.
+     */
     private String formatName = StringUtils.EMPTY;
 
     public GraphStoreFixture( boolean keepStatistics, String formatName )
@@ -118,11 +118,6 @@ public GraphStoreFixture( String formatName )
         this( false, formatName );
     }
 
-    public GraphStoreFixture()
-    {
-        this( false, StandardV3_0.NAME );
-    }
-
     public void apply( Transaction transaction ) throws TransactionFailureException
     {
         applyTransaction( transaction );
@@ -134,10 +129,9 @@ public DirectStoreAccess directStoreAccess()
         {
             DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
             PageCache pageCache = getPageCache( fileSystem );
-            Config config = new Config( stringMap( GraphDatabaseSettings.record_format.name(), formatName ) );
-            neoStore = new StoreFactory( fileSystem, directory, pageCache,
-                    RecordFormatSelector.autoSelectFormat(config,  NullLogService.getInstance() ),
-                    NullLogProvider.getInstance() ).openAllNeoStores();
+            LogProvider logProvider = NullLogProvider.getInstance();
+            StoreFactory storeFactory = new StoreFactory( directory, pageCache, fileSystem, logProvider );
+            neoStore = storeFactory.openAllNeoStores();
             StoreAccess nativeStores;
             if ( keepStatistics )
             {
@@ -428,9 +422,7 @@ protected void applyTransaction( Transaction transaction ) throws TransactionFai
         // and the next startup of the store would do recovery where the transaction would have been
         // applied and all would have been well.
 
-        GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( directory )
-                             .setConfig( GraphDatabaseSettings.record_format, formatName );
-        GraphDatabaseAPI database = (GraphDatabaseAPI) builder.newGraphDatabase();
+        GraphDatabaseAPI database = (GraphDatabaseAPI) new TestGraphDatabaseFactory().newEmbeddedDatabase( directory );
         try
         {
             DependencyResolver dependencyResolver = database.getDependencyResolver();

community/consistency-check/src/test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java

@@ -41,14 +41,12 @@
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.MyRelTypes;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.RelationshipStore;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
 import org.neo4j.kernel.impl.store.StoreType;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
-import org.neo4j.kernel.impl.store.format.RecordFormats;
+import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
 import org.neo4j.kernel.impl.store.record.RelationshipRecord;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
@@ -76,8 +74,6 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         // GIVEN a database which lots of relationships
         GraphDatabaseAPI db = getGraphDatabaseAPI();
         Sabotage sabotage;
-        RecordFormats selectedRecordFormat = RecordFormatSelector.autoSelectFormat( getTuningConfiguration(),
-                NullLogService.getInstance() );
         try
         {
             Node[] nodes = new Node[1_000];
@@ -99,8 +95,8 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
             // WHEN sabotaging a random relationship
             DependencyResolver resolver = db.getDependencyResolver();
             PageCache pageCache = resolver.resolveDependency( PageCache.class );
-            StoreFactory storeFactory = new StoreFactory( fileSystem, directory.directory(), pageCache,
-                    selectedRecordFormat, NullLogProvider.getInstance() );
+
+            StoreFactory storeFactory = newStoreFactory( pageCache );
 
             try ( NeoStores neoStores = storeFactory.openNeoStores( false, StoreType.RELATIONSHIP ) )
             {
@@ -120,8 +116,7 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         {
             DependencyResolver resolver = db.getDependencyResolver();
             PageCache pageCache = resolver.resolveDependency( PageCache.class );
-            StoreFactory storeFactory = new StoreFactory( fileSystem, directory.directory(), pageCache,
-                    selectedRecordFormat, NullLogProvider.getInstance() );
+            StoreFactory storeFactory = newStoreFactory( pageCache );
 
             try ( NeoStores neoStores = storeFactory.openAllNeoStores() )
             {
@@ -149,6 +144,12 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         }
     }
 
+    private StoreFactory newStoreFactory( PageCache pageCache )
+    {
+        return new StoreFactory( directory.directory(), getTuningConfiguration(),
+                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() );
+    }
+
     private Config getTuningConfiguration()
     {
         return new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m",

community/consistency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java

@@ -44,7 +44,6 @@
 import org.neo4j.test.rule.TargetDirectory;
 
 import static org.junit.Assert.assertEquals;
-import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL;
 
 public class RelationshipChainExplorerTest
@@ -143,8 +142,9 @@ private StoreAccess createStoreWithOneHighDegreeNodeAndSeveralDegreeTwoNodes( in
         }
         database.shutdown();
         PageCache pageCache = pageCacheRule.getPageCache( new DefaultFileSystemAbstraction() );
-        Config config = new Config( stringMap( GraphDatabaseSettings.record_format.name(), getRecordFormatName() ) );
-        return new StoreAccess( new DefaultFileSystemAbstraction(), pageCache, storeDirectory, config ).initialize();
+        StoreAccess storeAccess = new StoreAccess( new DefaultFileSystemAbstraction(), pageCache, storeDirectory,
+                Config.empty() );
+        return storeAccess.initialize();
     }
 
     protected String getRecordFormatName()

community/consistency-check/src/test/java/org/neo4j/consistency/store/StoreAssertions.java

@@ -42,14 +42,7 @@ private StoreAssertions()
 
     public static void assertConsistentStore( File dir ) throws ConsistencyCheckIncompleteException, IOException
     {
-        assertConsistentStore( dir, Config.empty() );
-    }
-
-    public static void assertConsistentStore( File dir, Config config ) throws ConsistencyCheckIncompleteException,
-            IOException
-    {
-        Map<String,String> params = config.getParams();
-        params.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" );
+        Map<String,String> params = stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m" );
         final Config configuration = new Config( params, GraphDatabaseSettings.class, ConsistencyCheckSettings.class );
 
         final ConsistencyCheckService.Result result = new ConsistencyCheckService().runFullConsistencyCheck(

community/consistency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java

@@ -19,7 +19,6 @@
  */
 package org.neo4j.unsafe.impl.batchimport;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.junit.Rule;
 import org.junit.Test;
@@ -58,6 +57,7 @@
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.NullLogService;
+import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.logging.NullLogProvider;
 import org.neo4j.test.Randoms;
 import org.neo4j.test.TestGraphDatabaseFactory;
@@ -72,6 +72,7 @@
 import org.neo4j.unsafe.impl.batchimport.input.Inputs;
 import org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator;
 import org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -180,7 +181,6 @@ public void shouldImportCsvData() throws Exception
             // THEN
             GraphDatabaseService db = new TestGraphDatabaseFactory()
                     .newEmbeddedDatabaseBuilder( directory.graphDbDir() )
-                    .setConfig( GraphDatabaseSettings.record_format, getFormatName() )
                     .newGraphDatabase();
             try ( Transaction tx = db.beginTx() )
             {
@@ -228,8 +228,7 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete
     {
         ConsistencyCheckService consistencyChecker = new ConsistencyCheckService();
         Result result = consistencyChecker.runFullConsistencyCheck( storeDir,
-                new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m",
-                        GraphDatabaseSettings.record_format.name(), getFormatName()) ),
+                new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m" ) ),
                 ProgressMonitorFactory.NONE,
                 NullLogProvider.getInstance(), false );
         assertTrue( "Database contains inconsistencies, there should be a report in " + storeDir,
@@ -238,7 +237,7 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete
 
     protected String getFormatName()
     {
-        return StringUtils.EMPTY;
+        return StandardV3_0.NAME;
     }
 
     public abstract static class InputIdGenerator

community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java

@@ -259,7 +259,6 @@ boolean applicable( DiagnosticsPhase phase )
     private final Log msgLog;
     private final LogService logService;
     private final AutoIndexing autoIndexing;
-    private final RecordFormats formats;
     private final LogProvider logProvider;
     private final DependencyResolver dependencyResolver;
     private final TokenNameLookup tokenNameLookup;
@@ -335,8 +334,7 @@ public NeoStoreDataSource(
             Monitors monitors,
             Tracers tracers,
             Procedures procedures,
-            IOLimiter ioLimiter,
-            RecordFormats formats )
+            IOLimiter ioLimiter )
     {
         this.storeDir = storeDir;
         this.config = config;
@@ -367,9 +365,6 @@ public NeoStoreDataSource(
         this.procedures = procedures;
         this.ioLimiter = ioLimiter;
 
-        this.formats = RecordFormatSelector.select( config, formats, logService );
-        new RecordFormatPropertyConfigurator( this.formats, this.config ).configure();
-
         readOnly = config.get( Configuration.read_only );
         msgLog = logProvider.getLog( getClass() );
         this.lockService = new ReentrantLockService();
@@ -430,6 +425,7 @@ public void start() throws IOException
         life.add( new Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
 
         // Upgrade the store before we begin
+        RecordFormats formats = selectStoreFormats( config, storeDir, fs, pageCache, logService );
         upgradeStore( formats );
 
         // Build all modules and their services
@@ -443,7 +439,7 @@ public void start() throws IOException
 
             storageEngine = buildStorageEngine(
                     propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup,
-                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering, formats );
+                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering );
 
             LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader =
                     new VersionAwareLogEntryReader<>( storageEngine.commandReaderFactory() );
@@ -531,6 +527,15 @@ public void start() throws IOException
         databaseHealth.healed();
     }
 
+    private static RecordFormats selectStoreFormats( Config config, File storeDir, FileSystemAbstraction fs,
+            PageCache pageCache, LogService logService )
+    {
+        LogProvider logging = logService.getInternalLogProvider();
+        RecordFormats formats = RecordFormatSelector.selectNewestFormat( config, storeDir, fs, pageCache, logging );
+        new RecordFormatPropertyConfigurator( formats, config ).configure();
+        return formats;
+    }
+
     private void upgradeStore( RecordFormats format )
     {
         LabelScanStoreProvider labelScanStoreProvider =
@@ -555,8 +560,7 @@ private StorageEngine buildStorageEngine(
             PropertyKeyTokenHolder propertyKeyTokenHolder, LabelTokenHolder labelTokens,
             RelationshipTypeTokenHolder relationshipTypeTokens,
             LegacyIndexProviderLookup legacyIndexProviderLookup, IndexConfigStore indexConfigStore,
-            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering,
-            RecordFormats format )
+            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering )
     {
         LabelScanStoreProvider labelScanStore = dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
                 HighestSelectionStrategy.getInstance() );
@@ -569,7 +573,7 @@ private StorageEngine buildStorageEngine(
                 fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaStateChangeCallback,
                 constraintSemantics, scheduler, tokenNameLookup, lockService, schemaIndexProvider,
                 indexingServiceMonitor, databaseHealth, labelScanStore, legacyIndexProviderLookup, indexConfigStore,
-                legacyIndexTransactionOrdering, transactionSnapshotSupplier, format );
+                legacyIndexTransactionOrdering, transactionSnapshotSupplier );
 
         // We pretend that the storage engine abstract hides all details within it. Whereas that's mostly
         // true it's not entirely true for the time being. As long as we need this call below, which

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java

@@ -47,7 +47,6 @@
 import org.neo4j.kernel.impl.locking.ResourceTypes;
 import org.neo4j.kernel.impl.locking.community.CommunityLockManger;
 import org.neo4j.kernel.impl.logging.LogService;
-import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
@@ -110,8 +109,6 @@ public CommunityEditionModule( PlatformModule platformModule )
 
         ioLimiter = IOLimiter.unlimited();
 
-        formats = StandardV3_0.RECORD_FORMATS;
-
         registerRecovery( platformModule.databaseInfo, life, dependencies );
 
         publishEditionInfo( dependencies.resolveDependency( UsageData.class ), platformModule.databaseInfo, config );

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java

@@ -201,8 +201,7 @@ public DataSourceModule( final GraphDatabaseFacadeFactory.Dependencies dependenc
                 platformModule.monitors,
                 platformModule.tracers,
                 procedures,
-                editionModule.ioLimiter,
-                editionModule.formats ) );
+                editionModule.ioLimiter ) );
         dataSourceManager.register( neoStoreDataSource );
 
         life.add( new MonitorGc( config, logging.getInternalLog( MonitorGc.class ) ) );

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/EditionModule.java

@@ -37,7 +37,6 @@
 import org.neo4j.kernel.impl.locking.Locks;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.proc.Procedures;
-import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.info.DiagnosticsManager;
@@ -86,8 +85,6 @@ public interface SPI
 
     public IOLimiter ioLimiter;
 
-    public RecordFormats formats;
-
     protected void doAfterRecoveryAndStartup( DatabaseInfo databaseInfo, DependencyResolver dependencyResolver )
     {
         DiagnosticsManager diagnosticsManager = dependencyResolver.resolveDependency( DiagnosticsManager.class );