Automatic detection of record format

This commit removes the need to always explicitly configure record format, if it is something else than default. This means that previously, tools like consistency checked and backup always needed an explicit config parameter `GraphDatabaseSettings.record_format` when applied to a high_limit store. Detection of format happens inside `NeoStoreDataSource` and `StoreFactory`. Later is often directly used by tools like consistency checker.
neo4j · Jun 3, 2016 · 0e4cac0 · 0e4cac0
1 parent b8d6b07
commit 0e4cac0
Show file tree

Hide file tree

Showing 100 changed files with 1,284 additions and 706 deletions.
diff --git a/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java b/community/consistency-check/src/main/java/org/neo4j/consistency/ConsistencyCheckService.java
@@ -51,12 +51,10 @@
 import org.neo4j.kernel.configuration.Settings;
 import org.neo4j.kernel.impl.api.index.IndexStoreView;
 import org.neo4j.kernel.impl.locking.LockService;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.pagecache.ConfiguringPageCacheFactory;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
 import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.state.NeoStoreIndexStoreView;
 import org.neo4j.logging.DuplicatingLog;
@@ -125,9 +123,7 @@ public Result runFullConsistencyCheck( final File storeDir, Config tuningConfigu
         Config consistencyCheckerConfig = tuningConfiguration.with(
                 MapUtil.stringMap( GraphDatabaseSettings.read_only.name(), Settings.TRUE ) );
         StoreFactory factory = new StoreFactory( storeDir, consistencyCheckerConfig,
-                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem,
+                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, logProvider );
-                RecordFormatSelector.autoSelectFormat( consistencyCheckerConfig, NullLogService.getInstance() ),
-                logProvider );
 
         ConsistencySummaryStatistics summary;
         // With the added neo4j_home config the logs directory will end up in db location

diff --git a/...ity/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java b/...ity/consistency-check/src/test/java/org/neo4j/consistency/checking/GraphStoreFixture.java
@@ -46,20 +46,16 @@
 import org.neo4j.kernel.api.impl.index.storage.DirectoryFactory;
 import org.neo4j.kernel.api.impl.schema.LuceneSchemaIndexProvider;
 import org.neo4j.kernel.api.index.SchemaIndexProvider;
-import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.api.TransactionRepresentationCommitProcess;
 import org.neo4j.kernel.impl.api.TransactionToApply;
 import org.neo4j.kernel.impl.api.index.IndexStoreView;
 import org.neo4j.kernel.impl.locking.LockService;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.NodeLabelsField;
 import org.neo4j.kernel.impl.store.NodeStore;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
-import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.kernel.impl.store.record.DynamicRecord;
 import org.neo4j.kernel.impl.store.record.NeoStoreRecord;
 import org.neo4j.kernel.impl.store.record.NodeRecord;
@@ -73,6 +69,7 @@
 import org.neo4j.kernel.impl.transaction.tracing.CommitEvent;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
 import org.neo4j.logging.FormattedLogProvider;
+import org.neo4j.logging.LogProvider;
 import org.neo4j.logging.NullLog;
 import org.neo4j.logging.NullLogProvider;
 import org.neo4j.storageengine.api.StorageEngine;
@@ -84,7 +81,6 @@
 
 import static java.lang.System.currentTimeMillis;
 import static org.neo4j.consistency.ConsistencyCheckService.defaultConsistencyCheckThreadsNumber;
-import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider.fullStoreLabelUpdateStream;
 
 public abstract class GraphStoreFixture extends PageCacheRule implements TestRule
@@ -105,6 +101,10 @@ public abstract class GraphStoreFixture extends PageCacheRule implements TestRul
     private long arrayPropId;
     private int relTypeId;
     private int propKeyId;
+
+    /**
+     * Record format used to generate initial database.
+     */
     private String formatName = StringUtils.EMPTY;
 
     public GraphStoreFixture( boolean keepStatistics, String formatName )
@@ -118,11 +118,6 @@ public GraphStoreFixture( String formatName )
         this( false, formatName );
     }
 
-    public GraphStoreFixture()
-    {
-        this( false, StandardV3_0.NAME );
-    }
-
     public void apply( Transaction transaction ) throws TransactionFailureException
     {
         applyTransaction( transaction );
@@ -134,10 +129,9 @@ public DirectStoreAccess directStoreAccess()
         {
             DefaultFileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
             PageCache pageCache = getPageCache( fileSystem );
-            Config config = new Config( stringMap( GraphDatabaseSettings.record_format.name(), formatName ) );
+            LogProvider logProvider = NullLogProvider.getInstance();
-            neoStore = new StoreFactory( fileSystem, directory, pageCache,
+            StoreFactory storeFactory = new StoreFactory( directory, pageCache, fileSystem, logProvider );
-                    RecordFormatSelector.autoSelectFormat(config,  NullLogService.getInstance() ),
+            neoStore = storeFactory.openAllNeoStores();
-                    NullLogProvider.getInstance() ).openAllNeoStores();
             StoreAccess nativeStores;
             if ( keepStatistics )
             {
@@ -428,9 +422,7 @@ protected void applyTransaction( Transaction transaction ) throws TransactionFai
         // and the next startup of the store would do recovery where the transaction would have been
         // applied and all would have been well.
 
-        GraphDatabaseBuilder builder = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( directory )
+        GraphDatabaseAPI database = (GraphDatabaseAPI) new TestGraphDatabaseFactory().newEmbeddedDatabase( directory );
-                             .setConfig( GraphDatabaseSettings.record_format, formatName );
-        GraphDatabaseAPI database = (GraphDatabaseAPI) builder.newGraphDatabase();
         try
         {
             DependencyResolver dependencyResolver = database.getDependencyResolver();

diff --git a/...test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java b/...test/java/org/neo4j/consistency/checking/full/DetectAllRelationshipInconsistenciesIT.java
@@ -41,14 +41,12 @@
 import org.neo4j.kernel.api.labelscan.LabelScanStore;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.MyRelTypes;
-import org.neo4j.kernel.impl.logging.NullLogService;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.RelationshipStore;
 import org.neo4j.kernel.impl.store.StoreAccess;
 import org.neo4j.kernel.impl.store.StoreFactory;
 import org.neo4j.kernel.impl.store.StoreType;
-import org.neo4j.kernel.impl.store.format.RecordFormatSelector;
+import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
-import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
 import org.neo4j.kernel.impl.store.record.RelationshipRecord;
 import org.neo4j.kernel.internal.GraphDatabaseAPI;
@@ -76,8 +74,6 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         // GIVEN a database which lots of relationships
         GraphDatabaseAPI db = getGraphDatabaseAPI();
         Sabotage sabotage;
-        RecordFormats selectedRecordFormat = RecordFormatSelector.autoSelectFormat( getTuningConfiguration(),
-                NullLogService.getInstance() );
         try
         {
             Node[] nodes = new Node[1_000];
@@ -99,8 +95,8 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
             // WHEN sabotaging a random relationship
             DependencyResolver resolver = db.getDependencyResolver();
             PageCache pageCache = resolver.resolveDependency( PageCache.class );
-            StoreFactory storeFactory = new StoreFactory( fileSystem, directory.directory(), pageCache,
+
-                    selectedRecordFormat, NullLogProvider.getInstance() );
+            StoreFactory storeFactory = newStoreFactory( pageCache );
 
             try ( NeoStores neoStores = storeFactory.openNeoStores( false, StoreType.RELATIONSHIP ) )
             {
@@ -120,8 +116,7 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         {
             DependencyResolver resolver = db.getDependencyResolver();
             PageCache pageCache = resolver.resolveDependency( PageCache.class );
-            StoreFactory storeFactory = new StoreFactory( fileSystem, directory.directory(), pageCache,
+            StoreFactory storeFactory = newStoreFactory( pageCache );
-                    selectedRecordFormat, NullLogProvider.getInstance() );
 
             try ( NeoStores neoStores = storeFactory.openAllNeoStores() )
             {
@@ -149,6 +144,12 @@ public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception
         }
     }
 
+    private StoreFactory newStoreFactory( PageCache pageCache )
+    {
+        return new StoreFactory( directory.directory(), getTuningConfiguration(),
+                new DefaultIdGeneratorFactory( fileSystem ), pageCache, fileSystem, NullLogProvider.getInstance() );
+    }
+
     private Config getTuningConfiguration()
     {
         return new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m",

diff --git a/...tency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java b/...tency-check/src/test/java/org/neo4j/consistency/repair/RelationshipChainExplorerTest.java
@@ -44,7 +44,6 @@
 import org.neo4j.test.rule.TargetDirectory;
 
 import static org.junit.Assert.assertEquals;
-import static org.neo4j.helpers.collection.MapUtil.stringMap;
 import static org.neo4j.kernel.impl.store.record.RecordLoad.NORMAL;
 
 public class RelationshipChainExplorerTest
@@ -143,8 +142,9 @@ private StoreAccess createStoreWithOneHighDegreeNodeAndSeveralDegreeTwoNodes( in
         }
         database.shutdown();
         PageCache pageCache = pageCacheRule.getPageCache( new DefaultFileSystemAbstraction() );
-        Config config = new Config( stringMap( GraphDatabaseSettings.record_format.name(), getRecordFormatName() ) );
+        StoreAccess storeAccess = new StoreAccess( new DefaultFileSystemAbstraction(), pageCache, storeDirectory,
-        return new StoreAccess( new DefaultFileSystemAbstraction(), pageCache, storeDirectory, config ).initialize();
+                Config.empty() );
+        return storeAccess.initialize();
     }
 
     protected String getRecordFormatName()

diff --git a/community/consistency-check/src/test/java/org/neo4j/consistency/store/StoreAssertions.java b/community/consistency-check/src/test/java/org/neo4j/consistency/store/StoreAssertions.java
@@ -42,14 +42,7 @@ private StoreAssertions()
 
     public static void assertConsistentStore( File dir ) throws ConsistencyCheckIncompleteException, IOException
     {
-        assertConsistentStore( dir, Config.empty() );
+        Map<String,String> params = stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m" );
-    }
-
-    public static void assertConsistentStore( File dir, Config config ) throws ConsistencyCheckIncompleteException,
-            IOException
-    {
-        Map<String,String> params = config.getParams();
-        params.put( GraphDatabaseSettings.pagecache_memory.name(), "8m" );
         final Config configuration = new Config( params, GraphDatabaseSettings.class, ConsistencyCheckSettings.class );
 
         final ConsistencyCheckService.Result result = new ConsistencyCheckService().runFullConsistencyCheck(

diff --git a/...ency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java b/...ency-check/src/test/java/org/neo4j/unsafe/impl/batchimport/ParallelBatchImporterTest.java
@@ -19,7 +19,6 @@
  */
 package org.neo4j.unsafe.impl.batchimport;
 
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.mutable.MutableLong;
 import org.junit.Rule;
 import org.junit.Test;
@@ -58,6 +57,7 @@
 import org.neo4j.io.fs.DefaultFileSystemAbstraction;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.NullLogService;
+import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.logging.NullLogProvider;
 import org.neo4j.test.Randoms;
 import org.neo4j.test.TestGraphDatabaseFactory;
@@ -72,6 +72,7 @@
 import org.neo4j.unsafe.impl.batchimport.input.Inputs;
 import org.neo4j.unsafe.impl.batchimport.input.SimpleInputIterator;
 import org.neo4j.unsafe.impl.batchimport.staging.ExecutionMonitor;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
@@ -180,7 +181,6 @@ public void shouldImportCsvData() throws Exception
             // THEN
             GraphDatabaseService db = new TestGraphDatabaseFactory()
                     .newEmbeddedDatabaseBuilder( directory.graphDbDir() )
-                    .setConfig( GraphDatabaseSettings.record_format, getFormatName() )
                     .newGraphDatabase();
             try ( Transaction tx = db.beginTx() )
             {
@@ -228,8 +228,7 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete
     {
         ConsistencyCheckService consistencyChecker = new ConsistencyCheckService();
         Result result = consistencyChecker.runFullConsistencyCheck( storeDir,
-                new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m",
+                new Config( stringMap( GraphDatabaseSettings.pagecache_memory.name(), "8m" ) ),
-                        GraphDatabaseSettings.record_format.name(), getFormatName()) ),
                 ProgressMonitorFactory.NONE,
                 NullLogProvider.getInstance(), false );
         assertTrue( "Database contains inconsistencies, there should be a report in " + storeDir,
@@ -238,7 +237,7 @@ private void assertConsistent( File storeDir ) throws ConsistencyCheckIncomplete
 
     protected String getFormatName()
     {
-        return StringUtils.EMPTY;
+        return StandardV3_0.NAME;
     }
 
     public abstract static class InputIdGenerator

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java
@@ -259,7 +259,6 @@ boolean applicable( DiagnosticsPhase phase )
     private final Log msgLog;
     private final LogService logService;
     private final AutoIndexing autoIndexing;
-    private final RecordFormats formats;
     private final LogProvider logProvider;
     private final DependencyResolver dependencyResolver;
     private final TokenNameLookup tokenNameLookup;
@@ -335,8 +334,7 @@ public NeoStoreDataSource(
             Monitors monitors,
             Tracers tracers,
             Procedures procedures,
-            IOLimiter ioLimiter,
+            IOLimiter ioLimiter )
-            RecordFormats formats )
     {
         this.storeDir = storeDir;
         this.config = config;
@@ -367,9 +365,6 @@ public NeoStoreDataSource(
         this.procedures = procedures;
         this.ioLimiter = ioLimiter;
 
-        this.formats = RecordFormatSelector.select( config, formats, logService );
-        new RecordFormatPropertyConfigurator( this.formats, this.config ).configure();
-
         readOnly = config.get( Configuration.read_only );
         msgLog = logProvider.getLog( getClass() );
         this.lockService = new ReentrantLockService();
@@ -430,6 +425,7 @@ public void start() throws IOException
         life.add( new Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
 
         // Upgrade the store before we begin
+        RecordFormats formats = selectStoreFormats( config, storeDir, fs, pageCache, logService );
         upgradeStore( formats );
 
         // Build all modules and their services
@@ -443,7 +439,7 @@ public void start() throws IOException
 
             storageEngine = buildStorageEngine(
                     propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup,
-                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering, formats );
+                    indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering );
 
             LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader =
                     new VersionAwareLogEntryReader<>( storageEngine.commandReaderFactory() );
@@ -531,6 +527,15 @@ public void start() throws IOException
         databaseHealth.healed();
     }
 
+    private static RecordFormats selectStoreFormats( Config config, File storeDir, FileSystemAbstraction fs,
+            PageCache pageCache, LogService logService )
+    {
+        LogProvider logging = logService.getInternalLogProvider();
+        RecordFormats formats = RecordFormatSelector.selectNewestFormat( config, storeDir, fs, pageCache, logging );
+        new RecordFormatPropertyConfigurator( formats, config ).configure();
+        return formats;
+    }
+
     private void upgradeStore( RecordFormats format )
     {
         LabelScanStoreProvider labelScanStoreProvider =
@@ -555,8 +560,7 @@ private StorageEngine buildStorageEngine(
             PropertyKeyTokenHolder propertyKeyTokenHolder, LabelTokenHolder labelTokens,
             RelationshipTypeTokenHolder relationshipTypeTokens,
             LegacyIndexProviderLookup legacyIndexProviderLookup, IndexConfigStore indexConfigStore,
-            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering,
+            Runnable schemaStateChangeCallback, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering )
-            RecordFormats format )
     {
         LabelScanStoreProvider labelScanStore = dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
                 HighestSelectionStrategy.getInstance() );
@@ -569,7 +573,7 @@ private StorageEngine buildStorageEngine(
                 fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaStateChangeCallback,
                 constraintSemantics, scheduler, tokenNameLookup, lockService, schemaIndexProvider,
                 indexingServiceMonitor, databaseHealth, labelScanStore, legacyIndexProviderLookup, indexConfigStore,
-                legacyIndexTransactionOrdering, transactionSnapshotSupplier, format );
+                legacyIndexTransactionOrdering, transactionSnapshotSupplier );
 
         // We pretend that the storage engine abstract hides all details within it. Whereas that's mostly
         // true it's not entirely true for the time being. As long as we need this call below, which

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java
@@ -47,7 +47,6 @@
 import org.neo4j.kernel.impl.locking.ResourceTypes;
 import org.neo4j.kernel.impl.locking.community.CommunityLockManger;
 import org.neo4j.kernel.impl.logging.LogService;
-import org.neo4j.kernel.impl.store.format.standard.StandardV3_0;
 import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
@@ -110,8 +109,6 @@ public CommunityEditionModule( PlatformModule platformModule )
 
         ioLimiter = IOLimiter.unlimited();
 
-        formats = StandardV3_0.RECORD_FORMATS;
-
         registerRecovery( platformModule.databaseInfo, life, dependencies );
 
         publishEditionInfo( dependencies.resolveDependency( UsageData.class ), platformModule.databaseInfo, config );

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java
@@ -201,8 +201,7 @@ public DataSourceModule( final GraphDatabaseFacadeFactory.Dependencies dependenc
                 platformModule.monitors,
                 platformModule.tracers,
                 procedures,
-                editionModule.ioLimiter,
+                editionModule.ioLimiter ) );
-                editionModule.formats ) );
         dataSourceManager.register( neoStoreDataSource );
 
         life.add( new MonitorGc( config, logging.getInternalLog( MonitorGc.class ) ) );

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/EditionModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/EditionModule.java
@@ -37,7 +37,6 @@
 import org.neo4j.kernel.impl.locking.Locks;
 import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.proc.Procedures;
-import org.neo4j.kernel.impl.store.format.RecordFormats;
 import org.neo4j.kernel.impl.store.id.IdGeneratorFactory;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.info.DiagnosticsManager;
@@ -86,8 +85,6 @@ public interface SPI
 
     public IOLimiter ioLimiter;
 
-    public RecordFormats formats;
-
     protected void doAfterRecoveryAndStartup( DatabaseInfo databaseInfo, DependencyResolver dependencyResolver )
     {
         DiagnosticsManager diagnosticsManager = dependencyResolver.resolveDependency( DiagnosticsManager.class );