Introduce DatabaseMigrator as concept responsible for whole store mig…

…ration. Introduce DatabaseMigrator as holder and builder of StoreUpgrader and all of it participants. Update progress monitor to be able to monitor whole store migration and be propagated to each migration participant. Cleanup StoreMigrationParticipant interface from any external service dependencies. Any dependency that requeired by any particular participant should be passed through constructor to that particular instance.
neo4j · Dec 5, 2015 · dfd2a22 · dfd2a22
1 parent ed079ff
commit dfd2a22
Show file tree

Hide file tree

Showing 42 changed files with 482 additions and 647 deletions.
diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java
@@ -25,7 +25,6 @@
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.LockSupport;
@@ -83,19 +82,16 @@
 import org.neo4j.kernel.impl.locking.LockService;
 import org.neo4j.kernel.impl.locking.Locks;
 import org.neo4j.kernel.impl.locking.ReentrantLockService;
+import org.neo4j.kernel.impl.logging.LogService;
 import org.neo4j.kernel.impl.storageengine.StorageEngine;
 import org.neo4j.kernel.impl.storageengine.impl.recordstorage.RecordStorageEngine;
 import org.neo4j.kernel.impl.store.MetaDataStore;
 import org.neo4j.kernel.impl.store.NeoStores;
 import org.neo4j.kernel.impl.store.StoreId;
 import org.neo4j.kernel.impl.store.UnderlyingStorageException;
-import org.neo4j.kernel.impl.store.record.SchemaRule;
-import org.neo4j.kernel.impl.storemigration.LegacyIndexMigrator;
-import org.neo4j.kernel.impl.storemigration.StoreMigrator;
-import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
-import org.neo4j.kernel.impl.storemigration.StoreVersionCheck;
-import org.neo4j.kernel.impl.storemigration.UpgradableDatabase;
-import org.neo4j.kernel.impl.storemigration.legacystore.LegacyStoreVersionCheck;
+import org.neo4j.kernel.impl.storemigration.DatabaseMigrator;
+import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor;
+import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.impl.transaction.TransactionMonitor;
 import org.neo4j.kernel.impl.transaction.log.BatchingTransactionAppender;
@@ -282,6 +278,7 @@ boolean applicable( DiagnosticsPhase phase )
     private final Tracers tracers;
 
     private final Log msgLog;
+    private final LogService logService;
     private final LogProvider logProvider;
     private final DependencyResolver dependencyResolver;
     private final TokenNameLookup tokenNameLookup;
@@ -297,8 +294,6 @@ boolean applicable( DiagnosticsPhase phase )
     private final LockService lockService;
     private final IndexingService.Monitor indexingServiceMonitor;
     private final FileSystemAbstraction fs;
-    private final StoreUpgrader storeMigrationProcess;
-    private StoreMigrator storeMigrator;
     private final TransactionMonitor transactionMonitor;
     private final DatabaseHealth databaseHealth;
     private final PhysicalLogFile.Monitor physicalLogMonitor;
@@ -324,19 +319,6 @@ boolean applicable( DiagnosticsPhase phase )
     private KernelModule kernelModule;
 
     /**
-     * Creates a <CODE>NeoStoreXaDataSource</CODE> using configuration from
-     * <CODE>params</CODE>. First the map is checked for the parameter
-     * <CODE>config</CODE>.
-     * If that parameter exists a config file with that value is loaded (via
-     * {@link Properties#load}). Any parameter that exist in the config file
-     * and in the map passed into this constructor will take the value from the
-     * map.
-     * <p>
-     * If <CODE>config</CODE> parameter is set but file doesn't exist an
-     * <CODE>IOException</CODE> is thrown. If any problem is found with that
-     * configuration file or Neo4j store can't be loaded an <CODE>IOException is
-     * thrown</CODE>.
-     * <p>
      * Note that the tremendous number of dependencies for this class, clearly, is an architecture smell. It is part
      * of the ongoing work on introducing the Kernel API, where components that were previously spread throughout the
      * core API are now slowly accumulating in the Kernel implementation. Over time, these components should be
@@ -346,7 +328,7 @@ public NeoStoreDataSource(
             File storeDir,
             Config config,
             IdGeneratorFactory idGeneratorFactory,
-            LogProvider logProvider,
+            LogService logService,
             JobScheduler scheduler,
             TokenNameLookup tokenNameLookup,
             DependencyResolver dependencyResolver,
@@ -358,8 +340,6 @@ public NeoStoreDataSource(
             TransactionEventHandlers transactionEventHandlers,
             IndexingService.Monitor indexingServiceMonitor,
             FileSystemAbstraction fs,
-            StoreUpgrader storeMigrationProcess,
-            StoreMigrator storeMigrator,
             TransactionMonitor transactionMonitor,
             DatabaseHealth databaseHealth,
             PhysicalLogFile.Monitor physicalLogMonitor,
@@ -378,7 +358,8 @@ public NeoStoreDataSource(
         this.tokenNameLookup = tokenNameLookup;
         this.dependencyResolver = dependencyResolver;
         this.scheduler = scheduler;
-        this.logProvider = logProvider;
+        this.logService = logService;
+        this.logProvider = logService.getInternalLogProvider();
         this.propertyKeyTokenHolder = propertyKeyTokens;
         this.labelTokens = labelTokens;
         this.relationshipTypeTokens = relationshipTypeTokens;
@@ -387,8 +368,6 @@ public NeoStoreDataSource(
         this.transactionEventHandlers = transactionEventHandlers;
         this.indexingServiceMonitor = indexingServiceMonitor;
         this.fs = fs;
-        this.storeMigrationProcess = storeMigrationProcess;
-        this.storeMigrator = storeMigrator;
         this.transactionMonitor = transactionMonitor;
         this.databaseHealth = databaseHealth;
         this.physicalLogMonitor = physicalLogMonitor;
@@ -448,10 +427,6 @@ public void start() throws IOException
         schemaIndexProvider = dependencyResolver.resolveDependency( SchemaIndexProvider.class,
                 HighestSelectionStrategy.getInstance() );
 
-        LabelScanStoreProvider labelScanStoreProvider =
-                dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
-                        HighestSelectionStrategy.getInstance() );
-
         IndexConfigStore indexConfigStore = new IndexConfigStore( storeDir, fs );
         dependencies.satisfyDependency( lockService );
         dependencies.satisfyDependency( indexConfigStore );
@@ -462,11 +437,10 @@ public void start() throws IOException
         monitors.addMonitorListener( loggingLogMonitor );
         monitors.addMonitorListener( (RecoveryVisitor.Monitor) txId -> recoveredCount.incrementAndGet() );
 
-        life.add( new Lifecycle.Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
+        life.add( new Delegate( Lifecycles.multiple( indexProviders.values() ) ) );
 
         // Upgrade the store before we begin
-        upgradeStore( storeDir, storeMigrationProcess, schemaIndexProvider, labelScanStoreProvider, indexProviders,
-                logProvider );
+        upgradeStore();
 
         // Build all modules and their services
         StorageEngine storageEngine = null;
@@ -570,19 +544,23 @@ public void start() throws IOException
         databaseHealth.healed();
     }
 
-    // Startup sequence
-    // By doing this sequence of method calls we can ensure that no dependency cycles exist, and get a clearer view
-    // of the dependency tree, starting at the bottom
-    private void upgradeStore( File storeDir, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider,
-            LabelScanStoreProvider labelScanStoreProvider, Map<String,IndexImplementation> indexProviders,
-            LogProvider logProvider )
+    private void upgradeStore()
     {
-        UpgradableDatabase upgradableDatabase =
-                new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) );
-        storeMigrationProcess.addParticipant( indexProvider.storeMigrationParticipant( fs, pageCache ) );
-        storeMigrationProcess.addParticipant( new LegacyIndexMigrator( fs, indexProviders, logProvider) );
-        storeMigrationProcess.addParticipant( storeMigrator );
-        storeMigrationProcess.migrateIfNeeded( storeDir, upgradableDatabase, indexProvider, labelScanStoreProvider );
+        LabelScanStoreProvider labelScanStoreProvider =
+                dependencyResolver.resolveDependency( LabelScanStoreProvider.class,
+                        HighestSelectionStrategy.getInstance() );
+
+        VisibleMigrationProgressMonitor progressMonitor =
+                new VisibleMigrationProgressMonitor( logService.getInternalLog( StoreMigrator.class ) );
+        new DatabaseMigrator(
+                progressMonitor,
+                fs,
+                config,
+                logService,
+                schemaIndexProvider,
+                labelScanStoreProvider,
+                indexProviders,
+                pageCache ).migrate( storeDir );
     }
 
     private StorageEngine buildStorageEngine(

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java b/community/kernel/src/main/java/org/neo4j/kernel/api/index/SchemaIndexProvider.java
@@ -21,23 +21,17 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 
-import org.neo4j.graphdb.DependencyResolver.SelectionStrategy;
 import org.neo4j.graphdb.ResourceIterator;
 import org.neo4j.helpers.collection.IteratorUtil;
 import org.neo4j.io.fs.FileSystemAbstraction;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.impl.api.index.IndexingService;
 import org.neo4j.kernel.impl.api.index.sampling.IndexSamplingConfig;
+import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider;
 import org.neo4j.kernel.impl.storemigration.StoreMigrationParticipant;
 import org.neo4j.kernel.lifecycle.LifecycleAdapter;
 
-import static org.neo4j.helpers.collection.IteratorUtil.addToCollection;
-import static org.neo4j.kernel.extension.KernelExtensionUtil.servicesClassPathEntryInformation;
-
 /**
  * Contract for implementing an index in Neo4j.
  *
@@ -125,7 +119,8 @@ public InternalIndexState getInitialState( long indexId )
                 }
 
                 @Override
-                public StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache )
+                public StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs,
+                        PageCache pageCache, LabelScanStoreProvider labelScanStoreProvider )
                 {
                     return StoreMigrationParticipant.NOT_PARTICIPATING;
                 }
@@ -220,7 +215,8 @@ public File getStoreDirectory( File storeDir )
         return new File( new File( new File( storeDir, "schema" ), "index" ), getProviderDescriptor().getKey() );
     }
 
-    public abstract StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache );
+    public abstract StoreMigrationParticipant storeMigrationParticipant( FileSystemAbstraction fs, PageCache pageCache,
+            LabelScanStoreProvider labelScanStoreProvider );
 
     /**
      * Provides a snapshot of meta files about this index provider, not the indexes themselves.

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java
@@ -51,7 +51,6 @@
 import org.neo4j.kernel.impl.locking.ResourceTypes;
 import org.neo4j.kernel.impl.locking.community.CommunityLockManger;
 import org.neo4j.kernel.impl.logging.LogService;
-import org.neo4j.kernel.impl.storemigration.ConfigMapUpgradeConfiguration;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.impl.transaction.state.DataSourceManager;
 import org.neo4j.kernel.lifecycle.LifeSupport;
@@ -105,8 +104,6 @@ public CommunityEditionModule( PlatformModule platformModule )
 
         transactionStartTimeout = config.get( GraphDatabaseSettings.transaction_start_timeout );
 
-        upgradeConfiguration = new ConfigMapUpgradeConfiguration( config );
-
         constraintSemantics = createSchemaRuleVerifier();
 
         registerRecovery( config.get( GraphDatabaseFacadeFactory.Configuration.editionName), life, dependencies );

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java
@@ -66,9 +66,6 @@
 import org.neo4j.kernel.impl.query.QueryEngineProvider;
 import org.neo4j.kernel.impl.query.QueryExecutionEngine;
 import org.neo4j.kernel.impl.store.StoreId;
-import org.neo4j.kernel.impl.storemigration.StoreMigrator;
-import org.neo4j.kernel.impl.storemigration.StoreUpgrader;
-import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor;
 import org.neo4j.kernel.impl.transaction.log.PhysicalLogFile;
 import org.neo4j.kernel.impl.transaction.state.DataSourceManager;
 import org.neo4j.kernel.info.DiagnosticsManager;
@@ -163,14 +160,6 @@ public GraphDatabaseService getGraphDatabaseService()
 
         SchemaWriteGuard schemaWriteGuard = deps.satisfyDependency( editionModule.schemaWriteGuard );
 
-        StoreUpgrader storeMigrationProcess = new StoreUpgrader( editionModule.upgradeConfiguration, fileSystem,
-                platformModule.monitors.newMonitor( StoreUpgrader.Monitor.class ), logging.getInternalLogProvider() );
-
-        VisibleMigrationProgressMonitor progressMonitor =
-                new VisibleMigrationProgressMonitor( logging.getInternalLog( StoreMigrator.class ) );
-
-        StoreMigrator storeMigrator = new StoreMigrator( progressMonitor, fileSystem, pageCache, config, logging );
-
         Guard guard = config.get( execution_guard_enabled ) ?
                       deps.satisfyDependency( new Guard( logging.getInternalLog( Guard.class ) ) ) :
                       null;
@@ -184,13 +173,13 @@ public GraphDatabaseService getGraphDatabaseService()
                 logging.getInternalLog( DatabaseHealth.class ) ) );
 
         neoStoreDataSource = deps.satisfyDependency( new NeoStoreDataSource( storeDir, config,
-                editionModule.idGeneratorFactory, logging.getInternalLogProvider(), platformModule.jobScheduler,
+                editionModule.idGeneratorFactory, logging, platformModule.jobScheduler,
                 new NonTransactionalTokenNameLookup( editionModule.labelTokenHolder,
                         editionModule.relationshipTypeTokenHolder, editionModule.propertyKeyTokenHolder ),
                 deps, editionModule.propertyKeyTokenHolder, editionModule.labelTokenHolder, relationshipTypeTokenHolder,
                 editionModule.lockManager, schemaWriteGuard, transactionEventHandlers,
                 platformModule.monitors.newMonitor( IndexingService.Monitor.class ), fileSystem,
-                storeMigrationProcess, storeMigrator, platformModule.transactionMonitor, databaseHealth,
+                platformModule.transactionMonitor, databaseHealth,
                 platformModule.monitors.newMonitor( PhysicalLogFile.Monitor.class ),
                 editionModule.headerInformationFactory, startupStatistics, nodeManager, guard,
                 editionModule.commitProcessFactory, pageCache, editionModule.constraintSemantics,

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/EditionModule.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/EditionModule.java
@@ -30,7 +30,6 @@
 import org.neo4j.kernel.impl.core.PropertyKeyTokenHolder;
 import org.neo4j.kernel.impl.core.RelationshipTypeTokenHolder;
 import org.neo4j.kernel.impl.locking.Locks;
-import org.neo4j.kernel.impl.storemigration.UpgradeConfiguration;
 import org.neo4j.kernel.impl.transaction.TransactionHeaderInformationFactory;
 import org.neo4j.kernel.info.DiagnosticsManager;
 
@@ -58,8 +57,6 @@ public abstract class EditionModule
 
     public SchemaWriteGuard schemaWriteGuard;
 
-    public UpgradeConfiguration upgradeConfiguration;
-
     public ConstraintSemantics constraintSemantics;
 
     protected void doAfterRecoveryAndStartup( String editionName, DependencyResolver dependencyResolver)

diff --git a/...nel/src/main/java/org/neo4j/kernel/impl/storemigration/ConfigMapUpgradeConfiguration.java b/...nel/src/main/java/org/neo4j/kernel/impl/storemigration/ConfigMapUpgradeConfiguration.java