Merge pull request #6618 from lutovich/3.0-migration-progress

Add progress monitor to DirectRecordStoreMigrator
neo4j · Mar 9, 2016 · b85dad3 · b85dad3
2 parents a6efd14 + 39c1be9
commit b85dad3
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 17 deletions.
diff --git a/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java b/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/DirectRecordStoreMigrator.java
@@ -37,6 +37,7 @@
 import org.neo4j.kernel.impl.store.id.DefaultIdGeneratorFactory;
 import org.neo4j.kernel.impl.store.record.AbstractBaseRecord;
 import org.neo4j.kernel.impl.store.record.RecordLoad;
+import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor;
 import org.neo4j.logging.NullLogProvider;
 
 import static org.neo4j.helpers.ArrayUtil.contains;
@@ -59,9 +60,11 @@ public DirectRecordStoreMigrator( PageCache pageCache, FileSystemAbstraction fs,
     }
 
     public void migrate( File fromStoreDir, RecordFormats fromFormat, File toStoreDir, RecordFormats toFormat,
-            StoreType[] types, StoreType... additionalTypesToOpen )
+            MigrationProgressMonitor.Section progressMonitor, StoreType[] types, StoreType... additionalTypesToOpen )
     {
         StoreType[] storesToOpen = ArrayUtil.concat( types, additionalTypesToOpen );
+        progressMonitor.start( storesToOpen.length );
+
         try (
             NeoStores fromStores = new StoreFactory( fromStoreDir, config, new DefaultIdGeneratorFactory( fs ),
                     pageCache, fs, NullLogProvider.getInstance(), fromFormat ).openNeoStores( true, storesToOpen );
@@ -76,6 +79,7 @@ public void migrate( File fromStoreDir, RecordFormats fromFormat, File toStoreDi
                 if ( type.isRecordStore() )
                 {
                     migrate( fromStores.getRecordStore( type ), toStores.getRecordStore( type ) );
+                    progressMonitor.progress( 1 );
                 }
             }
         }

diff --git a/.../java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java b/.../java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java
@@ -21,7 +21,7 @@
 
 public class SilentMigrationProgressMonitor implements MigrationProgressMonitor
 {
-    private static final Section SECTION = new Section()
+    public static final Section NO_OP_SECTION = new Section()
     {
         @Override
         public void progress( long add )
@@ -47,7 +47,7 @@ public void started()
     @Override
     public Section startSection( String name )
     {
-        return SECTION;
+        return NO_OP_SECTION;
     }
 
     @Override

diff --git a/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java
@@ -72,6 +72,7 @@
 import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogs;
 import org.neo4j.kernel.impl.storemigration.legacystore.v21.propertydeduplication.PropertyDeduplicator;
 import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor;
+import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor;
 import org.neo4j.kernel.impl.transaction.log.LogPosition;
 import org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles;
 import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
@@ -96,7 +97,6 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static java.util.Arrays.asList;
 import static java.util.Arrays.stream;
-
 import static org.neo4j.helpers.collection.Iterables.iterable;
 import static org.neo4j.kernel.impl.store.MetaDataStore.DEFAULT_NAME;
 import static org.neo4j.kernel.impl.store.StoreType.META_DATA;
@@ -163,7 +163,7 @@ public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor.
             if ( newFormat.hasSameCapabilities( oldFormat, CapabilityType.FORMAT ) )
             {
                 // Do direct migration
-                migrateWithDirectMigration( storeDir, migrationDir, oldFormat, newFormat );
+                migrateWithDirectMigration( storeDir, migrationDir, oldFormat, newFormat, progressMonitor );
             }
             else
             {
@@ -186,14 +186,14 @@ public void migrate( File storeDir, File migrationDir, MigrationProgressMonitor.
     }
 
     private void migrateWithDirectMigration( File storeDir, File migrationDir,
-            RecordFormats oldFormat, RecordFormats newFormat )
+            RecordFormats oldFormat, RecordFormats newFormat, MigrationProgressMonitor.Section progressMonitor )
     {
         DirectRecordStoreMigrator migrator = new DirectRecordStoreMigrator( pageCache, fileSystem, config );
         StoreType[] stores = stream( StoreType.values() )
                 // Not interested in migrating MetaData store.
                 .filter( type -> type != META_DATA )
                 .toArray( StoreType[]::new );
-        migrator.migrate( storeDir, oldFormat, migrationDir, newFormat, stores );
+        migrator.migrate( storeDir, oldFormat, migrationDir, newFormat, progressMonitor, stores );
     }
 
     private void writeLastTxChecksum( File migrationDir, long lastTxChecksum ) throws IOException
@@ -451,16 +451,18 @@ private void prepareBatchImportMigration( File storeDir, File migrationDir, Reco
         {
             // Migrate all token stores, schema store and dynamic node label ids, keeping their ids intact
             DirectRecordStoreMigrator migrator = new DirectRecordStoreMigrator( pageCache, fileSystem, config );
-            migrator.migrate( storeDir, oldFormat, migrationDir, newFormat,
-                    new StoreType[] {
-                        StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME,
-                        StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME,
-                        StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME,
-                        StoreType.NODE_LABEL,
-                        StoreType.SCHEMA },
-                    new StoreType[] {
-                        StoreType.NODE,
-                    } );
+
+            StoreType[] storesToMigrate = {
+                    StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME,
+                    StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME,
+                    StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME,
+                    StoreType.NODE_LABEL,
+                    StoreType.SCHEMA};
+
+            // Migrate these stores silently because they are usually very small
+            MigrationProgressMonitor.Section section = SilentMigrationProgressMonitor.NO_OP_SECTION;
+
+            migrator.migrate( storeDir, oldFormat, migrationDir, newFormat, section, storesToMigrate, StoreType.NODE );
         }
     }