Enhance logging of count migration process.

neo4j · Jul 5, 2017 · a086d27 · a086d27
1 parent b28a467
commit a086d27
Show file tree

Hide file tree

Showing 11 changed files with 56 additions and 109 deletions.
diff --git a/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java b/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java
@@ -72,7 +72,8 @@ void moveMigratedFiles( File migrationDir, File storeDir, String versionToMigrat
      * @param versionToMigrateFrom the version we have migrated from
      * @param versionToMigrateTo   @throws IOException if unable to move one or more files.
      * */
-    void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException;
+    void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom,
+            String versionToMigrateTo ) throws IOException;
 
     /**
      * Delete any file from {@code migrationDir} produced during migration.

diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java
@@ -69,6 +69,7 @@ public class StoreUpgrader
     public static final String MIGRATION_DIRECTORY = "upgrade";
     public static final String MIGRATION_LEFT_OVERS_DIRECTORY = "upgrade_backup";
     private static final String MIGRATION_STATUS_FILE = "_status";
+    public static final int COUNT_STORE_REBUILD_STEPS = 2;
 
     private final UpgradableDatabase upgradableDatabase;
     private final MigrationProgressMonitor progressMonitor;
@@ -120,7 +121,7 @@ public void migrateIfNeeded( File storeDirectory )
         }
 
         // One or more participants would like to do migration
-        progressMonitor.started();
+        progressMonitor.started( participants.size() + COUNT_STORE_REBUILD_STEPS );
 
         MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile );
         String versionToMigrateFrom = null;
@@ -232,8 +233,7 @@ private void migrateToIsolatedDirectory( File storeDir, File migrationDirectory,
             int index = 1;
             for ( StoreMigrationParticipant participant : participants )
             {
-                Section section = progressMonitor.startSection(
-                        format( "%s (%d/%d)", participant.getName(), index, participants.size() ) );
+                Section section = progressMonitor.startSection( participant.getName() );
                 participant.migrate( storeDir, migrationDirectory, section, versionToMigrateFrom,
                         upgradableDatabase.currentVersion() );
                 section.completed();

diff --git a/...c/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java b/...c/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java
@@ -23,8 +23,9 @@ public interface MigrationProgressMonitor
 {
     /**
      * Signals that the migration process has started.
+     * @param numStages
      */
-    void started();
+    void started( int numStages );
 
     /**
      * Signals that migration goes into section with given {@code name}.

diff --git a/.../java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java b/.../java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java
@@ -40,7 +40,7 @@ public void completed()
     };
 
     @Override
-    public void started()
+    public void started( int numStages )
     {
     }
 

diff --git a/...java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java b/...java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java
@@ -29,29 +29,36 @@ public class VisibleMigrationProgressMonitor implements MigrationProgressMonitor
     static final String MESSAGE_COMPLETED = "Successfully finished upgrade of database";
 
     private final Log log;
+    private int numStages;
+    private int currentStage;
 
     public VisibleMigrationProgressMonitor( Log log )
     {
         this.log = log;
     }
 
     @Override
-    public void started()
+    public void started( int numStages )
     {
+        this.numStages = numStages;
         log.info( MESSAGE_STARTED );
     }
 
     @Override
     public Section startSection( String name )
     {
-        log.info( "Migrating " + name + ":" );
+        log.info( format( "Migrating %s (%d/%d):", name, ++currentStage, numStages ) );
 
         return new ProgressSection();
     }
 
     @Override
     public void completed()
     {
+        if ( currentStage < numStages )
+        {
+            log.info( format( "%d stages were not required and have been skipped.", numStages - currentStage ) );
+        }
         log.info( MESSAGE_COMPLETED );
     }
 

diff --git a/...a/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java b/...a/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java
@@ -53,7 +53,8 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT
     }
 
     @Override
-    public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException
+    public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom,
+            String versionToMigrateTo ) throws IOException
     {
     }
 

diff --git a/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/.../kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java
@@ -299,7 +299,8 @@ TransactionId extractTransactionIdInformation( File neoStore, File storeDir, lon
         }
         // The legacy store we're migrating doesn't have this record in neostore so try to extract it from tx log
 
-        Optional<TransactionId> transactionInformation = legacyLogs.getTransactionInformation( storeDir, lastTransactionId );
+        Optional<TransactionId> transactionInformation = legacyLogs
+                .getTransactionInformation( storeDir, lastTransactionId );
         return transactionInformation.orElseGet( specificTransactionInformationSupplier( lastTransactionId ) );
     }
 
@@ -688,8 +689,8 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT
     }
 
     @Override
-    public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws
-            IOException
+    public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom,
+            String versionToMigrateTo ) throws IOException
     {
         if ( countStoreRebuildRequired( versionToMigrateFrom ) )
         {

diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java
@@ -19,8 +19,6 @@
  */
 package org.neo4j.unsafe.impl.batchimport;
 
-import javax.swing.*;
-
 import org.neo4j.kernel.api.ReadOperations;
 import org.neo4j.kernel.impl.api.CountsAccessor;
 import org.neo4j.kernel.impl.store.NodeLabelsField;

diff --git a/.../org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java b/.../org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java
@@ -36,7 +36,7 @@ public void shouldReportAllPercentageSteps() throws Exception
         AssertableLogProvider logProvider = new AssertableLogProvider();
         Log log = logProvider.getLog( getClass() );
         VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log );
-        monitor.started();
+        monitor.started( 1 );
 
         // WHEN
         monitorSection( monitor, "First", 100, 40, 25, 23 /*these are too far*/ , 10, 50 );
@@ -53,7 +53,7 @@ public void progressNeverReportMoreThenHundredPercent()
         Log log = logProvider.getLog( getClass() );
         VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log );
 
-        monitor.started();
+        monitor.started( 1 );
         monitorSection( monitor, "First", 100, 1, 10, 99, 170 );
         monitor.completed();
 

diff --git a/community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java b/community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java
diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java
@@ -61,10 +61,12 @@
 import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogFilenames;
 import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor;
 import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor;
+import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor;
 import org.neo4j.kernel.impl.storemigration.participant.AbstractStoreMigrationParticipant;
 import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator;
 import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator;
 import org.neo4j.kernel.monitoring.Monitors;
+import org.neo4j.logging.AssertableLogProvider;
 import org.neo4j.logging.NullLogProvider;
 import org.neo4j.test.rule.NeoStoreDataSourceRule;
 import org.neo4j.test.rule.PageCacheRule;
@@ -358,6 +360,28 @@ public void upgradeShouldNotLeaveLeftoverAndMigrationDirs() throws Exception
         assertThat( migrationHelperDirs(), is( emptyCollectionOf( File.class ) ) );
     }
 
+    @Test
+    public void upgradeShouldGiveProgressMonitorProgressMessages() throws Exception
+    {
+        // Given
+        fileSystem.deleteFile( new File( dbDirectory, INTERNAL_LOG_FILE ) );
+        PageCache pageCache = pageCacheRule.getPageCache( fileSystem );
+        UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, new StoreVersionCheck( pageCache ),
+                getRecordFormats() );
+
+        // When
+        AssertableLogProvider logProvider = new AssertableLogProvider();
+        newUpgrader( upgradableDatabase, pageCache, allowMigrateConfig,
+                new VisibleMigrationProgressMonitor( logProvider.getLog( "test" ) ) ).migrateIfNeeded( dbDirectory );
+
+        // Then
+        logProvider.assertContainsLogCallContaining( "Store files" );
+        logProvider.assertContainsLogCallContaining( "Indexes" );
+        logProvider.assertContainsLogCallContaining( "node count" );
+        logProvider.assertContainsLogCallContaining( "relationship count" );
+        logProvider.assertContainsLogCallContaining( "Successfully finished" );
+    }
+
     @Test
     public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Exception
     {
@@ -412,13 +436,17 @@ private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCa
     }
 
     private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config )
-            throws IOException
     {
         SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor();
 
+        return newUpgrader( upgradableDatabase, pageCache, config, progressMonitor );
+    }
+
+    private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config,
+            MigrationProgressMonitor progressMonitor )
+    {
         NullLogService instance = NullLogService.getInstance();
-        StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance
-        );
+        StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance );
         SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fileSystem, schemaIndexProvider );
 
         StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fileSystem, pageCache,