From a086d2791db0ace3cb63f42710317396202ef74c Mon Sep 17 00:00:00 2001 From: Ragnar Mellbin Date: Mon, 12 Jun 2017 09:11:45 +0200 Subject: [PATCH] Enhance logging of count migration process. --- .../StoreMigrationParticipant.java | 3 +- .../impl/storemigration/StoreUpgrader.java | 6 +- .../monitoring/MigrationProgressMonitor.java | 3 +- .../SilentMigrationProgressMonitor.java | 2 +- .../VisibleMigrationProgressMonitor.java | 11 ++- .../AbstractStoreMigrationParticipant.java | 3 +- .../participant/StoreMigrator.java | 7 +- .../impl/batchimport/NodeCountsProcessor.java | 2 - .../VisibleMigrationProgressMonitorTest.java | 4 +- ...stAccumulatorMigrationProgressMonitor.java | 90 ------------------- .../test/java/upgrade/StoreUpgraderTest.java | 34 ++++++- 11 files changed, 56 insertions(+), 109 deletions(-) delete mode 100644 community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java index 3899bdc0a2860..4170844710dfc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java @@ -72,7 +72,8 @@ void moveMigratedFiles( File migrationDir, File storeDir, String versionToMigrat * @param versionToMigrateFrom the version we have migrated from * @param versionToMigrateTo @throws IOException if unable to move one or more files. * */ - void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException; + void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, + String versionToMigrateTo ) throws IOException; /** * Delete any file from {@code migrationDir} produced during migration. diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java index f97adaf8d2259..a22561d007b85 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreUpgrader.java @@ -69,6 +69,7 @@ public class StoreUpgrader public static final String MIGRATION_DIRECTORY = "upgrade"; public static final String MIGRATION_LEFT_OVERS_DIRECTORY = "upgrade_backup"; private static final String MIGRATION_STATUS_FILE = "_status"; + public static final int COUNT_STORE_REBUILD_STEPS = 2; private final UpgradableDatabase upgradableDatabase; private final MigrationProgressMonitor progressMonitor; @@ -120,7 +121,7 @@ public void migrateIfNeeded( File storeDirectory ) } // One or more participants would like to do migration - progressMonitor.started(); + progressMonitor.started( participants.size() + COUNT_STORE_REBUILD_STEPS ); MigrationStatus migrationStatus = MigrationStatus.readMigrationStatus( fileSystem, migrationStateFile ); String versionToMigrateFrom = null; @@ -232,8 +233,7 @@ private void migrateToIsolatedDirectory( File storeDir, File migrationDirectory, int index = 1; for ( StoreMigrationParticipant participant : participants ) { - Section section = progressMonitor.startSection( - format( "%s (%d/%d)", participant.getName(), index, participants.size() ) ); + Section section = progressMonitor.startSection( participant.getName() ); participant.migrate( storeDir, migrationDirectory, section, versionToMigrateFrom, upgradableDatabase.currentVersion() ); section.completed(); diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java index 77f27ba9e4809..b62d29b25f8bb 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/MigrationProgressMonitor.java @@ -23,8 +23,9 @@ public interface MigrationProgressMonitor { /** * Signals that the migration process has started. + * @param numStages */ - void started(); + void started( int numStages ); /** * Signals that migration goes into section with given {@code name}. diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java index fea85cc3bc595..b7dc8c6ad5677 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/SilentMigrationProgressMonitor.java @@ -40,7 +40,7 @@ public void completed() }; @Override - public void started() + public void started( int numStages ) { } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java index a65c450f3d9d1..3aaf3a4318f98 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitor.java @@ -29,6 +29,8 @@ public class VisibleMigrationProgressMonitor implements MigrationProgressMonitor static final String MESSAGE_COMPLETED = "Successfully finished upgrade of database"; private final Log log; + private int numStages; + private int currentStage; public VisibleMigrationProgressMonitor( Log log ) { @@ -36,15 +38,16 @@ public VisibleMigrationProgressMonitor( Log log ) } @Override - public void started() + public void started( int numStages ) { + this.numStages = numStages; log.info( MESSAGE_STARTED ); } @Override public Section startSection( String name ) { - log.info( "Migrating " + name + ":" ); + log.info( format( "Migrating %s (%d/%d):", name, ++currentStage, numStages ) ); return new ProgressSection(); } @@ -52,6 +55,10 @@ public Section startSection( String name ) @Override public void completed() { + if ( currentStage < numStages ) + { + log.info( format( "%d stages were not required and have been skipped.", numStages - currentStage ) ); + } log.info( MESSAGE_COMPLETED ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java index 5f57c7c998540..f086dae3ef15c 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/AbstractStoreMigrationParticipant.java @@ -53,7 +53,8 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } @Override - public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws IOException + public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, + String versionToMigrateTo ) throws IOException { } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java index 788c922e747dd..8109185eba425 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/participant/StoreMigrator.java @@ -299,7 +299,8 @@ TransactionId extractTransactionIdInformation( File neoStore, File storeDir, lon } // The legacy store we're migrating doesn't have this record in neostore so try to extract it from tx log - Optional transactionInformation = legacyLogs.getTransactionInformation( storeDir, lastTransactionId ); + Optional transactionInformation = legacyLogs + .getTransactionInformation( storeDir, lastTransactionId ); return transactionInformation.orElseGet( specificTransactionInformationSupplier( lastTransactionId ) ); } @@ -688,8 +689,8 @@ public void moveMigratedFiles( File migrationDir, File storeDir, String versionT } @Override - public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, String versionToMigrateTo ) throws - IOException + public void rebuildCounts( File storeDir, MigrationProgressMonitor progressMonitor, String versionToMigrateFrom, + String versionToMigrateTo ) throws IOException { if ( countStoreRebuildRequired( versionToMigrateFrom ) ) { diff --git a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java index dc80a4940568e..d9db906aa4dc8 100644 --- a/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java +++ b/community/kernel/src/main/java/org/neo4j/unsafe/impl/batchimport/NodeCountsProcessor.java @@ -19,8 +19,6 @@ */ package org.neo4j.unsafe.impl.batchimport; -import javax.swing.*; - import org.neo4j.kernel.api.ReadOperations; import org.neo4j.kernel.impl.api.CountsAccessor; import org.neo4j.kernel.impl.store.NodeLabelsField; diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java index 388da0dcb2563..17a90b71ba4c2 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/monitoring/VisibleMigrationProgressMonitorTest.java @@ -36,7 +36,7 @@ public void shouldReportAllPercentageSteps() throws Exception AssertableLogProvider logProvider = new AssertableLogProvider(); Log log = logProvider.getLog( getClass() ); VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log ); - monitor.started(); + monitor.started( 1 ); // WHEN monitorSection( monitor, "First", 100, 40, 25, 23 /*these are too far*/ , 10, 50 ); @@ -53,7 +53,7 @@ public void progressNeverReportMoreThenHundredPercent() Log log = logProvider.getLog( getClass() ); VisibleMigrationProgressMonitor monitor = new VisibleMigrationProgressMonitor( log ); - monitor.started(); + monitor.started( 1 ); monitorSection( monitor, "First", 100, 1, 10, 99, 170 ); monitor.completed(); diff --git a/community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java b/community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java deleted file mode 100644 index ab5f81fe80db7..0000000000000 --- a/community/neo4j/src/test/java/upgrade/ListAccumulatorMigrationProgressMonitor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2002-2017 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package upgrade; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - -import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; - -public class ListAccumulatorMigrationProgressMonitor implements MigrationProgressMonitor -{ - private final Map events = new HashMap<>(); - private boolean started; - private boolean finished; - - @Override - public void started() - { - started = true; - } - - @Override - public Section startSection( String name ) - { - final AtomicLong progress = new AtomicLong(); - assert events.put( name, progress ) == null; - return new Section() - { - @Override - public void progress( long add ) - { - progress.addAndGet( add ); - } - - @Override - public void start( long max ) - { - } - - @Override - public void completed() - { - } - }; - } - - @Override - public void completed() - { - finished = true; - } - - public Map progresses() - { - Map result = new HashMap<>(); - for ( Map.Entry entry : events.entrySet() ) - { - result.put( entry.getKey(), entry.getValue().longValue() ); - } - return result; - } - - public boolean isStarted() - { - return started; - } - - public boolean isFinished() - { - return finished; - } -} diff --git a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java index 322c6c51395a2..bfafcd1566af0 100644 --- a/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java +++ b/community/neo4j/src/test/java/upgrade/StoreUpgraderTest.java @@ -61,10 +61,12 @@ import org.neo4j.kernel.impl.storemigration.legacylogs.LegacyLogFilenames; import org.neo4j.kernel.impl.storemigration.monitoring.MigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.monitoring.SilentMigrationProgressMonitor; +import org.neo4j.kernel.impl.storemigration.monitoring.VisibleMigrationProgressMonitor; import org.neo4j.kernel.impl.storemigration.participant.AbstractStoreMigrationParticipant; import org.neo4j.kernel.impl.storemigration.participant.SchemaIndexMigrator; import org.neo4j.kernel.impl.storemigration.participant.StoreMigrator; import org.neo4j.kernel.monitoring.Monitors; +import org.neo4j.logging.AssertableLogProvider; import org.neo4j.logging.NullLogProvider; import org.neo4j.test.rule.NeoStoreDataSourceRule; import org.neo4j.test.rule.PageCacheRule; @@ -358,6 +360,28 @@ public void upgradeShouldNotLeaveLeftoverAndMigrationDirs() throws Exception assertThat( migrationHelperDirs(), is( emptyCollectionOf( File.class ) ) ); } + @Test + public void upgradeShouldGiveProgressMonitorProgressMessages() throws Exception + { + // Given + fileSystem.deleteFile( new File( dbDirectory, INTERNAL_LOG_FILE ) ); + PageCache pageCache = pageCacheRule.getPageCache( fileSystem ); + UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fileSystem, new StoreVersionCheck( pageCache ), + getRecordFormats() ); + + // When + AssertableLogProvider logProvider = new AssertableLogProvider(); + newUpgrader( upgradableDatabase, pageCache, allowMigrateConfig, + new VisibleMigrationProgressMonitor( logProvider.getLog( "test" ) ) ).migrateIfNeeded( dbDirectory ); + + // Then + logProvider.assertContainsLogCallContaining( "Store files" ); + logProvider.assertContainsLogCallContaining( "Indexes" ); + logProvider.assertContainsLogCallContaining( "node count" ); + logProvider.assertContainsLogCallContaining( "relationship count" ); + logProvider.assertContainsLogCallContaining( "Successfully finished" ); + } + @Test public void upgraderShouldCleanupLegacyLeftoverAndMigrationDirs() throws Exception { @@ -412,13 +436,17 @@ private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCa } private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config ) - throws IOException { SilentMigrationProgressMonitor progressMonitor = new SilentMigrationProgressMonitor(); + return newUpgrader( upgradableDatabase, pageCache, config, progressMonitor ); + } + + private StoreUpgrader newUpgrader( UpgradableDatabase upgradableDatabase, PageCache pageCache, Config config, + MigrationProgressMonitor progressMonitor ) + { NullLogService instance = NullLogService.getInstance(); - StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance - ); + StoreMigrator defaultMigrator = new StoreMigrator( fileSystem, pageCache, getTuningConfig(), instance ); SchemaIndexMigrator indexMigrator = new SchemaIndexMigrator( fileSystem, schemaIndexProvider ); StoreUpgrader upgrader = new StoreUpgrader( upgradableDatabase, progressMonitor, config, fileSystem, pageCache,