From dd27c4acac18b432bc206507d6572190a1501c9f Mon Sep 17 00:00:00 2001 From: Przemek Hugh Kaznowski Date: Wed, 15 Nov 2017 09:58:43 +0000 Subject: [PATCH] Backups on blockdevices Adds support/fixes tests for performing backups on blockdevices --- .../neo4j/commandline/admin/AdminTool.java | 1 - .../neo4j/commandline/dbms/DumpCommand.java | 4 +- .../impl/factory/GraphDatabaseFacade.java | 4 +- .../org/neo4j/backup/BackupCopyService.java | 47 ++- .../backup/BackupModuleResolveAtRuntime.java | 7 - ...ow.java => BackupStrategyCoordinator.java} | 6 +- ... => BackupStrategyCoordinatorFactory.java} | 37 ++- .../neo4j/backup/BackupStrategyWrapper.java | 37 ++- .../org/neo4j/backup/OnlineBackupCommand.java | 16 +- .../backup/OnlineBackupCommandProvider.java | 2 +- .../backup/ParametrisedOutsideWorld.java | 138 +++++++++ .../neo4j/backup/BackupCopyServiceTest.java | 83 ++++++ ...ava => BackupStrategyCoordinatorTest.java} | 34 +-- .../backup/BackupStrategyWrapperTest.java | 128 +++++++- .../neo4j/backup/OnlineBackupCommandTest.java | 8 +- .../neo4j/com/storecopy/FileMoveAction.java | 7 +- .../com/storecopy/FileMoveActionInformer.java | 26 ++ .../neo4j/com/storecopy/FileMoveProvider.java | 159 ++++++++++ .../neo4j/com/storecopy/StoreCopyClient.java | 91 +++--- .../com/storecopy/FileMoveProviderTest.java | 275 ++++++++++++++++++ .../ha/HighlyAvailableGraphDatabase.java | 2 +- .../neo4j/kernel/impl/ha/ClusterManager.java | 6 +- 22 files changed, 973 insertions(+), 145 deletions(-) rename enterprise/backup/src/main/java/org/neo4j/backup/{BackupFlow.java => BackupStrategyCoordinator.java} (95%) rename enterprise/backup/src/main/java/org/neo4j/backup/{BackupFlowFactory.java => BackupStrategyCoordinatorFactory.java} (52%) create mode 100644 enterprise/backup/src/main/java/org/neo4j/backup/ParametrisedOutsideWorld.java create mode 100644 enterprise/backup/src/test/java/org/neo4j/backup/BackupCopyServiceTest.java rename enterprise/backup/src/test/java/org/neo4j/backup/{BackupFlowTest.java => BackupStrategyCoordinatorTest.java} (88%) create mode 100644 enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveActionInformer.java create mode 100644 enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java create mode 100644 enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java diff --git a/community/command-line/src/main/java/org/neo4j/commandline/admin/AdminTool.java b/community/command-line/src/main/java/org/neo4j/commandline/admin/AdminTool.java index f6c5b977a10b9..854bbbacf4481 100644 --- a/community/command-line/src/main/java/org/neo4j/commandline/admin/AdminTool.java +++ b/community/command-line/src/main/java/org/neo4j/commandline/admin/AdminTool.java @@ -29,7 +29,6 @@ import org.neo4j.helpers.Args; import static java.lang.String.format; - import static org.neo4j.commandline.Util.neo4jVersion; public class AdminTool diff --git a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java index 4549164726262..e2da7a8d29a27 100644 --- a/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java +++ b/community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java @@ -95,9 +95,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed } catch ( CannotWriteException e ) { - throw new CommandFailed( - "you do not have permission to dump the database -- is Neo4j running as a " + "different user?", - e ); + throw new CommandFailed( "you do not have permission to dump the database -- is Neo4j running as a different user?", e ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java index f3fda2cfe7399..62241d48777e0 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java @@ -378,9 +378,9 @@ public Schema schema() } @Override - public boolean isAvailable( long timeout ) + public boolean isAvailable( long timeoutMillis ) { - return spi.databaseIsAvailable( timeout ); + return spi.databaseIsAvailable( timeoutMillis ); } @Override diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupCopyService.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupCopyService.java index 962ed3ac5c797..11e54ac71f372 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupCopyService.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupCopyService.java @@ -21,36 +21,45 @@ import java.io.File; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; +import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; -import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.neo4j.com.storecopy.FileMoveAction; +import org.neo4j.com.storecopy.FileMoveProvider; import org.neo4j.commandline.admin.CommandFailed; -import org.neo4j.commandline.admin.OutsideWorld; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.configuration.Config; import static java.lang.String.format; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path; public class BackupCopyService { private static final int MAX_OLD_BACKUPS = 1000; - private final OutsideWorld outsideWorld; + private final PageCache pageCache; - public BackupCopyService( OutsideWorld outsideWorld ) + private final FileMoveProvider fileMoveProvider; + + public BackupCopyService( PageCache pageCache, FileMoveProvider fileMoveProvider ) { - this.outsideWorld = outsideWorld; + this.pageCache = pageCache; + this.fileMoveProvider = fileMoveProvider; } public void moveBackupLocation( File oldLocation, File newLocation ) throws CommandFailed { try { - outsideWorld.fileSystem().renameFile( oldLocation, newLocation ); + Iterator moves = fileMoveProvider.traverseGenerateMoveActions( oldLocation ).iterator(); + while ( moves.hasNext() ) + { + moves.next().move( newLocation ); + } } catch ( IOException e ) { @@ -58,9 +67,15 @@ public void moveBackupLocation( File oldLocation, File newLocation ) throws Comm } } + public void clearLogs( File neo4jHome ) + { + File logsDirectory = Config.defaults( logs_directory, neo4jHome.getPath() ).get( store_internal_log_path ); + logsDirectory.delete(); + } + boolean backupExists( File destination ) { - File[] listFiles = outsideWorld.fileSystem().listFiles( destination ); + File[] listFiles = pageCache.getCachedFileSystem().listFiles( destination ); return listFiles != null && listFiles.length > 0; } @@ -74,6 +89,13 @@ File findAnAvailableLocationForNewFullBackup( File desiredBackupLocation ) return findAnAvailableBackupLocation( desiredBackupLocation, "%s.temp.%d" ); } + /** + * Given a desired file name, find an available name that is similar to the given one that doesn't conflict with already existing backups + * + * @param file desired ideal file name + * @param pattern pattern to follow if desired name is taken (requires %s for original name, and %d for iteration) + * @return the resolved file name which can be the original desired, or a variation that matches the pattern + */ private File findAnAvailableBackupLocation( File file, String pattern ) { if ( backupExists( file ) ) @@ -84,7 +106,7 @@ private File findAnAvailableBackupLocation( File file, String pattern ) return availableAlternativeNames( file, pattern ) .peek( countNumberOfFilesProcessedForPotentialErrorMessage ) - .filter( f -> !backupExists(f) ) + .filter( f -> !backupExists( f ) ) .findFirst() .orElseThrow( () -> new RuntimeException( String.format( "Unable to find a free backup location for the provided %s. Number of iterations %d", file, counter.get() ) ) ); @@ -100,8 +122,7 @@ private static Stream availableAlternativeNames( File originalBackupDirect private static File alteredBackupDirectoryName( String pattern, File directory, int iteration ) { - return directory - .toPath() + return directory.toPath() .resolveSibling( format( pattern, directory.getName(), iteration ) ) .toFile(); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupModuleResolveAtRuntime.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupModuleResolveAtRuntime.java index 9b2658507dff9..bbb073c20a94d 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupModuleResolveAtRuntime.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupModuleResolveAtRuntime.java @@ -36,7 +36,6 @@ public class BackupModuleResolveAtRuntime private final Monitors monitors; private final Clock clock; private final TransactionLogCatchUpFactory transactionLogCatchUpFactory; - private final BackupCopyService backupCopyService; /** * Dependencies that can be resolved immediately after launching the backup tool @@ -53,7 +52,6 @@ public BackupModuleResolveAtRuntime( OutsideWorld outsideWorld, LogProvider logP this.clock = Clock.systemDefaultZone(); this.transactionLogCatchUpFactory = new TransactionLogCatchUpFactory(); this.fileSystemAbstraction = outsideWorld.fileSystem(); - this.backupCopyService = new BackupCopyService( outsideWorld ); } public LogProvider getLogProvider() @@ -85,9 +83,4 @@ public OutsideWorld getOutsideWorld() { return outsideWorld; } - - public BackupCopyService getBackupCopyService() - { - return backupCopyService; - } } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupFlow.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinator.java similarity index 95% rename from enterprise/backup/src/main/java/org/neo4j/backup/BackupFlow.java rename to enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinator.java index 2cbee4a547428..05837a8c6b6b1 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupFlow.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinator.java @@ -40,7 +40,7 @@ * when none of the backups worked. * Also handles the consistency check */ -class BackupFlow +class BackupStrategyCoordinator { private static final int STATUS_CC_ERROR = 2; private static final int STATUS_CC_INCONSISTENT = 3; @@ -51,7 +51,7 @@ class BackupFlow private final ProgressMonitorFactory progressMonitorFactory; private final List strategies; - BackupFlow( ConsistencyCheckService consistencyCheckService, OutsideWorld outsideWorld, LogProvider logProvider, + BackupStrategyCoordinator( ConsistencyCheckService consistencyCheckService, OutsideWorld outsideWorld, LogProvider logProvider, ProgressMonitorFactory progressMonitorFactory, List strategies ) { this.consistencyCheckService = consistencyCheckService; @@ -68,7 +68,7 @@ class BackupFlow * @param onlineBackupContext filesystem, command arguments and configuration * @throws CommandFailed when backup failed or there were issues with consistency checks */ - void performBackup( OnlineBackupContext onlineBackupContext ) throws CommandFailed + public void performBackup( OnlineBackupContext onlineBackupContext ) throws CommandFailed { // Convenience OnlineBackupRequiredArguments requiredArgs = onlineBackupContext.getRequiredArguments(); diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupFlowFactory.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinatorFactory.java similarity index 52% rename from enterprise/backup/src/main/java/org/neo4j/backup/BackupFlowFactory.java rename to enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinatorFactory.java index 9119fadf82d89..595e917118bb7 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupFlowFactory.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinatorFactory.java @@ -23,6 +23,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.neo4j.com.storecopy.FileMoveProvider; import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.consistency.ConsistencyCheckService; import org.neo4j.helpers.progress.ProgressMonitorFactory; @@ -30,40 +31,46 @@ import org.neo4j.logging.LogProvider; /* -Backup flows are iterate through backup strategies and make sure at least one of them is a valid backup. Handles cases when that isn't possible. -This factory helps in the construction of them + * Backup strategy coordinators iterate through backup strategies and make sure at least one of them can perform a valid backup. + * Handles cases when individual backups aren't possible. */ -class BackupFlowFactory +class BackupStrategyCoordinatorFactory { private final LogProvider logProvider; private final ConsistencyCheckService consistencyCheckService; private final AddressResolutionHelper addressResolutionHelper; - private final BackupCopyService backupCopyService; private final OutsideWorld outsideWorld; - BackupFlowFactory( BackupModuleResolveAtRuntime backupModuleResolveAtRuntime ) + BackupStrategyCoordinatorFactory( BackupModuleResolveAtRuntime backupModuleResolveAtRuntime ) { this.logProvider = backupModuleResolveAtRuntime.getLogProvider(); this.outsideWorld = backupModuleResolveAtRuntime.getOutsideWorld(); - this.backupCopyService = backupModuleResolveAtRuntime.getBackupCopyService(); this.consistencyCheckService = new ConsistencyCheckService(); this.addressResolutionHelper = new AddressResolutionHelper(); } - BackupFlow backupFlow( OnlineBackupContext onlineBackupContext, BackupProtocolService backupProtocolService, BackupDelegator backupDelegator, - PageCache pageCache ) + /** + * Construct a wrapper of supported backup strategies + * + * @param onlineBackupContext the input of the backup tool, such as CLI arguments, config etc. + * @param backupProtocolService the underlying backup implementation for HA and single node instances + * @param backupDelegator the backup implementation used for CC backups + * @param pageCache the page cache used moving files + * @return strategy coordinator that handles the which backup strategies are tried and establishes if a backup was successful or not + */ + BackupStrategyCoordinator backupStrategyCoordinator( OnlineBackupContext onlineBackupContext, BackupProtocolService backupProtocolService, + BackupDelegator backupDelegator, PageCache pageCache ) { + BackupCopyService backupCopyService = new BackupCopyService( pageCache, new FileMoveProvider( pageCache ) ); ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() ); - List strategies = Stream - .of( - new CausalClusteringBackupStrategy( backupDelegator, addressResolutionHelper ), + List strategies = Stream.of( new CausalClusteringBackupStrategy( backupDelegator, addressResolutionHelper ), new HaBackupStrategy( backupProtocolService, addressResolutionHelper, onlineBackupContext.getRequiredArguments().getTimeout() ) ) - .map( strategy -> new BackupStrategyWrapper( strategy, backupCopyService, pageCache, onlineBackupContext.getConfig(), - new BackupRecoveryService() ) ) - .collect( Collectors.toList() ); + .map( strategy -> new BackupStrategyWrapper( strategy, backupCopyService, pageCache, onlineBackupContext.getConfig(), + new BackupRecoveryService(), logProvider ) ) + .collect( Collectors.toList() ); - return new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, strategies ); + return new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, strategies ); } } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java index 6495b32fbf6f1..88e19947bdd30 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java @@ -26,24 +26,32 @@ import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.lifecycle.LifeSupport; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; +/** + * Individual backup strategies can perform incremental backups and full backups. The logic of how and when to perform full/incremental is identical. + * This class describes the behaviour of a single strategy and is used to wrap an interface providing incremental/full backup functionality + */ class BackupStrategyWrapper { private final BackupStrategy backupStrategy; private final BackupCopyService backupCopyService; private final BackupRecoveryService backupRecoveryService; + private final Log log; private final PageCache pageCache; private final Config config; BackupStrategyWrapper( BackupStrategy backupStrategy, BackupCopyService backupCopyService, PageCache pageCache, Config config, - BackupRecoveryService backupRecoveryService ) + BackupRecoveryService backupRecoveryService, LogProvider logProvider ) { this.backupStrategy = backupStrategy; this.backupCopyService = backupCopyService; this.pageCache = pageCache; this.config = config; this.backupRecoveryService = backupRecoveryService; + this.log = logProvider.getLog( BackupStrategyWrapper.class ); } /** @@ -70,8 +78,10 @@ private PotentiallyErroneousState performBackupWithoutLif final OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress(); final Config config = onlineBackupContext.getConfig(); - if ( backupCopyService.backupExists( backupLocation ) ) + boolean previousBackupExists = backupCopyService.backupExists( backupLocation ); + if ( previousBackupExists ) { + log.info( "Previous backup found, trying incremental backup." ); PotentiallyErroneousState state = backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress ); boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() ); @@ -79,6 +89,7 @@ private PotentiallyErroneousState performBackupWithoutLif if ( fullBackupWontWork || incrementalWasSuccessful ) { + backupCopyService.clearLogs( backupLocation ); return describeOutcome( state ); } if ( !onlineBackupContext.getRequiredArguments().isFallbackToFull() ) @@ -86,9 +97,28 @@ private PotentiallyErroneousState performBackupWithoutLif return describeOutcome( state ); } } - return describeOutcome( fullBackupWithTemporaryFolderResolutions( onlineBackupContext ) ); + if ( onlineBackupContext.getRequiredArguments().isFallbackToFull() ) + { + if ( !previousBackupExists ) + { + log.info( "Previous backup not found, a new full backup will be performed." ); + } + return describeOutcome( fullBackupWithTemporaryFolderResolutions( onlineBackupContext ) ); + } + return new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, null ); } + /** + * This will perform a full backup with some directory renaming if necessary. + *

+ * If there is no existing backup, then no renaming will occur. + * Otherwise the full backup will be done into a temporary directory and renaming + * will occur if everything was successful. + *

+ * + * @param onlineBackupContext command line arguments, config etc. + * @return outcome of full backup + */ private PotentiallyErroneousState fullBackupWithTemporaryFolderResolutions( OnlineBackupContext onlineBackupContext ) { final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName(); @@ -113,6 +143,7 @@ private PotentiallyErroneousState fullBackupWithTemporaryFol return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, commandFailed ); } } + backupCopyService.clearLogs( userSpecifiedBackupLocation ); } return state; } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommand.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommand.java index d6d1794ad4613..68b0dd9c98ec5 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommand.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommand.java @@ -25,13 +25,12 @@ import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.IncorrectUsage; import org.neo4j.commandline.admin.OutsideWorld; -import org.neo4j.io.pagecache.PageCache; public class OnlineBackupCommand implements AdminCommand { private final OutsideWorld outsideWorld; private final OnlineBackupContextLoader onlineBackupContextLoader; - private final BackupFlowFactory backupFlowFactory; + private final BackupStrategyCoordinatorFactory backupStrategyCoordinatorFactory; private final AbstractBackupSupportingClassesFactory backupSupportingClassesFactory; /** @@ -40,15 +39,15 @@ public class OnlineBackupCommand implements AdminCommand * @param outsideWorld provides a way to interact with the filesystem and output streams * @param onlineBackupContextLoader helper class to validate, process and return a grouped result of processing the command line arguments * @param backupSupportingClassesFactory necessary for constructing the strategy for backing up over the causal clustering transaction protocol - * @param backupFlowFactory class that actually handles the logic of performing a backup + * @param backupStrategyCoordinatorFactory class that actually handles the logic of performing a backup */ OnlineBackupCommand( OutsideWorld outsideWorld, OnlineBackupContextLoader onlineBackupContextLoader, - AbstractBackupSupportingClassesFactory backupSupportingClassesFactory, BackupFlowFactory backupFlowFactory ) + AbstractBackupSupportingClassesFactory backupSupportingClassesFactory, BackupStrategyCoordinatorFactory backupStrategyCoordinatorFactory ) { this.outsideWorld = outsideWorld; this.onlineBackupContextLoader = onlineBackupContextLoader; this.backupSupportingClassesFactory = backupSupportingClassesFactory; - this.backupFlowFactory = backupFlowFactory; + this.backupStrategyCoordinatorFactory = backupStrategyCoordinatorFactory; } @Override @@ -62,10 +61,11 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed checkDestination( onlineBackupContext.getRequiredArguments().getFolder() ); checkDestination( onlineBackupContext.getRequiredArguments().getReportDir() ); - BackupFlow backupFlow = backupFlowFactory.backupFlow( onlineBackupContext, backupSupportingClasses.getBackupProtocolService(), - backupSupportingClasses.getBackupDelegator(), backupSupportingClasses.getPageCache() ); + BackupStrategyCoordinator backupStrategyCoordinator = + backupStrategyCoordinatorFactory.backupStrategyCoordinator( onlineBackupContext, backupSupportingClasses.getBackupProtocolService(), + backupSupportingClasses.getBackupDelegator(), backupSupportingClasses.getPageCache() ); - backupFlow.performBackup( onlineBackupContext ); + backupStrategyCoordinator.performBackup( onlineBackupContext ); outsideWorld.stdOutLine( "Backup complete." ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandProvider.java b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandProvider.java index 7bdef8db334a5..def35c1303d4d 100644 --- a/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandProvider.java +++ b/enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommandProvider.java @@ -95,7 +95,7 @@ public AdminCommand create( Path homeDir, Path configDir, OutsideWorld outsideWo BackupSupportingClassesFactoryProvider.findBestProvider() .orElseThrow( noProviderException() ) .getFactory( backupModuleResolveAtRuntime ), - new BackupFlowFactory( backupModuleResolveAtRuntime ) + new BackupStrategyCoordinatorFactory( backupModuleResolveAtRuntime ) ); } diff --git a/enterprise/backup/src/main/java/org/neo4j/backup/ParametrisedOutsideWorld.java b/enterprise/backup/src/main/java/org/neo4j/backup/ParametrisedOutsideWorld.java new file mode 100644 index 0000000000000..37df665b3dae0 --- /dev/null +++ b/enterprise/backup/src/main/java/org/neo4j/backup/ParametrisedOutsideWorld.java @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.backup; + +import java.io.Console; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; + +import org.neo4j.commandline.admin.OutsideWorld; +import org.neo4j.io.IOUtils; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.fs.FileSystemAbstraction; + +/** + * An outside world where you can pick and choose which input/output are dummies. + */ +public class ParametrisedOutsideWorld implements OutsideWorld +{ + + private final PrintStream stdout; + private final PrintStream stderr; + private final Console stdin; + private final FileSystemAbstraction fileSystemAbstraction; + + public ParametrisedOutsideWorld( StringBuilder stdout, StringBuilder stderr ) + { + this( System.console(), streamFromBuilder( stdout ), streamFromBuilder( stderr ), new DefaultFileSystemAbstraction() ); + } + + private static OutputStream streamFromBuilder( StringBuilder stringBuilder ) + { + return new OutputStream() + { + @Override + public void write( int i ) throws IOException + { + stringBuilder.append( (char) i ); + } + }; + } + + public ParametrisedOutsideWorld( Console stdin, OutputStream stdout, OutputStream stderr, FileSystemAbstraction fileSystemAbstraction ) + { + this.stdout = new PrintStream( stdout ); + this.stderr = new PrintStream( stderr ); + this.fileSystemAbstraction = fileSystemAbstraction; + this.stdin = stdin; + } + + public ParametrisedOutsideWorld( PrintStream stdout, PrintStream stderr ) + { + this( System.console(), stdout, stderr, new DefaultFileSystemAbstraction() ); + } + + @Override + public void stdOutLine( String text ) + { + stdout.println( text ); + } + + @Override + public void stdErrLine( String text ) + { + stderr.println( text ); + } + + @Override + public String readLine() + { + return stdin.readLine(); + } + + @Override + public String promptLine( String fmt, Object... args ) + { + return stdin.readLine( fmt, args ); + } + + @Override + public char[] promptPassword( String fmt, Object... args ) + { + return stdin.readPassword( fmt, args ); + } + + @Override + public void exit( int status ) + { + IOUtils.closeAllSilently( this ); + } + + @Override + public void printStacktrace( Exception exception ) + { + exception.printStackTrace( stderr ); + } + + @Override + public FileSystemAbstraction fileSystem() + { + return fileSystemAbstraction; + } + + @Override + public PrintStream errorStream() + { + return stderr; + } + + @Override + public PrintStream outStream() + { + return stdout; + } + + @Override + public void close() throws IOException + { + fileSystemAbstraction.close(); + } +} diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupCopyServiceTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupCopyServiceTest.java new file mode 100644 index 0000000000000..c69ba9fb65b53 --- /dev/null +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupCopyServiceTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.backup; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.stream.Stream; + +import org.neo4j.com.storecopy.FileMoveAction; +import org.neo4j.com.storecopy.FileMoveProvider; +import org.neo4j.commandline.admin.CommandFailed; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.test.rule.TestDirectory; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class BackupCopyServiceTest +{ + private PageCache pageCache; + private FileMoveProvider fileMoveProvider; + + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory(); + + BackupCopyService subject; + + @Before + public void setup() + { + pageCache = mock( PageCache.class ); + fileMoveProvider = mock( FileMoveProvider.class ); + subject = new BackupCopyService( pageCache, fileMoveProvider ); + } + + @Test + public void logicForMovingBackupsIsDelegatedToFileMovePropagator() throws CommandFailed, IOException + { + // given + File parentDirectory = testDirectory.directory( "parent" ); + File oldLocation = new File( parentDirectory, "oldLocation" ); + oldLocation.mkdir(); + File newLocation = new File( parentDirectory, "newLocation" ); + + // and + FileMoveAction fileOneMoveAction = mock( FileMoveAction.class ); + FileMoveAction fileTwoMoveAction = mock( FileMoveAction.class ); + when( fileMoveProvider.traverseGenerateMoveActions( any() ) ).thenReturn( Stream.of( fileOneMoveAction, fileTwoMoveAction ) ); + + // when + subject.moveBackupLocation( oldLocation, newLocation ); + + // then file move propagator was requested with correct source and baseDirectory + verify( fileMoveProvider ).traverseGenerateMoveActions( oldLocation ); + + // and files were moved to correct target directory + verify( fileOneMoveAction ).move( newLocation ); + verify( fileTwoMoveAction ).move( newLocation ); + } +} diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyCoordinatorTest.java similarity index 88% rename from enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java rename to enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyCoordinatorTest.java index 8a013c24ec798..9ea2b75473eff 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupFlowTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyCoordinatorTest.java @@ -48,7 +48,7 @@ import static org.mockito.Mockito.when; import static org.neo4j.backup.ExceptionMatchers.exceptionContainsSuppressedThrowable; -public class BackupFlowTest +public class BackupStrategyCoordinatorTest { @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -61,7 +61,7 @@ public class BackupFlowTest private final BackupStrategyWrapper firstStrategy = mock( BackupStrategyWrapper.class ); private final BackupStrategyWrapper secondStrategy = mock( BackupStrategyWrapper.class ); - BackupFlow subject; + BackupStrategyCoordinator subject; // test method parameter mocks private final OnlineBackupContext onlineBackupContext = mock( OnlineBackupContext.class ); @@ -78,7 +78,7 @@ public void setup() when( outsideWorld.fileSystem() ).thenReturn( fileSystem ); when( onlineBackupContext.getRequiredArguments() ).thenReturn( requiredArguments ); when( requiredArguments.getReportDir() ).thenReturn( reportDir ); - subject = new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, + subject = new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, Arrays.asList( firstStrategy, secondStrategy ) ); } @@ -157,10 +157,8 @@ public void consistencyCheckIsRunIfSpecified() throws CommandFailed, IOException // given anyStrategyPasses(); when( requiredArguments.isDoConsistencyCheck() ).thenReturn( true ); - when( consistencyCheckService.runFullConsistencyCheck( any(), any(), - eq(progressMonitorFactory), any( LogProvider.class ), any(), - eq(false), any(), any() ) ) - .thenReturn( consistencyCheckResult ); + when( consistencyCheckService.runFullConsistencyCheck( any(), any(), eq( progressMonitorFactory ), any( LogProvider.class ), any(), eq( false ), any(), + any() ) ).thenReturn( consistencyCheckResult ); when( consistencyCheckResult.isSuccessful() ).thenReturn( true ); // when @@ -193,10 +191,8 @@ public void allFailureCausesAreCollectedAndAttachedToCommandFailedException() th RuntimeException secondCause = new RuntimeException( "Second cause" ); // and strategies fail with given causes - when( firstStrategy.doBackup( any() ) ) - .thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, firstCause ) ); - when( secondStrategy.doBackup( any() ) ) - .thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, secondCause ) ); + when( firstStrategy.doBackup( any() ) ).thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, firstCause ) ); + when( secondStrategy.doBackup( any() ) ).thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, secondCause ) ); // then the command failed exception contains the specified causes expectedException.expect( exceptionContainsSuppressedThrowable( firstCause ) ); @@ -214,10 +210,8 @@ public void errorsDuringConsistencyCheckAreWrappedAsCommandFailed() throws Comma // given anyStrategyPasses(); when( requiredArguments.isDoConsistencyCheck() ).thenReturn( true ); - when( consistencyCheckService.runFullConsistencyCheck( any(), any(), - eq(progressMonitorFactory), any( LogProvider.class ), any(), - eq(false), any(), any() ) ) - .thenThrow( new IOException( "Predictable message" ) ); + when( consistencyCheckService.runFullConsistencyCheck( any(), any(), eq( progressMonitorFactory ), any( LogProvider.class ), any(), eq( false ), any(), + any() ) ).thenThrow( new IOException( "Predictable message" ) ); // then expectedException.expect( CommandFailed.class ); @@ -234,10 +228,8 @@ public void commandFailedWhenConsistencyCheckFails() throws IOException, Consist anyStrategyPasses(); when( requiredArguments.isDoConsistencyCheck() ).thenReturn( true ); when( consistencyCheckResult.isSuccessful() ).thenReturn( false ); - when( consistencyCheckService.runFullConsistencyCheck( any(), any(), - eq(progressMonitorFactory), any( LogProvider.class ), any(), - eq(false), any(), any() ) ) - .thenReturn( consistencyCheckResult ); + when( consistencyCheckService.runFullConsistencyCheck( any(), any(), eq( progressMonitorFactory ), any( LogProvider.class ), any(), eq( false ), any(), + any() ) ).thenReturn( consistencyCheckResult ); // then expectedException.expect( CommandFailed.class ); @@ -251,7 +243,7 @@ public void commandFailedWhenConsistencyCheckFails() throws IOException, Consist public void havingNoStrategiesCausesAllSolutionsFailedException() throws CommandFailed { // given there are no strategies in the solution - subject = new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, Collections.emptyList() ); + subject = new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, Collections.emptyList() ); // then we want a predictable exception (instead of NullPointer) expectedException.expect( CommandFailed.class ); @@ -267,6 +259,6 @@ public void havingNoStrategiesCausesAllSolutionsFailedException() throws Command private void anyStrategyPasses() { when( firstStrategy.doBackup( any() ) ).thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.SUCCESS, null ) ); - when( secondStrategy.doBackup( any() )).thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.SUCCESS, null ) ); + when( secondStrategy.doBackup( any() ) ).thenReturn( new PotentiallyErroneousState<>( BackupStrategyOutcome.SUCCESS, null ) ); } } diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyWrapperTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyWrapperTest.java index 0352eb11183bc..3569b688f3272 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyWrapperTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/BackupStrategyWrapperTest.java @@ -32,6 +32,8 @@ import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.io.pagecache.PageCache; import org.neo4j.kernel.configuration.Config; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -63,6 +65,8 @@ public class BackupStrategyWrapperTest private PotentiallyErroneousState FAILURE = new PotentiallyErroneousState<>( BackupStageOutcome.FAILURE, null ); private PageCache pageCache = mock( PageCache.class ); private BackupRecoveryService backupRecoveryService = mock( BackupRecoveryService.class ); + private LogProvider logProvider = mock( LogProvider.class ); + private Log log = mock( Log.class ); @Before public void setup() @@ -73,8 +77,9 @@ public void setup() when( backupCopyService.findNewBackupLocationForBrokenExisting( any() ) ).thenReturn( availableOldBackupLocation ); when( onlineBackupContext.getRequiredArguments() ).thenReturn( requiredArguments ); when( backupStrategyImplementation.performFullBackup( any(), any(), any() ) ).thenReturn( SUCCESS ); + when( logProvider.getLog( (Class) any() ) ).thenReturn( log ); - subject = new BackupStrategyWrapper( backupStrategyImplementation, backupCopyService, pageCache, config, backupRecoveryService ); + subject = new BackupStrategyWrapper( backupStrategyImplementation, backupCopyService, pageCache, config, backupRecoveryService, logProvider ); } @Test @@ -96,6 +101,9 @@ public void fullBackupIsPerformedWhenNoOtherBackupExists() // given when( backupCopyService.backupExists( any() ) ).thenReturn( false ); + // and fallback is set to true + when( requiredArguments.isFallbackToFull() ).thenReturn( true ); + // when subject.doBackup( onlineBackupContext ); @@ -103,6 +111,25 @@ public void fullBackupIsPerformedWhenNoOtherBackupExists() verify( backupStrategyImplementation ).performFullBackup( any(), any(), any() ); } + @Test + public void fullBackupIsIgnoredIfNoOtherBackupAndNotFallback() + { + // given there is an existing backup + when( backupCopyService.backupExists( any() ) ).thenReturn( false ); + + // and we don't want to fallback to full backups + when( requiredArguments.isFallbackToFull() ).thenReturn( false ); + + // and incremental backup fails because it's a different store + when( backupStrategyImplementation.performIncrementalBackup( any(), any(), any() ) ).thenReturn( FAILURE ); + + // when + subject.doBackup( onlineBackupContext ); + + // then full backup wasnt performed + verify( backupStrategyImplementation, never() ).performFullBackup( any(), any(), any() ); + } + @Test public void fullBackupIsNotPerformedWhenAnIncrementalBackupIsSuccessful() { @@ -207,10 +234,10 @@ public void successfulFullBackupsMoveExistingBackup() throws CommandFailed PotentiallyErroneousState state = subject.doBackup( onlineBackupContext ); // then original existing backup is moved to err directory - verify( backupCopyService ).moveBackupLocation( desiredBackupLocation, newLocationForExistingBackup ); + verify( backupCopyService ).moveBackupLocation( eq( desiredBackupLocation ), eq( newLocationForExistingBackup ) ); // and new successful backup is renamed to original expected name - verify( backupCopyService ).moveBackupLocation( temporaryFullBackupLocation, desiredBackupLocation ); + verify( backupCopyService ).moveBackupLocation( eq( temporaryFullBackupLocation ), eq( desiredBackupLocation ) ); // and backup was successful assertEquals( BackupStrategyOutcome.SUCCESS, state.getState() ); @@ -250,6 +277,9 @@ public void failureDuringMoveCausesAbsoluteFailure() throws CommandFailed @Test public void performingFullBackupInvokesRecovery() { + // given full backup flag is set + when( requiredArguments.isFallbackToFull() ).thenReturn( true ); + // when subject.doBackup( onlineBackupContext ); @@ -332,8 +362,100 @@ public void recoveryIsPerformedBeforeRename() throws CommandFailed recoveryBeforeRenameOrder.verify( backupCopyService ).moveBackupLocation( eq( availableFreshBackupLocation ), eq( desiredBackupLocation ) ); } + @Test + public void logsAreClearedAfterIncrementalBackup() + { + // given backup exists + when( backupCopyService.backupExists( any() ) ).thenReturn( true ); + + // and + incrementalBackupIsSuccessful( true ); + + // when + subject.doBackup( onlineBackupContext ); + + // then + verify( backupCopyService ).clearLogs( any() ); + } + + @Test + public void logsAreNotClearedWhenIncrementalNotSuccessful() + { + // given backup exists + when( backupCopyService.backupExists( any() ) ).thenReturn( true ); + + // and incremental is not successful + incrementalBackupIsSuccessful( false ); + + // when backups are performed + subject.doBackup( onlineBackupContext ); + + // then do not + verify( backupCopyService, never() ).clearLogs( any() ); + } + + @Test + public void logsAreClearedWhenFullBackupIsSuccessful() + { + // given a backup doesn't exist + when( backupCopyService.backupExists( any() ) ).thenReturn( false ); + + // and + fallbackToFullPasses(); + + // when + subject.doBackup( onlineBackupContext ); + + // then + verify( backupCopyService ).clearLogs( any() ); + } + + @Test + public void logsAreNotClearedWhenFullBackupIsNotSuccessful() + { + // given a backup doesn't exist + when( backupCopyService.backupExists( any() ) ).thenReturn( false ); + + // and + bothBackupsFail(); + + // when + subject.doBackup( onlineBackupContext ); + + // then + verify( backupCopyService, never() ).clearLogs( any() ); + } + + @Test + public void logsWhenIncrementalFailsAndFallbackToFull() + { + // given backup exists + when( backupCopyService.backupExists( any() ) ).thenReturn( false ); + + // and fallback to full + fallbackToFullPasses(); + + // when + subject.doBackup( onlineBackupContext ); + + // then + verify( log ).info( "Previous backup not found, a new full backup will be performed." ); + } + // ==================================================================================================== + private void incrementalBackupIsSuccessful( boolean isSuccessful ) + { + if ( isSuccessful ) + { + when( backupStrategyImplementation.performIncrementalBackup( any(), any(), any() ) ).thenReturn( + new PotentiallyErroneousState<>( BackupStageOutcome.SUCCESS, null ) ); + return; + } + when( backupStrategyImplementation.performIncrementalBackup( any(), any(), any() ) ).thenReturn( + new PotentiallyErroneousState<>( BackupStageOutcome.FAILURE, null ) ); + } + private void bothBackupsFail() { when( requiredArguments.isFallbackToFull() ).thenReturn( true ); diff --git a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandTest.java b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandTest.java index 6a606d7580b8f..2ed07b7a64651 100644 --- a/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandTest.java +++ b/enterprise/backup/src/test/java/org/neo4j/backup/OnlineBackupCommandTest.java @@ -51,11 +51,11 @@ public class OnlineBackupCommandTest // Dependencies private OutsideWorld outsideWorld = mock( OutsideWorld.class ); - private BackupFlowFactory backupFlowFactory = mock( BackupFlowFactory.class ); + private BackupStrategyCoordinatorFactory backupStrategyCoordinatorFactory = mock( BackupStrategyCoordinatorFactory.class ); // Behaviour dependencies private FileSystemAbstraction fileSystemAbstraction = mock( FileSystemAbstraction.class ); - private BackupFlow backupFlow = mock( BackupFlow.class ); + private BackupStrategyCoordinator backupStrategyCoordinator = mock( BackupStrategyCoordinator.class ); // Parameters and helpers private final Config config = mock( Config.class ); @@ -83,9 +83,9 @@ public void setup() throws Exception when( requiredArguments.getFolder() ).thenReturn( backupDirectory ); when( requiredArguments.getReportDir() ).thenReturn( reportDirectory ); when( requiredArguments.getName() ).thenReturn( "backup name" ); - when( backupFlowFactory.backupFlow( any(), any(), any(), any() ) ).thenReturn( backupFlow ); + when( backupStrategyCoordinatorFactory.backupStrategyCoordinator( any(), any(), any(), any() ) ).thenReturn( backupStrategyCoordinator ); - subject = new OnlineBackupCommand( outsideWorld, onlineBackupContextLoader, backupSupportingClassesFactory, backupFlowFactory ); + subject = new OnlineBackupCommand( outsideWorld, onlineBackupContextLoader, backupSupportingClassesFactory, backupStrategyCoordinatorFactory ); } @Test diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java index ec606137e8a44..f6f807a463b18 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveAction.java @@ -39,12 +39,12 @@ static FileMoveAction copyViaPageCache( File file, PageCache pageCache ) { return new FileMoveAction() { - @Override public void move( File toDir, CopyOption... copyOptions ) throws IOException { Optional handle = pageCache.getCachedFileSystem().streamFilesRecursive( file ).findAny(); - if ( handle.isPresent() ) + boolean directoryExistsInCachedSystem = handle.isPresent(); + if ( directoryExistsInCachedSystem ) { handle.get().rename( new File( toDir, file.getName() ), copyOptions ); } @@ -60,12 +60,12 @@ public File file() static FileMoveAction copyViaFileSystem( File file, File basePath ) { + Path base = basePath.toPath(); return new FileMoveAction() { @Override public void move( File toDir, CopyOption... copyOptions ) throws IOException { - Path base = basePath.toPath(); Path originalPath = file.toPath(); Path relativePath = base.relativize( originalPath ); Path resolvedPath = toDir.toPath().resolve( relativePath ); @@ -80,5 +80,4 @@ public File file() } }; } - } diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveActionInformer.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveActionInformer.java new file mode 100644 index 0000000000000..3c828c1e3d748 --- /dev/null +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveActionInformer.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.com.storecopy; + +@FunctionalInterface +public interface FileMoveActionInformer +{ + boolean shouldBeManagedByPageCache( String storeFileName ); +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java new file mode 100644 index 0000000000000..8d0561e425999 --- /dev/null +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/FileMoveProvider.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.com.storecopy; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.kernel.impl.store.StoreType; + +import static org.neo4j.helpers.collection.Iterables.asList; + +public class FileMoveProvider +{ + + private final FileMoveActionInformer fileMoveActionInformer; + private final PageCache pageCache; + + public FileMoveProvider( PageCache pageCache ) + { + this( pageCache, StoreType::shouldBeManagedByPageCache ); + } + + public FileMoveProvider( PageCache pageCache, FileMoveActionInformer fileMoveActionInformer ) + { + this.pageCache = pageCache; + this.fileMoveActionInformer = fileMoveActionInformer; + } + + /** + * Construct a stream of files that are to be moved + * + * @param dir the source location of the move action + * @return a stream of the entire contents of the source location that can be applied to a target location to perform a move + */ + public Stream traverseGenerateMoveActions( File dir ) + { + return traverseGenerateMoveActions( dir, dir ); + } + + /** + * Copies the contents from the directory to the base target path. + *

+ * This is confusing, so here is an example + *

+ *

+ * + * +Parent
+ * |+--directoryA
+ * |...+--fileA
+ * |...+--fileB
+ * + *

+ * Suppose we want to move to move Parent/directoryA to Parent/directoryB.
+ *

+ * + * File directoryA = new File("Parent/directoryA");
+ * Stream fileMoveActions = new FileMoveProvider(pageCache).traverseGenerateMoveActions(directoryA, directoryA);
+ * + *

+ * In the above we clearly generate actions for moving all the files contained in directoryA. directoryA is mentioned twice due to a implementation detail, + * hence the public method with only one parameter. We then actually perform the moves by applying the base target directory that we want to move to. + *

+ * + * File directoryB = new File("Parent/directoryB");
+ * fileMoveActions.forEach( action -> action.move( directoryB ) ); + * + *

+ * + * @param dir this directory and all the child paths under it are subject to move + * @param basePath this is the parent of your intended target directory. + * @return a stream of individual move actions which can be iterated and applied whenever + */ + Stream traverseGenerateMoveActions( File dir, File basePath ) + { + // Note that flatMap is an *intermediate operation* and therefor always lazy. + // It is very important that the stream we return only *lazily* calls out to expandTraverseFiles! + return Stream.of( dir ).flatMap( d -> expandTraverseFiles( d, basePath ) ); + } + + private Stream expandTraverseFiles( File dir, File basePath ) + { + List listing = listFiles( dir ); + if ( listing.isEmpty() ) + { +// return Stream.of( copyFileCorrectly( dir, basePath ) ); + return Stream.empty(); + } + Stream files = listing.stream().filter( this::isFile ); + Stream dirs = listing.stream().filter( this::isDirectory ); + Stream moveFiles = files.map( f -> copyFileCorrectly( f, basePath ) ); + Stream traverseDirectories = dirs.flatMap( d -> traverseGenerateMoveActions( d, basePath ) ); + return Stream.concat( moveFiles, traverseDirectories ); + } + + private boolean isFile( File file ) + { + if ( fileMoveActionInformer.shouldBeManagedByPageCache( file.getName() ) ) + { + return !pageCache.getCachedFileSystem().isDirectory( file ); + } + return file.isFile(); + } + + private boolean isDirectory( File file ) + { + if ( fileMoveActionInformer.shouldBeManagedByPageCache( file.getName() ) ) + { + return pageCache.getCachedFileSystem().isDirectory( file ); + } + return file.isDirectory(); + } + + private List listFiles( File dir ) + { + List pageCacheFiles = asList( safeArray( pageCache.getCachedFileSystem().listFiles( dir ) ) ); + List fsFiles = safeArray( dir.listFiles() ); + return Stream.of( pageCacheFiles, fsFiles ).flatMap( List::stream ).collect( Collectors.toList() ); + } + + private List safeArray( File[] files ) + { + return Arrays.asList( Optional.ofNullable( files ).orElse( new File[]{} ) ); + } + + /** + * Some files are handled via page cache for CAPI flash, others are only used on the default file system. This contains the logic for handling files between + * the 2 systems + */ + private FileMoveAction copyFileCorrectly( File fileToMove, File basePath ) + { + if ( fileMoveActionInformer.shouldBeManagedByPageCache( fileToMove.getName() ) ) + { + return FileMoveAction.copyViaPageCache( fileToMove, pageCache ); + } + return FileMoveAction.copyViaFileSystem( fileToMove, basePath ); + } +} diff --git a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java index 72d549eb4880d..84349bb199fe9 100644 --- a/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java +++ b/enterprise/com/src/main/java/org/neo4j/com/storecopy/StoreCopyClient.java @@ -24,7 +24,6 @@ import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; @@ -151,10 +150,16 @@ public interface StoreCopyRequester private final PageCache pageCache; private final Monitor monitor; private final boolean forensics; + private final FileMoveProvider fileMoveProvider; - public StoreCopyClient( File storeDir, Config config, Iterable> kernelExtensions, - LogProvider logProvider, FileSystemAbstraction fs, - PageCache pageCache, Monitor monitor, boolean forensics ) + public StoreCopyClient( File storeDir, Config config, Iterable> kernelExtensions, LogProvider logProvider, + FileSystemAbstraction fs, PageCache pageCache, Monitor monitor, boolean forensics ) + { + this( storeDir, config, kernelExtensions, logProvider, fs, pageCache, monitor, forensics, new FileMoveProvider( pageCache ) ); + } + + public StoreCopyClient( File storeDir, Config config, Iterable> kernelExtensions, LogProvider logProvider, + FileSystemAbstraction fs, PageCache pageCache, Monitor monitor, boolean forensics, FileMoveProvider fileMoveProvider ) { this.storeDir = storeDir; this.config = config; @@ -164,17 +169,17 @@ public StoreCopyClient( File storeDir, Config config, Iterable storeFileMoveActions = new ArrayList<>(); @@ -182,8 +187,8 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell // Request store files and transactions that will need recovery monitor.startReceivingStoreFiles(); - try ( Response response = requester.copyStore( decorateWithProgressIndicator( - new ToFileStoreWriter( tempStore, fs, monitor, pageCache, storeFileMoveActions ) ) ) ) + try ( Response response = requester.copyStore( + decorateWithProgressIndicator( new ToFileStoreWriter( tempStore, fs, monitor, pageCache, storeFileMoveActions ) ) ) ) { monitor.finishReceivingStoreFiles(); // Update highest archived log id @@ -199,23 +204,13 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell checkCancellation( cancellationRequest, tempStore ); // Run recovery, so that the transactions we just wrote into the active log will be applied. - monitor.startRecoveringStore(); - GraphDatabaseService graphDatabaseService = newTempDatabase( tempStore ); - graphDatabaseService.shutdown(); - monitor.finishRecoveringStore(); - - LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ) - .withConfig( config ) - .build(); + recoverDatabase( tempStore ); + // All is well, move the streamed files to the real store directory. // Start with the files written through the page cache. Should only be record store files. // Note that the stream is lazy, so the file system traversal won't happen until *after* the store files // have been moved. Thus we ensure that we only attempt to move them once. - Stream moveActionStream = Stream.concat( - storeFileMoveActions.stream(), traverseGenerateMoveActions( tempStore, tempStore ) ); - Function destinationMapper = - file -> logFiles.isLogFile( file ) ? logFiles.logFilesDirectory() : storeDir; - moveAfterCopy.move( moveActionStream, tempStore, destinationMapper ); + moveFromTemporaryLocationToCorrect( storeFileMoveActions, tempStore, moveAfterCopy ); } finally { @@ -224,28 +219,22 @@ public void copyStore( StoreCopyRequester requester, CancellationRequest cancell } } - private static Stream traverseGenerateMoveActions( File dir, File basePath ) + private void moveFromTemporaryLocationToCorrect( List storeFileMoveActions, File tempStore, MoveAfterCopy moveAfterCopy ) throws Exception { - // Note that flatMap is an *intermediate operation* and therefor always lazy. - // It is very important that the stream we return only *lazily* calls out to expandTraverseFiles! - return Stream.of( dir ).flatMap( d -> expandTraverseFiles( d, basePath ) ); + LogFiles logFiles = LogFilesBuilder.activeFilesBuilder( storeDir, fs, pageCache ).withConfig( config ).build(); + + Stream moveActionStream = + Stream.concat( storeFileMoveActions.stream(), fileMoveProvider.traverseGenerateMoveActions( tempStore, tempStore ) ); + Function destinationMapper = file -> logFiles.isLogFile( file ) ? logFiles.logFilesDirectory() : storeDir; + moveAfterCopy.move( moveActionStream, tempStore, destinationMapper ); } - private static Stream expandTraverseFiles( File dir, File basePath ) + private void recoverDatabase( File tempStore ) { - File[] listing = dir.listFiles(); - if ( listing == null ) - { - // Weird, we somehow listed files for something that is no longer a directory. It's either a file, - // or doesn't exists. If the pathname no longer exists, then we are safe to return null here, - // because the flatMap in traverseGenerateMoveActions will just ignore it. - return dir.isFile() ? Stream.of( FileMoveAction.copyViaFileSystem( dir, basePath ) ) : null; - } - Stream files = Arrays.stream( listing ).filter( File::isFile ); - Stream dirs = Arrays.stream( listing ).filter( File::isDirectory ); - Stream moveFiles = files.map( f -> FileMoveAction.copyViaFileSystem( f, basePath ) ); - Stream traverseDirectories = dirs.flatMap( d -> traverseGenerateMoveActions( d, basePath ) ); - return Stream.concat( moveFiles, traverseDirectories ); + monitor.startRecoveringStore(); + GraphDatabaseService graphDatabaseService = newTempDatabase( tempStore ); + graphDatabaseService.shutdown(); + monitor.finishRecoveringStore(); } private void writeTransactionsToActiveLogFile( File tempStoreDir, Response response ) throws Exception @@ -263,8 +252,7 @@ private void writeTransactionsToActiveLogFile( File tempStoreDir, Response re // transactions that goes some time back, before the last committed transaction id. So we cannot // use a TransactionAppender, since it has checks for which transactions one can append. FlushableChannel channel = logFiles.getLogFile().getWriter(); - final TransactionLogWriter writer = new TransactionLogWriter( - new LogEntryWriter( channel ) ); + final TransactionLogWriter writer = new TransactionLogWriter( new LogEntryWriter( channel ) ); final AtomicLong firstTxId = new AtomicLong( BASE_TX_ID ); response.accept( new Response.Handler() @@ -315,11 +303,7 @@ public Visitor transactions() // last closed transaction offset will not overcome old one. Till that happens it will be // impossible for recovery process to restore the store File neoStore = new File( tempStoreDir, MetaDataStore.DEFAULT_NAME ); - MetaDataStore.setRecord( - pageCache, - neoStore, - MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, - LOG_HEADER_SIZE ); + MetaDataStore.setRecord( pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, LOG_HEADER_SIZE ); } } finally @@ -330,18 +314,15 @@ public Visitor transactions() private GraphDatabaseService newTempDatabase( File tempStore ) { - ExternallyManagedPageCache.GraphDatabaseFactoryWithPageCacheFactory factory = - ExternallyManagedPageCache.graphDatabaseFactoryWithPageCache( pageCache ); - return factory - .setKernelExtensions( kernelExtensions ) + ExternallyManagedPageCache.GraphDatabaseFactoryWithPageCacheFactory factory = ExternallyManagedPageCache.graphDatabaseFactoryWithPageCache( pageCache ); + return factory.setKernelExtensions( kernelExtensions ) .setUserLogProvider( NullLogProvider.getInstance() ) .newEmbeddedDatabaseBuilder( tempStore.getAbsoluteFile() ) .setConfig( "dbms.backup.enabled", Settings.FALSE ) .setConfig( GraphDatabaseSettings.logs_directory, tempStore.getAbsolutePath() ) .setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE ) .setConfig( GraphDatabaseSettings.logical_logs_location, tempStore.getAbsolutePath() ) - .setConfig( GraphDatabaseSettings.allow_upgrade, - config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) + .setConfig( GraphDatabaseSettings.allow_upgrade, config.get( GraphDatabaseSettings.allow_upgrade ).toString() ) .newGraphDatabase(); } @@ -352,8 +333,8 @@ private StoreWriter decorateWithProgressIndicator( final StoreWriter actual ) private int totalFiles; @Override - public long write( String path, ReadableByteChannel data, ByteBuffer temporaryBuffer, - boolean hasData, int requiredElementAlignment ) throws IOException + public long write( String path, ReadableByteChannel data, ByteBuffer temporaryBuffer, boolean hasData, int requiredElementAlignment ) + throws IOException { log.info( "Copying %s", path ); long written = actual.write( path, data, temporaryBuffer, hasData, requiredElementAlignment ); diff --git a/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java b/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java new file mode 100644 index 0000000000000..1bee9fb056b97 --- /dev/null +++ b/enterprise/com/src/test/java/org/neo4j/com/storecopy/FileMoveProviderTest.java @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2002-2017 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.com.storecopy; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.neo4j.graphdb.mockfs.EphemeralFileSystemAbstraction; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.io.fs.StoreChannel; +import org.neo4j.io.pagecache.PageCache; +import org.neo4j.test.rule.PageCacheRule; +import org.neo4j.test.rule.TestDirectory; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class FileMoveProviderTest +{ + private DefaultFileSystemAbstraction defaultFileSystemAbstraction = new DefaultFileSystemAbstraction(); + private EphemeralFileSystemAbstraction ephemeralFileSystemAbstraction = new EphemeralFileSystemAbstraction(); + + @Rule + public TestDirectory testDirectory = TestDirectory.testDirectory( defaultFileSystemAbstraction ); + + private FileMoveProvider subject; + + public PageCacheRule pageCacheRule = new PageCacheRule(); + private PageCache pageCache; + private FileMoveActionInformer fileMoveActionInformer; + + @Before + public void setup() + { + pageCache = pageCacheRule.getPageCache( ephemeralFileSystemAbstraction ); + fileMoveActionInformer = mock( FileMoveActionInformer.class ); + subject = new FileMoveProvider( pageCache, fileMoveActionInformer ); + } + + @Test + public void moveSingleFiles() + { + // given + File sharedParent = testDirectory.directory( "shared_parent" ); + File sourceParent = createDirectory( new File( sharedParent, "source" ) ); + File sourceFile = createFile( new File( sourceParent, "file.txt" ) ); + writeToFile( sourceFile, "Garbage data" ); + File targetParent = createDirectory( new File( sharedParent, "target" ) ); + File targetFile = new File( targetParent, "file.txt" ); + + // when + subject.traverseGenerateMoveActions( sourceFile ).forEach( moveToDirectory( targetParent ) ); + + // then + assertEquals( "Garbage data", readFromFile( targetFile ) ); + } + + private interface RunnableThrowable + { + void run() throws Throwable; + } + + private static Runnable runnableFromThrowable( RunnableThrowable runnableThrowable ) + { + return () -> + { + try + { + runnableThrowable.run(); + } + catch ( Throwable throwable ) + { + throw new RuntimeException( throwable ); + } + }; + } + + @Test + public void singleDirectoriesAreNotMoved() + { + // given + File sharedParent = testDirectory.directory( "shared_parent" ); + File sourceParent = createDirectory( new File( sharedParent, "source" ) ); + File sourceDirectory = createDirectory( new File( sourceParent, "directory" ) ); + + // and + File targetParent = createDirectory( new File( sharedParent, "target" ) ); + File targetDirectory = new File( targetParent, "directory" ); + assertFalse( targetDirectory.exists() ); + + // when + subject.traverseGenerateMoveActions( sourceParent ).forEach( moveToDirectory( targetDirectory ) ); + + // then + assertTrue( sourceDirectory.exists() ); + assertFalse( targetDirectory.exists() ); + } + + @Test + public void moveNestedFiles() + { + // given + File sharedParent = testDirectory.directory( "shared_parent" ); + File sourceParent = createDirectory( new File( sharedParent, "source" ) ); + File targetParent = createDirectory( new File( sharedParent, "target" ) ); + + // and + File nestedFileOne = createFile( new File( createDirectory( new File( sourceParent, "A" ) ), "file.txt" ) ); + File nestedFileTwo = createFile( new File( createDirectory( new File( sourceParent, "B" ) ), "file.txt" ) ); + writeToFile( nestedFileOne, "This is the file contained in directory A" ); + writeToFile( nestedFileTwo, "This is the file contained in directory B" ); + + // and + File targetFileOne = new File( targetParent, "A/file.txt" ); + File targetFileTwo = new File( targetParent, "B/file.txt" ); + + // when + subject.traverseGenerateMoveActions( sourceParent ).forEach( moveToDirectory( targetParent ) ); + + // then + assertEquals( "This is the file contained in directory A", readFromFile( targetFileOne ) ); + assertEquals( "This is the file contained in directory B", readFromFile( targetFileTwo ) ); + } + + @Test + public void filesAreMovedViaPageCacheWhenNecessary() throws IOException + { + // given there is a file on the default file system + File parentDirectory = createDirectory( testDirectory.directory( "parent" ) ); + File aNormalFile = createFile( new File( parentDirectory, "aNormalFile.A" ) ); + + // and we have an expected target directory + File targetDirectory = createDirectory( testDirectory.directory( "targetDirectory" ) ); + pageCache.getCachedFileSystem().mkdirs( targetDirectory ); + + // and there is also a file on the block device + File aPageCacheFile = new File( parentDirectory, "aBlockCopyFile.B" ); + pageCache.getCachedFileSystem().mkdirs( parentDirectory ); + StoreChannel storeChannel = pageCache.getCachedFileSystem().create( aPageCacheFile ); + storeChannel.write( ByteBuffer.allocate( 20 ).putChar( 'a' ).putChar( 'b' ) ); + + // and some of these files are handled by the page cache + when( fileMoveActionInformer.shouldBeManagedByPageCache( any() ) ).thenReturn( false ); + when( fileMoveActionInformer.shouldBeManagedByPageCache( eq( aPageCacheFile.getName() ) ) ).thenReturn( true ); + + // when the files are copied to target location + List moveActions = + subject.traverseGenerateMoveActions( parentDirectory ).collect( Collectors.toList() );//.forEach( moveToDirectory( targetDirectory ) ); + moveActions.forEach( moveToDirectory( targetDirectory ) ); + + // then some files are copied over the default file system + File expectedNormalCopy = new File( targetDirectory, aNormalFile.getName() ); + assertTrue( expectedNormalCopy.exists() ); + + // and correct files are copied over the page cache + File expectedPageCacheCopy = new File( targetDirectory, aPageCacheFile.getName() ); + assertTrue( expectedPageCacheCopy.toString(), pageCache.getCachedFileSystem().fileExists( expectedPageCacheCopy ) ); + } + + @Test + public void filesAreMovedBeforeDirectories() // TODO doesnt test anything maybe + { + // given there is a file contained in a directory + File parentDirectory = createDirectory( testDirectory.directory( "parent" ) ); + File sourceDirectory = createDirectory( new File( parentDirectory, "source" ) ); + File childFile = createFile( new File( sourceDirectory, "child" ) ); + writeToFile( childFile, "Content" ); + + // and we have an expected target directory + File targetDirectory = createDirectory( new File( parentDirectory, "target" ) ); + + // when + subject.traverseGenerateMoveActions( sourceDirectory ).forEach( moveToDirectory( targetDirectory ) ); + + // then no exception due to files happening before empty target directory + } + + private Supplier failure() + { + return () -> new RuntimeException( "Fail" ); + } + + private List safeList( File dir ) + { + return Arrays.asList( Optional.ofNullable( dir ).map( File::listFiles ).orElse( new File[]{} ) ); + } + + private Consumer moveToDirectory( File fileToMove ) + { + return fileMoveAction -> runnableFromThrowable( () -> fileMoveAction.move( fileToMove ) ).run(); + } + + private String readFromFile( File input ) + { + try + { + BufferedReader fileReader = new BufferedReader( new FileReader( input ) ); + StringBuilder stringBuilder = new StringBuilder(); + char[] data = new char[32]; + int read; + while ( (read = fileReader.read( data )) != -1 ) + { + stringBuilder.append( data, 0, read ); + } + return stringBuilder.toString(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } + + private File createDirectory( File file ) + { + runnableFromThrowable( file::mkdirs ).run(); + return file; + } + + private File createFile( File file ) + { + runnableFromThrowable( file::createNewFile ).run(); + return file; + } + + private void writeToFile( File output, String input ) + { + try + { + BufferedWriter bw = new BufferedWriter( new FileWriter( output ) ); + bw.write( input ); + bw.close(); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + } +} diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java index 295e6a89f27ea..0d84e2aa63196 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java @@ -68,7 +68,7 @@ public String role() public boolean isMaster() { - return HighAvailabilityModeSwitcher.MASTER.equals( role() ); + return HighAvailabilityModeSwitcher.MASTER.equalsIgnoreCase( role() ); } public File getStoreDirectory() diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/impl/ha/ClusterManager.java b/enterprise/ha/src/test/java/org/neo4j/kernel/impl/ha/ClusterManager.java index 70d9c86e26c15..c54bb2b337c55 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/impl/ha/ClusterManager.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/impl/ha/ClusterManager.java @@ -949,9 +949,13 @@ public HostnamePort getBackupAddress( HighlyAvailableGraphDatabase hagdb ) */ public HighlyAvailableGraphDatabase getMaster() { + int instance=0; for ( HighlyAvailableGraphDatabase graphDatabaseService : getAllMembers() ) { - if ( graphDatabaseService.isAvailable( 0 ) && graphDatabaseService.isMaster() ) + boolean available = graphDatabaseService.isAvailable( 1000 ); + boolean master = graphDatabaseService.isMaster(); + System.out.printf( "Instance %d: available=%b and master=%b [i.e. %s]\n", instance++, available, master, graphDatabaseService.role() ); + if ( available && master ) { return graphDatabaseService; }