Backups on blockdevices

Adds support/fixes tests for performing backups on blockdevices

community/command-line/src/main/java/org/neo4j/commandline/admin/AdminTool.java

@@ -29,7 +29,6 @@
 import org.neo4j.helpers.Args;
 
 import static java.lang.String.format;
-
 import static org.neo4j.commandline.Util.neo4jVersion;
 
 public class AdminTool

community/dbms/src/main/java/org/neo4j/commandline/dbms/DumpCommand.java

@@ -95,9 +95,7 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
         }
         catch ( CannotWriteException e )
         {
-            throw new CommandFailed(
-                    "you do not have permission to dump the database -- is Neo4j running as a " + "different user?",
-                    e );
+            throw new CommandFailed( "you do not have permission to dump the database -- is Neo4j running as a different user?", e );
         }
     }
 

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/GraphDatabaseFacade.java

@@ -378,9 +378,9 @@ public Schema schema()
     }
 
     @Override
-    public boolean isAvailable( long timeout )
+    public boolean isAvailable( long timeoutMillis )
     {
-        return spi.databaseIsAvailable( timeout );
+        return spi.databaseIsAvailable( timeoutMillis );
     }
 
     @Override

enterprise/backup/src/main/java/org/neo4j/backup/BackupCopyService.java

@@ -21,46 +21,61 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
+import java.util.Iterator;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
+import org.neo4j.com.storecopy.FileMoveAction;
+import org.neo4j.com.storecopy.FileMoveProvider;
 import org.neo4j.commandline.admin.CommandFailed;
-import org.neo4j.commandline.admin.OutsideWorld;
+import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.configuration.Config;
 
 import static java.lang.String.format;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory;
+import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path;
 
 public class BackupCopyService
 {
     private static final int MAX_OLD_BACKUPS = 1000;
 
-    private final OutsideWorld outsideWorld;
+    private final PageCache pageCache;
 
-    public BackupCopyService( OutsideWorld outsideWorld )
+    private final FileMoveProvider fileMoveProvider;
+
+    public BackupCopyService( PageCache pageCache, FileMoveProvider fileMoveProvider )
     {
-        this.outsideWorld = outsideWorld;
+        this.pageCache = pageCache;
+        this.fileMoveProvider = fileMoveProvider;
     }
 
     public void moveBackupLocation( File oldLocation, File newLocation ) throws CommandFailed
     {
         try
         {
-            outsideWorld.fileSystem().renameFile( oldLocation, newLocation );
+            Iterator<FileMoveAction> moves = fileMoveProvider.traverseGenerateMoveActions( oldLocation ).iterator();
+            while ( moves.hasNext() )
+            {
+                moves.next().move( newLocation );
+            }
         }
         catch ( IOException e )
         {
             throw new CommandFailed( "Failed to move old backup out of the way: " + e.getMessage(), e );
         }
     }
 
+    public void clearLogs( File neo4jHome )
+    {
+        File logsDirectory = Config.defaults( logs_directory, neo4jHome.getPath() ).get( store_internal_log_path );
+        logsDirectory.delete();
+    }
+
     boolean backupExists( File destination )
     {
-        File[] listFiles = outsideWorld.fileSystem().listFiles( destination );
+        File[] listFiles = pageCache.getCachedFileSystem().listFiles( destination );
         return listFiles != null && listFiles.length > 0;
     }
 
@@ -74,6 +89,13 @@ File findAnAvailableLocationForNewFullBackup( File desiredBackupLocation )
         return findAnAvailableBackupLocation( desiredBackupLocation, "%s.temp.%d" );
     }
 
+    /**
+     * Given a desired file name, find an available name that is similar to the given one that doesn't conflict with already existing backups
+     *
+     * @param file desired ideal file name
+     * @param pattern pattern to follow if desired name is taken (requires %s for original name, and %d for iteration)
+     * @return the resolved file name which can be the original desired, or a variation that matches the pattern
+     */
     private File findAnAvailableBackupLocation( File file, String pattern )
     {
         if ( backupExists( file ) )
@@ -84,7 +106,7 @@ private File findAnAvailableBackupLocation( File file, String pattern )
 
             return availableAlternativeNames( file, pattern )
                     .peek( countNumberOfFilesProcessedForPotentialErrorMessage )
-                    .filter( f -> !backupExists(f) )
+                    .filter( f -> !backupExists( f ) )
                     .findFirst()
                     .orElseThrow( () -> new RuntimeException(
                             String.format( "Unable to find a free backup location for the provided %s. Number of iterations %d", file, counter.get() ) ) );
@@ -100,8 +122,7 @@ private static Stream<File> availableAlternativeNames( File originalBackupDirect
 
     private static File alteredBackupDirectoryName( String pattern, File directory, int iteration )
     {
-        return directory
-                .toPath()
+        return directory.toPath()
                 .resolveSibling( format( pattern, directory.getName(), iteration ) )
                 .toFile();
     }

enterprise/backup/src/main/java/org/neo4j/backup/BackupModuleResolveAtRuntime.java

@@ -36,7 +36,6 @@ public class BackupModuleResolveAtRuntime
     private final Monitors monitors;
     private final Clock clock;
     private final TransactionLogCatchUpFactory transactionLogCatchUpFactory;
-    private final BackupCopyService backupCopyService;
 
     /**
      * Dependencies that can be resolved immediately after launching the backup tool
@@ -53,7 +52,6 @@ public BackupModuleResolveAtRuntime( OutsideWorld outsideWorld, LogProvider logP
         this.clock = Clock.systemDefaultZone();
         this.transactionLogCatchUpFactory = new TransactionLogCatchUpFactory();
         this.fileSystemAbstraction = outsideWorld.fileSystem();
-        this.backupCopyService = new BackupCopyService( outsideWorld );
     }
 
     public LogProvider getLogProvider()
@@ -85,9 +83,4 @@ public OutsideWorld getOutsideWorld()
     {
         return outsideWorld;
     }
-
-    public BackupCopyService getBackupCopyService()
-    {
-        return backupCopyService;
-    }
 }

enterprise/backup/src/main/java/org/neo4j/backup/BackupFlow.java → enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinator.java

@@ -40,7 +40,7 @@
  * when none of the backups worked.
  * Also handles the consistency check
  */
-class BackupFlow
+class BackupStrategyCoordinator
 {
     private static final int STATUS_CC_ERROR = 2;
     private static final int STATUS_CC_INCONSISTENT = 3;
@@ -51,7 +51,7 @@ class BackupFlow
     private final ProgressMonitorFactory progressMonitorFactory;
     private final List<BackupStrategyWrapper> strategies;
 
-    BackupFlow( ConsistencyCheckService consistencyCheckService, OutsideWorld outsideWorld, LogProvider logProvider,
+    BackupStrategyCoordinator( ConsistencyCheckService consistencyCheckService, OutsideWorld outsideWorld, LogProvider logProvider,
             ProgressMonitorFactory progressMonitorFactory, List<BackupStrategyWrapper> strategies )
     {
         this.consistencyCheckService = consistencyCheckService;
@@ -68,7 +68,7 @@ class BackupFlow
      * @param onlineBackupContext filesystem, command arguments and configuration
      * @throws CommandFailed when backup failed or there were issues with consistency checks
      */
-    void performBackup( OnlineBackupContext onlineBackupContext ) throws CommandFailed
+    public void performBackup( OnlineBackupContext onlineBackupContext ) throws CommandFailed
     {
         // Convenience
         OnlineBackupRequiredArguments requiredArgs = onlineBackupContext.getRequiredArguments();

enterprise/backup/src/main/java/org/neo4j/backup/BackupFlowFactory.java → enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyCoordinatorFactory.java

@@ -23,47 +23,54 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import org.neo4j.com.storecopy.FileMoveProvider;
 import org.neo4j.commandline.admin.OutsideWorld;
 import org.neo4j.consistency.ConsistencyCheckService;
 import org.neo4j.helpers.progress.ProgressMonitorFactory;
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.logging.LogProvider;
 
 /*
-Backup flows are iterate through backup strategies and make sure at least one of them is a valid backup. Handles cases when that isn't possible.
-This factory helps in the construction of them
+ * Backup strategy coordinators iterate through backup strategies and make sure at least one of them can perform a valid backup.
+ * Handles cases when individual backups aren't  possible.
  */
-class BackupFlowFactory
+class BackupStrategyCoordinatorFactory
 {
     private final LogProvider logProvider;
     private final ConsistencyCheckService consistencyCheckService;
     private final AddressResolutionHelper addressResolutionHelper;
-    private final BackupCopyService backupCopyService;
     private final OutsideWorld outsideWorld;
 
-    BackupFlowFactory( BackupModuleResolveAtRuntime backupModuleResolveAtRuntime )
+    BackupStrategyCoordinatorFactory( BackupModuleResolveAtRuntime backupModuleResolveAtRuntime )
     {
         this.logProvider = backupModuleResolveAtRuntime.getLogProvider();
         this.outsideWorld = backupModuleResolveAtRuntime.getOutsideWorld();
-        this.backupCopyService = backupModuleResolveAtRuntime.getBackupCopyService();
 
         this.consistencyCheckService = new ConsistencyCheckService();
         this.addressResolutionHelper = new AddressResolutionHelper();
     }
 
-    BackupFlow backupFlow( OnlineBackupContext onlineBackupContext, BackupProtocolService backupProtocolService, BackupDelegator backupDelegator,
-            PageCache pageCache )
+    /**
+     * Construct a wrapper of supported backup strategies
+     *
+     * @param onlineBackupContext the input of the backup tool, such as CLI arguments, config etc.
+     * @param backupProtocolService the underlying backup implementation for HA and single node instances
+     * @param backupDelegator the backup implementation used for CC backups
+     * @param pageCache the page cache used moving files
+     * @return strategy coordinator that handles the which backup strategies are tried and establishes if a backup was successful or not
+     */
+    BackupStrategyCoordinator backupStrategyCoordinator( OnlineBackupContext onlineBackupContext, BackupProtocolService backupProtocolService,
+            BackupDelegator backupDelegator, PageCache pageCache )
     {
+        BackupCopyService backupCopyService = new BackupCopyService( pageCache, new FileMoveProvider( pageCache ) );
         ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() );
 
-        List<BackupStrategyWrapper> strategies = Stream
-            .of(
-                new CausalClusteringBackupStrategy( backupDelegator, addressResolutionHelper ),
+        List<BackupStrategyWrapper> strategies = Stream.of( new CausalClusteringBackupStrategy( backupDelegator, addressResolutionHelper ),
                 new HaBackupStrategy( backupProtocolService, addressResolutionHelper, onlineBackupContext.getRequiredArguments().getTimeout() ) )
-            .map( strategy -> new BackupStrategyWrapper( strategy, backupCopyService, pageCache, onlineBackupContext.getConfig(),
-                    new BackupRecoveryService() ) )
-            .collect( Collectors.toList() );
+                .map( strategy -> new BackupStrategyWrapper( strategy, backupCopyService, pageCache, onlineBackupContext.getConfig(),
+                        new BackupRecoveryService(), logProvider ) )
+                .collect( Collectors.toList() );
 
-        return new BackupFlow( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, strategies );
+        return new BackupStrategyCoordinator( consistencyCheckService, outsideWorld, logProvider, progressMonitorFactory, strategies );
     }
 }

enterprise/backup/src/main/java/org/neo4j/backup/BackupStrategyWrapper.java

@@ -26,24 +26,32 @@
 import org.neo4j.io.pagecache.PageCache;
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.lifecycle.LifeSupport;
+import org.neo4j.logging.Log;
+import org.neo4j.logging.LogProvider;
 
+/**
+ * Individual backup strategies can perform incremental backups and full backups. The logic of how and when to perform full/incremental is identical.
+ * This class describes the behaviour of a single strategy and is used to wrap an interface providing incremental/full backup functionality
+ */
 class BackupStrategyWrapper
 {
     private final BackupStrategy backupStrategy;
     private final BackupCopyService backupCopyService;
     private final BackupRecoveryService backupRecoveryService;
+    private final Log log;
 
     private final PageCache pageCache;
     private final Config config;
 
     BackupStrategyWrapper( BackupStrategy backupStrategy, BackupCopyService backupCopyService, PageCache pageCache, Config config,
-            BackupRecoveryService backupRecoveryService )
+            BackupRecoveryService backupRecoveryService, LogProvider logProvider )
     {
         this.backupStrategy = backupStrategy;
         this.backupCopyService = backupCopyService;
         this.pageCache = pageCache;
         this.config = config;
         this.backupRecoveryService = backupRecoveryService;
+        this.log = logProvider.getLog( BackupStrategyWrapper.class );
     }
 
     /**
@@ -70,25 +78,47 @@ private PotentiallyErroneousState<BackupStrategyOutcome> performBackupWithoutLif
         final OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress();
         final Config config = onlineBackupContext.getConfig();
 
-        if ( backupCopyService.backupExists( backupLocation ) )
+        boolean previousBackupExists = backupCopyService.backupExists( backupLocation );
+        if ( previousBackupExists )
         {
+            log.info( "Previous backup found, trying incremental backup." );
             PotentiallyErroneousState<BackupStageOutcome> state =
                     backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress );
             boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() );
             boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() );
 
             if ( fullBackupWontWork || incrementalWasSuccessful )
             {
+                backupCopyService.clearLogs( backupLocation );
                 return describeOutcome( state );
             }
             if ( !onlineBackupContext.getRequiredArguments().isFallbackToFull() )
             {
                 return describeOutcome( state );
             }
         }
-        return describeOutcome( fullBackupWithTemporaryFolderResolutions( onlineBackupContext ) );
+        if ( onlineBackupContext.getRequiredArguments().isFallbackToFull() )
+        {
+            if ( !previousBackupExists )
+            {
+                log.info( "Previous backup not found, a new full backup will be performed." );
+            }
+            return describeOutcome( fullBackupWithTemporaryFolderResolutions( onlineBackupContext ) );
+        }
+        return new PotentiallyErroneousState<>( BackupStrategyOutcome.INCORRECT_STRATEGY, null );
     }
 
+    /**
+     * This will perform a full backup with some directory renaming if necessary.
+     * <p>
+     * If there is no existing backup, then no renaming will occur.
+     * Otherwise the full backup will be done into a temporary directory and renaming
+     * will occur if everything was successful.
+     * </p>
+     *
+     * @param onlineBackupContext command line arguments, config etc.
+     * @return outcome of full backup
+     */
     private PotentiallyErroneousState<BackupStageOutcome> fullBackupWithTemporaryFolderResolutions( OnlineBackupContext onlineBackupContext )
     {
         final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName();
@@ -113,6 +143,7 @@ private PotentiallyErroneousState<BackupStageOutcome> fullBackupWithTemporaryFol
                     return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, commandFailed );
                 }
             }
+            backupCopyService.clearLogs( userSpecifiedBackupLocation );
         }
         return state;
     }

enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackupCommand.java

@@ -25,13 +25,12 @@
 import org.neo4j.commandline.admin.CommandFailed;
 import org.neo4j.commandline.admin.IncorrectUsage;
 import org.neo4j.commandline.admin.OutsideWorld;
-import org.neo4j.io.pagecache.PageCache;
 
 public class OnlineBackupCommand implements AdminCommand
 {
     private final OutsideWorld outsideWorld;
     private final OnlineBackupContextLoader onlineBackupContextLoader;
-    private final BackupFlowFactory backupFlowFactory;
+    private final BackupStrategyCoordinatorFactory backupStrategyCoordinatorFactory;
     private final AbstractBackupSupportingClassesFactory backupSupportingClassesFactory;
 
     /**
@@ -40,15 +39,15 @@ public class OnlineBackupCommand implements AdminCommand
      * @param outsideWorld provides a way to interact with the filesystem and output streams
      * @param onlineBackupContextLoader helper class to validate, process and return a grouped result of processing the command line arguments
      * @param backupSupportingClassesFactory necessary for constructing the strategy for backing up over the causal clustering transaction protocol
-     * @param backupFlowFactory class that actually handles the logic of performing a backup
+     * @param backupStrategyCoordinatorFactory class that actually handles the logic of performing a backup
      */
     OnlineBackupCommand( OutsideWorld outsideWorld, OnlineBackupContextLoader onlineBackupContextLoader,
-            AbstractBackupSupportingClassesFactory backupSupportingClassesFactory, BackupFlowFactory backupFlowFactory )
+            AbstractBackupSupportingClassesFactory backupSupportingClassesFactory, BackupStrategyCoordinatorFactory backupStrategyCoordinatorFactory )
     {
         this.outsideWorld = outsideWorld;
         this.onlineBackupContextLoader = onlineBackupContextLoader;
         this.backupSupportingClassesFactory = backupSupportingClassesFactory;
-        this.backupFlowFactory = backupFlowFactory;
+        this.backupStrategyCoordinatorFactory = backupStrategyCoordinatorFactory;
     }
 
     @Override
@@ -62,10 +61,11 @@ public void execute( String[] args ) throws IncorrectUsage, CommandFailed
         checkDestination( onlineBackupContext.getRequiredArguments().getFolder() );
         checkDestination( onlineBackupContext.getRequiredArguments().getReportDir() );
 
-        BackupFlow backupFlow = backupFlowFactory.backupFlow( onlineBackupContext, backupSupportingClasses.getBackupProtocolService(),
-                backupSupportingClasses.getBackupDelegator(), backupSupportingClasses.getPageCache() );
+        BackupStrategyCoordinator backupStrategyCoordinator =
+                backupStrategyCoordinatorFactory.backupStrategyCoordinator( onlineBackupContext, backupSupportingClasses.getBackupProtocolService(),
+                        backupSupportingClasses.getBackupDelegator(), backupSupportingClasses.getPageCache() );
 
-        backupFlow.performBackup( onlineBackupContext );
+        backupStrategyCoordinator.performBackup( onlineBackupContext );
         outsideWorld.stdOutLine( "Backup complete." );
     }