Skip to content

Commit

Permalink
Addressed Martin's and Andrew's comments
Browse files Browse the repository at this point in the history
  • Loading branch information
phughk committed Oct 13, 2017
1 parent efb93f3 commit 294b3ed
Show file tree
Hide file tree
Showing 15 changed files with 281 additions and 183 deletions.
Expand Up @@ -73,7 +73,7 @@ BackupSupportingClasses createSupportingClassesForBackupStrategies( Config confi
{ {
PageCache pageCache = createPageCache( fileSystemAbstraction, config ); PageCache pageCache = createPageCache( fileSystemAbstraction, config );
return new BackupSupportingClasses( return new BackupSupportingClasses(
backupDelegatorFormConfig( pageCache, config ), backupDelegatorFromConfig( pageCache, config ),
haFromConfig( pageCache ), haFromConfig( pageCache ),
pageCache ); pageCache );
} }
Expand All @@ -83,7 +83,7 @@ private BackupProtocolService haFromConfig( PageCache pageCache )
return new BackupProtocolService( () -> fileSystemAbstraction, logProvider, logDestination, monitors, pageCache ); return new BackupProtocolService( () -> fileSystemAbstraction, logProvider, logDestination, monitors, pageCache );
} }


private BackupDelegator backupDelegatorFormConfig( PageCache pageCache, Config config ) private BackupDelegator backupDelegatorFromConfig( PageCache pageCache, Config config )
{ {
PipelineHandlerAppenderFactory pipelineHandlerAppenderFactory = getPipelineHandlerAppenderFactory(); PipelineHandlerAppenderFactory pipelineHandlerAppenderFactory = getPipelineHandlerAppenderFactory();
PipelineHandlerAppender pipelineHandlerAppender = pipelineHandlerAppenderFactory.create( config, getDependencies(), logProvider ); PipelineHandlerAppender pipelineHandlerAppender = pipelineHandlerAppenderFactory.create( config, getDependencies(), logProvider );
Expand Down
Expand Up @@ -21,8 +21,12 @@


import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import java.util.stream.Stream; import java.util.stream.Stream;


Expand Down
Expand Up @@ -22,21 +22,17 @@
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.function.Supplier; import java.util.function.Supplier;


import org.neo4j.commandline.admin.CommandFailed; import org.neo4j.commandline.admin.CommandFailed;
import org.neo4j.commandline.admin.OutsideWorld; import org.neo4j.commandline.admin.OutsideWorld;
import org.neo4j.consistency.ConsistencyCheckService; import org.neo4j.consistency.ConsistencyCheckService;
import org.neo4j.consistency.checking.full.ConsistencyFlags; import org.neo4j.consistency.checking.full.ConsistencyFlags;
import org.neo4j.helpers.progress.ProgressMonitorFactory; import org.neo4j.helpers.progress.ProgressMonitorFactory;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.internal.GraphDatabaseAPI;
import org.neo4j.logging.LogProvider; import org.neo4j.logging.LogProvider;


import static java.lang.String.format; import static java.lang.String.format;
import static org.neo4j.backup.BackupProtocolService.startTemporaryDb;


/** /**
* Controls the outcome of the backup tool. * Controls the outcome of the backup tool.
Expand Down
Expand Up @@ -80,6 +80,7 @@
import static org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.DEFAULT_BATCH_SIZE; import static org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.DEFAULT_BATCH_SIZE;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.logs_directory;
import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_path;
import static org.neo4j.helpers.Exceptions.launderedException;
import static org.neo4j.helpers.Exceptions.rootCause; import static org.neo4j.helpers.Exceptions.rootCause;
import static org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory.createPageCache; import static org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory.createPageCache;


Expand All @@ -93,6 +94,8 @@ class BackupProtocolService
" perform a full backup at this point. You can modify this time interval by setting the '" + " perform a full backup at this point. You can modify this time interval by setting the '" +
GraphDatabaseSettings.keep_logical_logs.name() + "' configuration on the database to a higher value."; GraphDatabaseSettings.keep_logical_logs.name() + "' configuration on the database to a higher value.";


static final String DIFFERENT_STORE_MESSAGE = "Target directory contains full backup of a logically different store.";

private final Supplier<FileSystemAbstraction> fileSystemSupplier; private final Supplier<FileSystemAbstraction> fileSystemSupplier;
private final LogProvider logProvider; private final LogProvider logProvider;
private final Log log; private final Log log;
Expand Down Expand Up @@ -368,7 +371,7 @@ private long incrementalWithContext( String sourceHostNameOrIp, int sourcePort,
} }
catch ( MismatchingStoreIdException e ) catch ( MismatchingStoreIdException e )
{ {
throw new ExistingBackupWithDifferentStoreException( e ); throw new RuntimeException( DIFFERENT_STORE_MESSAGE, e );
} }
catch ( RuntimeException | IOException e ) catch ( RuntimeException | IOException e )
{ {
Expand Down
Expand Up @@ -70,15 +70,17 @@ PotentiallyErroneousState<BackupStrategyOutcome> doBackup( OnlineBackupContext o
private PotentiallyErroneousState<BackupStrategyOutcome> performBackupWithoutLifecycle( OnlineBackupContext onlineBackupContext ) private PotentiallyErroneousState<BackupStrategyOutcome> performBackupWithoutLifecycle( OnlineBackupContext onlineBackupContext )
{ {
File backupLocation = onlineBackupContext.getResolvedLocationFromName(); File backupLocation = onlineBackupContext.getResolvedLocationFromName();
PotentiallyErroneousState<BackupStageOutcome> state;
final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName(); final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName();
final OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress(); final OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress();
final Config config = onlineBackupContext.getConfig(); final Config config = onlineBackupContext.getConfig();

if ( backupCopyService.backupExists( backupLocation ) ) if ( backupCopyService.backupExists( backupLocation ) )
{ {
state = backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress ); PotentiallyErroneousState<BackupStageOutcome> state =
backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress );
boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() ); boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() );
boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() ); boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() );

if ( fullBackupWontWork || incrementalWasSuccessful ) if ( fullBackupWontWork || incrementalWasSuccessful )
{ {
return describeOutcome( state ); return describeOutcome( state );
Expand All @@ -95,28 +97,37 @@ private PotentiallyErroneousState<BackupStageOutcome> fullBackupWithTemporaryFol
{ {
final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName(); final File userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName();
File temporaryFullBackupLocation = backupCopyService.findAnAvailableLocationForNewFullBackup( userSpecifiedBackupLocation ); File temporaryFullBackupLocation = backupCopyService.findAnAvailableLocationForNewFullBackup( userSpecifiedBackupLocation );
PotentiallyErroneousState<BackupStageOutcome> state = PotentiallyErroneousState<BackupStageOutcome> state = backupStrategy.performFullBackup( temporaryFullBackupLocation, onlineBackupContext.getConfig(),
backupStrategy.performFullBackup( temporaryFullBackupLocation, onlineBackupContext.getConfig(), onlineBackupContext.getRequiredArguments().getAddress() );
onlineBackupContext.getRequiredArguments().getAddress() );


boolean aBackupAlreadyExisted = userSpecifiedBackupLocation.equals( temporaryFullBackupLocation ); boolean aBackupAlreadyExisted =
if ( BackupStageOutcome.SUCCESS.equals( state.getState() ) && !aBackupAlreadyExisted ) userSpecifiedBackupLocation.equals( temporaryFullBackupLocation ); // NOTE temporaryFullBackupLocation can be equal to desired

if ( BackupStageOutcome.SUCCESS.equals( state.getState() ) )
{ {
File newBackupLocationForPreExistingBackup = backupCopyService.findNewBackupLocationForBrokenExisting( userSpecifiedBackupLocation ); backupRecoveryService.recoverWithDatabase( temporaryFullBackupLocation, pageCache, config );
try if ( !aBackupAlreadyExisted )
{
backupCopyService.moveBackupLocation( userSpecifiedBackupLocation, newBackupLocationForPreExistingBackup );
backupCopyService.moveBackupLocation( temporaryFullBackupLocation, userSpecifiedBackupLocation );
}
catch ( CommandFailed commandFailed )
{ {
return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, commandFailed ); try
{
renameTemporaryBackupToExpected( temporaryFullBackupLocation, userSpecifiedBackupLocation );
}
catch ( CommandFailed commandFailed )
{
return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, commandFailed );
}
} }
} }
backupRecoveryService.recoverWithDatabase( userSpecifiedBackupLocation, pageCache, config );
return state; return state;
} }


private void renameTemporaryBackupToExpected( File temporaryFullBackupLocation, File userSpecifiedBackupLocation ) throws CommandFailed
{
File newBackupLocationForPreExistingBackup = backupCopyService.findNewBackupLocationForBrokenExisting( userSpecifiedBackupLocation );
backupCopyService.moveBackupLocation( userSpecifiedBackupLocation, newBackupLocationForPreExistingBackup );
backupCopyService.moveBackupLocation( temporaryFullBackupLocation, userSpecifiedBackupLocation );
}

private PotentiallyErroneousState<BackupStrategyOutcome> describeOutcome( PotentiallyErroneousState<BackupStageOutcome> strategyStageOutcome ) private PotentiallyErroneousState<BackupStrategyOutcome> describeOutcome( PotentiallyErroneousState<BackupStageOutcome> strategyStageOutcome )
{ {
BackupStageOutcome stageOutcome = strategyStageOutcome.getState(); BackupStageOutcome stageOutcome = strategyStageOutcome.getState();
Expand Down

This file was deleted.

Expand Up @@ -25,6 +25,7 @@
import org.neo4j.helpers.HostnamePort; import org.neo4j.helpers.HostnamePort;
import org.neo4j.helpers.OptionalHostnamePort; import org.neo4j.helpers.OptionalHostnamePort;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.store.MismatchingStoreIdException;
import org.neo4j.kernel.lifecycle.LifecycleAdapter; import org.neo4j.kernel.lifecycle.LifecycleAdapter;


public class HaBackupStrategy extends LifecycleAdapter implements BackupStrategy public class HaBackupStrategy extends LifecycleAdapter implements BackupStrategy
Expand All @@ -50,10 +51,9 @@ public PotentiallyErroneousState<BackupStageOutcome> performIncrementalBackup( F
config ); config );
return new PotentiallyErroneousState<>( BackupStageOutcome.SUCCESS, null ); return new PotentiallyErroneousState<>( BackupStageOutcome.SUCCESS, null );
} }
catch ( ExistingBackupWithDifferentStoreException e ) catch ( MismatchingStoreIdException e )
{ {
ExistingBackupWithDifferentStoreException exceptionWithFilename = new ExistingBackupWithDifferentStoreException( e, backupDestination ); return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, e );
return new PotentiallyErroneousState<>( BackupStageOutcome.UNRECOVERABLE_FAILURE, exceptionWithFilename );
} }
catch ( IncrementalBackupNotPossibleException e ) catch ( IncrementalBackupNotPossibleException e )
{ {
Expand Down
26 changes: 14 additions & 12 deletions enterprise/backup/src/main/java/org/neo4j/backup/OnlineBackup.java
Expand Up @@ -99,8 +99,8 @@ public OnlineBackup backup( String targetDirectory )
*/ */
public OnlineBackup backup( File targetDirectory ) public OnlineBackup backup( File targetDirectory )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, getConsistencyCheck( true ),
getConsistencyCheck( true ), defaultConfig(), timeoutMillis, forensics ); defaultConfig(), timeoutMillis, forensics );
return this; return this;
} }


Expand Down Expand Up @@ -133,8 +133,9 @@ public OnlineBackup backup( String targetDirectory, boolean verification )
*/ */
public OnlineBackup backup( File targetDirectory, boolean verification ) public OnlineBackup backup( File targetDirectory, boolean verification )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, outcome =
getConsistencyCheck( verification ), defaultConfig(), timeoutMillis, forensics ); new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, getConsistencyCheck( verification ),
defaultConfig(), timeoutMillis, forensics );
return this; return this;
} }


Expand Down Expand Up @@ -166,8 +167,8 @@ public OnlineBackup backup( String targetDirectory, Config tuningConfiguration )
*/ */
public OnlineBackup backup( File targetDirectory, Config tuningConfiguration ) public OnlineBackup backup( File targetDirectory, Config tuningConfiguration )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, getConsistencyCheck( true ),
getConsistencyCheck( true ), tuningConfiguration, timeoutMillis, forensics ); tuningConfiguration, timeoutMillis, forensics );
return this; return this;
} }


Expand Down Expand Up @@ -202,8 +203,9 @@ public OnlineBackup backup( String targetDirectory, Config tuningConfiguration,
*/ */
public OnlineBackup backup( File targetDirectory, Config tuningConfiguration, boolean verification ) public OnlineBackup backup( File targetDirectory, Config tuningConfiguration, boolean verification )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, outcome =
getConsistencyCheck( verification ), tuningConfiguration, timeoutMillis, forensics ); new BackupProtocolService( out ).doIncrementalBackupOrFallbackToFull( hostNameOrIp, port, targetDirectory, getConsistencyCheck( verification ),
tuningConfiguration, timeoutMillis, forensics );
return this; return this;
} }


Expand Down Expand Up @@ -308,8 +310,8 @@ public OnlineBackup full( String targetDirectory, boolean verification, Config t
@Deprecated @Deprecated
public OnlineBackup incremental( String targetDirectory ) public OnlineBackup incremental( String targetDirectory )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackup( hostNameOrIp, port, new File( targetDirectory ), outcome = new BackupProtocolService( out ).doIncrementalBackup( hostNameOrIp, port, new File( targetDirectory ), getConsistencyCheck( false ),
getConsistencyCheck( false ), timeoutMillis, defaultConfig() ); timeoutMillis, defaultConfig() );
return this; return this;
} }


Expand All @@ -330,8 +332,8 @@ public OnlineBackup incremental( String targetDirectory )
@Deprecated @Deprecated
public OnlineBackup incremental( String targetDirectory, boolean verification ) public OnlineBackup incremental( String targetDirectory, boolean verification )
{ {
outcome = new BackupProtocolService( out ).doIncrementalBackup( hostNameOrIp, port, new File( targetDirectory ), outcome = new BackupProtocolService( out ).doIncrementalBackup( hostNameOrIp, port, new File( targetDirectory ), getConsistencyCheck( verification ),
getConsistencyCheck( verification ), timeoutMillis, defaultConfig() ); timeoutMillis, defaultConfig() );
return this; return this;
} }


Expand Down
Expand Up @@ -55,13 +55,11 @@
import org.neo4j.io.fs.FileUtils; import org.neo4j.io.fs.FileUtils;
import org.neo4j.io.pagecache.IOLimiter; import org.neo4j.io.pagecache.IOLimiter;
import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.impl.muninn.MuninnPageCache;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.configuration.Settings; import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings; import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings;
import org.neo4j.kernel.impl.factory.DatabaseInfo; import org.neo4j.kernel.impl.factory.DatabaseInfo;
import org.neo4j.kernel.impl.logging.LogService; import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.pagecache.ConfigurableStandalonePageCacheFactory;
import org.neo4j.kernel.impl.spi.SimpleKernelContext; import org.neo4j.kernel.impl.spi.SimpleKernelContext;
import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.store.MetaDataStore.Position; import org.neo4j.kernel.impl.store.MetaDataStore.Position;
Expand Down Expand Up @@ -1011,7 +1009,7 @@ public void incrementalBackupShouldFailWhenTargetDirContainsDifferentStore() thr
catch ( RuntimeException e ) catch ( RuntimeException e )
{ {
// Then // Then
assertThat( e.getMessage(), equalTo( ExistingBackupWithDifferentStoreException.DIFFERENT_STORE_MESSAGE ) ); assertThat( e.getMessage(), equalTo( BackupProtocolService.DIFFERENT_STORE_MESSAGE ) );
assertThat( e.getCause(), instanceOf( MismatchingStoreIdException.class ) ); assertThat( e.getCause(), instanceOf( MismatchingStoreIdException.class ) );
} }
} }
Expand Down

0 comments on commit 294b3ed

Please sign in to comment.