Skip to content

Commit

Permalink
Backup?
Browse files Browse the repository at this point in the history
  • Loading branch information
@klaren @MishaDemianenko
klaren authored and MishaDemianenko committed Jul 23, 2018
1 parent 28e97d2 commit 8be58b7
Show file tree
Hide file tree
Showing 23 changed files with 417 additions and 561 deletions.
2 changes: 1 addition & 1 deletion community/command-line/src/main/java/org/neo4j/commandline/Util.java
View file
Expand Up @@ -81,7 +81,7 @@ public static boolean isSameOrChildPath( Path parent, Path candidate )
public static void checkLock( Path databaseDirectory ) throws CommandFailed public static void checkLock( Path databaseDirectory ) throws CommandFailed
{ {
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction(); try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.toFile() ) ) StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.getParent().toFile() ) )
{ {
storeLocker.checkLock(); storeLocker.checkLock();
} }
Expand Down
4 changes: 2 additions & 2 deletions community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java
View file
Expand Up @@ -114,8 +114,8 @@ public class GraphDatabaseSettings implements LoadableConfig


@Internal @Internal
public static final Setting<File> database_path = derivedSetting( "unsupported.dbms.directories.database", public static final Setting<File> database_path = derivedSetting( "unsupported.dbms.directories.database",
data_directory, active_database, data_directory,
( data, current ) -> new File( new File( data, "databases" ), current ), data -> new File( data, "databases" ),
PATH ); PATH );


@Title( "Read only database" ) @Title( "Read only database" )
Expand Down
10 changes: 5 additions & 5 deletions community/kernel/src/main/java/org/neo4j/kernel/impl/util/Validators.java
View file
Expand Up @@ -102,13 +102,13 @@ static List<File> matchingFiles( File fileWithRegexInName )
} }
}; };


public static final Validator<File> CONTAINS_EXISTING_DATABASE = value -> public static final Validator<File> CONTAINS_EXISTING_DATABASE = dbDir ->
{ {
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction() ) try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction() )
{ {
if ( !isExistingDatabase( fileSystem, value ) ) if ( !isExistingDatabase( fileSystem, dbDir ) )
{ {
throw new IllegalArgumentException( "Directory '" + value + "' does not contain a database" ); throw new IllegalArgumentException( "Directory '" + dbDir + "' does not contain a database" );
} }
} }
catch ( IOException e ) catch ( IOException e )
Expand All @@ -117,9 +117,9 @@ static List<File> matchingFiles( File fileWithRegexInName )
} }
}; };


private static boolean isExistingDatabase( FileSystemAbstraction fileSystem, File value ) private static boolean isExistingDatabase( FileSystemAbstraction fileSystem, File dbDir )
{ {
return fileSystem.fileExists( new File( value, StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ) ); return fileSystem.fileExists( new File( dbDir, StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ) );
} }


public static Validator<String> inList( String[] validStrings ) public static Validator<String> inList( String[] validStrings )
Expand Down
2 changes: 1 addition & 1 deletion community/kernel/src/main/java/org/neo4j/kernel/internal/locker/StoreLocker.java
View file
Expand Up @@ -110,7 +110,7 @@ StoreLockException unableToObtainLockException()
return storeLockException( message, null ); return storeLockException( message, null );
} }


private StoreLockException storeLockException( String message, Exception e ) private static StoreLockException storeLockException( String message, Exception e )
{ {
String help = "Please ensure no other process is using this database, and that the directory is writable " + String help = "Please ensure no other process is using this database, and that the directory is writable " +
"(required even for read-only access)"; "(required even for read-only access)";
Expand Down
9 changes: 3 additions & 6 deletions enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupCopyService.java
View file
Expand Up @@ -48,18 +48,15 @@ class BackupCopyService
private static final int MAX_OLD_BACKUPS = 1000; private static final int MAX_OLD_BACKUPS = 1000;


private final FileSystemAbstraction fs; private final FileSystemAbstraction fs;
private final PageCache pageCache;
private final FileMoveProvider fileMoveProvider; private final FileMoveProvider fileMoveProvider;


BackupCopyService( FileSystemAbstraction fs, PageCache pageCache, BackupCopyService( FileSystemAbstraction fs, FileMoveProvider fileMoveProvider )
FileMoveProvider fileMoveProvider )
{ {
this.fs = fs; this.fs = fs;
this.pageCache = pageCache;
this.fileMoveProvider = fileMoveProvider; this.fileMoveProvider = fileMoveProvider;
} }


public void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOException void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOException
{ {
try try
{ {
Expand All @@ -77,7 +74,7 @@ public void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOEx
} }
} }


public void clearIdFiles( Path backupLocation ) throws IOException void clearIdFiles( Path backupLocation ) throws IOException
{ {
IOException exception = null; IOException exception = null;
File targetDirectory = backupLocation.toFile(); File targetDirectory = backupLocation.toFile();
Expand Down
16 changes: 10 additions & 6 deletions enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupProtocolService.java
View file
Expand Up @@ -210,11 +210,13 @@ private BackupOutcome incrementalBackup( FileSystemAbstraction fileSystem, Strin
throw new RuntimeException( targetDirectory + " doesn't contain a database" ); throw new RuntimeException( targetDirectory + " doesn't contain a database" );
} }


Map<String,String> temporaryDbConfig = getTemporaryDbConfig(); Path storeDir = targetDirectory.getParent();

Map<String,String> temporaryDbConfig = getTemporaryDbConfig( storeDir, targetDirectory.getFileName().toString() );
config.augment( temporaryDbConfig ); config.augment( temporaryDbConfig );


Map<String,String> configParams = config.getRaw(); Map<String,String> configParams = config.getRaw();
GraphDatabaseAPI targetDb = startTemporaryDb( targetDirectory, pageCache, configParams ); GraphDatabaseAPI targetDb = startTemporaryDb( storeDir, pageCache, configParams );
long backupStartTime = System.currentTimeMillis(); long backupStartTime = System.currentTimeMillis();
long lastCommittedTx; long lastCommittedTx;
try try
Expand Down Expand Up @@ -255,13 +257,15 @@ private boolean checkDbConsistency( FileSystemAbstraction fileSystem, Path targe
return consistent; return consistent;
} }


private Map<String,String> getTemporaryDbConfig() private static Map<String,String> getTemporaryDbConfig( Path storeDir, String databaseName )
{ {
Map<String,String> tempDbConfig = new HashMap<>(); Map<String,String> tempDbConfig = new HashMap<>();
tempDbConfig.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.FALSE ); tempDbConfig.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.FALSE );
// In case someone deleted the logical log from a full backup // In case someone deleted the logical log from a full backup
tempDbConfig.put( GraphDatabaseSettings.keep_logical_logs.name(), Settings.TRUE ); tempDbConfig.put( GraphDatabaseSettings.keep_logical_logs.name(), Settings.TRUE );
tempDbConfig.put( GraphDatabaseSettings.pagecache_warmup_enabled.name(), Settings.FALSE ); tempDbConfig.put( GraphDatabaseSettings.pagecache_warmup_enabled.name(), Settings.FALSE );
tempDbConfig.put( GraphDatabaseSettings.active_database.name(), databaseName );
tempDbConfig.put( GraphDatabaseSettings.database_path.name(), storeDir.toString() );
return tempDbConfig; return tempDbConfig;
} }


Expand Down Expand Up @@ -328,19 +332,19 @@ public BackupOutcome doIncrementalBackup( String sourceHostNameOrIp, int sourceP
return new BackupOutcome( lastCommittedTransaction, true ); return new BackupOutcome( lastCommittedTransaction, true );
} }


private RequestContext slaveContextOf( GraphDatabaseAPI graphDb ) private static RequestContext slaveContextOf( GraphDatabaseAPI graphDb )
{ {
TransactionIdStore transactionIdStore = graphDb.getDependencyResolver().resolveDependency( TransactionIdStore transactionIdStore = graphDb.getDependencyResolver().resolveDependency(
TransactionIdStore.class ); TransactionIdStore.class );
return anonymous( transactionIdStore.getLastCommittedTransactionId() ); return anonymous( transactionIdStore.getLastCommittedTransactionId() );
} }


private boolean directoryContainsDb( Path targetDirectory ) private static boolean directoryContainsDb( Path targetDirectory )
{ {
return Files.isRegularFile( targetDirectory.resolve( MetaDataStore.DEFAULT_NAME ) ); return Files.isRegularFile( targetDirectory.resolve( MetaDataStore.DEFAULT_NAME ) );
} }


private boolean directoryIsEmpty( Path dir ) throws IOException private static boolean directoryIsEmpty( Path dir ) throws IOException
{ {
return Files.notExists( dir ) || Files.isDirectory( dir ) && FileUtils.countFilesInDirectoryPath( dir ) == 0; return Files.notExists( dir ) || Files.isDirectory( dir ) && FileUtils.countFilesInDirectoryPath( dir ) == 0;
} }
Expand Down
2 changes: 1 addition & 1 deletion enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyCoordinatorFactory.java
View file
Expand Up @@ -68,7 +68,7 @@ BackupStrategyCoordinator backupStrategyCoordinator(
BackupDelegator backupDelegator, PageCache pageCache ) BackupDelegator backupDelegator, PageCache pageCache )
{ {
FileSystemAbstraction fs = outsideWorld.fileSystem(); FileSystemAbstraction fs = outsideWorld.fileSystem();
BackupCopyService copyService = new BackupCopyService( fs, pageCache, new FileMoveProvider( fs ) ); BackupCopyService copyService = new BackupCopyService( fs, new FileMoveProvider( fs ) );
ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() ); ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() );
BackupRecoveryService recoveryService = new BackupRecoveryService(); BackupRecoveryService recoveryService = new BackupRecoveryService();
long timeout = onlineBackupContext.getRequiredArguments().getTimeout(); long timeout = onlineBackupContext.getRequiredArguments().getTimeout();
Expand Down
11 changes: 6 additions & 5 deletions enterprise/backup/src/main/java/org/neo4j/backup/impl/BackupStrategyWrapper.java
View file
Expand Up @@ -25,6 +25,7 @@
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;


import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.pagecache.PageCache; import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.util.OptionalHostnamePort; import org.neo4j.kernel.impl.util.OptionalHostnamePort;
Expand Down Expand Up @@ -78,7 +79,6 @@ private Fallible<BackupStrategyOutcome> performBackupWithoutLifecycle(
OnlineBackupContext onlineBackupContext ) OnlineBackupContext onlineBackupContext )
{ {
Path backupLocation = onlineBackupContext.getResolvedLocationFromName(); Path backupLocation = onlineBackupContext.getResolvedLocationFromName();
Path userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName();
OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress(); OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress();
log.debug( "User specified address is %s:%s", userSpecifiedAddress.getHostname().toString(), userSpecifiedAddress.getPort().toString() ); log.debug( "User specified address is %s:%s", userSpecifiedAddress.getHostname().toString(), userSpecifiedAddress.getPort().toString() );
Config config = onlineBackupContext.getConfig(); Config config = onlineBackupContext.getConfig();
Expand All @@ -88,7 +88,7 @@ private Fallible<BackupStrategyOutcome> performBackupWithoutLifecycle(
{ {
log.info( "Previous backup found, trying incremental backup." ); log.info( "Previous backup found, trying incremental backup." );
Fallible<BackupStageOutcome> state = Fallible<BackupStageOutcome> state =
backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress ); backupStrategy.performIncrementalBackup( backupLocation, config, userSpecifiedAddress );
boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() ); boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() );
boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() ); boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() );


Expand Down Expand Up @@ -146,12 +146,13 @@ private Fallible<BackupStageOutcome> fullBackupWithTemporaryFolderResolutions(
Fallible<BackupStageOutcome> state = backupStrategy.performFullBackup( temporaryFullBackupLocation, config, address ); Fallible<BackupStageOutcome> state = backupStrategy.performFullBackup( temporaryFullBackupLocation, config, address );


// NOTE temporaryFullBackupLocation can be equal to desired // NOTE temporaryFullBackupLocation can be equal to desired
boolean aBackupAlreadyExisted = userSpecifiedBackupLocation.equals( temporaryFullBackupLocation ); boolean aBackupAlreadyExisted = !userSpecifiedBackupLocation.equals( temporaryFullBackupLocation );


if ( BackupStageOutcome.SUCCESS.equals( state.getState() ) ) if ( BackupStageOutcome.SUCCESS.equals( state.getState() ) )
{ {
backupRecoveryService.recoverWithDatabase( temporaryFullBackupLocation, pageCache, config ); config.augment( GraphDatabaseSettings.active_database, temporaryFullBackupLocation.getFileName().toString() ); // TODO: Refactor
if ( !aBackupAlreadyExisted ) backupRecoveryService.recoverWithDatabase( temporaryFullBackupLocation.getParent(), pageCache, config ); // TODO: getParent()
if ( aBackupAlreadyExisted )
{ {
try try
{ {
Expand Down
41 changes: 20 additions & 21 deletions enterprise/backup/src/main/java/org/neo4j/restore/RestoreDatabaseCommand.java
View file
Expand Up @@ -41,56 +41,55 @@
public class RestoreDatabaseCommand public class RestoreDatabaseCommand
{ {
private FileSystemAbstraction fs; private FileSystemAbstraction fs;
private final File fromPath; private final File fromDatabasePath;
private final File databaseDir; private final File toDatabaseDir;
private final File transactionLogsDirectory; private final File transactionLogsDirectory;
private String databaseName; private String toDatabaseName;
private boolean forceOverwrite; private boolean forceOverwrite;


public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, String databaseName, public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromDatabasePath, Config config, String toDatabaseName,
boolean forceOverwrite ) boolean forceOverwrite )
{ {
this.fs = fs; this.fs = fs;
this.fromPath = fromPath; this.fromDatabasePath = fromDatabasePath;
this.databaseName = databaseName;
this.forceOverwrite = forceOverwrite; this.forceOverwrite = forceOverwrite;
this.databaseDir = config.get( database_path ).getAbsoluteFile(); this.toDatabaseName = toDatabaseName;
this.toDatabaseDir = new File(config.get( database_path ), toDatabaseName ).getAbsoluteFile();
this.transactionLogsDirectory = config.get( GraphDatabaseSettings.logical_logs_location ).getAbsoluteFile(); this.transactionLogsDirectory = config.get( GraphDatabaseSettings.logical_logs_location ).getAbsoluteFile();
} }


public void execute() throws IOException, CommandFailed public void execute() throws IOException, CommandFailed
{ {
if ( !fs.fileExists( fromPath ) ) if ( !fs.fileExists( fromDatabasePath ) )
{ {
throw new IllegalArgumentException( format( "Source directory does not exist [%s]", fromPath ) ); throw new IllegalArgumentException( format( "Source directory does not exist [%s]", fromDatabasePath ) );
} }


try try
{ {
Validators.CONTAINS_EXISTING_DATABASE.validate( fromPath ); Validators.CONTAINS_EXISTING_DATABASE.validate( fromDatabasePath );
} }
catch ( IllegalArgumentException e ) catch ( IllegalArgumentException e )
{ {
throw new IllegalArgumentException( throw new IllegalArgumentException(
format( "Source directory is not a database backup [%s]", fromPath ) ); format( "Source directory is not a database backup [%s]", fromDatabasePath ) );
} }


if ( fs.fileExists( databaseDir ) && !forceOverwrite ) if ( fs.fileExists( toDatabaseDir ) && !forceOverwrite )
{ {
throw new IllegalArgumentException( format( "Database with name [%s] already exists at %s", throw new IllegalArgumentException( format( "Database with name [%s] already exists at %s", toDatabaseName, toDatabaseDir ) );
databaseName, databaseDir ) );
} }


checkLock( databaseDir.toPath() ); checkLock( toDatabaseDir.toPath() );


fs.deleteRecursively( databaseDir ); fs.deleteRecursively( toDatabaseDir );


if ( !isSameOrChildFile( databaseDir, transactionLogsDirectory ) ) if ( !isSameOrChildFile( toDatabaseDir, transactionLogsDirectory ) )
{ {
fs.deleteRecursively( transactionLogsDirectory ); fs.deleteRecursively( transactionLogsDirectory );
} }
LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fs ).build(); LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromDatabasePath, fs ).build();
restoreDatabaseFiles( backupLogFiles, fromPath.listFiles() ); restoreDatabaseFiles( backupLogFiles, fromDatabasePath.listFiles() );
} }


private void restoreDatabaseFiles( LogFiles backupLogFiles, File[] files ) throws IOException private void restoreDatabaseFiles( LogFiles backupLogFiles, File[] files ) throws IOException
Expand All @@ -101,13 +100,13 @@ private void restoreDatabaseFiles( LogFiles backupLogFiles, File[] files ) throw
{ {
if ( file.isDirectory() ) if ( file.isDirectory() )
{ {
File destination = new File( databaseDir, file.getName() ); File destination = new File( toDatabaseDir, file.getName() );
fs.mkdirs( destination ); fs.mkdirs( destination );
fs.copyRecursively( file, destination ); fs.copyRecursively( file, destination );
} }
else else
{ {
fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : databaseDir ); fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : toDatabaseDir );
} }
} }
} }
Expand Down

0 comments on commit 8be58b7

Please sign in to comment.