Skip to content

Commit

Permalink
Backup?
Browse files Browse the repository at this point in the history
  • Loading branch information
klaren authored and MishaDemianenko committed Jul 23, 2018
1 parent 28e97d2 commit 8be58b7
Show file tree
Hide file tree
Showing 23 changed files with 417 additions and 561 deletions.
Expand Up @@ -81,7 +81,7 @@ public static boolean isSameOrChildPath( Path parent, Path candidate )
public static void checkLock( Path databaseDirectory ) throws CommandFailed
{
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.toFile() ) )
StoreLocker storeLocker = new GlobalStoreLocker( fileSystem, databaseDirectory.getParent().toFile() ) )
{
storeLocker.checkLock();
}
Expand Down
Expand Up @@ -114,8 +114,8 @@ public class GraphDatabaseSettings implements LoadableConfig

@Internal
public static final Setting<File> database_path = derivedSetting( "unsupported.dbms.directories.database",
data_directory, active_database,
( data, current ) -> new File( new File( data, "databases" ), current ),
data_directory,
data -> new File( data, "databases" ),
PATH );

@Title( "Read only database" )
Expand Down
Expand Up @@ -102,13 +102,13 @@ static List<File> matchingFiles( File fileWithRegexInName )
}
};

public static final Validator<File> CONTAINS_EXISTING_DATABASE = value ->
public static final Validator<File> CONTAINS_EXISTING_DATABASE = dbDir ->
{
try ( FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction() )
{
if ( !isExistingDatabase( fileSystem, value ) )
if ( !isExistingDatabase( fileSystem, dbDir ) )
{
throw new IllegalArgumentException( "Directory '" + value + "' does not contain a database" );
throw new IllegalArgumentException( "Directory '" + dbDir + "' does not contain a database" );
}
}
catch ( IOException e )
Expand All @@ -117,9 +117,9 @@ static List<File> matchingFiles( File fileWithRegexInName )
}
};

private static boolean isExistingDatabase( FileSystemAbstraction fileSystem, File value )
private static boolean isExistingDatabase( FileSystemAbstraction fileSystem, File dbDir )
{
return fileSystem.fileExists( new File( value, StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ) );
return fileSystem.fileExists( new File( dbDir, StoreFileType.STORE.augment( MetaDataStore.DEFAULT_NAME ) ) );
}

public static Validator<String> inList( String[] validStrings )
Expand Down
Expand Up @@ -110,7 +110,7 @@ StoreLockException unableToObtainLockException()
return storeLockException( message, null );
}

private StoreLockException storeLockException( String message, Exception e )
private static StoreLockException storeLockException( String message, Exception e )
{
String help = "Please ensure no other process is using this database, and that the directory is writable " +
"(required even for read-only access)";
Expand Down
Expand Up @@ -48,18 +48,15 @@ class BackupCopyService
private static final int MAX_OLD_BACKUPS = 1000;

private final FileSystemAbstraction fs;
private final PageCache pageCache;
private final FileMoveProvider fileMoveProvider;

BackupCopyService( FileSystemAbstraction fs, PageCache pageCache,
FileMoveProvider fileMoveProvider )
BackupCopyService( FileSystemAbstraction fs, FileMoveProvider fileMoveProvider )
{
this.fs = fs;
this.pageCache = pageCache;
this.fileMoveProvider = fileMoveProvider;
}

public void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOException
void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOException
{
try
{
Expand All @@ -77,7 +74,7 @@ public void moveBackupLocation( Path oldLocation, Path newLocation ) throws IOEx
}
}

public void clearIdFiles( Path backupLocation ) throws IOException
void clearIdFiles( Path backupLocation ) throws IOException
{
IOException exception = null;
File targetDirectory = backupLocation.toFile();
Expand Down
Expand Up @@ -210,11 +210,13 @@ private BackupOutcome incrementalBackup( FileSystemAbstraction fileSystem, Strin
throw new RuntimeException( targetDirectory + " doesn't contain a database" );
}

Map<String,String> temporaryDbConfig = getTemporaryDbConfig();
Path storeDir = targetDirectory.getParent();

Map<String,String> temporaryDbConfig = getTemporaryDbConfig( storeDir, targetDirectory.getFileName().toString() );
config.augment( temporaryDbConfig );

Map<String,String> configParams = config.getRaw();
GraphDatabaseAPI targetDb = startTemporaryDb( targetDirectory, pageCache, configParams );
GraphDatabaseAPI targetDb = startTemporaryDb( storeDir, pageCache, configParams );
long backupStartTime = System.currentTimeMillis();
long lastCommittedTx;
try
Expand Down Expand Up @@ -255,13 +257,15 @@ private boolean checkDbConsistency( FileSystemAbstraction fileSystem, Path targe
return consistent;
}

private Map<String,String> getTemporaryDbConfig()
private static Map<String,String> getTemporaryDbConfig( Path storeDir, String databaseName )
{
Map<String,String> tempDbConfig = new HashMap<>();
tempDbConfig.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.FALSE );
// In case someone deleted the logical log from a full backup
tempDbConfig.put( GraphDatabaseSettings.keep_logical_logs.name(), Settings.TRUE );
tempDbConfig.put( GraphDatabaseSettings.pagecache_warmup_enabled.name(), Settings.FALSE );
tempDbConfig.put( GraphDatabaseSettings.active_database.name(), databaseName );
tempDbConfig.put( GraphDatabaseSettings.database_path.name(), storeDir.toString() );
return tempDbConfig;
}

Expand Down Expand Up @@ -328,19 +332,19 @@ public BackupOutcome doIncrementalBackup( String sourceHostNameOrIp, int sourceP
return new BackupOutcome( lastCommittedTransaction, true );
}

private RequestContext slaveContextOf( GraphDatabaseAPI graphDb )
private static RequestContext slaveContextOf( GraphDatabaseAPI graphDb )
{
TransactionIdStore transactionIdStore = graphDb.getDependencyResolver().resolveDependency(
TransactionIdStore.class );
return anonymous( transactionIdStore.getLastCommittedTransactionId() );
}

private boolean directoryContainsDb( Path targetDirectory )
private static boolean directoryContainsDb( Path targetDirectory )
{
return Files.isRegularFile( targetDirectory.resolve( MetaDataStore.DEFAULT_NAME ) );
}

private boolean directoryIsEmpty( Path dir ) throws IOException
private static boolean directoryIsEmpty( Path dir ) throws IOException
{
return Files.notExists( dir ) || Files.isDirectory( dir ) && FileUtils.countFilesInDirectoryPath( dir ) == 0;
}
Expand Down
Expand Up @@ -68,7 +68,7 @@ BackupStrategyCoordinator backupStrategyCoordinator(
BackupDelegator backupDelegator, PageCache pageCache )
{
FileSystemAbstraction fs = outsideWorld.fileSystem();
BackupCopyService copyService = new BackupCopyService( fs, pageCache, new FileMoveProvider( fs ) );
BackupCopyService copyService = new BackupCopyService( fs, new FileMoveProvider( fs ) );
ProgressMonitorFactory progressMonitorFactory = ProgressMonitorFactory.textual( outsideWorld.errorStream() );
BackupRecoveryService recoveryService = new BackupRecoveryService();
long timeout = onlineBackupContext.getRequiredArguments().getTimeout();
Expand Down
Expand Up @@ -25,6 +25,7 @@
import java.io.IOException;
import java.nio.file.Path;

import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.impl.util.OptionalHostnamePort;
Expand Down Expand Up @@ -78,7 +79,6 @@ private Fallible<BackupStrategyOutcome> performBackupWithoutLifecycle(
OnlineBackupContext onlineBackupContext )
{
Path backupLocation = onlineBackupContext.getResolvedLocationFromName();
Path userSpecifiedBackupLocation = onlineBackupContext.getResolvedLocationFromName();
OptionalHostnamePort userSpecifiedAddress = onlineBackupContext.getRequiredArguments().getAddress();
log.debug( "User specified address is %s:%s", userSpecifiedAddress.getHostname().toString(), userSpecifiedAddress.getPort().toString() );
Config config = onlineBackupContext.getConfig();
Expand All @@ -88,7 +88,7 @@ private Fallible<BackupStrategyOutcome> performBackupWithoutLifecycle(
{
log.info( "Previous backup found, trying incremental backup." );
Fallible<BackupStageOutcome> state =
backupStrategy.performIncrementalBackup( userSpecifiedBackupLocation, config, userSpecifiedAddress );
backupStrategy.performIncrementalBackup( backupLocation, config, userSpecifiedAddress );
boolean fullBackupWontWork = BackupStageOutcome.WRONG_PROTOCOL.equals( state.getState() );
boolean incrementalWasSuccessful = BackupStageOutcome.SUCCESS.equals( state.getState() );

Expand Down Expand Up @@ -146,12 +146,13 @@ private Fallible<BackupStageOutcome> fullBackupWithTemporaryFolderResolutions(
Fallible<BackupStageOutcome> state = backupStrategy.performFullBackup( temporaryFullBackupLocation, config, address );

// NOTE temporaryFullBackupLocation can be equal to desired
boolean aBackupAlreadyExisted = userSpecifiedBackupLocation.equals( temporaryFullBackupLocation );
boolean aBackupAlreadyExisted = !userSpecifiedBackupLocation.equals( temporaryFullBackupLocation );

if ( BackupStageOutcome.SUCCESS.equals( state.getState() ) )
{
backupRecoveryService.recoverWithDatabase( temporaryFullBackupLocation, pageCache, config );
if ( !aBackupAlreadyExisted )
config.augment( GraphDatabaseSettings.active_database, temporaryFullBackupLocation.getFileName().toString() ); // TODO: Refactor
backupRecoveryService.recoverWithDatabase( temporaryFullBackupLocation.getParent(), pageCache, config ); // TODO: getParent()
if ( aBackupAlreadyExisted )
{
try
{
Expand Down
Expand Up @@ -41,56 +41,55 @@
public class RestoreDatabaseCommand
{
private FileSystemAbstraction fs;
private final File fromPath;
private final File databaseDir;
private final File fromDatabasePath;
private final File toDatabaseDir;
private final File transactionLogsDirectory;
private String databaseName;
private String toDatabaseName;
private boolean forceOverwrite;

public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromPath, Config config, String databaseName,
public RestoreDatabaseCommand( FileSystemAbstraction fs, File fromDatabasePath, Config config, String toDatabaseName,
boolean forceOverwrite )
{
this.fs = fs;
this.fromPath = fromPath;
this.databaseName = databaseName;
this.fromDatabasePath = fromDatabasePath;
this.forceOverwrite = forceOverwrite;
this.databaseDir = config.get( database_path ).getAbsoluteFile();
this.toDatabaseName = toDatabaseName;
this.toDatabaseDir = new File(config.get( database_path ), toDatabaseName ).getAbsoluteFile();
this.transactionLogsDirectory = config.get( GraphDatabaseSettings.logical_logs_location ).getAbsoluteFile();
}

public void execute() throws IOException, CommandFailed
{
if ( !fs.fileExists( fromPath ) )
if ( !fs.fileExists( fromDatabasePath ) )
{
throw new IllegalArgumentException( format( "Source directory does not exist [%s]", fromPath ) );
throw new IllegalArgumentException( format( "Source directory does not exist [%s]", fromDatabasePath ) );
}

try
{
Validators.CONTAINS_EXISTING_DATABASE.validate( fromPath );
Validators.CONTAINS_EXISTING_DATABASE.validate( fromDatabasePath );
}
catch ( IllegalArgumentException e )
{
throw new IllegalArgumentException(
format( "Source directory is not a database backup [%s]", fromPath ) );
format( "Source directory is not a database backup [%s]", fromDatabasePath ) );
}

if ( fs.fileExists( databaseDir ) && !forceOverwrite )
if ( fs.fileExists( toDatabaseDir ) && !forceOverwrite )
{
throw new IllegalArgumentException( format( "Database with name [%s] already exists at %s",
databaseName, databaseDir ) );
throw new IllegalArgumentException( format( "Database with name [%s] already exists at %s", toDatabaseName, toDatabaseDir ) );
}

checkLock( databaseDir.toPath() );
checkLock( toDatabaseDir.toPath() );

fs.deleteRecursively( databaseDir );
fs.deleteRecursively( toDatabaseDir );

if ( !isSameOrChildFile( databaseDir, transactionLogsDirectory ) )
if ( !isSameOrChildFile( toDatabaseDir, transactionLogsDirectory ) )
{
fs.deleteRecursively( transactionLogsDirectory );
}
LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromPath, fs ).build();
restoreDatabaseFiles( backupLogFiles, fromPath.listFiles() );
LogFiles backupLogFiles = LogFilesBuilder.logFilesBasedOnlyBuilder( fromDatabasePath, fs ).build();
restoreDatabaseFiles( backupLogFiles, fromDatabasePath.listFiles() );
}

private void restoreDatabaseFiles( LogFiles backupLogFiles, File[] files ) throws IOException
Expand All @@ -101,13 +100,13 @@ private void restoreDatabaseFiles( LogFiles backupLogFiles, File[] files ) throw
{
if ( file.isDirectory() )
{
File destination = new File( databaseDir, file.getName() );
File destination = new File( toDatabaseDir, file.getName() );
fs.mkdirs( destination );
fs.copyRecursively( file, destination );
}
else
{
fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : databaseDir );
fs.copyToDirectory( file, backupLogFiles.isLogFile( file ) ? transactionLogsDirectory : toDatabaseDir );
}
}
}
Expand Down

0 comments on commit 8be58b7

Please sign in to comment.