Skip to content

Commit

Permalink
Add ConsistencyChecker argument to backuptool
Browse files Browse the repository at this point in the history
Backuptool can now take an argument "-consistency-checker", which specifies which consistency checker to use.
  • Loading branch information
spacecowboy committed Oct 1, 2015
1 parent c2ca785 commit e22a7be
Show file tree
Hide file tree
Showing 9 changed files with 365 additions and 105 deletions.
31 changes: 13 additions & 18 deletions enterprise/backup/src/main/java/org/neo4j/backup/BackupService.java
Expand Up @@ -37,8 +37,6 @@
import org.neo4j.com.storecopy.StoreCopyClient;
import org.neo4j.com.storecopy.StoreWriter;
import org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker;
import org.neo4j.consistency.ConsistencyCheckService;
import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException;
import org.neo4j.graphdb.DependencyResolver;
import org.neo4j.graphdb.factory.GraphDatabaseFactory;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
Expand Down Expand Up @@ -126,15 +124,14 @@ public boolean isConsistent()
}

BackupOutcome doFullBackup( final String sourceHostNameOrIp, final int sourcePort, File targetDirectory,
boolean checkConsistency, Config tuningConfiguration, final long timeout, final boolean forensics )
ConsistencyCheck consistencyCheck, Config tuningConfiguration, final long timeout, final boolean forensics )
{
if ( directoryContainsDb( targetDirectory ) )
{
throw new RuntimeException( targetDirectory + " already contains a database" );
}
long timestamp = System.currentTimeMillis();
long lastCommittedTx = -1;
boolean consistent = !checkConsistency; // default to true if we're not checking consistency
try ( PageCache pageCache = createPageCache( fileSystem ) )
{
StoreCopyClient storeCopier = new StoreCopyClient( targetDirectory, tuningConfiguration, loadKernelExtensions(),
Expand Down Expand Up @@ -164,18 +161,16 @@ public void done()
}, CancellationRequest.NEVER_CANCELLED );

bumpMessagesDotLogFile( targetDirectory, timestamp );
if ( checkConsistency )
boolean consistent = false;
try
{
try
{
consistent = new ConsistencyCheckService().runFullConsistencyCheck(
targetDirectory, tuningConfiguration, ProgressMonitorFactory.textual( System.err ),
logProvider, fileSystem, pageCache, false ).isSuccessful();
}
catch ( ConsistencyCheckIncompleteException e )
{
log.error( "Consistency check incomplete", e );
}
consistent = consistencyCheck.runFull( targetDirectory, tuningConfiguration,
ProgressMonitorFactory.textual( System.err ),
logProvider, fileSystem, pageCache, false );
}
catch ( ConsistencyCheck.ConsistencyCheckFailedException e )
{
log.error( "Consistency check incomplete", e );
}
clearIdFiles( targetDirectory );
return new BackupOutcome( lastCommittedTx, consistent );
Expand Down Expand Up @@ -229,11 +224,11 @@ private Map<String,String> getTemporaryDbConfig()
}

BackupOutcome doIncrementalBackupOrFallbackToFull( String sourceHostNameOrIp, int sourcePort,
File targetDirectory, boolean verification, Config config, long timeout, boolean forensics )
File targetDirectory, ConsistencyCheck consistencyCheck, Config config, long timeout, boolean forensics )
{
if ( !directoryContainsDb( targetDirectory ) )
{
return doFullBackup( sourceHostNameOrIp, sourcePort, targetDirectory, verification, config, timeout,
return doFullBackup( sourceHostNameOrIp, sourcePort, targetDirectory, consistencyCheck, config, timeout,
forensics );
}
try
Expand All @@ -247,7 +242,7 @@ BackupOutcome doIncrementalBackupOrFallbackToFull( String sourceHostNameOrIp, in
log.warn( "Attempt to do incremental backup failed.", e );
log.info( "Existing backup is too far out of date, a new full backup will be performed." );
FileUtils.deleteRecursively( targetDirectory );
return doFullBackup( sourceHostNameOrIp, sourcePort, targetDirectory, verification,
return doFullBackup( sourceHostNameOrIp, sourcePort, targetDirectory, consistencyCheck,
config, timeout, forensics );
}
catch ( Exception fullBackupFailure )
Expand Down
59 changes: 36 additions & 23 deletions enterprise/backup/src/main/java/org/neo4j/backup/BackupTool.java
Expand Up @@ -39,13 +39,13 @@
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.kernel.DefaultFileSystemAbstraction;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.logging.FormattedLogProvider;
import org.neo4j.kernel.impl.logging.SimpleLogService;
import org.neo4j.kernel.impl.store.MismatchingStoreIdException;
import org.neo4j.kernel.impl.storemigration.LogFiles;
import org.neo4j.kernel.impl.storemigration.StoreFile;
import org.neo4j.kernel.impl.storemigration.StoreFileType;
import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException;
import org.neo4j.logging.FormattedLogProvider;
import org.neo4j.logging.NullLogProvider;

import static org.neo4j.helpers.collection.MapUtil.stringMap;
Expand All @@ -63,23 +63,24 @@ public class BackupTool
private static final String VERIFY = "verify";
private static final String CONFIG = "config";

private static final String CONSISTENCY_CHECKER = "consistency-checker";

private static final String TIMEOUT = "timeout";
private static final String FORENSICS = "gather-forensics";
public static final String DEFAULT_SCHEME = "single";
static final String MISMATCHED_STORE_ID = "You tried to perform a backup from database %s, " +
"but the target directory contained a backup from database %s. ";
"but the target directory contained a backup from database %s. ";

static final String WRONG_FROM_ADDRESS_SYNTAX = "Please properly specify a location to backup in the" +
" form " + dash( HOST ) + " <host> " + dash( PORT ) + " <port>";
" form " + dash( HOST ) + " <host> " + dash( PORT ) + " <port>";

static final String UNKNOWN_SCHEMA_MESSAGE_PATTERN = "%s was specified as a backup module but it was not found. " +
"Please make sure that the implementing service is on the classpath.";

static final String NO_SOURCE_SPECIFIED = "Please specify " + dash( HOST ) + " and optionally " + dash( PORT ) +
", examples:\n" +
" " + dash( HOST ) + " 192.168.1.34\n" +
" " + dash( HOST ) + " 192.168.1.34 " + dash( PORT ) + " 1234";
", examples:\n" +
" " + dash( HOST ) + " 192.168.1.34\n" +
" " + dash( HOST ) + " 192.168.1.34 " + dash( PORT ) + " 1234";

public static void main( String[] args )
{
Expand Down Expand Up @@ -142,27 +143,39 @@ private BackupOutcome runBackupWithLegacyArgs( Args args ) throws ToolFailureExc
{
String from = args.get( FROM ).trim();
String to = args.get( TO ).trim();
boolean verify = args.getBoolean( VERIFY, true, true );
Config tuningConfiguration = readTuningConfiguration( args );
boolean forensics = args.getBoolean( FORENSICS, false, true );
ConsistencyCheck consistencyCheck = parseConsistencyChecker( args );

long timeout = args.getDuration(TIMEOUT, BackupClient.BIG_READ_TIMEOUT);
long timeout = args.getDuration( TIMEOUT, BackupClient.BIG_READ_TIMEOUT );

URI backupURI = resolveBackupUri( from, args, tuningConfiguration );

HostnamePort hostnamePort = newHostnamePort( backupURI );

return executeBackup( hostnamePort, new File( to ), verify, tuningConfiguration, timeout, forensics );
return executeBackup( hostnamePort, new File( to ), consistencyCheck, tuningConfiguration, timeout, forensics );
}

private static ConsistencyCheck parseConsistencyChecker( Args args )
{
boolean verify = args.getBoolean( VERIFY, true, true );
if (verify)
{
String consistencyCheckerName = args.get( CONSISTENCY_CHECKER, ConsistencyCheck.DEFAULT.toString(),
ConsistencyCheck.DEFAULT.toString() );
return ConsistencyCheck.fromString( consistencyCheckerName );
}
return ConsistencyCheck.NONE;
}

private BackupOutcome runBackup( Args args ) throws ToolFailureException
{
String host = args.get( HOST ).trim();
int port = args.getNumber( PORT, BackupServer.DEFAULT_PORT ).intValue();
String to = args.get( TO ).trim();
boolean verify = args.getBoolean( VERIFY, true, true );
Config tuningConfiguration = readTuningConfiguration( args );
boolean forensics = args.getBoolean( FORENSICS, false, true );
ConsistencyCheck consistencyCheck = parseConsistencyChecker( args );

if ( host.contains( ":" ) )
{
Expand All @@ -176,22 +189,22 @@ private BackupOutcome runBackup( Args args ) throws ToolFailureException
}
}

long timeout = args.getDuration(TIMEOUT, BackupClient.BIG_READ_TIMEOUT);
long timeout = args.getDuration( TIMEOUT, BackupClient.BIG_READ_TIMEOUT );

URI backupURI = newURI( DEFAULT_SCHEME + "://" + host + ":" + port ); // a bit of validation

HostnamePort hostnamePort = newHostnamePort( backupURI );

return executeBackup( hostnamePort, new File( to ), verify, tuningConfiguration, timeout, forensics );
return executeBackup( hostnamePort, new File( to ), consistencyCheck, tuningConfiguration, timeout, forensics );
}

private BackupOutcome executeBackup( HostnamePort hostnamePort, File to, boolean verify,
Config tuningConfiguration, long timeout, boolean forensics ) throws ToolFailureException
private BackupOutcome executeBackup( HostnamePort hostnamePort, File to, ConsistencyCheck consistencyCheck,
Config tuningConfiguration, long timeout, boolean forensics ) throws ToolFailureException
{
try
{
systemOut.println( "Performing backup from '" + hostnamePort + "'" );
return doBackup( hostnamePort, to, verify, tuningConfiguration, timeout, forensics );
return doBackup( hostnamePort, to, consistencyCheck, tuningConfiguration, timeout, forensics );
}
catch ( TransactionFailureException tfe )
{
Expand All @@ -200,35 +213,35 @@ private BackupOutcome executeBackup( HostnamePort hostnamePort, File to, boolean
try
{
systemOut.println( "The database present in the target directory is of an older version. " +
"Backing that up in target and performing a full backup from source" );
"Backing that up in target and performing a full backup from source" );
moveExistingDatabase( fs, to );

}
catch ( IOException e )
{
throw new ToolFailureException( "There was a problem moving the old database out of the way" +
" - cannot continue, aborting.", e );
" - cannot continue, aborting.", e );
}

return doBackup( hostnamePort, to, verify, tuningConfiguration, timeout, forensics );
return doBackup( hostnamePort, to, consistencyCheck, tuningConfiguration, timeout, forensics );
}
else
{
throw new ToolFailureException( "TransactionFailureException " +
"from existing backup at '" + hostnamePort + "'.", tfe );
"from existing backup at '" + hostnamePort + "'.", tfe );
}
}
}

private BackupOutcome doBackup( HostnamePort hostnamePort, File to, boolean checkConsistency,
Config config, long timeout, boolean forensics ) throws ToolFailureException
private BackupOutcome doBackup( HostnamePort hostnamePort, File to, ConsistencyCheck consistencyCheck,
Config config, long timeout, boolean forensics ) throws ToolFailureException
{
try
{
String host = hostnamePort.getHost();
int port = hostnamePort.getPort();

BackupOutcome outcome = backupService.doIncrementalBackupOrFallbackToFull( host, port, to, checkConsistency,
BackupOutcome outcome = backupService.doIncrementalBackupOrFallbackToFull( host, port, to, consistencyCheck,
config, timeout, forensics );
systemOut.println( "Done" );
return outcome;
Expand All @@ -246,7 +259,7 @@ private BackupOutcome doBackup( HostnamePort hostnamePort, File to, boolean chec

private static Config readTuningConfiguration( Args arguments ) throws ToolFailureException
{
Map<String, String> specifiedProperties = stringMap();
Map<String,String> specifiedProperties = stringMap();

String propertyFilePath = arguments.get( CONFIG, null );
if ( propertyFilePath != null )
Expand Down
121 changes: 121 additions & 0 deletions enterprise/backup/src/main/java/org/neo4j/backup/ConsistencyCheck.java
@@ -0,0 +1,121 @@
/*
* Copyright (c) 2002-2015 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.backup;

import java.io.File;
import java.util.Arrays;

import org.neo4j.helpers.progress.ProgressMonitorFactory;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.logging.LogProvider;

public enum ConsistencyCheck
{
NONE
{
@Override
public boolean runFull( File storeDir, Config tuningConfiguration,
ProgressMonitorFactory progressFactory, LogProvider logProvider,
FileSystemAbstraction fileSystem, PageCache pageCache, boolean verbose )
{
return true;
}
},
DEFAULT
{
@Override
public boolean runFull( File storeDir, Config tuningConfiguration,
ProgressMonitorFactory progressFactory, LogProvider logProvider,
FileSystemAbstraction fileSystem, PageCache pageCache, boolean verbose )
throws ConsistencyCheckFailedException
{
return EXPERIMENTAL.runFull( storeDir, tuningConfiguration,
progressFactory, logProvider, fileSystem, pageCache, verbose );
}
},
EXPERIMENTAL
{
@Override
public boolean runFull( File storeDir, Config tuningConfiguration,
ProgressMonitorFactory progressFactory, LogProvider logProvider,
FileSystemAbstraction fileSystem, PageCache pageCache, boolean verbose )
throws ConsistencyCheckFailedException
{
try
{
return new org.neo4j.consistency.ConsistencyCheckService().runFullConsistencyCheck(
storeDir, tuningConfiguration, progressFactory,
logProvider, fileSystem, pageCache, verbose ).isSuccessful();
}
catch ( org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException e )
{
throw new ConsistencyCheckFailedException( e );
}
}
},
LEGACY
{
@Override
public boolean runFull( File storeDir, Config tuningConfiguration,
ProgressMonitorFactory progressFactory, LogProvider logProvider,
FileSystemAbstraction fileSystem, PageCache pageCache, boolean verbose )
throws ConsistencyCheckFailedException
{
try
{
return new org.neo4j.legacy.consistency.ConsistencyCheckService().runFullConsistencyCheck(
storeDir, tuningConfiguration, progressFactory,
logProvider, fileSystem, pageCache ).isSuccessful();
}
catch ( org.neo4j.legacy.consistency.checking.full.ConsistencyCheckIncompleteException e )
{
throw new ConsistencyCheckFailedException( e );
}
}
};

public abstract boolean runFull( File storeDir, Config tuningConfiguration,
ProgressMonitorFactory progressFactory, LogProvider logProvider,
FileSystemAbstraction fileSystem, PageCache pageCache, boolean verbose )
throws ConsistencyCheckFailedException;

public static ConsistencyCheck fromString( String name )
{
for ( ConsistencyCheck consistencyCheck : values() )
{
if ( consistencyCheck.toString().equalsIgnoreCase( name ) )
{
return consistencyCheck;
}
}
throw new IllegalArgumentException( "Unknown consistency check name: " + name +
". Supported values: " + Arrays.toString( values() ) );
}

public static class ConsistencyCheckFailedException extends Exception
{
public ConsistencyCheckFailedException( Throwable cause )
{
super( cause );
}
}
}

0 comments on commit e22a7be

Please sign in to comment.