Fail loudly when doing backup against an old version backup

Incremental backup has the following scenarios, when run against an
existing backup B:

1. B is "up to date" (incremental backup possible)
2. B is "too far out of date" (full backup required)
3. B is a backup of an older database store (because you just updated
   to 3.1 and your backup is of 3.0)
4. B is a backup of a different store but the same version

Previously, 3 and 4 were intended to behave differently.

4 would correctly fail with an error to avoid deleting someone's
precious backup files.

3 failed with an error due a bug in the error handling logic, the
intention was to move the old backup and then do a full
backup. However, another bug (masked by the error handling bug) would
have deleted that directory.

Regardless, the correct thing is to fail for 3. The primary reason
being that it is impossible (or unreasonably hard) to determine if
both 3 and 4 apply (e.g., it's a backup of a different store which
also has a different version). This because we start an embedded
database server as part of the backup, and that will always fail if
the store is of a different version.

Conflicts:
	enterprise/backup/src/main/java/org/neo4j/backup/BackupTool.java

community/kernel/src/main/java/org/neo4j/kernel/impl/store/UnexpectedStoreVersionException.java

@@ -25,6 +25,11 @@ public class UnexpectedStoreVersionException extends StoreFailureException
 
     public UnexpectedStoreVersionException( String actualStoreVersion, String expectedStoreVersion )
     {
-        super( String.format( MESSAGE, actualStoreVersion, expectedStoreVersion ) );
+        this( String.format( MESSAGE, actualStoreVersion, expectedStoreVersion ) );
+    }
+
+    public UnexpectedStoreVersionException( String msg )
+    {
+        super( msg );
     }
 }

enterprise/backup/src/main/java/org/neo4j/backup/BackupService.java

@@ -58,7 +58,9 @@
 import org.neo4j.kernel.impl.store.MetaDataStore;
 import org.neo4j.kernel.impl.store.MismatchingStoreIdException;
 import org.neo4j.kernel.impl.store.StoreId;
+import org.neo4j.kernel.impl.store.UnexpectedStoreVersionException;
 import org.neo4j.kernel.impl.store.id.IdGeneratorImpl;
+import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException;
 import org.neo4j.kernel.impl.transaction.log.MissingLogDataException;
 import org.neo4j.kernel.impl.transaction.log.TransactionIdStore;
 import org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader;
@@ -72,6 +74,7 @@
 
 import static org.neo4j.com.RequestContext.anonymous;
 import static org.neo4j.com.storecopy.TransactionCommittingResponseUnpacker.DEFAULT_BATCH_SIZE;
+import static org.neo4j.helpers.Exceptions.rootCause;
 import static org.neo4j.kernel.impl.pagecache.StandalonePageCacheFactory.createPageCache;
 
 /**
@@ -257,6 +260,18 @@ BackupOutcome doIncrementalBackupOrFallbackToFull( String sourceHostNameOrIp, in
                         "but that failed as well: '" + fullBackupFailure.getMessage() + "'.", fullBackupFailure );
             }
         }
+        catch ( RuntimeException e )
+        {
+            Throwable cause = e.getCause() == null ? e : rootCause( e );
+
+            if ( cause instanceof UpgradeNotAllowedByConfigurationException )
+            {
+                throw new UnexpectedStoreVersionException( "Failed to perform backup because existing backup is from " +
+                        "a different version." );
+            }
+
+            throw e;
+        }
     }
 
     BackupOutcome doIncrementalBackup( String sourceHostNameOrIp, int sourcePort, GraphDatabaseAPI targetDb,

enterprise/backup/src/main/java/org/neo4j/backup/BackupTool.java

@@ -30,7 +30,6 @@
 import org.neo4j.backup.BackupService.BackupOutcome;
 import org.neo4j.com.ComException;
 import org.neo4j.consistency.ConsistencyCheckSettings;
-import org.neo4j.graphdb.TransactionFailureException;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.Args;
 import org.neo4j.helpers.HostnamePort;
@@ -41,16 +40,11 @@
 import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.logging.SimpleLogService;
 import org.neo4j.kernel.impl.store.MismatchingStoreIdException;
-import org.neo4j.kernel.impl.storemigration.ExistingTargetStrategy;
-import org.neo4j.kernel.impl.storemigration.LogFiles;
-import org.neo4j.kernel.impl.storemigration.StoreFile;
-import org.neo4j.kernel.impl.storemigration.StoreFileType;
-import org.neo4j.kernel.impl.storemigration.UpgradeNotAllowedByConfigurationException;
+import org.neo4j.kernel.impl.store.UnexpectedStoreVersionException;
 import org.neo4j.logging.FormattedLogProvider;
 import org.neo4j.logging.NullLogProvider;
 
 import static org.neo4j.helpers.collection.MapUtil.stringMap;
-import static org.neo4j.kernel.impl.storemigration.FileOperation.MOVE;
 
 public class BackupTool
 {
@@ -195,46 +189,12 @@ private BackupOutcome runBackup( Args args ) throws ToolFailureException
         return executeBackup( hostnamePort, new File( to ), consistencyCheck, tuningConfiguration, timeout, forensics );
     }
 
-    private BackupOutcome executeBackup( HostnamePort hostnamePort, File to, ConsistencyCheck consistencyCheck,
-            Config tuningConfiguration, long timeout, boolean forensics ) throws ToolFailureException
-    {
-        try
-        {
-            systemOut.println( "Performing backup from '" + hostnamePort + "'" );
-            return doBackup( hostnamePort, to, consistencyCheck, tuningConfiguration, timeout, forensics );
-        }
-        catch ( TransactionFailureException tfe )
-        {
-            if ( tfe.getCause() instanceof UpgradeNotAllowedByConfigurationException )
-            {
-                try
-                {
-                    systemOut.println( "The database present in the target directory is of an older version. " +
-                            "Backing that up in target and performing a full backup from source" );
-                    moveExistingDatabase( fs, to );
-
-                }
-                catch ( IOException e )
-                {
-                    throw new ToolFailureException( "There was a problem moving the old database out of the way" +
-                            " - cannot continue, aborting.", e );
-                }
-
-                return doBackup( hostnamePort, to, consistencyCheck, tuningConfiguration, timeout, forensics );
-            }
-            else
-            {
-                throw new ToolFailureException( "TransactionFailureException " +
-                        "from existing backup at '" + hostnamePort + "'.", tfe );
-            }
-        }
-    }
-
-    private BackupOutcome doBackup( HostnamePort hostnamePort, File to, ConsistencyCheck consistencyCheck,
+    BackupOutcome executeBackup( HostnamePort hostnamePort, File to, ConsistencyCheck consistencyCheck,
             Config config, long timeout, boolean forensics ) throws ToolFailureException
     {
         try
         {
+            systemOut.println( "Performing backup from '" + hostnamePort + "'" );
             String host = hostnamePort.getHost();
             int port = hostnamePort.getPort();
 
@@ -243,6 +203,10 @@ private BackupOutcome doBackup( HostnamePort hostnamePort, File to, ConsistencyC
             systemOut.println( "Done" );
             return outcome;
         }
+        catch ( UnexpectedStoreVersionException e )
+        {
+            throw new ToolFailureException( e.getMessage() );
+        }
         catch ( MismatchingStoreIdException e )
         {
             throw new ToolFailureException( String.format( MISMATCHED_STORE_ID, e.getExpected(), e.getEncountered() ) );
@@ -357,19 +321,6 @@ private static HostnamePort newHostnamePort( URI backupURI ) throws ToolFailureE
         return new HostnamePort( host, port );
     }
 
-    private static void moveExistingDatabase( FileSystemAbstraction fs, File toDir ) throws IOException
-    {
-        File backupDir = new File( toDir, "old-version" );
-        if ( !fs.mkdir( backupDir ) )
-        {
-            throw new IOException( "Trouble making target backup directory " + backupDir.getAbsolutePath() );
-        }
-        StoreFile.fileOperation( MOVE, fs, toDir, backupDir, StoreFile.currentStoreFiles(), false,
-                ExistingTargetStrategy.FAIL,
-                StoreFileType.values() );
-        LogFiles.move( fs, toDir, backupDir );
-    }
-
     private static String dash( String name )
     {
         return "-" + name;

enterprise/backup/src/test/java/org/neo4j/backup/BackupToolIT.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.backup;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.nio.file.Path;
+
+import org.neo4j.helpers.HostnamePort;
+import org.neo4j.io.fs.DefaultFileSystemAbstraction;
+import org.neo4j.io.pagecache.PageCache;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.kernel.impl.pagecache.StandalonePageCacheFactory;
+import org.neo4j.kernel.impl.store.MetaDataStore;
+import org.neo4j.kernel.impl.store.format.standard.StandardV2_3;
+import org.neo4j.test.EmbeddedDatabaseRule;
+import org.neo4j.test.TargetDirectory;
+
+import static org.mockito.Mockito.mock;
+import static org.neo4j.kernel.impl.store.MetaDataStore.Position.STORE_VERSION;
+
+public class BackupToolIT
+{
+    @Rule
+    public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() );
+
+    @Rule
+    public ExpectedException expected = ExpectedException.none();
+
+    @Rule
+    public EmbeddedDatabaseRule dbRule = new EmbeddedDatabaseRule( getClass() ).startLazily();
+
+    private DefaultFileSystemAbstraction fs;
+    private PageCache pageCache;
+    private Path backupDir;
+    private BackupTool backupTool;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        backupDir = testDirectory.directory( "backups/graph.db" ).toPath();
+        fs = new DefaultFileSystemAbstraction();
+        pageCache = StandalonePageCacheFactory.createPageCache( fs );
+        backupTool = new BackupTool( new BackupService(), mock( PrintStream.class ) );
+    }
+
+    @After
+    public void teardown() throws Exception
+    {
+        pageCache.close();
+    }
+
+    @Test
+    public void oldIncompatibleBackupsAreMovedToSubDir() throws Exception
+    {
+        // Prepare an "old" backup
+        prepareNeoStoreFile( StandardV2_3.STORE_VERSION );
+
+        // Start database to backup
+        dbRule.getGraphDatabaseAPI();
+
+        expected.expect( BackupTool.ToolFailureException.class );
+        expected.expectMessage( "Failed to perform backup because existing backup is from a different version." );
+
+        // Perform backup
+        backupTool.executeBackup( new HostnamePort( "localhost", 6362 ), backupDir.toFile(),
+                ConsistencyCheck.NONE, Config.defaults(), 20L * 60L * 1000L, false );
+    }
+
+    private void prepareNeoStoreFile( String storeVersion ) throws Exception
+    {
+        File neoStoreFile = createNeoStoreFile();
+        long value = MetaDataStore.versionStringToLong( storeVersion );
+        MetaDataStore.setRecord( pageCache, neoStoreFile, STORE_VERSION, value );
+    }
+
+    private File createNeoStoreFile() throws Exception
+    {
+        fs.mkdirs( backupDir.toFile() );
+        File neoStoreFile = new File( backupDir.toFile(), MetaDataStore.DEFAULT_NAME );
+        fs.create( neoStoreFile ).close();
+        return neoStoreFile;
+    }
+}