Cluster seeding tests

enterprise/backup/src/test/java/org/neo4j/causalclustering/BackupUtil.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.neo4j.causalclustering.discovery.CoreClusterMember;
+import org.neo4j.commandline.admin.CommandFailed;
+import org.neo4j.io.fs.FileSystemAbstraction;
+import org.neo4j.kernel.configuration.Config;
+import org.neo4j.restore.RestoreDatabaseCommand;
+
+import static org.junit.Assert.assertEquals;
+import static org.neo4j.causalclustering.BackupCoreIT.backupArguments;
+import static org.neo4j.util.TestHelpers.runBackupToolFromOtherJvmToGetExitCode;
+
+public class BackupUtil
+{
+    private static String backupAddress( CoreClusterMember core )
+    {
+        return core.settingValue( "causal_clustering.transaction_listen_address" );
+    }
+
+    public static File createBackupFromCore( CoreClusterMember core, String backupName, File baseBackupDir ) throws Exception
+    {
+        String[] args = backupArguments( backupAddress( core ), baseBackupDir, backupName );
+        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( baseBackupDir, args ) );
+        return new File( baseBackupDir, backupName );
+    }
+
+    static void restoreFromBackup( File backup, FileSystemAbstraction fsa, CoreClusterMember coreClusterMember ) throws IOException, CommandFailed
+    {
+        Config config = coreClusterMember.config();
+        RestoreDatabaseCommand restoreDatabaseCommand = new RestoreDatabaseCommand( fsa, backup, config, "graph-db", true );
+        restoreDatabaseCommand.execute();
+    }
+}

enterprise/backup/src/test/java/org/neo4j/causalclustering/ClusterSeedingIT.java

@@ -21,62 +21,73 @@
 
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.RuleChain;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import java.io.File;
+import java.util.Optional;
 
-import org.neo4j.causalclustering.core.CoreGraphDatabase;
+import org.neo4j.causalclustering.backup_stores.BackupStore;
+import org.neo4j.causalclustering.backup_stores.BackupStoreWithSomeData;
+import org.neo4j.causalclustering.backup_stores.BackupStoreWithSomeDataButNoTransactionLogs;
+import org.neo4j.causalclustering.backup_stores.EmptyBackupStore;
+import org.neo4j.causalclustering.backup_stores.NoStore;
 import org.neo4j.causalclustering.discovery.Cluster;
 import org.neo4j.causalclustering.discovery.CoreClusterMember;
 import org.neo4j.causalclustering.discovery.IpFamily;
 import org.neo4j.causalclustering.discovery.SharedDiscoveryService;
-import org.neo4j.graphdb.factory.GraphDatabaseSettings;
-import org.neo4j.io.fs.FileSystemAbstraction;
-import org.neo4j.kernel.configuration.Config;
-import org.neo4j.kernel.impl.enterprise.configuration.OnlineBackupSettings;
 import org.neo4j.kernel.impl.store.format.standard.Standard;
-import org.neo4j.kernel.monitoring.Monitors;
-import org.neo4j.restore.RestoreDatabaseCommand;
 import org.neo4j.test.DbRepresentation;
 import org.neo4j.test.rule.SuppressOutput;
 import org.neo4j.test.rule.TestDirectory;
 import org.neo4j.test.rule.fs.DefaultFileSystemRule;
 
 import static java.util.Collections.emptyMap;
 import static org.junit.Assert.assertEquals;
-import static org.neo4j.causalclustering.BackupCoreIT.backupAddress;
+import static org.neo4j.causalclustering.BackupUtil.restoreFromBackup;
 import static org.neo4j.causalclustering.discovery.Cluster.dataMatchesEventually;
-import static org.neo4j.causalclustering.helpers.DataCreator.createEmptyNodes;
-import static org.neo4j.util.TestHelpers.runBackupToolFromOtherJvmToGetExitCode;
 
+@RunWith( Parameterized.class )
 public class ClusterSeedingIT
 {
-    private Cluster backupCluster;
-    private Cluster cluster;
-    private FileSystemAbstraction fsa;
+    @Parameterized.Parameter()
+    public BackupStore initialStore;
+
+    @Parameterized.Parameter( 1 )
+    public boolean shouldStoreCopy;
+
+    private SuppressOutput suppressOutput = SuppressOutput.suppressAll();
+    private TestDirectory testDir = TestDirectory.testDirectory();
+    private DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();
 
-    public TestDirectory testDir = TestDirectory.testDirectory();
-    public DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();
-    public SuppressOutput suppressOutput = SuppressOutput.suppressAll();
     @Rule
     public RuleChain rules = RuleChain.outerRule( fileSystemRule ).around( testDir ).around( suppressOutput );
 
+    private Cluster backupCluster;
+    private Cluster cluster;
+    private FileCopyDetector fileCopyDetector;
     private File baseBackupDir;
 
+    @Parameterized.Parameters( name = "{0}" )
+    public static Object[][] data() throws Exception
+    {
+        return new Object[][]{{new NoStore(), true}, {new EmptyBackupStore(), false}, {new BackupStoreWithSomeData(), false},
+                {new BackupStoreWithSomeDataButNoTransactionLogs(), false}};
+    }
+
     @Before
-    public void setup()
+    public void setup() throws Exception
     {
-        fsa = fileSystemRule.get();
-        backupCluster = new Cluster( testDir.directory( "cluster-for-backup" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), Standard
-                .LATEST_NAME, IpFamily.IPV4, false );
+        this.fileCopyDetector = new FileCopyDetector();
 
-        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), Standard.LATEST_NAME,
-                IpFamily.IPV4, false );
+        backupCluster = new Cluster( testDir.directory( "cluster-for-backup" ), 3, 0, new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(),
+                emptyMap(), Standard.LATEST_NAME, IpFamily.IPV4, false );
+
+        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0, new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(), emptyMap(),
+                Standard.LATEST_NAME, IpFamily.IPV4, false );
 
         baseBackupDir = testDir.directory( "backups" );
     }
@@ -94,110 +105,33 @@ public void after()
         }
     }
 
-    private File createBackupUsingAnotherCluster() throws Exception
-    {
-        backupCluster.start();
-        CoreGraphDatabase db = BackupCoreIT.createSomeData( backupCluster );
-
-        File backup = createBackup( backupCluster, "some-backup" );
-        backupCluster.shutdown();
-
-        return backup;
-    }
-
-    private File createBackup( Cluster cluster, String backupName ) throws Exception
-    {
-        String[] args = BackupCoreIT.backupArguments( backupAddress( cluster ), baseBackupDir, backupName );
-        assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( testDir.absolutePath(), args ) );
-        return new File( baseBackupDir, backupName );
-    }
-
     @Test
-    public void shouldRestoreBySeedingAllMembers() throws Throwable
+    public void shouldSeedNewCluster() throws Exception
     {
         // given
-        File backupDir = createBackupUsingAnotherCluster();
-        Config config = Config.defaults( OnlineBackupSettings.online_backup_enabled, Boolean.FALSE.toString() );
-        DbRepresentation before = DbRepresentation.of( backupDir, config );
+        backupCluster.start();
+        Optional<File> backup = initialStore.generate( baseBackupDir, backupCluster );
+        backupCluster.shutdown();
 
-        // when
-        for ( CoreClusterMember coreClusterMember : cluster.coreMembers() )
+        if ( backup.isPresent() )
         {
-            String databaseName = coreClusterMember
-                    .getMemberConfig().get( GraphDatabaseSettings.active_database );
-            new RestoreDatabaseCommand( fsa, backupDir, coreClusterMember.getMemberConfig(), databaseName, true )
-                    .execute();
+            for ( CoreClusterMember coreClusterMember : cluster.coreMembers() )
+            {
+                restoreFromBackup( backup.get(), fileSystemRule.get(), coreClusterMember );
+            }
         }
-        cluster.start();
-
-        // then
-        dataMatchesEventually( before, cluster.coreMembers() );
-    }
 
-    @Test
-    public void shouldSeedNewMemberFromEmptyIdleCluster() throws Throwable
-    {
-        // given
-        Monitors monitors = new Monitors();
-        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), Standard.LATEST_NAME,
-                IpFamily.IPV4, false );
-        cluster.start();
-
-        // when: creating a backup
-        File backupDir = createBackup( cluster, "the-backup" );
-
-        // and: seeding new member with said backup
-        CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 );
-        String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database );
-        new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute();
-        newMember.start();
-
-        // then
-        dataMatchesEventually( DbRepresentation.of( newMember.database() ), cluster.coreMembers() );
-    }
-
-    @Test
-    public void shouldSeedNewMemberFromNonEmptyIdleCluster() throws Throwable
-    {
-        // given
-        Monitors monitors = new Monitors();
-        cluster = new Cluster( testDir.directory( "cluster-b" ), 3, 0,
-                new SharedDiscoveryService(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), Standard.LATEST_NAME,
-                IpFamily.IPV4, false );
-
-        cluster.start();
-        createEmptyNodes( cluster, 100 );
-
-        // when: creating a backup
-        File backupDir = createBackup( cluster, "the-backup" );
-
-        // and: seeding new member with said backup
-        CoreClusterMember newMember = cluster.addCoreMemberWithId( 3 );
-        String databaseName = newMember.getMemberConfig().get( GraphDatabaseSettings.active_database );
-        new RestoreDatabaseCommand( fsa, backupDir, newMember.getMemberConfig(), databaseName, true ).execute();
-        newMember.start();
-
-        // then
-        dataMatchesEventually( DbRepresentation.of( newMember.database() ), cluster.coreMembers() );
-    }
-
-    @Test
-    @Ignore( "need to seed all members for now" )
-    public void shouldRestoreBySeedingSingleMember() throws Throwable
-    {
-        // given
-        File backupDir = createBackupUsingAnotherCluster();
-        DbRepresentation before = DbRepresentation.of( backupDir );
+        // we want the cluster to seed from backup. No instance should delete and re-copy the store.
+        cluster.coreMembers().forEach( ccm -> ccm.monitors().addMonitorListener( fileCopyDetector ) );
 
         // when
-        fsa.copyRecursively( backupDir, cluster.getCoreMemberById( 0 ).storeDir() );
-        cluster.getCoreMemberById( 0 ).start();
-        Thread.sleep( 2_000 );
-        cluster.getCoreMemberById( 1 ).start();
-        cluster.getCoreMemberById( 2 ).start();
+        cluster.start();
 
         // then
-        dataMatchesEventually( before, cluster.coreMembers() );
+        if ( backup.isPresent() )
+        {
+            dataMatchesEventually( DbRepresentation.of( backup.get() ), cluster.coreMembers() );
+        }
+        assertEquals( shouldStoreCopy, fileCopyDetector.hasDetectedAnyFileCopied() );
     }
 }

enterprise/backup/src/test/java/org/neo4j/causalclustering/FileCopyDetector.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2002-2018 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.causalclustering;
+
+import java.io.File;
+
+import org.neo4j.causalclustering.catchup.tx.FileCopyMonitor;
+
+public class FileCopyDetector implements FileCopyMonitor
+{
+    private volatile boolean hasCopiedFile;
+
+    @Override
+    public void copyFile( File file )
+    {
+        hasCopiedFile = true;
+    }
+
+    boolean hasDetectedAnyFileCopied()
+    {
+        return hasCopiedFile;
+    }
+}