diff --git a/community/io/src/test/java/org/neo4j/test/TargetDirectory.java b/community/io/src/test/java/org/neo4j/test/TargetDirectory.java index aaecf7329e63e..b9690ab060c68 100644 --- a/community/io/src/test/java/org/neo4j/test/TargetDirectory.java +++ b/community/io/src/test/java/org/neo4j/test/TargetDirectory.java @@ -89,7 +89,18 @@ public File directory( String name ) return dir; } - public File graphDbDir() { + public File cleanDirectory( String name ) throws IOException + { + File directory = directory( name ); + for ( File file : fileSystem.listFiles( directory ) ) + { + fileSystem.deleteRecursively( file ); + } + return directory; + } + + public File graphDbDir() + { return directory( "graph-db" ); } diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/impl/ha/ClusterManager.java b/enterprise/ha/src/main/java/org/neo4j/kernel/impl/ha/ClusterManager.java index 66e6590103c80..433f8b90d312e 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/impl/ha/ClusterManager.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/impl/ha/ClusterManager.java @@ -28,7 +28,6 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -41,6 +40,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; + import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -57,9 +57,11 @@ import org.neo4j.cluster.member.ClusterMemberListener; import org.neo4j.cluster.protocol.election.NotElectableElectionCredentialsProvider; import org.neo4j.function.Function; +import org.neo4j.function.IntFunction; import org.neo4j.function.Predicate; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.TransactionFailureException; +import org.neo4j.graphdb.config.Setting; import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.HighlyAvailableGraphDatabaseFactory; @@ -89,8 +91,8 @@ import static java.lang.String.format; import static java.util.Arrays.asList; -import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; + import static org.neo4j.helpers.collection.Iterables.count; import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.io.fs.FileUtils.copyRecursively; @@ -123,56 +125,46 @@ public interface RepairKit HighlyAvailableGraphDatabase repair() throws Throwable; } + public static IntFunction constant( final String value ) + { + return new IntFunction() + { + @Override + public String apply( int ignored ) + { + return value; + } + }; + } + private final File root; - private final Map commonConfig; - private final Map> instanceConfig; + private final Map> commonConfig; private final Map clusterMap = new HashMap<>(); private final Provider clustersProvider; private final HighlyAvailableGraphDatabaseFactory dbFactory; private final StoreDirInitializer storeDirInitializer; LifeSupport life; - public ClusterManager( Provider clustersProvider, File root, Map commonConfig, - Map> instanceConfig, - HighlyAvailableGraphDatabaseFactory dbFactory ) - { - this.clustersProvider = clustersProvider; - this.root = root; - this.commonConfig = withDefaults( commonConfig ); - this.instanceConfig = instanceConfig; - this.dbFactory = dbFactory; - this.storeDirInitializer = null; - } - - public ClusterManager( Builder builder ) + private ClusterManager( Builder builder ) { this.clustersProvider = builder.provider; this.root = builder.root; this.commonConfig = withDefaults( builder.commonConfig ); - this.instanceConfig = builder.instanceConfig; this.dbFactory = builder.factory; this.storeDirInitializer = builder.initializer; } - private Map withDefaults( Map commonConfig ) + private Map> withDefaults( Map> commonConfig ) { - Map result = new HashMap<>( CONFIG_FOR_SINGLE_JVM_CLUSTER ); + Map> result = new HashMap<>(); + for ( Map.Entry conf : CONFIG_FOR_SINGLE_JVM_CLUSTER.entrySet() ) + { + result.put( conf.getKey(), constant( conf.getValue() ) ); + } result.putAll( commonConfig ); return result; } - public ClusterManager( Provider clustersProvider, File root, Map commonConfig, - Map> instanceConfig ) - { - this( clustersProvider, root, commonConfig, instanceConfig, new HighlyAvailableGraphDatabaseFactory() ); - } - - public ClusterManager( Provider clustersProvider, File root, Map commonConfig ) - { - this( clustersProvider, root, commonConfig, Collections.>emptyMap(), - new HighlyAvailableGraphDatabaseFactory() ); - } - /** * Provider pointing out an XML file to read. * @@ -576,12 +568,49 @@ protected void insertInitialData( GraphDatabaseService db, String name, Instance { } - public static class Builder + public interface ClusterBuilder { - private final File root; - private final Map> instanceConfig = new HashMap<>(); + SELF withRootDirectory( File root ); + + SELF withSeedDir( File seedDir ); + + SELF withStoreDirInitializer( StoreDirInitializer initializer ); + + SELF withDbFactory( HighlyAvailableGraphDatabaseFactory dbFactory ); + + SELF withProvider( Provider provider ); + + /** + * Supplies configuration where config values, as opposed to {@link #withSharedConfig(Map)}, + * are a function of (one-based) server id. The function may return {@code null} which means + * that the particular member doesn't have that config value, or at least not specifically + * set, such that any default value would be used. + */ + SELF withInstanceConfig( Map> commonConfig ); + + /** + * Like {@link #withInstanceConfig(Map)}, but for individual settings, conveniently using + * {@link Setting} instance as key as well. + */ + SELF withInstanceSetting( Setting setting, IntFunction valueFunction ); + + /** + * Supplies configuration where config values are shared with all instances in the cluster. + */ + SELF withSharedConfig( Map commonConfig ); + + /** + * Like {@link #withInstanceSetting(Setting, IntFunction)}, but for individual settings, conveniently using + * {@link Setting} instance as key as well. + */ + SELF withSharedSetting( Setting setting, String value ); + } + + public static class Builder implements ClusterBuilder + { + private File root; private Provider provider = clusterOfSize( 3 ); - private Map commonConfig = emptyMap(); + private final Map> commonConfig = new HashMap<>(); private HighlyAvailableGraphDatabaseFactory factory = new HighlyAvailableGraphDatabaseFactory(); private StoreDirInitializer initializer; @@ -590,6 +619,20 @@ public Builder( File root ) this.root = root; } + public Builder() + { + // We want this, at least in the ClusterRule case where we fill this Builder instances + // with all our behavior, but we don't know about the root directory until we evaluate the rule. + } + + @Override + public Builder withRootDirectory( File root ) + { + this.root = root; + return this; + } + + @Override public Builder withSeedDir( final File seedDir ) { return withStoreDirInitializer( new StoreDirInitializer() @@ -602,36 +645,58 @@ public void initializeStoreDir( int serverId, File storeDir ) throws IOException } ); } + @Override public Builder withStoreDirInitializer( StoreDirInitializer initializer ) { this.initializer = initializer; return this; } + @Override public Builder withDbFactory( HighlyAvailableGraphDatabaseFactory dbFactory ) { this.factory = dbFactory; return this; } + @Override public Builder withProvider( Provider provider ) { this.provider = provider; return this; } - public Builder withCommonConfig( Map commonConfig ) + @Override + public Builder withInstanceConfig( Map> commonConfig ) { - this.commonConfig = commonConfig; + this.commonConfig.putAll( commonConfig ); return this; } - public Builder withInstanceConfig( int instanceNr, Map instanceConfig ) + @Override + public Builder withInstanceSetting( Setting setting, IntFunction valueFunction ) { - this.instanceConfig.put( instanceNr, instanceConfig ); + this.commonConfig.put( setting.name(), valueFunction ); return this; } + @Override + public Builder withSharedConfig( Map commonConfig ) + { + Map> dynamic = new HashMap<>(); + for ( Map.Entry entry : commonConfig.entrySet() ) + { + dynamic.put( entry.getKey(), constant( entry.getValue() ) ); + } + return withInstanceConfig( dynamic ); + } + + @Override + public Builder withSharedSetting( Setting setting, String value ) + { + return withInstanceSetting( setting, constant( value ) ); + } + public ClusterManager build() { return new ClusterManager( this ); @@ -982,10 +1047,9 @@ private void startMember( InstanceId serverId ) throws URISyntaxException, IOExc builder.setConfig( ClusterSettings.cluster_server, "0.0.0.0:" + clusterPort ); builder.setConfig( HaSettings.ha_server, ":" + haPort ); builder.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ); - builder.setConfig( commonConfig ); - if ( instanceConfig.containsKey( serverId.toIntegerIndex() ) ) + for ( Map.Entry> conf : commonConfig.entrySet() ) { - builder.setConfig( instanceConfig.get( serverId.toIntegerIndex() ) ); + builder.setConfig( conf.getKey(), conf.getValue().apply( serverId.toIntegerIndex() ) ); } config( builder, name, serverId ); diff --git a/enterprise/ha/src/test/java/jmx/HaBeanIT.java b/enterprise/ha/src/test/java/jmx/HaBeanIT.java index f95e15cff0152..f7dc8cca99212 100644 --- a/enterprise/ha/src/test/java/jmx/HaBeanIT.java +++ b/enterprise/ha/src/test/java/jmx/HaBeanIT.java @@ -19,24 +19,19 @@ */ package jmx; -import org.junit.After; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.TestName; import java.net.URI; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; -import org.neo4j.cluster.InstanceId; import org.neo4j.function.Function; import org.neo4j.function.Predicate; import org.neo4j.graphdb.Transaction; -import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.jmx.Kernel; import org.neo4j.jmx.impl.JmxKernelExtension; import org.neo4j.kernel.ha.HaSettings; @@ -50,7 +45,7 @@ import org.neo4j.management.ClusterMemberInfo; import org.neo4j.management.HighAvailability; import org.neo4j.management.Neo4jManager; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -59,43 +54,21 @@ import static java.util.concurrent.TimeUnit.SECONDS; +import static org.neo4j.helpers.Settings.STRING; +import static org.neo4j.helpers.Settings.setting; import static org.neo4j.helpers.collection.Iterables.filter; import static org.neo4j.helpers.collection.Iterables.first; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesMembers; +import static org.neo4j.test.ha.ClusterRule.intBase; +import static org.neo4j.test.ha.ClusterRule.stringWithIntBase; public class HaBeanIT { - @Rule - public final TestName testName = new TestName(); - @Rule - public TargetDirectory.TestDirectory dir = TargetDirectory.testDirForTest( HaBeanIT.class ); - private ManagedCluster cluster; - private ClusterManager clusterManager; - - public void startCluster( int size ) throws Throwable - { - clusterManager = new ClusterManager( clusterOfSize( size ), dir.directory( testName.getMethodName() ), - MapUtil.stringMap() ) - { - @Override - protected void config( GraphDatabaseBuilder builder, String clusterName, InstanceId serverId ) - { - builder.setConfig( "jmx.port", "" + ( 9912 + serverId.toIntegerIndex() ) ); - builder.setConfig( HaSettings.ha_server, ":" + ( 1136 + serverId.toIntegerIndex() ) ); - builder.setConfig( GraphDatabaseSettings.forced_kernel_id, testName.getMethodName() + serverId ); - } - }; - clusterManager.start(); - cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - } - - @After - public void stopCluster() throws Throwable - { - clusterManager.stop(); - } + @ClassRule + public static final ClusterRule clusterRule = new ClusterRule( HaBeanIT.class ) + .withInstanceSetting( setting( "jmx.port", STRING, (String) null ), intBase( 9912 ) ) + .withInstanceSetting( HaSettings.ha_server, stringWithIntBase( ":", 1136 ) ) + .withInstanceSetting( GraphDatabaseSettings.forced_kernel_id, stringWithIntBase( "kernel", 0 ) ); public Neo4jManager beans( HighlyAvailableGraphDatabase db ) { @@ -111,7 +84,7 @@ public HighAvailability ha( HighlyAvailableGraphDatabase db ) @Test public void canGetHaBean() throws Throwable { - startCluster( 1 ); + ManagedCluster cluster = clusterRule.startCluster(); HighAvailability ha = ha( cluster.getMaster() ); assertNotNull( "could not get ha bean", ha ); assertMasterInformation( ha ); @@ -128,7 +101,7 @@ private void assertMasterInformation( HighAvailability ha ) @Test public void testLatestTxInfoIsCorrect() throws Throwable { - startCluster( 1 ); + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase db = cluster.getMaster(); HighAvailability masterHa = ha( db ); long lastCommitted = masterHa.getLastCommittedTxId(); @@ -143,7 +116,7 @@ public void testLatestTxInfoIsCorrect() throws Throwable @Test public void testUpdatePullWorksAndUpdatesLastUpdateTime() throws Throwable { - startCluster( 2 ); + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase master = cluster.getMaster(); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); try (Transaction tx = master.beginTx()) @@ -162,15 +135,19 @@ public void testUpdatePullWorksAndUpdatesLastUpdateTime() throws Throwable @Test public void testAfterGentleMasterSwitchClusterInfoIsCorrect() throws Throwable { - startCluster( 3 ); + ManagedCluster cluster = clusterRule.startCluster(); RepairKit masterShutdown = cluster.shutdown( cluster.getMaster() ); - cluster.await( ClusterManager.masterAvailable() ); - cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 1 ) ); - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) + try + { + for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) + { + assertEquals( 2, ha( db ).getInstancesInCluster().length ); + } + } + finally { - assertEquals( 2, ha( db ).getInstancesInCluster().length ); + masterShutdown.repair(); } - masterShutdown.repair(); cluster.await( ClusterManager.allSeesAllAsAvailable() ); for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { @@ -205,20 +182,24 @@ public void testAfterGentleMasterSwitchClusterInfoIsCorrect() throws Throwable @Test public void testAfterHardMasterSwitchClusterInfoIsCorrect() throws Throwable { - startCluster( 3 ); + ManagedCluster cluster = clusterRule.startCluster(); RepairKit masterShutdown = cluster.fail( cluster.getMaster() ); - cluster.await( ClusterManager.masterAvailable() ); - cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 1 ) ); - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) + try { - if ( db.getInstanceState() == HighAvailabilityMemberState.PENDING ) + for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { - continue; + if ( db.getInstanceState() == HighAvailabilityMemberState.PENDING ) + { + continue; + } + // Instance that was hard killed will still be in the cluster + assertEquals( 3, ha( db ).getInstancesInCluster().length ); } - // Instance that was hard killed will still be in the cluster - assertEquals( 3, ha( db ).getInstancesInCluster().length ); } - masterShutdown.repair(); + finally + { + masterShutdown.repair(); + } cluster.await( ClusterManager.masterAvailable() ); cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 2 ) ); for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) @@ -246,7 +227,7 @@ public void testAfterHardMasterSwitchClusterInfoIsCorrect() throws Throwable @Test public void canGetBranchedStoreBean() throws Throwable { - startCluster( 1 ); + ManagedCluster cluster = clusterRule.startCluster(); BranchedStore bs = beans( cluster.getMaster() ).getBranchedStoreBean(); assertNotNull( "could not get branched store bean", bs ); assertEquals( "no branched stores for new db", 0, @@ -256,11 +237,11 @@ public void canGetBranchedStoreBean() throws Throwable @Test public void joinedInstanceShowsUpAsSlave() throws Throwable { - startCluster( 2 ); + ManagedCluster cluster = clusterRule.startCluster(); ClusterMemberInfo[] instancesInCluster = ha( cluster.getMaster() ).getInstancesInCluster(); - assertEquals( 2, instancesInCluster.length ); + assertEquals( 3, instancesInCluster.length ); ClusterMemberInfo[] secondInstancesInCluster = ha( cluster.getAnySlave() ).getInstancesInCluster(); - assertEquals( 2, secondInstancesInCluster.length ); + assertEquals( 3, secondInstancesInCluster.length ); assertMasterAndSlaveInformation( instancesInCluster ); assertMasterAndSlaveInformation( secondInstancesInCluster ); } @@ -270,30 +251,41 @@ public void leftInstanceDisappearsFromMemberList() throws Throwable { // Start the second db and make sure it's visible in the member list. // Then shut it down to see if it disappears from the member list again. - startCluster( 3 ); + ManagedCluster cluster = clusterRule.startCluster(); assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length ); - cluster.shutdown( cluster.getAnySlave() ); + RepairKit repair = cluster.shutdown( cluster.getAnySlave() ); - cluster.await( masterSeesMembers( 2 ) ); - - assertEquals( 2, ha( cluster.getMaster() ).getInstancesInCluster().length ); - assertMasterInformation( ha( cluster.getMaster() ) ); + try + { + cluster.await( masterSeesMembers( 2 ) ); + assertEquals( 2, ha( cluster.getMaster() ).getInstancesInCluster().length ); + assertMasterInformation( ha( cluster.getMaster() ) ); + } + finally + { + repair.repair(); + } } @Test public void failedMemberIsStillInMemberListAlthoughFailed() throws Throwable { - startCluster( 3 ); + ManagedCluster cluster = clusterRule.startCluster(); assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length ); // Fail the instance HighlyAvailableGraphDatabase failedDb = cluster.getAnySlave(); RepairKit dbFailure = cluster.fail( failedDb ); - await( ha( cluster.getMaster() ), dbAlive( false ) ); - await( ha( cluster.getAnySlave( failedDb )), dbAlive( false ) ); - - // Repair the failure and come back - dbFailure.repair(); + try + { + await( ha( cluster.getMaster() ), dbAlive( false ) ); + await( ha( cluster.getAnySlave( failedDb ) ), dbAlive( false ) ); + } + finally + { + // Repair the failure and come back + dbFailure.repair(); + } for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { await( ha( db ), dbAvailability( true ) ); diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/BackupHaIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/BackupHaIT.java index 7d01acca0841f..48b7d7c4826b5 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/BackupHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/BackupHaIT.java @@ -19,93 +19,52 @@ */ package org.neo4j.ha; -import org.apache.commons.io.FileUtils; -import org.junit.After; -import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; import java.io.File; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.neo4j.backup.OnlineBackupSettings; -import org.neo4j.cluster.InstanceId; -import org.neo4j.graphdb.factory.GraphDatabaseBuilder; +import org.neo4j.function.IntFunction; import org.neo4j.helpers.Settings; -import org.neo4j.helpers.collection.MapUtil; -import org.neo4j.kernel.impl.ha.ClusterManager; import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.test.DbRepresentation; import org.neo4j.test.SuppressOutput; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; -import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertNotEquals; import static org.neo4j.backup.BackupEmbeddedIT.createSomeData; import static org.neo4j.backup.BackupEmbeddedIT.runBackupToolFromOtherJvmToGetExitCode; -import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; public class BackupHaIT { - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( BackupHaIT.class ) + .withSharedSetting( OnlineBackupSettings.online_backup_enabled, Settings.TRUE ) + .withInstanceSetting( OnlineBackupSettings.online_backup_server, new IntFunction() + { + @Override + public String apply( int serverId ) + { + return (":" + (4444 + serverId)); + } + } ); @Rule public final SuppressOutput suppressOutput = SuppressOutput.suppressAll(); - private File path; - private File backupPath; - - private DbRepresentation representation; - private ClusterManager clusterManager; - private ManagedCluster cluster; - - @Before - public void setup() throws Throwable - { - path = testDirectory.directory( "db" ); - backupPath = testDirectory.directory( "backup-db" ); - FileUtils.deleteDirectory( path ); - FileUtils.deleteDirectory( backupPath ); - - startCluster(); - - // Really doesn't matter which instance - representation = createSomeData( cluster.getMaster() ); - } + private static File backupPath; - private void startCluster() throws Throwable + @BeforeClass + public static void setup() throws Exception { - clusterManager = new ClusterManager( ClusterManager.fromXml( getClass().getResource( "/threeinstances.xml" ) - .toURI() ), - path, MapUtil.stringMap( OnlineBackupSettings.online_backup_enabled.name(), - Settings.TRUE ) ) - { - @Override - protected void config( GraphDatabaseBuilder builder, String clusterName, InstanceId serverId ) - { - builder.setConfig( OnlineBackupSettings.online_backup_server, (":"+(4444 + serverId.toIntegerIndex()) )); - } - }; - clusterManager.start(); - cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - } - - @After - public void stopCluster() throws Throwable - { - clusterManager.stop(); - clusterManager.shutdown(); - } - - @Test - public void makeSureBackupCanBePerformedFromClusterWithDefaultName() throws Throwable - { - testBackupFromCluster( null ); + backupPath = clusterRule.cleanDirectory( "backup-db" ); + createSomeData( clusterRule.startCluster().getMaster() ); } @Test @@ -116,84 +75,51 @@ public void makeSureBackupCanBePerformedFromWronglyNamedCluster() throws Throwab } @Test - public void makeSureBackupCanBeRestored() throws Throwable + public void makeSureBackupCanBePerformed() throws Throwable { // Run backup + ManagedCluster cluster = clusterRule.startCluster(); + DbRepresentation beforeChange = DbRepresentation.of( cluster.getMaster() ); assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( backupArguments( "localhost:4445", backupPath.getPath(), null ) ) ); // Add some new data - DbRepresentation changedData = createSomeData( cluster.getMaster() ); - - stopCluster(); - - cleanData(); + DbRepresentation afterChange = createSomeData( cluster.getMaster() ); + cluster.sync(); - copyBackup(); - - startCluster(); - - // Verify that old data is back - assertThat( changedData.equals( DbRepresentation.of( cluster.getMaster() ) ), equalTo(false) ); + // Verify that backed up database can be started and compare representation + DbRepresentation backupRepresentation = DbRepresentation.of( backupPath ); + assertEquals( beforeChange, backupRepresentation ); + assertNotEquals( backupRepresentation, afterChange ); } @Test public void makeSureBackupCanBePerformedFromAnyInstance() throws Throwable { + ManagedCluster cluster = clusterRule.startCluster(); Integer[] backupPorts = {4445, 4446, 4447}; for ( Integer port : backupPorts ) { // Run backup + DbRepresentation beforeChange = DbRepresentation.of( cluster.getMaster() ); assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( backupArguments( "localhost:" + port, backupPath.getPath(), null ) ) ); // Add some new data - DbRepresentation changedData = createSomeData( cluster.getMaster() ); - - stopCluster(); - - cleanData(); - - copyBackup(); - - startCluster(); + DbRepresentation afterChange = createSomeData( cluster.getMaster() ); + cluster.sync(); // Verify that old data is back - assertThat( changedData.equals( DbRepresentation.of( cluster.getMaster() ) ), equalTo(false) ); + DbRepresentation backupRepresentation = DbRepresentation.of( backupPath ); + assertEquals( beforeChange, backupRepresentation ); + assertNotEquals( backupRepresentation, afterChange ); } } - - private void copyBackup() throws IOException - { - FileUtils.copyDirectory( backupPath, new File( path, "neo4j.ha/server1" ) ); - FileUtils.copyDirectory( backupPath, new File( path, "neo4j.ha/server2") ); - FileUtils.copyDirectory( backupPath, new File( path, "neo4j.ha/server3" ) ); - } - - private void cleanData() throws IOException - { - FileUtils.cleanDirectory( new File( path, "neo4j.ha/server1" ) ); - FileUtils.cleanDirectory( new File( path, "neo4j.ha/server2")); - FileUtils.cleanDirectory( new File( path, "neo4j.ha/server3" ) ); - } - - private void testBackupFromCluster( String askForCluster ) throws Throwable - { - assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( - backupArguments( "localhost:4445", backupPath.getPath(), askForCluster ) ) ); - assertEquals( representation, DbRepresentation.of( backupPath ) ); - ManagedCluster cluster = clusterManager.getCluster( askForCluster == null ? "neo4j.ha" : askForCluster ); - DbRepresentation newRepresentation = createSomeData( cluster.getAnySlave() ); - assertEquals( 0, runBackupToolFromOtherJvmToGetExitCode( - backupArguments( "localhost:4445", backupPath.getPath(), askForCluster ) ) ); - assertEquals( newRepresentation, DbRepresentation.of( backupPath ) ); - } - private String[] backupArguments( String from, String to, String clusterName ) { - List args = new ArrayList(); + List args = new ArrayList<>(); args.add( "-from" ); args.add( from ); args.add( "-to" ); diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/ClusterTransactionTest.java b/enterprise/ha/src/test/java/org/neo4j/ha/ClusterTransactionTest.java index 19e073900aabc..e06f41f5c11eb 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/ClusterTransactionTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/ClusterTransactionTest.java @@ -51,9 +51,9 @@ public class ClusterTransactionTest public void givenClusterWhenShutdownMasterThenCannotStartTransactionOnSlave() throws Throwable { final ClusterManager.ManagedCluster cluster = - clusterRule.provider( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ) ) - .config( HaSettings.ha_server, ":6001-6005" ) - .config( HaSettings.tx_push_factor, "2" ).startCluster(); + clusterRule.withProvider( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ) ) + .withSharedSetting( HaSettings.ha_server, ":6001-6005" ) + .withSharedSetting( HaSettings.tx_push_factor, "2" ).startCluster(); cluster.await( ClusterManager.allSeesAllAsAvailable() ); diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/HaCacheIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/HaCacheIT.java deleted file mode 100644 index eb980086f0f9a..0000000000000 --- a/enterprise/ha/src/test/java/org/neo4j/ha/HaCacheIT.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright (c) 2002-2015 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.ha; - -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.neo4j.consistency.ConsistencyCheckService; -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Relationship; -import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.progress.ProgressMonitorFactory; -import org.neo4j.io.fs.FileUtils; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.logging.NullLogProvider; -import org.neo4j.test.TargetDirectory; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import static org.neo4j.graphdb.DynamicRelationshipType.withName; -import static org.neo4j.graphdb.factory.GraphDatabaseSettings.dense_node_threshold; -import static org.neo4j.helpers.collection.IteratorUtil.count; -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.ha.HaSettings.tx_push_factor; -import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; - -public class HaCacheIT -{ - private static final int DENSE_NODE = 10; - - @Rule - public TargetDirectory.TestDirectory root = TargetDirectory.testDirForTest( getClass() ); - - @Test - public void shouldUpdateSlaveCacheWhenRemovingRelationshipGroupFromDenseNode() throws Throwable - { - ClusterManager manager = new ClusterManager( clusterOfSize( 3 ), root.directory(), - stringMap( tx_push_factor.name(), "2", - dense_node_threshold.name(), "" + DENSE_NODE ) ); - try - { - // given - manager.start(); - ClusterManager.ManagedCluster cluster = manager.getDefaultCluster(); - cluster.await( ClusterManager.masterAvailable() ); - cluster.await( ClusterManager.masterSeesAllSlavesAsAvailable() ); - HighlyAvailableGraphDatabase master = cluster.getMaster(); - long nodeId; // a dense node - try ( Transaction tx = master.beginTx() ) - { - Node node = master.createNode(); - for ( int i = 0; i < DENSE_NODE; i++ ) - { - node.createRelationshipTo( master.createNode(), withName( "FOO" ) ); - } - master.createNode().createRelationshipTo( node, withName( "BAR" ) ); - - tx.success(); - nodeId = node.getId(); - } - // fully cache node on all instances - int count = 0; - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - try ( Transaction tx = db.beginTx() ) - { - int these = count( db.getNodeById( nodeId ).getRelationships() ); - assertTrue( String.format( "expected=%s, count here=%s", count, these ), - these != 0 && (count == 0 || these == count) ); - count = these; - tx.success(); - } - } - - // when - try ( Transaction tx = master.beginTx() ) - { - for ( Relationship relationship : master.getNodeById( nodeId ).getRelationships( withName( "BAR" ) ) ) - { - relationship.delete(); - } - tx.success(); - } - - // then - HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); - try ( Transaction tx = slave.beginTx() ) - { - List relationships = new ArrayList<>(); - for ( Relationship relationship : slave.getNodeById( nodeId ).getRelationships() ) - { - relationships.add( String.format( "(%d)-[%d:%s]->(%d)", - relationship.getStartNode().getId(), - relationship.getId(), relationship.getType().name(), - relationship.getEndNode().getId() ) ); - } - assertEquals( joinLines( relationships ), count - 1, relationships.size() ); - assertEquals( count - 1, count( slave.getNodeById( nodeId ).getRelationships() ) ); - - tx.success(); - } - } - finally - { - manager.shutdown(); - } - } - - /* - * This test has been introduced to reproduce an inconsistent issue present in 2.2.2 and it can reproduce the - * problem at every run. - * - * The problem details are the following: - * - adding a property to the node from a slave - * - changing such introduced property from the master - * - causes to introduce twice the property key in the property chain - * - * The original issue in 2.2.2 was caused by a cache poisoning (the cache on the master didn't contain the added - * property from the slave) which cannot be reproduced in later versions of the product. - * - * This test has been added only to make sure we do not regress and introduce this problem once again. - */ - @Test - public void duplicatePropertyWhenAddingChangingAPropertyFromSlaveAndMasterRespectively() throws Throwable - { - File storeDir = root.directory( "ha-cluster" ); - FileUtils.deleteRecursively( storeDir ); - ClusterManager clusterManager = new ClusterManager( - new ClusterManager.Builder( storeDir ).withProvider( clusterOfSize( 3 ) ) - ); - - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - - long id = init( cluster.getMaster() ); - cluster.sync(); - - GraphDatabaseService slave = cluster.getAnySlave(); - - // when - // adding a new property by writing on the slave - try ( Transaction tx = slave.beginTx() ) - { - Node node = slave.getNodeById( id ); - node.setProperty( "1", 1 ); - - tx.success(); - } - - Thread.sleep( 100 ); - - // and changing the introduced property on the master - GraphDatabaseService master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - Node node = master.getNodeById( id ); - node.setProperty( "1", 0 ); - tx.success(); - } - - clusterManager.stop(); - clusterManager.shutdown(); - - // then there should be no property key duplications in the property chain - File masterStoreDir = new File( storeDir, "neo4j.ha/server1" ); - ConsistencyCheckService.Result result = - new ConsistencyCheckService().runFullConsistencyCheck( masterStoreDir, new Config(), - ProgressMonitorFactory.NONE, NullLogProvider.getInstance(), false ); - - assertTrue( result.isSuccessful() ); - } - - private long init( GraphDatabaseService db ) - { - // create 2 prop keys before hand and "the node" used in the test - long id; - try ( Transaction tx = db.beginTx() ) - { - Node theNode = db.createNode(); - id = theNode.getId(); - for ( int i = 0; i < 1; i++ ) - { - Node node = db.createNode(); - node.setProperty( "" + i, "" + i ); - } - tx.success(); - } - - - // set one property on the node - try ( Transaction tx = db.beginTx() ) - { - Node node = db.getNodeById( id ); - node.setProperty( "0", 0 ); - tx.success(); - } - - return id; - } - - private String joinLines( Iterable lines ) - { - StringBuilder result = new StringBuilder(); - for ( String line : lines ) - { - result.append( "\n\t" ).append( line ); - } - return result.toString(); - } -} diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/MultipleClusterTest.java b/enterprise/ha/src/test/java/org/neo4j/ha/MultipleClusterTest.java deleted file mode 100644 index 4e9bab521f604..0000000000000 --- a/enterprise/ha/src/test/java/org/neo4j/ha/MultipleClusterTest.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2002-2015 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.ha; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import java.io.File; -import java.util.logging.Level; - -import org.neo4j.graphdb.GraphDatabaseService; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.collection.MapUtil; -import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.ha.UpdatePuller; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; -import org.neo4j.test.LoggerRule; -import org.neo4j.test.TargetDirectory; - -import static org.junit.Assert.assertEquals; - -import static org.neo4j.kernel.impl.ha.ClusterManager.fromXml; - -/** - * Verify that we can run multiple clusters simultaneously - */ -@Ignore("Fails too often") -public class MultipleClusterTest -{ - @Rule - public LoggerRule logging = new LoggerRule( Level.OFF ); - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - - @Test - public void runTwoClusters() throws Throwable - { - File root = testDirectory.directory( "cluster" ); - - ClusterManager clusterManager = new ClusterManager( - fromXml( getClass().getResource( "/twoclustertest.xml" ).toURI() ), root, MapUtil.stringMap() ); - - try - { - clusterManager.start(); - ManagedCluster cluster1 = clusterManager.getCluster( "neo4j.ha" ); - - long cluster1NodeId; - { - GraphDatabaseService master = cluster1.getMaster(); - logging.getLogger().info( "CREATE NODE" ); - Transaction tx = master.beginTx(); - Node node = master.createNode(); - node.setProperty( "cluster", "neo4j.ha" ); - cluster1NodeId = node.getId(); - logging.getLogger().info( "CREATED NODE" ); - tx.success(); - tx.close(); - } - - ManagedCluster cluster2 = clusterManager.getCluster( "neo4j.ha2" ); - long cluster2NodeId; - { - GraphDatabaseService master = cluster2.getMaster(); - logging.getLogger().info( "CREATE NODE" ); - Transaction tx = master.beginTx(); - Node node = master.createNode(); - node.setProperty( "cluster", "neo4j.ha2" ); - cluster2NodeId = node.getId(); - logging.getLogger().info( "CREATED NODE" ); - tx.success(); - tx.close(); - } - - // Verify properties in all cluster nodes - for ( HighlyAvailableGraphDatabase highlyAvailableGraphDatabase : cluster1.getAllMembers() ) - { - highlyAvailableGraphDatabase.getDependencyResolver().resolveDependency( UpdatePuller.class ).pullUpdates(); - - Transaction transaction = highlyAvailableGraphDatabase.beginTx(); - assertEquals( "neo4j.ha", highlyAvailableGraphDatabase.getNodeById( cluster1NodeId ).getProperty( - "cluster" ) ); - transaction.close(); - } - - for ( HighlyAvailableGraphDatabase highlyAvailableGraphDatabase : cluster2.getAllMembers() ) - { - highlyAvailableGraphDatabase.getDependencyResolver().resolveDependency( UpdatePuller.class ).pullUpdates(); - - Transaction transaction = highlyAvailableGraphDatabase.beginTx(); - assertEquals( "neo4j.ha2", highlyAvailableGraphDatabase.getNodeById( cluster2NodeId ).getProperty( - "cluster" ) ); - transaction.close(); - } - } - finally - { - clusterManager.stop(); - } - } - -} diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/PullStormIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/PullStormIT.java index d4a3e32781441..ec056f65f7241 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/PullStormIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/PullStormIT.java @@ -19,26 +19,25 @@ */ package org.neo4j.ha; -import static org.junit.Assert.assertEquals; - -import static org.neo4j.helpers.collection.MapUtil.stringMap; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.logging.Level; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.graphdb.Transaction; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; import org.neo4j.test.LoggerRule; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; + +import static org.junit.Assert.assertEquals; /** * This is a test for the Neo4j HA self-inflicted DDOS "pull storm" phenomenon. In a 2 instance setup, whereby @@ -50,108 +49,87 @@ @Ignore("A good idea but the test is too high level, is fragile and takes too long.") public class PullStormIT { - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( PullStormIT.class ) + .withSharedSetting( HaSettings.pull_interval, "0" ) + .withSharedSetting( HaSettings.tx_push_factor, "1" ); - @Rule - public LoggerRule logger = new LoggerRule( Level.OFF ); + @ClassRule + public static LoggerRule logger = new LoggerRule( Level.OFF ); @Test public void testPullStorm() throws Throwable { // given + ManagedCluster cluster = clusterRule.startCluster(); - ClusterManager clusterManager = new ClusterManager( ClusterManager.clusterWithAdditionalArbiters( 2, 1 ), - testDirectory.directory(), - stringMap( HaSettings.pull_interval.name(), "0", - HaSettings.tx_push_factor.name(), "1") ); - - clusterManager.start(); - - try + // Create data + final HighlyAvailableGraphDatabase master = cluster.getMaster(); { - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.masterAvailable( ) ); - cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 1 ) ); - - // Create data - final HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - try ( Transaction tx = master.beginTx() ) + for ( int i = 0; i < 1000; i++ ) { - for ( int i = 0; i < 1000; i++ ) - { - master.createNode().setProperty( "foo", "bar" ); - } - tx.success(); + master.createNode().setProperty( "foo", "bar" ); } + tx.success(); } + } - // Slave goes down - HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); - ClusterManager.RepairKit repairKit = cluster.fail( slave ); + // Slave goes down + HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); + ClusterManager.RepairKit repairKit = cluster.fail( slave ); - // Create more data - for ( int i = 0; i < 1000; i++ ) + // Create more data + for ( int i = 0; i < 1000; i++ ) + { { + try (Transaction tx = master.beginTx()) { - try (Transaction tx = master.beginTx()) + for ( int j = 0; j < 1000; j++ ) { - for ( int j = 0; j < 1000; j++ ) - { - master.createNode().setProperty( "foo", "bar" ); - master.createNode().setProperty( "foo", "bar" ); - } - tx.success(); + master.createNode().setProperty( "foo", "bar" ); + master.createNode().setProperty( "foo", "bar" ); } + tx.success(); } } + } - // Slave comes back online - repairKit.repair(); + // Slave comes back online + repairKit.repair(); - cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 1 ) ); + cluster.await( ClusterManager.masterSeesSlavesAsAvailable( 1 ) ); - // when + // when - // Create 20 concurrent transactions - System.out.println( "Pull storm" ); - ExecutorService executor = Executors.newFixedThreadPool( 20 ); - for ( int i = 0; i < 20; i++ ) + // Create 20 concurrent transactions + ExecutorService executor = Executors.newFixedThreadPool( 20 ); + for ( int i = 0; i < 20; i++ ) + { + executor.submit( new Runnable() { - executor.submit( new Runnable() + @Override + public void run() { - @Override - public void run() + // Transaction close should cause lots of concurrent calls to pullUpdate() + try ( Transaction tx = master.beginTx() ) { - // Transaction close should cause lots of concurrent calls to pullUpdate() - try ( Transaction tx = master.beginTx() ) - { - master.createNode().setProperty( "foo", "bar" ); - tx.success(); - } + master.createNode().setProperty( "foo", "bar" ); + tx.success(); } - } ); - } - - executor.shutdown(); - executor.awaitTermination( 1, TimeUnit.MINUTES ); - - System.out.println( "Pull storm done" ); + } + } ); + } - // then + executor.shutdown(); + executor.awaitTermination( 1, TimeUnit.MINUTES ); - long masterLastCommittedTxId = lastCommittedTxId( master ); - for ( HighlyAvailableGraphDatabase member : cluster.getAllMembers() ) - { - assertEquals( masterLastCommittedTxId, lastCommittedTxId( member ) ); - } - } - finally + // then + long masterLastCommittedTxId = lastCommittedTxId( master ); + for ( HighlyAvailableGraphDatabase member : cluster.getAllMembers() ) { - System.err.println( "Shutting down" ); - clusterManager.shutdown(); - System.err.println( "Shut down" ); + assertEquals( masterLastCommittedTxId, lastCommittedTxId( member ) ); } } diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/QuorumWritesIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/QuorumWritesIT.java index d17f6ab3666ee..94e3b1ba66de4 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/QuorumWritesIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/QuorumWritesIT.java @@ -19,6 +19,7 @@ */ package org.neo4j.ha; +import org.junit.ClassRule; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; @@ -37,125 +38,106 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.TransactionFailureException; import org.neo4j.graphdb.factory.TestHighlyAvailableGraphDatabaseFactory; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.ha.cluster.HighAvailabilityModeSwitcher; import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.kernel.impl.store.StoreId; import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; - @Ignore("To be rewritten") public class QuorumWritesIT { + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( QuorumWritesIT.class ) + .withSharedSetting( HaSettings.tx_push_factor, "2" ) + .withSharedSetting( HaSettings.state_switch_timeout, "5s" ); + @Test public void testMasterStopsWritesWhenMajorityIsUnavailable() throws Throwable { - File root = testDirectory.directory( "testMasterStopsWritesWhenMajorityIsUnavailable" ); - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 3 ), root, - MapUtil.stringMap( HaSettings.tx_push_factor.name(), "2", HaSettings.state_switch_timeout.name(), "5s" - ) ); - try - { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.masterAvailable( ) ); - cluster.await( ClusterManager.masterSeesAllSlavesAsAvailable() ); - - HighlyAvailableGraphDatabase master = cluster.getMaster(); + ManagedCluster cluster = clusterRule.startCluster(); + HighlyAvailableGraphDatabase master = cluster.getMaster(); - doTx( master ); + doTx( master ); - final CountDownLatch latch1 = new CountDownLatch( 1 ); - waitOnHeartbeatFail( master, latch1 ); + final CountDownLatch latch1 = new CountDownLatch( 1 ); + waitOnHeartbeatFail( master, latch1 ); - HighlyAvailableGraphDatabase slave1 = cluster.getAnySlave(); - cluster.fail( slave1 ); + HighlyAvailableGraphDatabase slave1 = cluster.getAnySlave(); + cluster.fail( slave1 ); - latch1.await(); - slave1.shutdown(); + latch1.await(); + slave1.shutdown(); - doTx( master ); + doTx( master ); - final CountDownLatch latch2 = new CountDownLatch( 1 ); - waitOnHeartbeatFail( master, latch2 ); + final CountDownLatch latch2 = new CountDownLatch( 1 ); + waitOnHeartbeatFail( master, latch2 ); - HighlyAvailableGraphDatabase slave2 = cluster.getAnySlave( slave1 ); - ClusterManager.RepairKit rk2 = cluster.fail( slave2 ); + HighlyAvailableGraphDatabase slave2 = cluster.getAnySlave( slave1 ); + ClusterManager.RepairKit rk2 = cluster.fail( slave2 ); - latch2.await(); + latch2.await(); - // The master should stop saying that it's master - assertFalse( master.isMaster() ); + // The master should stop saying that it's master + assertFalse( master.isMaster() ); - try - { - doTx( master ); - fail( "After both slaves fail txs should not go through" ); - } - catch ( TransactionFailureException e ) - { - assertEquals( "Timeout waiting for cluster to elect master", e.getMessage() ); - } + try + { + doTx( master ); + fail( "After both slaves fail txs should not go through" ); + } + catch ( TransactionFailureException e ) + { + assertEquals( "Timeout waiting for cluster to elect master", e.getMessage() ); + } - // This is not a hack, this simulates a period of inactivity in the cluster. - Thread.sleep( 120000 ); // TODO Define "inactivity" and await that condition instead of 120 seconds. + // This is not a hack, this simulates a period of inactivity in the cluster. + Thread.sleep( 120000 ); // TODO Define "inactivity" and await that condition instead of 120 seconds. - final CountDownLatch latch3 = new CountDownLatch( 1 ); - final CountDownLatch latch4 = new CountDownLatch( 1 ); - final CountDownLatch latch5 = new CountDownLatch( 1 ); - waitOnHeartbeatAlive( master, latch3 ); + final CountDownLatch latch3 = new CountDownLatch( 1 ); + final CountDownLatch latch4 = new CountDownLatch( 1 ); + final CountDownLatch latch5 = new CountDownLatch( 1 ); + waitOnHeartbeatAlive( master, latch3 ); // waitOnRoleIsAvailable( master, latch4, HighAvailabilityModeSwitcher.MASTER ); - waitOnRoleIsAvailable( master, latch5, HighAvailabilityModeSwitcher.SLAVE ); + waitOnRoleIsAvailable( master, latch5, HighAvailabilityModeSwitcher.SLAVE ); - rk2.repair(); + rk2.repair(); - latch3.await(); + latch3.await(); - cluster.await( ClusterManager.masterAvailable( slave1, slave2 ) ); + cluster.await( ClusterManager.masterAvailable( slave1, slave2 ) ); // latch4.await(); - latch5.await(); + latch5.await(); - cluster.await( ClusterManager.masterAvailable( ) ); + cluster.await( ClusterManager.masterAvailable( ) ); - assertTrue( master.isMaster() ); - assertFalse( slave2.isMaster() ); + assertTrue( master.isMaster() ); + assertFalse( slave2.isMaster() ); - Node finalNode = doTx( master ); + Node finalNode = doTx( master ); - try ( Transaction transaction = slave2.beginTx() ) - { - slave2.getNodeById( finalNode.getId() ); - transaction.success(); - } - } - finally + try ( Transaction transaction = slave2.beginTx() ) { - clusterManager.stop(); + slave2.getNodeById( finalNode.getId() ); + transaction.success(); } } @Test public void testInstanceCanBeReplacedToReestablishQuorum() throws Throwable { - File root = testDirectory.directory( "testInstanceCanBeReplacedToReestablishQuorum" ); - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 3 ), root, - MapUtil.stringMap( HaSettings.tx_push_factor.name(), "2", HaSettings.state_switch_timeout.name(), "5s" ) ); - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase master = cluster.getMaster(); - - cluster.await( ClusterManager.masterSeesAllSlavesAsAvailable() ); - doTx( master ); final CountDownLatch latch1 = new CountDownLatch( 1 ); @@ -202,7 +184,8 @@ public void testInstanceCanBeReplacedToReestablishQuorum() throws Throwable HighlyAvailableGraphDatabase replacement = (HighlyAvailableGraphDatabase) new TestHighlyAvailableGraphDatabaseFactory(). - newHighlyAvailableDatabaseBuilder( new File( root, "replacement" ).getAbsolutePath() ). + newHighlyAvailableDatabaseBuilder( new File( clusterRule.directory( "another" ), + "replacement" ).getAbsolutePath() ). setConfig( ClusterSettings.cluster_server, ":5010" ). setConfig( HaSettings.ha_server, ":6010" ). setConfig( ClusterSettings.server_id, "3" ). @@ -224,7 +207,6 @@ public void testInstanceCanBeReplacedToReestablishQuorum() throws Throwable replacement.getNodeById( finalNode.getId() ); } - clusterManager.stop(); replacement.shutdown(); } diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TestBranchedData.java b/enterprise/ha/src/test/java/org/neo4j/ha/TestBranchedData.java index 6cd0ba25ac126..b00821cb24a07 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TestBranchedData.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TestBranchedData.java @@ -51,9 +51,11 @@ import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.tooling.GlobalGraphOperations; -import static java.lang.String.format; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; + +import static java.lang.String.format; + import static org.neo4j.helpers.SillyUtils.nonNull; import static org.neo4j.helpers.collection.IteratorUtil.asSet; import static org.neo4j.helpers.collection.MapUtil.stringMap; @@ -97,7 +99,8 @@ public void shouldCopyStoreFromMasterIfBranched() throws Throwable { // GIVEN File dir = directory.directory(); - ClusterManager clusterManager = life.add( new ClusterManager( clusterOfSize( 2 ), dir, stringMap() ) ); + ClusterManager clusterManager = life.add( new ClusterManager.Builder( dir ) + .withProvider( clusterOfSize( 2 ) ).build() ); ManagedCluster cluster = clusterManager.getDefaultCluster(); cluster.await( allSeesAllAsAvailable() ); createNode( cluster.getMaster(), "A" ); @@ -130,10 +133,11 @@ public void shouldCopyStoreFromMasterIfBranchedInLiveScenario() throws Throwable // thor is whoever is the master to begin with // odin is whoever is picked as _the_ slave given thor as initial master File dir = directory.directory(); - ClusterManager clusterManager = life.add( new ClusterManager( clusterOfSize( 3 ), dir, stringMap( + ClusterManager clusterManager = life.add( new ClusterManager.Builder( dir ) + .withSharedConfig( stringMap( // Effectively disable automatic transaction propagation within the cluster HaSettings.tx_push_factor.name(), "0", - HaSettings.pull_interval.name(), "0" ) ) ); + HaSettings.pull_interval.name(), "0" ) ).build() ); ManagedCluster cluster = clusterManager.getDefaultCluster(); cluster.await( allSeesAllAsAvailable() ); HighlyAvailableGraphDatabase thor = cluster.getMaster(); diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterClientPadding.java b/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterClientPadding.java index 1dbfd3df45b7e..5283fb4bc1d94 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterClientPadding.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterClientPadding.java @@ -41,7 +41,7 @@ public class TestClusterClientPadding @Before public void setUp() throws Throwable { - cluster = clusterRule.provider( clusterWithAdditionalClients( 2, 1 ) ) + cluster = clusterRule.withProvider( clusterWithAdditionalClients( 2, 1 ) ) .availabilityChecks( masterAvailable(), masterSeesMembers( 3 ), allSeesAllAsJoined() ) .startCluster(); } diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterIndexDeletion.java b/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterIndexDeletion.java index 582e3b912dd80..97a1a7d85653e 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterIndexDeletion.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TestClusterIndexDeletion.java @@ -19,74 +19,57 @@ */ package org.neo4j.ha; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.GraphDatabaseAPI; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.test.TargetDirectory; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.test.ha.ClusterRule; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertThat; -import static org.neo4j.kernel.impl.ha.ClusterManager.fromXml; - public class TestClusterIndexDeletion { + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( TestClusterIndexDeletion.class ) + .withSharedSetting( HaSettings.ha_server, ":6001-6005" ) + .withSharedSetting( HaSettings.tx_push_factor, "2" ); + @Test public void givenClusterWithCreatedIndexWhenDeleteIndexOnMasterThenIndexIsDeletedOnSlave() throws Throwable { - ClusterManager clusterManager = - new ClusterManager( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ), - testDirectory.directory( "testCluster" ), - MapUtil.stringMap( HaSettings.ha_server.name(), ":6001-6005", - HaSettings.tx_push_factor.name(), "2" )); - try + ManagedCluster cluster = clusterRule.startCluster(); + GraphDatabaseAPI master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - // Given - clusterManager.start(); - - clusterManager.getDefaultCluster().await( ClusterManager.allSeesAllAsAvailable() ); - - GraphDatabaseAPI master = clusterManager.getDefaultCluster().getMaster(); - try ( Transaction tx = master.beginTx() ) - { - master.index().forNodes( "Test" ); - tx.success(); - } - - HighlyAvailableGraphDatabase aSlave = clusterManager.getDefaultCluster().getAnySlave(); - try ( Transaction tx = aSlave.beginTx() ) - { - assertThat( aSlave.index().existsForNodes( "Test" ), equalTo( true ) ); - tx.success(); - } + master.index().forNodes( "Test" ); + tx.success(); + } - // When - try ( Transaction tx = master.beginTx() ) - { - master.index().forNodes( "Test" ).delete(); - tx.success(); - } + HighlyAvailableGraphDatabase aSlave = cluster.getAnySlave(); + try ( Transaction tx = aSlave.beginTx() ) + { + assertThat( aSlave.index().existsForNodes( "Test" ), equalTo( true ) ); + tx.success(); + } - // Then - HighlyAvailableGraphDatabase anotherSlave = clusterManager.getDefaultCluster().getAnySlave(); - try ( Transaction tx = anotherSlave.beginTx() ) - { - assertThat( anotherSlave.index().existsForNodes( "Test" ), equalTo( false ) ); - tx.success(); - } + // When + try ( Transaction tx = master.beginTx() ) + { + master.index().forNodes( "Test" ).delete(); + tx.success(); } - finally + + // Then + HighlyAvailableGraphDatabase anotherSlave = cluster.getAnySlave(); + try ( Transaction tx = anotherSlave.beginTx() ) { - clusterManager.stop(); + assertThat( anotherSlave.index().existsForNodes( "Test" ), equalTo( false ) ); + tx.success(); } } - - @Rule - public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); } diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TestPullUpdates.java b/enterprise/ha/src/test/java/org/neo4j/ha/TestPullUpdates.java index 420bafca545a6..43efb0ddfcc73 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TestPullUpdates.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TestPullUpdates.java @@ -19,22 +19,21 @@ */ package org.neo4j.ha; -import java.io.File; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - import org.junit.After; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import java.io.File; +import java.net.URI; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + import org.neo4j.cluster.ClusterSettings; import org.neo4j.cluster.InstanceId; import org.neo4j.cluster.client.ClusterClient; import org.neo4j.cluster.protocol.cluster.ClusterListener; +import org.neo4j.function.IntFunction; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.NotFoundException; @@ -83,10 +82,11 @@ public void doAfter() throws Throwable public void makeSureUpdatePullerGetsGoingAfterMasterSwitch() throws Throwable { File root = testDirectory.directory( testName.getMethodName() ); - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 3 ), root, MapUtil.stringMap( + ClusterManager clusterManager = new ClusterManager.Builder( root ) + .withSharedConfig( MapUtil.stringMap( HaSettings.pull_interval.name(), PULL_INTERVAL+"ms", ClusterSettings.heartbeat_interval.name(), "2s", - ClusterSettings.heartbeat_timeout.name(), "30s") ); + ClusterSettings.heartbeat_timeout.name(), "30s") ).build(); clusterManager.start(); cluster = clusterManager.getDefaultCluster(); cluster.await( allSeesAllAsAvailable() ); @@ -124,18 +124,22 @@ public void makeSureUpdatePullerGetsGoingAfterMasterSwitch() throws Throwable public void pullUpdatesShellAppPullsUpdates() throws Throwable { File root = testDirectory.directory( testName.getMethodName() ); - Map> instanceConfig = new HashMap<>(); - for (int i = 1; i <= 2; i++) - { - Map thisInstance = - MapUtil.stringMap( ShellSettings.remote_shell_port.name(), "" + (SHELL_PORT + i) ); - instanceConfig.put( i, thisInstance ); - } - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 2 ), root, MapUtil.stringMap( - HaSettings.pull_interval.name(), "0", - HaSettings.tx_push_factor.name(), "0" , - ShellSettings.remote_shell_enabled.name(), "true" - ), instanceConfig ); + ClusterManager clusterManager = new ClusterManager.Builder( root ) + .withProvider( clusterOfSize( 2 ) ) + .withSharedConfig( MapUtil.stringMap( + HaSettings.pull_interval.name(), "0", + HaSettings.tx_push_factor.name(), "0" , + ShellSettings.remote_shell_enabled.name(), "true" ) ) + .withInstanceConfig( MapUtil.>genericMap( + ShellSettings.remote_shell_port.name(), new IntFunction() + { + @Override + public String apply( int oneBasedServerId ) + { + return oneBasedServerId >= 1 && oneBasedServerId <= 2 ? + "" + (SHELL_PORT + oneBasedServerId) : null; + } + } ) ).build(); clusterManager.start(); cluster = clusterManager.getDefaultCluster(); diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TestSlaveOnlyCluster.java b/enterprise/ha/src/test/java/org/neo4j/ha/TestSlaveOnlyCluster.java index 5dd8a3200a714..e5f8c470ae5e4 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TestSlaveOnlyCluster.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TestSlaveOnlyCluster.java @@ -19,106 +19,71 @@ */ package org.neo4j.ha; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; -import java.io.File; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.neo4j.cluster.InstanceId; import org.neo4j.cluster.client.ClusterClient; import org.neo4j.cluster.protocol.heartbeat.HeartbeatListener; +import org.neo4j.function.IntFunction; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.collection.MapUtil; +import org.neo4j.helpers.Settings; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.test.TargetDirectory; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit; +import org.neo4j.test.ha.ClusterRule; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertThat; - import static org.junit.Assert.assertTrue; + import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -import static org.neo4j.kernel.impl.ha.ClusterManager.fromXml; public class TestSlaveOnlyCluster { - @Rule - public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( TestSlaveOnlyCluster.class ) + .withInstanceSetting( HaSettings.slave_only, new IntFunction() + { + @Override + public String apply( int value ) + { + return value == 1 || value == 2 ? Settings.TRUE : Settings.FALSE; + } + } ); private static final String PROPERTY = "foo"; private static final String VALUE = "bar"; @Test public void testMasterElectionAfterMasterRecoversInSlaveOnlyCluster() throws Throwable { - final ClusterManager clusterManager = createCluster( "masterrecovery", 1, 2 ); + ManagedCluster cluster = clusterRule.startCluster(); + assertThat( cluster.getServerId( cluster.getMaster() ), equalTo( new InstanceId( 3 ) ) ); + HighlyAvailableGraphDatabase master = cluster.getMaster(); + CountDownLatch masterFailedLatch = createMasterFailLatch( cluster ); + RepairKit repairKit = cluster.fail( master ); try { - clusterManager.start(); - - final ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - - final HighlyAvailableGraphDatabase master = cluster.getMaster(); - final CountDownLatch masterFailedLatch = createMasterFailLatch( cluster ); - - final ClusterManager.RepairKit repairKit = cluster.fail( master ); - assertTrue( masterFailedLatch.await( 60, TimeUnit.SECONDS ) ); - - repairKit.repair(); - - cluster.await( allSeesAllAsAvailable() ); - - long nodeId = createNodeWithPropertyOn( cluster.getAnySlave(), PROPERTY, VALUE ); - - try ( Transaction ignore = master.beginTx() ) - { - assertThat( (String) master.getNodeById( nodeId ).getProperty( PROPERTY ), equalTo( VALUE ) ); - } } finally { - clusterManager.stop(); + repairKit.repair(); } - } - - @Test - public void testMasterElectionAfterSlaveOnlyInstancesStartFirst() throws Throwable - { - final ClusterManager clusterManager = createCluster( "slaveonly", 1, 2 ); - - try - { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - assertThat( cluster.getServerId( cluster.getMaster() ), equalTo( new InstanceId( 3 ) ) ); - } - finally - { - clusterManager.stop(); - } - } + cluster.await( allSeesAllAsAvailable() ); + long nodeId = createNodeWithPropertyOn( cluster.getAnySlave(), PROPERTY, VALUE ); - private ClusterManager createCluster( String dirname, int... slaveIds ) throws URISyntaxException - { - final File dir = testDirectory.directory( dirname ); - final ClusterManager.Provider provider = fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ); - final Map> instanceConfig = new HashMap<>( slaveIds.length ); - for ( int slaveId : slaveIds ) + try ( Transaction ignore = master.beginTx() ) { - instanceConfig.put( slaveId, MapUtil.stringMap( HaSettings.slave_only.name(), "true" ) ); + assertThat( (String) master.getNodeById( nodeId ).getProperty( PROPERTY ), equalTo( VALUE ) ); } - - return new ClusterManager( provider, dir, MapUtil.stringMap(), instanceConfig ); } private long createNodeWithPropertyOn( HighlyAvailableGraphDatabase db, String property, String value ) diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/TransactionConstraintsIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/TransactionConstraintsIT.java index 78c9a9dcfaa19..93e761006d6e0 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/TransactionConstraintsIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/TransactionConstraintsIT.java @@ -19,14 +19,14 @@ */ package org.neo4j.ha; -import java.util.concurrent.Callable; -import java.util.concurrent.Future; - import org.junit.Before; import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; + import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Label; @@ -48,15 +48,15 @@ import org.neo4j.test.TestGraphDatabaseFactory; import org.neo4j.test.ha.ClusterRule; -import static java.lang.System.currentTimeMillis; -import static java.util.concurrent.TimeUnit.MINUTES; -import static java.util.concurrent.TimeUnit.SECONDS; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static java.lang.System.currentTimeMillis; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + import static org.neo4j.helpers.collection.IteratorUtil.single; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable; @@ -65,7 +65,8 @@ public class TransactionConstraintsIT { @Rule - public final ClusterRule clusterRule = new ClusterRule(getClass()).config(HaSettings.pull_interval, "0"); + public final ClusterRule clusterRule = new ClusterRule( getClass() ) + .withSharedSetting( HaSettings.pull_interval, "0" ); protected ClusterManager.ManagedCluster cluster; diff --git a/enterprise/ha/src/test/java/org/neo4j/ha/UpdatePullerSwitchIT.java b/enterprise/ha/src/test/java/org/neo4j/ha/UpdatePullerSwitchIT.java index ebad36972050a..18974fb5d8444 100644 --- a/enterprise/ha/src/test/java/org/neo4j/ha/UpdatePullerSwitchIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/ha/UpdatePullerSwitchIT.java @@ -55,9 +55,9 @@ public class UpdatePullerSwitchIT @Before public void setup() throws Exception { - managedCluster = clusterRule.provider( ClusterManager.clusterOfSize( 2 ) ) - .config( tx_push_factor, "0" ) - .config( HaSettings.pull_interval, "100s" ) + managedCluster = clusterRule.withProvider( ClusterManager.clusterOfSize( 2 ) ) + .withSharedSetting( tx_push_factor, "0" ) + .withSharedSetting( HaSettings.pull_interval, "100s" ) .startCluster(); } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/HACountsPropagationTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/HACountsPropagationTest.java index d80f8b02c9f51..344874d5a38e4 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/HACountsPropagationTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/HACountsPropagationTest.java @@ -19,107 +19,76 @@ */ package org.neo4j.kernel; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; -import java.io.File; - import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.counts.CountsTracker; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertEquals; -import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; import static org.neo4j.register.Registers.newDoubleLongRegister; public class HACountsPropagationTest { private static final int PULL_INTERVAL = 100; - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( HACountsPropagationTest.class ) + .withSharedSetting( HaSettings.pull_interval, PULL_INTERVAL + "ms" ); @Test public void shouldPropagateNodeCountsInHA() throws Throwable { - File root = testDirectory.directory( "shouldPropagateNodeCountsInHA" ); - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 3 ), root, - MapUtil.stringMap( HaSettings.pull_interval.name(), PULL_INTERVAL + "ms" ) ); - clusterManager.start(); - try + ManagedCluster cluster = clusterRule.startCluster(); + HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - - HighlyAvailableGraphDatabase master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - master.createNode(); - master.createNode( DynamicLabel.label( "A" ) ); - tx.success(); - } + master.createNode(); + master.createNode( DynamicLabel.label( "A" ) ); + tx.success(); + } - waitForPullUpdates(); + waitForPullUpdates(); - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - CountsTracker counts = db.getDependencyResolver().resolveDependency( NeoStores.class ).getCounts(); - assertEquals( 2, counts.nodeCount( -1, newDoubleLongRegister() ).readSecond() ); - assertEquals( 1, counts.nodeCount( 0 /* A */, newDoubleLongRegister() ).readSecond() ); - } - } - finally + for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { - - clusterManager.shutdown(); + CountsTracker counts = db.getDependencyResolver().resolveDependency( NeoStores.class ).getCounts(); + assertEquals( 2, counts.nodeCount( -1, newDoubleLongRegister() ).readSecond() ); + assertEquals( 1, counts.nodeCount( 0 /* A */, newDoubleLongRegister() ).readSecond() ); } } @Test public void shouldPropagateRelationshipCountsInHA() throws Throwable { - File root = testDirectory.directory( "shouldPropagateRelationshipCountsInHA" ); - ClusterManager clusterManager = new ClusterManager( clusterOfSize( 3 ), root, - MapUtil.stringMap( HaSettings.pull_interval.name(), PULL_INTERVAL + "ms" ) ); - clusterManager.start(); - try + ManagedCluster cluster = clusterRule.startCluster(); + HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( allSeesAllAsAvailable() ); - - HighlyAvailableGraphDatabase master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - Node left = master.createNode(); - Node right = master.createNode( DynamicLabel.label( "A" ) ); - left.createRelationshipTo( right, DynamicRelationshipType.withName( "Type" ) ); - tx.success(); - } + Node left = master.createNode(); + Node right = master.createNode( DynamicLabel.label( "A" ) ); + left.createRelationshipTo( right, DynamicRelationshipType.withName( "Type" ) ); + tx.success(); + } - waitForPullUpdates(); + waitForPullUpdates(); - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - CountsTracker counts = db.getDependencyResolver().resolveDependency( NeoStores.class ).getCounts(); - assertEquals( 1, counts.relationshipCount( -1, -1, -1, newDoubleLongRegister() ).readSecond() ); - assertEquals( 1, counts.relationshipCount( -1, -1, 0, newDoubleLongRegister() ).readSecond() ); - assertEquals( 1, counts.relationshipCount( -1, 0, -1, newDoubleLongRegister() ).readSecond() ); - assertEquals( 1, counts.relationshipCount( -1, 0, 0, newDoubleLongRegister() ).readSecond() ); - } - } - finally + for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { - clusterManager.shutdown(); + CountsTracker counts = db.getDependencyResolver().resolveDependency( NeoStores.class ).getCounts(); + assertEquals( 1, counts.relationshipCount( -1, -1, -1, newDoubleLongRegister() ).readSecond() ); + assertEquals( 1, counts.relationshipCount( -1, -1, 0, newDoubleLongRegister() ).readSecond() ); + assertEquals( 1, counts.relationshipCount( -1, 0, -1, newDoubleLongRegister() ).readSecond() ); + assertEquals( 1, counts.relationshipCount( -1, 0, 0, newDoubleLongRegister() ).readSecond() ); } } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/ConstraintHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/ConstraintHaIT.java index c23731866bea1..a016727ae3991 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/ConstraintHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/ConstraintHaIT.java @@ -330,7 +330,7 @@ public void shouldNotAllowOldUncommittedTransactionsToResumeAndViolateConstraint { // Given ClusterManager.ManagedCluster cluster = - clusterRule.config( HaSettings.read_timeout, "4000s" ).startCluster(); + clusterRule.withSharedSetting( HaSettings.read_timeout, "4000s" ).startCluster(); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); HighlyAvailableGraphDatabase master = cluster.getMaster(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java index fa1500aacd87a..1d9831de88b1f 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/api/SchemaIndexHaIT.java @@ -72,10 +72,12 @@ import org.neo4j.test.DoubleLatch; import org.neo4j.test.ha.ClusterRule; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + +import static java.util.concurrent.TimeUnit.SECONDS; + import static org.neo4j.graphdb.DynamicLabel.label; import static org.neo4j.helpers.collection.IteratorUtil.asSet; import static org.neo4j.helpers.collection.IteratorUtil.asUniqueSet; @@ -129,7 +131,7 @@ public void indexPopulationJobsShouldContinueThroughRoleSwitch() throws Throwabl { // GIVEN a cluster of 3 ControlledGraphDatabaseFactory dbFactory = new ControlledGraphDatabaseFactory(); - ManagedCluster cluster = clusterRule.factory( dbFactory ).startCluster( ); + ManagedCluster cluster = clusterRule.withDbFactory( dbFactory ).startCluster(); HighlyAvailableGraphDatabase firstMaster = cluster.getMaster(); // where the master gets some data created as well as an index @@ -181,7 +183,7 @@ public void populatingSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterSto // GIVEN ControlledGraphDatabaseFactory dbFactory = new ControlledGraphDatabaseFactory( IS_MASTER ); - ManagedCluster cluster = clusterRule.factory( dbFactory ).startCluster( ); + ManagedCluster cluster = clusterRule.withDbFactory( dbFactory ).startCluster( ); try { @@ -245,7 +247,7 @@ public void onlineSchemaIndicesOnMasterShouldBeBroughtOnlineOnSlavesAfterStoreCo // GIVEN ControlledGraphDatabaseFactory dbFactory = new ControlledGraphDatabaseFactory(); - ManagedCluster cluster = clusterRule.factory( dbFactory ).startCluster( ); + ManagedCluster cluster = clusterRule.withDbFactory( dbFactory ).startCluster(); cluster.await( allSeesAllAsAvailable(), 120 ); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/BiggerThanLogTxIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/BiggerThanLogTxIT.java index 1c5e2def1568e..f01c3b07821c7 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/BiggerThanLogTxIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/BiggerThanLogTxIT.java @@ -19,12 +19,12 @@ */ package org.neo4j.kernel.ha; -import java.util.concurrent.TimeUnit; - import org.junit.Before; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; +import java.util.concurrent.TimeUnit; + import org.neo4j.function.Function; import org.neo4j.graphdb.GraphDatabaseService; import org.neo4j.graphdb.Node; @@ -44,9 +44,9 @@ public class BiggerThanLogTxIT { private static final String ROTATION_THRESHOLD = "1M"; - @Rule - public ClusterRule clusterRule = new ClusterRule( getClass() ) - .config( GraphDatabaseSettings.logical_log_rotation_threshold, ROTATION_THRESHOLD ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( BiggerThanLogTxIT.class ) + .withSharedSetting( GraphDatabaseSettings.logical_log_rotation_threshold, ROTATION_THRESHOLD ); protected ClusterManager.ManagedCluster cluster; diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java index 38f579fe0cb0b..1daa886650a2a 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java @@ -58,10 +58,12 @@ import org.neo4j.test.SuppressOutput; import org.neo4j.test.ha.ClusterRule; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; + +import static java.util.concurrent.TimeUnit.SECONDS; + import static org.neo4j.cluster.protocol.cluster.ClusterConfiguration.COORDINATOR; import static org.neo4j.function.Predicates.not; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; @@ -86,9 +88,9 @@ public class ClusterTopologyChangesIT public void setup() throws Exception { cluster = clusterRule - .config(HaSettings.read_timeout, "1s") - .config(HaSettings.state_switch_timeout, "2s") - .config(HaSettings.com_chunk_size, "1024") + .withSharedSetting( HaSettings.read_timeout, "1s" ) + .withSharedSetting( HaSettings.state_switch_timeout, "2s" ) + .withSharedSetting( HaSettings.com_chunk_size, "1024" ) .startCluster(); } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/DeletionTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/DeletionTest.java index f5a3861285591..c6eb3ca351b1a 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/DeletionTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/DeletionTest.java @@ -19,19 +19,17 @@ */ package org.neo4j.kernel.ha; -import org.junit.After; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.test.TargetDirectory; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertNotNull; import static org.neo4j.graphdb.DynamicRelationshipType.withName; -import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; /** @@ -56,20 +54,9 @@ */ public class DeletionTest { - private ClusterManager clusterManager; - - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - - @After - public void after() throws Throwable - { - if ( clusterManager != null ) - { - clusterManager.stop(); - clusterManager = null; - } - } + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( DeletionTest.class ) + .withProvider( clusterOfSize( 2 ) ); /** * The problem would manifest even if the transaction was performed on the Master, it would then occur when the @@ -80,12 +67,8 @@ public void after() throws Throwable public void shouldDeleteRecords() throws Throwable { // given - clusterManager = - new ClusterManager( clusterOfSize( 2 ), testDirectory.directory( "deleteRecords" ), stringMap() ); - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); + ManagedCluster cluster = clusterRule.startCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); HighlyAvailableGraphDatabase master = cluster.getMaster(); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaCountsIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaCountsIT.java index ca488d4a6a1fd..9b6d8bf253c32 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaCountsIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaCountsIT.java @@ -19,34 +19,82 @@ */ package org.neo4j.kernel.ha; -import org.junit.After; import org.junit.Before; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.schema.IndexDefinition; import org.neo4j.kernel.api.Statement; import org.neo4j.kernel.api.exceptions.KernelException; import org.neo4j.kernel.api.index.IndexDescriptor; import org.neo4j.kernel.impl.core.ThreadToStatementContextBridge; -import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.kernel.impl.store.NeoStores; import org.neo4j.kernel.impl.store.counts.CountsTracker; import org.neo4j.register.Register.DoubleLongRegister; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; import static org.neo4j.register.Registers.newDoubleLongRegister; +import static org.neo4j.tooling.GlobalGraphOperations.at; public class HaCountsIT { + private static final Label LABEL = DynamicLabel.label( "label" ); + private static final String PROPERTY_NAME = "prop"; + private static final String PROPERTY_VALUE = "value"; + + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( DeletionTest.class ); + + private ManagedCluster cluster; + private HighlyAvailableGraphDatabase master; + private HighlyAvailableGraphDatabase slave1; + private HighlyAvailableGraphDatabase slave2; + + @Before + public void setup() throws Exception + { + cluster = clusterRule.startCluster(); + master = cluster.getMaster(); + slave1 = cluster.getAnySlave(); + slave2 = cluster.getAnySlave( slave1 ); + clearDatabase(); + } + + private void clearDatabase() throws InterruptedException + { + try ( Transaction tx = master.beginTx() ) + { + for ( IndexDefinition index : master.schema().getIndexes() ) + { + index.drop(); + } + tx.success(); + } + + try ( Transaction tx = master.beginTx() ) + { + for ( Node node : at( master ).getAllNodes() ) + { + for ( Relationship relationship : node.getRelationships() ) + { + relationship.delete(); + } + node.delete(); + } + tx.success(); + } + cluster.sync(); + } + @Test public void shouldUpdateCountsOnSlavesWhenCreatingANodeOnMaster() throws Exception { @@ -219,34 +267,4 @@ private IndexDescriptor awaitOnline( HighlyAvailableGraphDatabase db, IndexDescr } throw new IllegalStateException( "Index did not become ONLINE within reasonable time" ); } - - private static final Label LABEL = DynamicLabel.label( "label" ); - private static final String PROPERTY_NAME = "prop"; - private static final String PROPERTY_VALUE = "value"; - - @Rule - public TargetDirectory.TestDirectory dir = TargetDirectory.testDirForTest( getClass() ); - private ClusterManager clusterManager; - private ClusterManager.ManagedCluster cluster; - private HighlyAvailableGraphDatabase master; - private HighlyAvailableGraphDatabase slave1; - private HighlyAvailableGraphDatabase slave2; - - @Before - public void setup() throws Throwable - { - clusterManager = new ClusterManager( clusterOfSize( 3 ), dir.graphDbDir(), stringMap() ); - clusterManager.start(); - cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - master = cluster.getMaster(); - slave1 = cluster.getAnySlave(); - slave2 = cluster.getAnySlave( slave1 ); - } - - @After - public void teardown() throws Throwable - { - clusterManager.shutdown(); - } } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaLoggingIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaLoggingIT.java index 5253b05e245f3..5701f220a1485 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaLoggingIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/HaLoggingIT.java @@ -31,8 +31,6 @@ import org.neo4j.kernel.impl.logging.StoreLogService; import org.neo4j.test.ha.ClusterRule; -import static java.util.Arrays.asList; - import static org.neo4j.helpers.collection.IteratorUtil.asIterable; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsJoined; import static org.neo4j.kernel.impl.ha.ClusterManager.clusterWithAdditionalClients; @@ -50,8 +48,8 @@ public class HaLoggingIT public void setup() throws Exception { cluster = clusterRule - .provider( clusterWithAdditionalClients( 2, 1 ) ) - .availabilityChecks( asList( masterAvailable(), masterSeesMembers( 3 ), allSeesAllAsJoined() ) ) + .withProvider( clusterWithAdditionalClients( 2, 1 ) ) + .availabilityChecks( masterAvailable(), masterSeesMembers( 3 ), allSeesAllAsJoined() ) .startCluster(); } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/MeasureUpdatePullingRecordAndIndexGap.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/MeasureUpdatePullingRecordAndIndexGap.java index 2821d79288add..7ba572c86f495 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/MeasureUpdatePullingRecordAndIndexGap.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/MeasureUpdatePullingRecordAndIndexGap.java @@ -51,7 +51,7 @@ public class MeasureUpdatePullingRecordAndIndexGap @Rule public final ClusterRule clusterRule = new ClusterRule( getClass() ) - .config( HaSettings.tx_push_factor, "0" ); + .withSharedSetting( HaSettings.tx_push_factor, "0" ); @Test public void shouldMeasureThatGap() throws Exception diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/PropertyConstraintsStressIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/PropertyConstraintsStressIT.java index 57c4e16869b61..3588c5bbb9de5 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/PropertyConstraintsStressIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/PropertyConstraintsStressIT.java @@ -113,7 +113,7 @@ public static Iterable params() @Before public void setup() throws Exception { - cluster = clusterRule.config( HaSettings.pull_interval, "0" ).startCluster(); + cluster = clusterRule.withSharedSetting( HaSettings.pull_interval, "0" ).startCluster(); } /* The different orders and delays in the below variations try to stress all known scenarios, as well as diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TestBasicHaOperations.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TestBasicHaOperations.java index e4713fb46eda1..61e84b24b6573 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TestBasicHaOperations.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TestBasicHaOperations.java @@ -19,8 +19,7 @@ */ package org.neo4j.kernel.ha; -import org.junit.After; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; import java.util.logging.Level; @@ -28,70 +27,56 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit; import org.neo4j.test.LoggerRule; -import org.neo4j.test.TargetDirectory; +import org.neo4j.test.ha.ClusterRule; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.ha.HaSettings.tx_push_factor; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; - public class TestBasicHaOperations { - @Rule - public LoggerRule logger = new LoggerRule( Level.OFF ); - @Rule - public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - private ClusterManager clusterManager; - - @After - public void after() throws Throwable - { - if ( clusterManager != null ) - { - clusterManager.stop(); - clusterManager = null; - } - } + @ClassRule + public static LoggerRule logger = new LoggerRule( Level.OFF ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( TestBasicHaOperations.class ) + .withSharedSetting( HaSettings.tx_push_factor, "2" ); @Test public void testBasicFailover() throws Throwable { // given - clusterManager = new ClusterManager( clusterOfSize( 3 ), testDirectory.directory( "failover" ), stringMap() ); - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase master = cluster.getMaster(); HighlyAvailableGraphDatabase slave1 = cluster.getAnySlave(); HighlyAvailableGraphDatabase slave2 = cluster.getAnySlave( slave1 ); // When long start = System.nanoTime(); - cluster.shutdown( master ); - logger.getLogger().warning( "Shut down master" ); - - cluster.await( ClusterManager.masterAvailable() ); - long end = System.nanoTime(); - - logger.getLogger().warning( "Failover took:" + (end - start) / 1000000 + "ms" ); - - // Then - boolean slave1Master = slave1.isMaster(); - boolean slave2Master = slave2.isMaster(); - - if ( slave1Master ) + RepairKit repair = cluster.shutdown( master ); + try { - assertFalse( slave2Master ); + logger.getLogger().warning( "Shut down master" ); + cluster.await( ClusterManager.masterAvailable() ); + long end = System.nanoTime(); + logger.getLogger().warning( "Failover took:" + (end - start) / 1000000 + "ms" ); + // Then + boolean slave1Master = slave1.isMaster(); + boolean slave2Master = slave2.isMaster(); + if ( slave1Master ) + { + assertFalse( slave2Master ); + } + else + { + assertTrue( slave2Master ); + } } - else + finally { - assertTrue( slave2Master ); + repair.repair(); } } @@ -99,20 +84,13 @@ public void testBasicFailover() throws Throwable public void testBasicPropagationFromSlaveToMaster() throws Throwable { // given - // a cluster of 2 - clusterManager = new ClusterManager( clusterOfSize( 2 ), testDirectory.directory( "propagation" ), - stringMap(HaSettings.tx_push_factor.name(), "1" ) ); - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase master = cluster.getMaster(); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); long nodeId; // a node with a property - try( Transaction tx = master.beginTx() ) + try ( Transaction tx = master.beginTx() ) { Node node = master.createNode(); nodeId = node.getId(); @@ -120,15 +98,7 @@ public void testBasicPropagationFromSlaveToMaster() throws Throwable tx.success(); } - try( Transaction tx = master.beginTx() ) - { - // make sure it's in the cache - master.getNodeById( nodeId ).getProperty( "foo" ); - tx.success(); - } - - // which has propagated to the slaves - slave.getDependencyResolver().resolveDependency( UpdatePuller.class ).pullUpdates(); + cluster.sync(); // when // the slave does a change @@ -151,13 +121,7 @@ public void testBasicPropagationFromSlaveToMaster() throws Throwable public void testBasicPropagationFromMasterToSlave() throws Throwable { // given - clusterManager = new ClusterManager( clusterOfSize( 3 ), testDirectory.directory( "propagation" ), - stringMap( tx_push_factor.name(), "2" ) ); - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - + ManagedCluster cluster = clusterRule.startCluster(); long nodeId = 4; HighlyAvailableGraphDatabase master = cluster.getMaster(); try ( Transaction tx = master.beginTx() ) diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TxPushStrategyConfigIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TxPushStrategyConfigIT.java index 90f7f03cdcda7..fb7163df236a4 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TxPushStrategyConfigIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/TxPushStrategyConfigIT.java @@ -39,12 +39,14 @@ import org.neo4j.test.SuppressOutput; import org.neo4j.test.ha.ClusterRule; -import static java.lang.Math.max; -import static java.lang.Math.min; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; + +import static java.lang.Math.max; +import static java.lang.Math.min; + import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable; @@ -172,9 +174,9 @@ public void slavesListGetsUpdatedWhenSlaveRageQuits() throws Throwable private ManagedCluster startCluster( int memberCount, final int pushFactor, final HaSettings.TxPushStrategy pushStrategy ) throws Exception { - ManagedCluster cluster = clusterRule.provider( clusterOfSize( memberCount ) ) - .config( HaSettings.tx_push_factor, "" + pushFactor ) - .config( HaSettings.tx_push_strategy, pushStrategy.name() ) + ManagedCluster cluster = clusterRule.withProvider( clusterOfSize( memberCount ) ) + .withSharedSetting( HaSettings.tx_push_factor, "" + pushFactor ) + .withSharedSetting( HaSettings.tx_push_strategy, pushStrategy.name() ) .startCluster(); mapMachineIds( cluster ); diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/member/HighAvailabilitySlavesIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/member/HighAvailabilitySlavesIT.java deleted file mode 100644 index 4a1c2268df2b7..0000000000000 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/member/HighAvailabilitySlavesIT.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2002-2015 "Neo Technology," - * Network Engine for Objects in Lund AB [http://neotechnology.com] - * - * This file is part of Neo4j. - * - * Neo4j is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -package org.neo4j.kernel.ha.cluster.member; - -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; - -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; -import org.neo4j.test.TargetDirectory; -import org.neo4j.tooling.GlobalGraphOperations; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.ha.HaSettings.tx_push_factor; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; - -public class HighAvailabilitySlavesIT -{ - @Rule - public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - private ClusterManager clusterManager; - - @After - public void after() throws Throwable - { - clusterManager.stop(); - } - - @Test - public void transactionsGetsPushedToSlaves() throws Throwable - { - // given - clusterManager = new ClusterManager( clusterOfSize( 3 ), testDirectory.directory( "dbs" ), - stringMap( tx_push_factor.name(), "2" ) ); - clusterManager.start(); - ManagedCluster cluster = clusterManager.getDefaultCluster(); - - // when - String name = "a node"; - long node = createNode( cluster.getMaster(), name ); - - // then - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) { - try (Transaction transaction = db.beginTx()) - { - assertEquals( node, getNodeByName( db, name ) ); - } - } - } - - private long getNodeByName( HighlyAvailableGraphDatabase db, String name ) - { - for ( Node node : GlobalGraphOperations.at( db ).getAllNodes() ) - if ( name.equals( node.getProperty( "name", null ) ) ) - return node.getId(); - fail( "No node '" + name + "' found in " + db ); - return 0; // Never called - } - - private long createNode( HighlyAvailableGraphDatabase db, String name ) - { - try ( Transaction tx = db.beginTx() ) - { - Node node = db.createNode(); - node.setProperty( "name", name ); - tx.success(); - return node.getId(); - } - } -} diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/index/AutoIndexConfigIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/index/AutoIndexConfigIT.java index de1f6eca001ce..c5dc53114a2ab 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/index/AutoIndexConfigIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/index/AutoIndexConfigIT.java @@ -19,54 +19,25 @@ */ package org.neo4j.kernel.index; -import org.junit.After; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; -import org.neo4j.cluster.InstanceId; import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.factory.GraphDatabaseBuilder; import org.neo4j.graphdb.index.AutoIndexer; -import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.test.TargetDirectory; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.test.ha.ClusterRule; import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; -import static org.neo4j.helpers.collection.MapUtil.stringMap; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable; public class AutoIndexConfigIT { - @Rule - public final TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - private ClusterManager.ManagedCluster cluster; - private ClusterManager clusterManager; - - public void startCluster( int size ) throws Throwable - { - clusterManager = new ClusterManager( clusterOfSize( size ), testDirectory.directory( "dbs" ), stringMap() ) - { - @Override - protected void config( GraphDatabaseBuilder builder, String clusterName, InstanceId serverId ) - { - builder.setConfig( "jmx.port", "" + (9912+serverId.toIntegerIndex()) ); - builder.setConfig( HaSettings.ha_server, ":" + (1136+serverId.toIntegerIndex()) ); - } - }; - clusterManager.start(); - cluster = clusterManager.getDefaultCluster(); - } - - @After - public void stopCluster() throws Throwable - { - clusterManager.stop(); - } + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( AutoIndexConfigIT.class ); @Test public void programmaticConfigShouldSurviveMasterSwitches() throws Throwable @@ -74,7 +45,7 @@ public void programmaticConfigShouldSurviveMasterSwitches() throws Throwable String propertyToIndex = "programmatic-property"; // Given - startCluster( 3 ); + ManagedCluster cluster = clusterRule.startCluster(); HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); AutoIndexer originalAutoIndex = slave.index().getNodeAutoIndexer(); @@ -88,8 +59,7 @@ public void programmaticConfigShouldSurviveMasterSwitches() throws Throwable // Then AutoIndexer newAutoIndex = slave.index().getNodeAutoIndexer(); - assertThat(newAutoIndex.isEnabled(), is(true)); + assertThat( newAutoIndex.isEnabled(), is( true ) ); assertThat( newAutoIndex.getAutoIndexedProperties(), hasItem( propertyToIndex ) ); } - } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/index/IndexOperationsIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/index/IndexOperationsIT.java index 47fcb354bc1f7..c8fcbd99fc051 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/index/IndexOperationsIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/index/IndexOperationsIT.java @@ -20,6 +20,7 @@ package org.neo4j.kernel.index; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; @@ -36,6 +37,7 @@ import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.ha.UpdatePuller; import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit; import org.neo4j.test.OtherThreadExecutor; import org.neo4j.test.OtherThreadExecutor.WorkerCommand; import org.neo4j.test.ha.ClusterRule; @@ -48,8 +50,8 @@ public class IndexOperationsIT { - @Rule - public ClusterRule clusterRule = new ClusterRule( getClass() ); + @ClassRule + public static ClusterRule clusterRule = new ClusterRule( IndexOperationsIT.class ); protected ClusterManager.ManagedCluster cluster; @@ -85,7 +87,7 @@ public void index_modifications_are_propagated() throws Exception } @Test - public void index_objects_can_be_reused_after_role_switch() throws Exception + public void index_objects_can_be_reused_after_role_switch() throws Throwable { // GIVEN // -- an existing index @@ -108,7 +110,7 @@ public void index_objects_can_be_reused_after_role_switch() throws Exception // WHEN // -- there's a master switch - cluster.shutdown( master ); + RepairKit repair = cluster.shutdown( master ); indexManagers.remove( master ); indexes.remove( master ); @@ -137,6 +139,7 @@ public void index_objects_can_be_reused_after_role_switch() throws Exception assertEquals( nodeId, index.get( key, value ).getSingle().getId() ); } } + repair.repair(); } @Test @@ -144,7 +147,7 @@ public void put_if_absent_works_across_instances() throws Exception { // GIVEN // -- two instances, each begin a transaction - String key = "key", value = "value"; + String key = "key2", value = "value2"; HighlyAvailableGraphDatabase db1 = cluster.getMaster(), db2 = cluster.getAnySlave(); long node = createNode( db1, key, value, false ); cluster.sync(); diff --git a/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterRule.java b/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterRule.java index 30c337b2e6d6e..212fb0a1d19d6 100644 --- a/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterRule.java +++ b/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterRule.java @@ -19,91 +19,148 @@ */ package org.neo4j.test.ha; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.rules.ExternalResource; import org.junit.runner.Description; import org.junit.runners.model.Statement; import java.io.File; -import java.util.ArrayList; +import java.io.IOException; import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; +import org.neo4j.function.IntFunction; import org.neo4j.function.Predicate; import org.neo4j.graphdb.config.Setting; -import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.HighlyAvailableGraphDatabaseFactory; -import org.neo4j.graphdb.factory.TestHighlyAvailableGraphDatabaseFactory; import org.neo4j.kernel.impl.ha.ClusterManager; -import org.neo4j.kernel.impl.ha.ClusterManager.Builder; +import org.neo4j.kernel.impl.ha.ClusterManager.ClusterBuilder; import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; +import org.neo4j.kernel.impl.ha.ClusterManager.Provider; +import org.neo4j.kernel.impl.ha.ClusterManager.StoreDirInitializer; import org.neo4j.test.TargetDirectory; import static java.util.Arrays.asList; import static org.neo4j.cluster.ClusterSettings.default_timeout; import static org.neo4j.graphdb.factory.GraphDatabaseSettings.pagecache_memory; -import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.store_internal_log_level; import static org.neo4j.kernel.ha.HaSettings.tx_push_factor; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; -import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; -public class ClusterRule extends ExternalResource +/** + * Starts, manages and in the end shuts down an HA cluster as a JUnit {@code Rule} or {@link ClassRule}. + * Basically this is {@link ClusterManager} in a JUnit {@link Rule} packaging. + */ +public class ClusterRule extends ExternalResource implements ClusterBuilder { + private ClusterManager.Builder clusterManagerBuilder; private ClusterManager clusterManager; private File storeDirectory; - - private ClusterManager.Provider provider = clusterOfSize( 3 ); - private final Map config = new HashMap<>(); - private HighlyAvailableGraphDatabaseFactory factory = new TestHighlyAvailableGraphDatabaseFactory(); private List> availabilityChecks = asList( allSeesAllAsAvailable() ); private final TargetDirectory.TestDirectory testDirectory; + private ManagedCluster cluster; public ClusterRule( Class testClass ) { this.testDirectory = TargetDirectory.testDirForTest( testClass ); - config.putAll( stringMap( - GraphDatabaseSettings.store_internal_log_level.name(), "DEBUG", - default_timeout.name(), "1s", - tx_push_factor.name(), "0", - pagecache_memory.name(), "8m" ) ); + this.clusterManagerBuilder = new ClusterManager.Builder() + .withSharedSetting( store_internal_log_level, "DEBUG" ) + .withSharedSetting( default_timeout, "1s" ) + .withSharedSetting( tx_push_factor, "0" ) + .withSharedSetting( pagecache_memory, "8m" ); + } + + @Override + public ClusterRule withRootDirectory( File root ) + { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterRule withSeedDir( final File seedDir ) + { + clusterManagerBuilder = clusterManagerBuilder.withSeedDir( seedDir ); + return this; } - public ClusterRule config( Setting setting, String value ) + @Override + public ClusterRule withStoreDirInitializer( StoreDirInitializer initializer ) { - config.put( setting.name(), value ); + clusterManagerBuilder = clusterManagerBuilder.withStoreDirInitializer( initializer ); return this; } - public ClusterRule provider( ClusterManager.Provider provider ) + @Override + public ClusterRule withDbFactory( HighlyAvailableGraphDatabaseFactory dbFactory ) { - this.provider = provider; + clusterManagerBuilder = clusterManagerBuilder.withDbFactory( dbFactory ); return this; } - public ClusterRule factory( HighlyAvailableGraphDatabaseFactory factory ) + @Override + public ClusterRule withProvider( Provider provider ) { - this.factory = factory; + clusterManagerBuilder = clusterManagerBuilder.withProvider( provider ); return this; } - @SafeVarargs - public final ClusterRule availabilityChecks( Predicate... checks ) + @Override + public ClusterRule withInstanceConfig( Map> commonConfig ) + { + clusterManagerBuilder = clusterManagerBuilder.withInstanceConfig( commonConfig ); + return this; + } + + @Override + public ClusterRule withInstanceSetting( Setting setting, IntFunction valueFunction ) + { + clusterManagerBuilder = clusterManagerBuilder.withInstanceSetting( setting, valueFunction ); + return this; + } + + @Override + public ClusterRule withSharedConfig( Map commonConfig ) + { + clusterManagerBuilder = clusterManagerBuilder.withSharedConfig( commonConfig ); + return this; + } + + @Override + public ClusterRule withSharedSetting( Setting setting, String value ) { - return availabilityChecks( Arrays.asList( checks ) ); + clusterManagerBuilder = clusterManagerBuilder.withSharedSetting( setting, value ); + return this; } - public ClusterRule availabilityChecks( List> checks ) + /** + * Specifies which availability checks are performed and awaited before considering the cluster + * up and running. + * + * @param checks availability checks to perform. There are default checks if no custom checks + * are provided. All previous checks will be replaced by these new ones. + * @return this {@link ClusterRule} instance, for builder convenience. + */ + @SafeVarargs + public final ClusterRule availabilityChecks( Predicate... checks ) { - availabilityChecks = new ArrayList<>( checks ); + availabilityChecks = Arrays.asList( checks ); return this; } + /** + * Starts cluster with the configuration provided at instantiation time. + */ public ClusterManager.ManagedCluster startCluster() throws Exception { - clusterManager = new Builder( storeDirectory ) - .withCommonConfig( config ).withProvider( provider ).withDbFactory( factory ).build(); + if ( cluster != null ) + { + return cluster; + } + + clusterManager = clusterManagerBuilder.withRootDirectory( storeDirectory ).build(); try { clusterManager.start(); @@ -117,7 +174,8 @@ public ClusterManager.ManagedCluster startCluster() throws Exception { cluster.await( availabilityCheck ); } - return cluster; + + return this.cluster = cluster; } @Override @@ -128,7 +186,11 @@ public Statement apply( final Statement base, final Description description ) @Override public void evaluate() throws Throwable { - storeDirectory = testDirectory.directory( description.getMethodName() ); + // If this is used as class rule then getMethodName() returns null, so use + // getClassName() instead. + String name = description.getMethodName() != null ? + description.getMethodName() : description.getClassName(); + storeDirectory = testDirectory.directory( name ); base.evaluate(); } }; @@ -153,4 +215,70 @@ protected void after() throwable.printStackTrace(); } } + + public File directory( String name ) + { + return testDirectory.directory( name ); + } + + public File cleanDirectory( String name ) throws IOException + { + return testDirectory.cleanDirectory( name ); + } + + /** + * Adapter for providing a static config value into a setting where per-instances dynamic config values + * are supplied. + * + * @param value static config value. + * @return this {@link ClusterRule} instance, for builder convenience. + */ + public static IntFunction constant( String value ) + { + return ClusterManager.constant( value ); + } + + /** + * Dynamic configuration value, of sorts. Can be used as input to {@link #config(Setting, IntFunction)}. + * Some configuration values are a function of server id of the cluster member and this is a utility + * for creating such dynamic configuration values. + * + * @param oneBasedServerId value onto which one-based server id is added. So for example + * a value of 10 would have cluster member with server id 2 that config value set to 12. + */ + public static IntFunction intBase( final int oneBasedServerId ) + { + return new IntFunction() + { + @Override + public String apply( int serverId ) + { + return String.valueOf( oneBasedServerId + serverId ); + } + }; + } + + /** + * Dynamic configuration value, of sorts. Can be used as input to {@link #config(Setting, IntFunction)}. + * Some configuration values are a function of server id of the cluster member and this is a utility + * for creating such dynamic configuration values. + * + * @param prefix string prefix for these config values. + * @param oneBasedServerId value onto which one-based server id is added. So for example + * a value of 10 would have cluster member with server id 2 that config value set to 12. + * @return a string which has a prefix and an integer part, where the integer part is a function of + * server id of the cluster member. Can be used to set config values like a host, where arguments could look + * something like: {@code prefix: "localhost:" oneBasedServerId: 5000}. + */ + public static IntFunction stringWithIntBase( final String prefix, final int oneBasedServerId ) + { + return new IntFunction() + { + @Override + public String apply( int serverId ) + { + return prefix + (oneBasedServerId + serverId); + } + }; + } } diff --git a/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterTest.java b/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterTest.java index e360204fc70fb..26d05d7c55cb0 100644 --- a/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/test/ha/ClusterTest.java @@ -33,7 +33,6 @@ import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.TransactionFailureException; import org.neo4j.graphdb.factory.TestHighlyAvailableGraphDatabaseFactory; -import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.impl.ha.ClusterManager; @@ -44,11 +43,13 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.neo4j.helpers.collection.MapUtil.entry; +import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; +import static org.neo4j.kernel.impl.ha.ClusterManager.clusterWithAdditionalArbiters; import static org.neo4j.kernel.impl.ha.ClusterManager.fromXml; import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable; import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesSlavesAsAvailable; +import static org.neo4j.kernel.impl.ha.ClusterManager.provided; public class ClusterTest { @@ -57,14 +58,14 @@ public class ClusterTest @Rule public TargetDirectory.TestDirectory testDirectory = TargetDirectory.testDirForTest( getClass() ); - @Test public void testCluster() throws Throwable { - ClusterManager clusterManager = new ClusterManager( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ), - testDirectory.directory( "testCluster" ), - entry( HaSettings.ha_server.name(), "localhost:6001-6005" ). - entry( HaSettings.tx_push_factor.name(), "2" ).create() ); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "testCluster" ) ) + .withProvider( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ) ) + .withSharedConfig( stringMap( + HaSettings.ha_server.name(), "localhost:6001-6005", + HaSettings.tx_push_factor.name(), "2" ) ).build(); try { clusterManager.start(); @@ -107,10 +108,11 @@ public void testClusterWithHostnames() throws Throwable final Clusters clusters = new Clusters(); clusters.getClusters().add( cluster ); - ClusterManager clusterManager = new ClusterManager( ClusterManager.provided( clusters ), - testDirectory.directory( "testCluster" ), - MapUtil.stringMap( HaSettings.ha_server.name(), "localhost:6001-6005", - HaSettings.tx_push_factor.name(), "2" )); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "testCluster" ) ) + .withProvider( provided( clusters ) ) + .withSharedConfig( stringMap( + HaSettings.ha_server.name(), "localhost:6001-6005", + HaSettings.tx_push_factor.name(), "2" ) ).build(); try { clusterManager.start(); @@ -152,10 +154,11 @@ public void testClusterWithWildcardIP() throws Throwable final Clusters clusters = new Clusters(); clusters.getClusters().add( cluster ); - ClusterManager clusterManager = new ClusterManager( ClusterManager.provided( clusters ), - testDirectory.directory( "testCluster" ), - MapUtil.stringMap( HaSettings.ha_server.name(), "0.0.0.0:6001-6005", - HaSettings.tx_push_factor.name(), "2" )); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "testCluster" ) ) + .withProvider( provided( clusters ) ) + .withSharedConfig( stringMap( + HaSettings.ha_server.name(), "0.0.0.0:6001-6005", + HaSettings.tx_push_factor.name(), "2" ) ).build(); try { clusterManager.start(); @@ -188,8 +191,8 @@ public void testClusterWithWildcardIP() throws Throwable @Test @Ignore("JH: Ignored for by CG in March 2013, needs revisit. I added @ignore instead of commenting out to list this in static analysis.") public void testArbiterStartsFirstAndThenTwoInstancesJoin() throws Throwable { - ClusterManager clusterManager = new ClusterManager( ClusterManager.clusterWithAdditionalArbiters( 2, 1 ), - testDirectory.directory( "testCluster" ), MapUtil.stringMap()); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "testCluster" ) ) + .withProvider( clusterWithAdditionalArbiters( 2, 1 ) ).build(); try { clusterManager.start(); @@ -297,8 +300,8 @@ public void testInstancesWithConflictingHaPorts() throws Throwable @Test public void given4instanceClusterWhenMasterGoesDownThenElectNewMaster() throws Throwable { - ClusterManager clusterManager = new ClusterManager( fromXml( getClass().getResource( "/fourinstances.xml" ).toURI() ), - testDirectory.directory( "4instances" ), MapUtil.stringMap() ); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "4instances" ) ) + .withProvider( fromXml( getClass().getResource( "/fourinstances.xml" ).toURI() ) ).build(); try { clusterManager.start(); @@ -351,8 +354,8 @@ public void givenEmptyHostListWhenClusterStartupThenFormClusterWithSingleInstanc @Test public void givenClusterWhenMasterGoesDownAndTxIsRunningThenDontWaitToSwitch() throws Throwable { - ClusterManager clusterManager = new ClusterManager( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ), - testDirectory.directory( "waitfortx" ), MapUtil.stringMap() ); + ClusterManager clusterManager = new ClusterManager.Builder( testDirectory.directory( "waitfortx" ) ) + .withProvider( fromXml( getClass().getResource( "/threeinstances.xml" ).toURI() ) ).build(); try { clusterManager.start(); diff --git a/enterprise/ha/src/test/java/org/neo4j/test/ha/ReadOnlySlaveTest.java b/enterprise/ha/src/test/java/org/neo4j/test/ha/ReadOnlySlaveTest.java index 5da44017966fe..b90d994185276 100644 --- a/enterprise/ha/src/test/java/org/neo4j/test/ha/ReadOnlySlaveTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/test/ha/ReadOnlySlaveTest.java @@ -19,28 +19,25 @@ */ package org.neo4j.test.ha; -import org.junit.Rule; +import org.junit.ClassRule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.util.Map; import org.neo4j.cluster.InstanceId; +import org.neo4j.function.IntFunction; import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.TransactionFailureException; -import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.Settings; import org.neo4j.kernel.api.exceptions.ReadOnlyDbException; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.impl.ha.ClusterManager; +import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertThat; -import static org.neo4j.helpers.collection.MapUtil.stringMap; +import static org.neo4j.graphdb.factory.GraphDatabaseSettings.read_only; import static org.neo4j.kernel.ha.HaSettings.tx_push_factor; /** @@ -48,95 +45,66 @@ */ public class ReadOnlySlaveTest { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + @ClassRule + public static final ClusterRule clusterRule = new ClusterRule( ReadOnlySlaveTest.class ) + .withSharedSetting( tx_push_factor, "2" ) + .withInstanceSetting( read_only, new IntFunction() + { + @Override + public String apply( int oneBasedServerId ) + { + return oneBasedServerId == 2 ? Settings.TRUE : null; + } + } ); @Test public void givenClusterWithReadOnlySlaveWhenWriteTxOnSlaveThenCommitFails() throws Throwable { - // Given - Map config = stringMap( - tx_push_factor.name(), "2" ); - ClusterManager clusterManager = new ClusterManager.Builder( folder.getRoot() ) - .withCommonConfig( config ) - .withInstanceConfig( 2, stringMap( - GraphDatabaseSettings.read_only - .name(), - Settings.TRUE ) ) - .build(); - try - { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); + // When + ManagedCluster cluster = clusterRule.startCluster(); + HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); - // When - HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); - - try ( Transaction tx = readOnlySlave.beginTx() ) - { - readOnlySlave.createNode(); - tx.success(); - } - catch ( TransactionFailureException ex ) - { - // Then - assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); - } + try ( Transaction tx = readOnlySlave.beginTx() ) + { + readOnlySlave.createNode(); + tx.success(); } - finally + catch ( TransactionFailureException ex ) { - clusterManager.shutdown(); + // Then + assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); } - } @Test public void givenClusterWithReadOnlySlaveWhenChangePropertyOnSlaveThenThrowException() throws Throwable { // Given - Map config = stringMap( - tx_push_factor.name(), "2" ); - ClusterManager clusterManager = new ClusterManager.Builder( folder.getRoot() ) - .withCommonConfig( config ).withInstanceConfig( 2, stringMap( GraphDatabaseSettings.read_only.name(), - Settings.TRUE ) ) - .build(); - try + ManagedCluster cluster = clusterRule.startCluster(); + Node node; + HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - - Node node; - HighlyAvailableGraphDatabase master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - node = master.createNode(); - tx.success(); - } - - // When - HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); + node = master.createNode(); + tx.success(); + } - try ( Transaction tx = readOnlySlave.beginTx() ) - { - Node slaveNode = readOnlySlave.getNodeById( node.getId() ); + // When + HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); + try ( Transaction tx = readOnlySlave.beginTx() ) + { + Node slaveNode = readOnlySlave.getNodeById( node.getId() ); - // Then - slaveNode.setProperty( "foo", "bar" ); - tx.success(); - } - catch ( TransactionFailureException ex ) - { - // Ok! - assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); - } + // Then + slaveNode.setProperty( "foo", "bar" ); + tx.success(); } - finally + catch ( TransactionFailureException ex ) { - clusterManager.shutdown(); + // Ok! + assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); } } @@ -144,101 +112,66 @@ public void givenClusterWithReadOnlySlaveWhenChangePropertyOnSlaveThenThrowExcep public void givenClusterWithReadOnlySlaveWhenAddNewLabelOnSlaveThenThrowException() throws Throwable { // Given - Map config = stringMap( - tx_push_factor.name(), "2" ); - ClusterManager clusterManager = new ClusterManager.Builder( folder.getRoot() ) - .withCommonConfig( config ).withInstanceConfig( 2, - stringMap( GraphDatabaseSettings.read_only.name(), Settings.TRUE ) ) - .build(); - try + ManagedCluster cluster = clusterRule.startCluster(); + Node node; + HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - - Node node; - HighlyAvailableGraphDatabase master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - node = master.createNode(); - tx.success(); - } + node = master.createNode(); + tx.success(); + } - // When - HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); + // When + HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); - try ( Transaction tx = readOnlySlave.beginTx() ) - { - Node slaveNode = readOnlySlave.getNodeById( node.getId() ); + try ( Transaction tx = readOnlySlave.beginTx() ) + { + Node slaveNode = readOnlySlave.getNodeById( node.getId() ); - // Then - slaveNode.addLabel( DynamicLabel.label( "FOO" ) ); - tx.success(); - } - catch ( TransactionFailureException ex ) - { - // Ok! - assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); - } + // Then + slaveNode.addLabel( DynamicLabel.label( "FOO" ) ); + tx.success(); } - finally + catch ( TransactionFailureException ex ) { - clusterManager.shutdown(); + // Ok! + assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); } - } @Test public void givenClusterWithReadOnlySlaveWhenAddNewRelTypeOnSlaveThenThrowException() throws Throwable { // Given - Map config = stringMap( - tx_push_factor.name(), "2" ); - ClusterManager clusterManager = new ClusterManager.Builder( folder.getRoot() ) - .withCommonConfig( config ) - .withInstanceConfig( 2, stringMap( GraphDatabaseSettings.read_only.name(), Settings.TRUE ) ) - .build(); - try + ManagedCluster cluster = clusterRule.startCluster(); + Node node; + Node node2; + HighlyAvailableGraphDatabase master = cluster.getMaster(); + try ( Transaction tx = master.beginTx() ) { - clusterManager.start(); - ClusterManager.ManagedCluster cluster = clusterManager.getDefaultCluster(); - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - - Node node; - Node node2; - HighlyAvailableGraphDatabase master = cluster.getMaster(); - try ( Transaction tx = master.beginTx() ) - { - node = master.createNode(); - node2 = master.createNode(); - tx.success(); - } - - // When - HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); + node = master.createNode(); + node2 = master.createNode(); + tx.success(); + } - try ( Transaction tx = readOnlySlave.beginTx() ) - { - Node slaveNode = readOnlySlave.getNodeById( node.getId() ); - Node slaveNode2 = readOnlySlave.getNodeById( node2.getId() ); + // When + HighlyAvailableGraphDatabase readOnlySlave = cluster.getMemberByServerId( new InstanceId( 2 ) ); + try ( Transaction tx = readOnlySlave.beginTx() ) + { + Node slaveNode = readOnlySlave.getNodeById( node.getId() ); + Node slaveNode2 = readOnlySlave.getNodeById( node2.getId() ); - // Then - slaveNode.createRelationshipTo( slaveNode2, DynamicRelationshipType.withName( "KNOWS" ) ); - tx.success(); - } - catch ( TransactionFailureException ex ) - { - // Ok! - assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); - } + // Then + slaveNode.createRelationshipTo( slaveNode2, DynamicRelationshipType.withName( "KNOWS" ) ); + tx.success(); } - finally + catch ( TransactionFailureException ex ) { - clusterManager.shutdown(); + // Ok! + assertThat( ex.getCause(), instanceOf( ReadOnlyDbException.class ) ); } - } } diff --git a/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java b/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java index 7d6510f91584f..2a8e14c79a0ce 100644 --- a/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java +++ b/integrationtests/src/test/java/org/neo4j/ha/HAClusterStartupIT.java @@ -19,13 +19,13 @@ */ package org.neo4j.ha; -import java.io.File; -import java.io.IOException; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import java.io.File; +import java.io.IOException; + import org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException; import org.neo4j.graphdb.Transaction; import org.neo4j.io.fs.FileUtils; @@ -36,7 +36,6 @@ import org.neo4j.test.TargetDirectory.TestDirectory; import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; -import static org.neo4j.helpers.collection.MapUtil.stringMap; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; import static org.neo4j.kernel.impl.ha.ClusterManager.clusterOfSize; @@ -52,7 +51,7 @@ public class HAClusterStartupIT @Before public void instantiateClusterManager() { - clusterManager = new ClusterManager( clusterOfSize( 3 ), dir.directory(), stringMap() ); + clusterManager = new ClusterManager.Builder( dir.directory() ).build(); } @Before @@ -220,9 +219,8 @@ public void aClusterShouldStartAndRunWhenSeededWithAStoreHavingNoLogicalLogFiles File newDir = new File( dir.directory(), "new" ); FileUtils.deleteRecursively( newDir ); - ClusterManager newClusterManager = new ClusterManager( - new ClusterManager.Builder( newDir ).withProvider( clusterOfSize( 3 ) ).withSeedDir( seedDir ) - ); + ClusterManager newClusterManager = new ClusterManager.Builder( newDir ) + .withProvider( clusterOfSize( 3 ) ).withSeedDir( seedDir ).build(); newClusterManager.start(); diff --git a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIntegrationTest.java b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIntegrationTest.java index 80e12f06eeb24..bfd7dd5d22931 100644 --- a/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIntegrationTest.java +++ b/integrationtests/src/test/java/org/neo4j/storeupgrade/StoreUpgradeIntegrationTest.java @@ -79,6 +79,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; + import static org.neo4j.consistency.store.StoreAssertions.assertConsistentStore; import static org.neo4j.helpers.collection.Iterables.concat; import static org.neo4j.helpers.collection.Iterables.count; @@ -236,9 +237,8 @@ public void migratingOlderDataAndThanStartAClusterUsingTheNewerDataShouldWork() // start the cluster with the db migrated from the old instance File haDir = new File( dir.getParentFile(), "ha-stuff" ); FileUtils.deleteRecursively( haDir ); - ClusterManager clusterManager = new ClusterManager( - new ClusterManager.Builder( haDir ).withSeedDir( dir ).withProvider( clusterOfSize( 2 ) ) - ); + ClusterManager clusterManager = new ClusterManager.Builder( haDir ) + .withSeedDir( dir ).withProvider( clusterOfSize( 2 ) ).build(); clusterManager.start();