From 213dea9e93d82b3ee3f10ab488ffaae0453c276c Mon Sep 17 00:00:00 2001 From: Mattias Persson Date: Tue, 13 Sep 2016 09:51:50 +0200 Subject: [PATCH] Converted most of HaBeanIT to unit test --- .../org/neo4j/jmx/impl/ManagementData.java | 2 +- .../org/neo4j/jmx/impl/ManagementSupport.java | 6 +- .../ha/cluster/member/ClusterMember.java | 2 +- enterprise/ha/src/test/java/jmx/HaBeanIT.java | 247 +------------- .../management/HighAvailabilityBeanTest.java | 304 ++++++++++++++++++ 5 files changed, 321 insertions(+), 240 deletions(-) create mode 100644 enterprise/ha/src/test/java/org/neo4j/kernel/ha/management/HighAvailabilityBeanTest.java diff --git a/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java b/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java index 294c23ab3de13..f9f742ec985e6 100644 --- a/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java +++ b/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java @@ -30,7 +30,7 @@ public final class ManagementData extends DependencyResolver.Adapter private final ManagementSupport support; final ManagementBeanProvider provider; - ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support ) + public ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support ) { this.provider = provider; this.kernel = kernel; diff --git a/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java b/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java index 80f980f7fa6ab..13a885eb12f4e 100644 --- a/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java +++ b/community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java @@ -35,7 +35,7 @@ public class ManagementSupport { - static final ManagementSupport load() + public static final ManagementSupport load() { ManagementSupport support = new ManagementSupport(); for ( ManagementSupport candidate : Service.load( ManagementSupport.class ) ) @@ -66,7 +66,7 @@ protected T makeProxy( KernelBean kernel, ObjectName name, Class beanInte final Collection getProxiesFor( Class beanInterface, KernelBean kernel ) { - Collection result = new ArrayList(); + Collection result = new ArrayList<>(); ObjectName query = createObjectNameQuery( kernel.getInstanceId(), beanInterface ); for ( ObjectName name : getMBeanServer().queryNames( query, null ) ) { @@ -115,7 +115,7 @@ protected String getBeanName( Class beanInterface ) protected ObjectName createObjectName( String instanceId, String beanName, boolean query, String... extraNaming ) { - Hashtable properties = new Hashtable(); + Hashtable properties = new Hashtable<>(); properties.put( "instance", "kernel#" + instanceId ); properties.put( "name", beanName ); for ( int i = 0; i < extraNaming.length; i++ ) diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/member/ClusterMember.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/member/ClusterMember.java index 6b9719132f2dd..e43da77ee0c83 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/member/ClusterMember.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/member/ClusterMember.java @@ -42,7 +42,7 @@ public ClusterMember( InstanceId instanceId ) this( instanceId, Collections.emptyMap(), StoreId.DEFAULT, true ); } - ClusterMember( InstanceId instanceId, Map roles, StoreId storeId, boolean alive ) + public ClusterMember( InstanceId instanceId, Map roles, StoreId storeId, boolean alive ) { this.instanceId = instanceId; this.roles = roles; diff --git a/enterprise/ha/src/test/java/jmx/HaBeanIT.java b/enterprise/ha/src/test/java/jmx/HaBeanIT.java index 736f0b5f05467..bbb016b1642c5 100644 --- a/enterprise/ha/src/test/java/jmx/HaBeanIT.java +++ b/enterprise/ha/src/test/java/jmx/HaBeanIT.java @@ -23,30 +23,21 @@ import org.junit.Test; import java.net.URI; -import java.text.DateFormat; -import java.text.SimpleDateFormat; import java.util.Arrays; -import java.util.function.Predicate; - -import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.helpers.collection.Iterables; import org.neo4j.jmx.Kernel; import org.neo4j.jmx.impl.JmxKernelExtension; import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; -import org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState; import org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher; -import org.neo4j.kernel.impl.ha.ClusterManager; import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; -import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit; import org.neo4j.management.BranchedStore; import org.neo4j.management.ClusterMemberInfo; import org.neo4j.management.HighAvailability; import org.neo4j.management.Neo4jManager; import org.neo4j.test.ha.ClusterRule; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -55,232 +46,42 @@ import static org.neo4j.helpers.collection.Iterables.firstOrNull; import static org.neo4j.kernel.configuration.Settings.STRING; import static org.neo4j.kernel.configuration.Settings.setting; -import static org.neo4j.kernel.impl.ha.ClusterManager.instanceEvicted; -import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable; -import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesMembers; -import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesSlavesAsAvailable; import static org.neo4j.test.ha.ClusterRule.intBase; import static org.neo4j.test.ha.ClusterRule.stringWithIntBase; public class HaBeanIT { @Rule - public final ClusterRule clusterRule = new ClusterRule( getClass() ) + public final ClusterRule clusterRule = new ClusterRule( HaBeanIT.class ) .withInstanceSetting( setting( "jmx.port", STRING, (String) null ), intBase( 9912 ) ) .withInstanceSetting( HaSettings.ha_server, stringWithIntBase( ":", 1136 ) ) .withInstanceSetting( GraphDatabaseSettings.forced_kernel_id, stringWithIntBase( "kernel", 0 ) ); @Test - public void canGetHaBean() throws Throwable + public void shouldAccessHaBeans() throws Throwable { ManagedCluster cluster = clusterRule.startCluster(); + + // High Availability bean HighAvailability ha = ha( cluster.getMaster() ); assertNotNull( "could not get ha bean", ha ); assertMasterInformation( ha ); - } - - private void assertMasterInformation( HighAvailability ha ) - { - assertTrue( "should be available", ha.isAvailable() ); - assertEquals( "should be master", HighAvailabilityModeSwitcher.MASTER, ha.getRole() ); - } - - @Test - public void testLatestTxInfoIsCorrect() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); - HighlyAvailableGraphDatabase db = cluster.getMaster(); - HighAvailability masterHa = ha( db ); - long lastCommitted = masterHa.getLastCommittedTxId(); - try ( Transaction tx = db.beginTx() ) - { - db.createNode(); - tx.success(); - } - assertEquals( lastCommitted + 1, masterHa.getLastCommittedTxId() ); - } - - @Test - public void testUpdatePullWorksAndUpdatesLastUpdateTime() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); - HighlyAvailableGraphDatabase master = cluster.getMaster(); - HighlyAvailableGraphDatabase slave = cluster.getAnySlave(); - try (Transaction tx = master.beginTx()) - { - master.createNode(); - tx.success(); - } - HighAvailability slaveBean = ha( slave ); - DateFormat format = new SimpleDateFormat( "yyyy-MM-DD kk:mm:ss.SSSZZZZ" ); - // To begin with, no updates - slaveBean.update(); - long timeUpdated = format.parse( slaveBean.getLastUpdateTime() ).getTime(); - assertTrue( timeUpdated > 0 ); - } - - @Test - public void testAfterGentleMasterSwitchClusterInfoIsCorrect() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); - HighlyAvailableGraphDatabase master = cluster.getMaster(); - RepairKit masterShutdown = cluster.shutdown( master ); - - cluster.await( masterAvailable( master ) ); - cluster.await( masterSeesSlavesAsAvailable( 1 ) ); - - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - assertEquals( 2, ha( db ).getInstancesInCluster().length ); - } - - masterShutdown.repair(); - - cluster.await( ClusterManager.allSeesAllAsAvailable() ); - - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - HighAvailability bean = ha( db ); - - assertEquals( 3, bean.getInstancesInCluster().length ); - for ( ClusterMemberInfo info : bean.getInstancesInCluster() ) - { - assertTrue( "every instance should be available", info.isAvailable() ); - assertTrue( "every instances should have at least one role", info.getRoles().length > 0 ); - if ( HighAvailabilityModeSwitcher.MASTER.equals( info.getRoles()[0] ) ) - { - assertEquals( "coordinator should be master", - HighAvailabilityModeSwitcher.MASTER, info.getHaRole() ); - } - else - { - assertEquals( "Either master or slave, no other way", - HighAvailabilityModeSwitcher.SLAVE, info.getRoles()[0] ); - assertEquals( "instance " + info.getInstanceId() + " is cluster slave but HA master", - HighAvailabilityModeSwitcher.SLAVE, info.getHaRole() ); - } - for ( String uri : info.getUris() ) - { - assertTrue( "roles should contain URIs", - uri.startsWith( "ha://" ) || uri.startsWith( "backup://" ) ); - } - } - } - } - - @Test - public void testAfterHardMasterSwitchClusterInfoIsCorrect() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); - - cluster.await( masterSeesSlavesAsAvailable( 2 ) ); - - HighlyAvailableGraphDatabase master = cluster.getMaster(); - RepairKit masterShutdown = cluster.fail( master ); - - cluster.await( instanceEvicted( master ) ); - - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - if ( db.getInstanceState() == HighAvailabilityMemberState.PENDING ) - { - continue; - } - // Instance that was hard killed will still be in the cluster - assertEquals( 3, ha( db ).getInstancesInCluster().length ); - } - - masterShutdown.repair(); - - cluster.await( ClusterManager.allSeesAllAsAvailable(), 180 ); - - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) + assertMasterAndSlaveInformation( ha.getInstancesInCluster() ); + for ( ClusterMemberInfo info : ha.getInstancesInCluster() ) { - int mastersFound = 0; - HighAvailability bean = ha( db ); - - assertEquals( 3, bean.getInstancesInCluster().length ); - for ( ClusterMemberInfo info : bean.getInstancesInCluster() ) - { - assertTrue( bean.getInstanceId() + ": every instance should be available: " + info.getInstanceId(), - info.isAvailable() ); - for ( String role : info.getRoles() ) - { - if ( role.equals( HighAvailabilityModeSwitcher.MASTER ) ) - { - mastersFound++; - } - } - } - assertEquals( 1, mastersFound ); + assertTrue( info.isAlive() ); + assertTrue( info.isAvailable() ); } - } - @Test - public void canGetBranchedStoreBean() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); + // Branched data bean BranchedStore bs = beans( cluster.getMaster() ).getBranchedStoreBean(); assertNotNull( "could not get branched store bean", bs ); } - @Test - public void joinedInstanceShowsUpAsSlave() throws Throwable - { - ManagedCluster cluster = clusterRule.startCluster(); - ClusterMemberInfo[] instancesInCluster = ha( cluster.getMaster() ).getInstancesInCluster(); - assertEquals( 3, instancesInCluster.length ); - ClusterMemberInfo[] secondInstancesInCluster = ha( cluster.getAnySlave() ).getInstancesInCluster(); - assertEquals( 3, secondInstancesInCluster.length ); - assertMasterAndSlaveInformation( instancesInCluster ); - assertMasterAndSlaveInformation( secondInstancesInCluster ); - } - - @Test - public void leftInstanceDisappearsFromMemberList() throws Throwable - { - // Start the cluster and make sure it's up. - // Then shut down one of the slaves to see if it disappears from the member list. - ManagedCluster cluster = clusterRule.startCluster(); - assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length ); - RepairKit repair = cluster.shutdown( cluster.getAnySlave() ); - - try - { - cluster.await( masterSeesMembers( 2 ) ); - HighAvailability haMaster = ha( cluster.getMaster() ); - assertEquals( 2, haMaster.getInstancesInCluster().length ); - } - finally - { - repair.repair(); - } - } - - @Test - public void failedMemberIsStillInMemberListAlthoughFailed() throws Throwable + private void assertMasterInformation( HighAvailability ha ) { - ManagedCluster cluster = clusterRule.startCluster(); - assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length ); - - // Fail the instance - HighlyAvailableGraphDatabase failedDb = cluster.getAnySlave(); - RepairKit dbFailure = cluster.fail( failedDb ); - try - { - await( ha( cluster.getMaster() ), dbAlive( false ) ); - await( ha( cluster.getAnySlave( failedDb ) ), dbAlive( false ) ); - } - finally - { - // Repair the failure and come back - dbFailure.repair(); - } - for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() ) - { - await( ha( db ), dbAvailability( true ) ); - await( ha( db ), dbAlive( true ) ); - } + assertTrue( "should be available", ha.isAvailable() ); + assertEquals( "should be master", HighAvailabilityModeSwitcher.MASTER, ha.getRole() ); } private Neo4jManager beans( HighlyAvailableGraphDatabase db ) @@ -324,28 +125,4 @@ private ClusterMemberInfo member( ClusterMemberInfo[] members, int instanceId ) members ) ); return null; // it will never get here. } - - private void await( HighAvailability ha, Predicate predicate ) throws InterruptedException - { - long end = System.currentTimeMillis() + SECONDS.toMillis( 300 ); - while ( System.currentTimeMillis() < end ) - { - if ( predicate.test( member( ha.getInstancesInCluster(), 2 ) ) ) - { - return; - } - Thread.sleep( 500 ); - } - fail( "Failed instance didn't show up as such in JMX" ); - } - - private Predicate dbAvailability( final boolean available ) - { - return item -> item.isAvailable() == available; - } - - private Predicate dbAlive( final boolean alive ) - { - return item -> item.isAlive() == alive; - } } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/management/HighAvailabilityBeanTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/management/HighAvailabilityBeanTest.java new file mode 100644 index 0000000000000..3539e7ce77463 --- /dev/null +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/management/HighAvailabilityBeanTest.java @@ -0,0 +1,304 @@ +package org.neo4j.kernel.ha.management; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.function.Predicate; + +import javax.management.NotCompliantMBeanException; + +import org.neo4j.cluster.InstanceId; +import org.neo4j.helpers.Format; +import org.neo4j.helpers.collection.Iterables; +import org.neo4j.helpers.collection.MapUtil; +import org.neo4j.io.fs.DefaultFileSystemAbstraction; +import org.neo4j.jmx.impl.ManagementData; +import org.neo4j.jmx.impl.ManagementSupport; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; +import org.neo4j.kernel.ha.LastUpdateTime; +import org.neo4j.kernel.ha.UpdatePuller; +import org.neo4j.kernel.ha.cluster.member.ClusterMember; +import org.neo4j.kernel.ha.cluster.member.ClusterMembers; +import org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher; +import org.neo4j.kernel.impl.core.LastTxIdGetter; +import org.neo4j.kernel.impl.util.Dependencies; +import org.neo4j.kernel.internal.GraphDatabaseAPI; +import org.neo4j.kernel.internal.KernelData; +import org.neo4j.kernel.internal.Version; +import org.neo4j.management.ClusterMemberInfo; +import org.neo4j.management.HighAvailability; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import static java.util.Arrays.asList; + +import static org.neo4j.helpers.collection.Iterables.filter; +import static org.neo4j.helpers.collection.Iterables.firstOrNull; +import static org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher.MASTER; +import static org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher.SLAVE; +import static org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher.UNKNOWN; +import static org.neo4j.kernel.impl.store.StoreId.DEFAULT; + +public class HighAvailabilityBeanTest +{ + private final GraphDatabaseAPI db = mock( HighlyAvailableGraphDatabase.class ); + private final Dependencies dependencies = new Dependencies(); + private final ClusterMembers clusterMembers = mock( ClusterMembers.class ); + private final HighAvailabilityBean bean = new HighAvailabilityBean(); + private final LastTxIdGetter lastTxIdGetter = mock( LastTxIdGetter.class ); + private final LastUpdateTime lastUpdateTime = mock( LastUpdateTime.class ); + private final ClusterDatabaseInfoProvider dbInfoProvider = + new ClusterDatabaseInfoProvider( clusterMembers, lastTxIdGetter, lastUpdateTime ); + private final KernelData kerneData = new HighlyAvailableKernelData( db, clusterMembers, dbInfoProvider, + new DefaultFileSystemAbstraction(), null, new File( "storeDir" ), Config.empty() ) + { + @Override + public Version version() + { + return Version.getKernel(); + } + + @Override + public GraphDatabaseAPI graphDatabase() + { + return db; + } + }; + private final ManagementData data = new ManagementData( bean, kerneData, ManagementSupport.load() ); + private HighAvailability haBean; + + @Before + public void setup() throws NotCompliantMBeanException + { + when( db.getDependencyResolver() ).thenReturn( dependencies ); + haBean = (HighAvailability) new HighAvailabilityBean().createMBean( data ); + } + + @Test + public void shouldPickUpOnLastCommittedTxId() throws Exception + { + // GIVEN + long txId = 101L; + when( lastTxIdGetter.getLastTxId() ).thenReturn( txId, txId + 1 ); + when( clusterMembers.getCurrentMember() ).thenReturn( clusterMember( 1, MASTER, 1010 ) ); + + // WHEN + long reportedTxId = haBean.getLastCommittedTxId(); + + // THEN + assertEquals( txId, reportedTxId ); + + // and WHEN + long nextReportedTxId = haBean.getLastCommittedTxId(); + + // THEN + assertEquals( txId + 1, nextReportedTxId ); + } + + @Test + public void shouldPickUpOnLastUpdateTime() throws Exception + { + // GIVEN + long updateTime = 123456789L; + when( lastUpdateTime.getLastUpdateTime() ).thenReturn( 0L, updateTime, updateTime + 1_000 ); + when( clusterMembers.getCurrentMember() ).thenReturn( clusterMember( 1, MASTER, 1010 ) ); + + // WHEN + String reportedUpdateTime = haBean.getLastUpdateTime(); + + // THEN + assertEquals( "N/A", reportedUpdateTime ); + + // and WHEN + String secondReportedUpdateTime = haBean.getLastUpdateTime(); + + // THEN + assertEquals( Format.date( updateTime ), secondReportedUpdateTime ); + + // and WHEN + String thirdReportedTxId = haBean.getLastUpdateTime(); + + // THEN + assertEquals( Format.date( updateTime + 1_000 ), thirdReportedTxId ); + } + + @Test + public void shouldSeeChangesInClusterMembers() throws Exception + { + // GIVEN + when( clusterMembers.getMembers() ).thenReturn( asList( + clusterMember( 1, MASTER, 1137 ), + clusterMember( 2, SLAVE, 1138 ), + clusterMember( 3, SLAVE, 1139 ) ) ); + + // THEN + assertMasterAndSlaveInformation( haBean.getInstancesInCluster() ); + + // and WHEN + when( clusterMembers.getMembers() ).thenReturn( asList( + clusterMember( 1, SLAVE, 1137 ), + clusterMember( 2, MASTER, 1138 ), + clusterMember( 3, SLAVE, 1139 ) ) ); + + // THEN + for ( ClusterMemberInfo info : haBean.getInstancesInCluster() ) + { + assertTrue( "every instance should be available", info.isAvailable() ); + assertTrue( "every instances should have at least one role", info.getRoles().length > 0 ); + if ( HighAvailabilityModeSwitcher.MASTER.equals( info.getRoles()[0] ) ) + { + assertEquals( "coordinator should be master", + HighAvailabilityModeSwitcher.MASTER, info.getHaRole() ); + } + else + { + assertEquals( "Either master or slave, no other way", + HighAvailabilityModeSwitcher.SLAVE, info.getRoles()[0] ); + assertEquals( "instance " + info.getInstanceId() + " is cluster slave but HA master", + HighAvailabilityModeSwitcher.SLAVE, info.getHaRole() ); + } + for ( String uri : info.getUris() ) + { + assertTrue( "roles should contain URIs", + uri.startsWith( "ha://" ) || uri.startsWith( "backup://" ) ); + } + } + } + + @Test + public void shouldSeeLeavingMemberDisappear() throws Exception + { + // GIVEN + when( clusterMembers.getMembers() ).thenReturn( asList( + clusterMember( 1, MASTER, 1137 ), + clusterMember( 2, SLAVE, 1138 ), + clusterMember( 3, SLAVE, 1139 ) ) ); + assertMasterAndSlaveInformation( haBean.getInstancesInCluster() ); + + // WHEN + when( clusterMembers.getMembers() ).thenReturn( asList( + clusterMember( 1, MASTER, 1137 ), + clusterMember( 3, SLAVE, 1139 ) ) ); + + // THEN + assertEquals( 2, haBean.getInstancesInCluster().length ); + } + + @Test + public void shouldSeeFailedMembersInMemberList() throws Exception + { + // GIVEN + when( clusterMembers.getMembers() ).thenReturn( asList( + clusterMember( 1, MASTER, 1137 ), + clusterMember( 2, SLAVE, 1138 ), + clusterMember( 3, UNKNOWN, 1139, false ) ) ); + + // WHEN + ClusterMemberInfo[] instances = haBean.getInstancesInCluster(); + + // THEN + assertEquals( 3, instances.length ); + assertEquals( 2, count( instances, instance -> instance.isAlive() ) ); + assertEquals( 2, count( instances, instance -> instance.isAvailable() ) ); + } + + @Test + public void shouldPullUpdates() throws Exception + { + // GIVEN + UpdatePuller updatePuller = mock( UpdatePuller.class ); + dependencies.satisfyDependency( updatePuller ); + + // WHEN + String result = haBean.update(); + + // THEN + verify( updatePuller ).pullUpdates(); + assertTrue( result, result.contains( "Update completed in" ) ); + } + + @Test + public void shouldReportFailedPullUpdates() throws Exception + { + // GIVEN + UpdatePuller updatePuller = mock( UpdatePuller.class ); + RuntimeException myException = new RuntimeException( "My test exception" ); + Mockito.doThrow( myException ).when( updatePuller ).pullUpdates(); + dependencies.satisfyDependency( updatePuller ); + + // WHEN + String result = haBean.update(); + + // THEN + verify( updatePuller ).pullUpdates(); + assertTrue( result, result.contains( myException.getMessage() ) ); + } + + private int count( ClusterMemberInfo[] instances, Predicate filter ) + { + int count = 0; + for ( ClusterMemberInfo instance : instances ) + { + if ( filter.test( instance ) ) + { + count++; + } + } + return count; + } + + private ClusterMember clusterMember( int serverId, String role, int port ) throws URISyntaxException + { + return clusterMember( serverId, role, port, true ); + } + + private ClusterMember clusterMember( int serverId, String role, int port, boolean alive ) throws URISyntaxException + { + URI uri = HighAvailabilityModeSwitcher.UNKNOWN.equals( role ) ? null : new URI( "ha://" + role + ":" + port ); + return new ClusterMember( new InstanceId( serverId ), + MapUtil.genericMap( role, uri ), DEFAULT, alive ); + } + + private void assertMasterAndSlaveInformation( ClusterMemberInfo[] instancesInCluster ) throws Exception + { + ClusterMemberInfo master = member( instancesInCluster, 1 ); + assertEquals( 1137, + getUriForScheme( "ha", Iterables.map( URI::create, Arrays.asList( master.getUris() ) ) ).getPort() ); + assertEquals( HighAvailabilityModeSwitcher.MASTER, master.getHaRole() ); + ClusterMemberInfo slave = member( instancesInCluster, 2 ); + assertEquals( 1138, + getUriForScheme( "ha", Iterables.map( URI::create, Arrays.asList( slave.getUris() ) ) ).getPort() ); + assertEquals( HighAvailabilityModeSwitcher.SLAVE, slave.getHaRole() ); + assertTrue( "Slave not available", slave.isAvailable() ); + } + + private static URI getUriForScheme( final String scheme, Iterable uris ) + { + return firstOrNull( filter( item -> item.getScheme().equals( scheme ), uris ) ); + } + + private ClusterMemberInfo member( ClusterMemberInfo[] members, int instanceId ) + { + for ( ClusterMemberInfo member : members ) + { + if ( member.getInstanceId().equals( Integer.toString( instanceId ) ) ) + { + return member; + } + } + fail( "Couldn't find cluster member with cluster URI port " + instanceId + " among " + Arrays.toString( + members ) ); + return null; // it will never get here. + } +}