From eea557c9a653de8c2b837d0e4c4578197dbde71c Mon Sep 17 00:00:00 2001 From: Mikhaylo Demianenko Date: Tue, 20 Oct 2015 18:41:21 +0200 Subject: [PATCH] Update handling of unavailability event for current member. Unavailability listener updated to work as following: as soon as we receive unavailability event about us - then something went wrong in a cluster and we need to perform new elections. Elections should be triggered for all states except HighAvailabilityMemberState.PENDING, since first of all there is nothing or we already made a switch and waiting election to start, so no reason to start them again. Simplify InvalidEpochExceptionHandler to only generate unavailability event. --- .../ha/HighlyAvailableGraphDatabase.java | 2 +- .../HighAvailabilityMemberStateMachine.java | 45 +- .../cluster/HighAvailabilityModeSwitcher.java | 18 +- .../slave/InvalidEpochExceptionHandler.java | 9 - .../kernel/ha/ClusterTopologyChangesIT.java | 45 +- ...ighAvailabilityMemberStateMachineTest.java | 524 +++++++++--------- .../HighAvailabilityModeSwitcherTest.java | 35 +- 7 files changed, 325 insertions(+), 353 deletions(-) diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java index 1f28a6f9b0f4a..1e6d5476d509d 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/HighlyAvailableGraphDatabase.java @@ -483,7 +483,7 @@ protected void createModeSwitcher() @Override public void handle() { - highAvailabilityModeSwitcher.forceElections(); + highAvailabilityModeSwitcher.postMemberUnavailable(); } }; diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachine.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachine.java index cbce5f53804ef..f58bfd7e0a2d0 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachine.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachine.java @@ -235,16 +235,49 @@ public void notify( HighAvailabilityMemberListener listener ) } } + + /** + * As soon as we receive an unavailability message and the instanceId belongs to us, depending on the current + * state we do the following: + * + * The assumption here is: as soon as we receive unavailability event about us - then something went wrong + * in a cluster and we need to perform new elections. + * Elections should be triggered for all states except {@link HighAvailabilityMemberState#PENDING}, since + * first of all there is nothing or we already made a switch and waiting election to start, so no reason to + * start them again. + *

+ * Listener invoked from sync block in {@link org.neo4j.cluster.member.paxos.PaxosClusterMemberEvents} so we + * should not have any racing here. + *

+ * @param role The role for which the member is unavailable + * @param unavailableId The id of the member which became unavailable for that role + */ @Override public void memberIsUnavailable( String role, InstanceId unavailableId ) { - if ( context.getMyId().equals( unavailableId ) && - HighAvailabilityModeSwitcher.SLAVE.equals( role ) && - state == HighAvailabilityMemberState.SLAVE ) + if ( context.getMyId().equals( unavailableId ) ) { - HighAvailabilityMemberState oldState = state; - changeStateToPending(); - logger.debug( "Got memberIsUnavailable(" + unavailableId + "), moved to " + state + " from " + oldState ); + if ( HighAvailabilityMemberState.PENDING != state ) + { + HighAvailabilityMemberState oldState = state; + changeStateToPending(); + logger.debug( "Got memberIsUnavailable(" + unavailableId + "), moved to " + state + " from " + + oldState ); + logger.debug( "Forcing new round of elections." ); + election.performRoleElections(); + } + else + { + logger.debug( "Got memberIsUnavailable(" + unavailableId + "), but already in " + + HighAvailabilityMemberState.PENDING + " state, will skip state change and " + + "new election."); + } } else { diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcher.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcher.java index 7ae54ec4a2e2f..c38cc84465425 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcher.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcher.java @@ -24,7 +24,6 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.neo4j.cluster.BindingListener; @@ -99,7 +98,6 @@ public static InstanceId getServerId( URI haUri ) private volatile URI me; private volatile Future modeSwitcherFuture; private volatile HighAvailabilityMemberState currentTargetState; - private final AtomicBoolean canAskForElections = new AtomicBoolean( true ); public HighAvailabilityModeSwitcher( SwitchToSlave switchToSlave, SwitchToMaster switchToMaster, @@ -209,13 +207,9 @@ public void removeModeSwitcher( ModeSwitcher modeSwitcher ) modeSwitchListeners = Listeners.removeListener( modeSwitcher, modeSwitchListeners ); } - public void forceElections() + public void postMemberUnavailable() { - if ( canAskForElections.compareAndSet( true, false ) ) - { - clusterMemberAvailability.memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE ); - election.performRoleElections(); - } + clusterMemberAvailability.memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE ); } private void stateChanged( HighAvailabilityMemberChangeEvent event ) @@ -239,12 +233,6 @@ private void stateChanged( HighAvailabilityMemberChangeEvent event ) switch ( event.getNewState() ) { case TO_MASTER: - - if ( event.getOldState().equals( HighAvailabilityMemberState.SLAVE ) ) - { - clusterMemberAvailability.memberIsUnavailable( SLAVE ); - } - switchToMaster(); break; case TO_SLAVE: @@ -300,7 +288,6 @@ public void notify( ModeSwitcher listener ) try { masterHaURI = switchToMaster.switchToMaster( haCommunicationLife, me ); - canAskForElections.set( true ); } catch ( Throwable e ) { @@ -380,7 +367,6 @@ public void notify( ModeSwitcher listener ) else { slaveHaURI = resultingSlaveHaURI; - canAskForElections.set( true ); } } catch ( HighAvailabilityStoreFailureException e ) diff --git a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/com/slave/InvalidEpochExceptionHandler.java b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/com/slave/InvalidEpochExceptionHandler.java index 33c3ee17d56c6..5520b779cda15 100644 --- a/enterprise/ha/src/main/java/org/neo4j/kernel/ha/com/slave/InvalidEpochExceptionHandler.java +++ b/enterprise/ha/src/main/java/org/neo4j/kernel/ha/com/slave/InvalidEpochExceptionHandler.java @@ -22,13 +22,4 @@ public interface InvalidEpochExceptionHandler { void handle(); - - InvalidEpochExceptionHandler NONE = new InvalidEpochExceptionHandler() - { - @Override - public void handle() - { - - } - }; } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java index 101113ff3b9c9..0b0bcd1146e61 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/ClusterTopologyChangesIT.java @@ -19,16 +19,15 @@ */ package org.neo4j.kernel.ha; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -import org.junit.After; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; - import org.neo4j.cluster.ClusterSettings; import org.neo4j.cluster.InstanceId; import org.neo4j.cluster.client.ClusterClient; @@ -42,25 +41,20 @@ import org.neo4j.graphdb.Node; import org.neo4j.graphdb.Transaction; import org.neo4j.graphdb.factory.GraphDatabaseSettings; -import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.MapUtil; import org.neo4j.kernel.InternalAbstractGraphDatabase; import org.neo4j.kernel.configuration.Config; import org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState; -import org.neo4j.kernel.ha.com.master.InvalidEpochException; import org.neo4j.kernel.impl.ha.ClusterManager; import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit; import org.neo4j.kernel.logging.DevNullLoggingService; import org.neo4j.kernel.monitoring.Monitors; import org.neo4j.test.CleanupRule; import org.neo4j.test.ha.ClusterRule; -import org.neo4j.tooling.GlobalGraphOperations; import static java.util.concurrent.TimeUnit.SECONDS; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; - import static org.neo4j.cluster.protocol.cluster.ClusterConfiguration.COORDINATOR; import static org.neo4j.helpers.Predicates.not; import static org.neo4j.kernel.impl.ha.ClusterManager.allSeesAllAsAvailable; @@ -75,7 +69,7 @@ public class ClusterTopologyChangesIT @Rule public final CleanupRule cleanup = new CleanupRule(); - protected ClusterManager.ManagedCluster cluster; + private ClusterManager.ManagedCluster cluster; @Before public void setup() throws Exception @@ -88,12 +82,7 @@ public void setup() throws Exception .startCluster(); } - @After - public void cleanup() - { - cluster = null; - } - + @Test public void masterRejoinsAfterFailureAndReelection() throws Throwable { @@ -242,14 +231,6 @@ public void enteredCluster( ClusterConfiguration clusterConfiguration ) assertEquals( new InstanceId( 2 ), coordinatorIdWhenReJoined.get() ); } - private static long nodeCountOn( HighlyAvailableGraphDatabase db ) - { - try ( Transaction ignored = db.beginTx() ) - { - return Iterables.count( GlobalGraphOperations.at( db ).getAllNodes() ); - } - } - private static ClusterClient clusterClientOf( HighlyAvailableGraphDatabase db ) { return db.getDependencyResolver().resolveDependency( ClusterClient.class ); @@ -298,18 +279,4 @@ private static void attemptTransactions( HighlyAvailableGraphDatabase... dbs ) } } } - - private static void assertHasInvalidEpoch( HighlyAvailableGraphDatabase db ) - { - InvalidEpochException invalidEpochException = null; - try - { - createNodeOn( db ); - } - catch ( InvalidEpochException e ) - { - invalidEpochException = e; - } - assertNotNull( "Expected InvalidEpochException was not thrown", invalidEpochException ); - } } diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachineTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachineTest.java index 98972b452e8c7..db086ff64697f 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachineTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityMemberStateMachineTest.java @@ -29,10 +29,8 @@ import java.net.URI; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -90,6 +88,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import static org.neo4j.kernel.ha.cluster.HighAvailabilityModeSwitcher.MASTER; import static org.neo4j.kernel.ha.cluster.HighAvailabilityModeSwitcher.SLAVE; @@ -100,93 +99,130 @@ public class HighAvailabilityMemberStateMachineTest public void shouldStartFromPending() throws Exception { // Given - HighAvailabilityMemberContext context = mock( HighAvailabilityMemberContext.class ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); - ClusterMemberEvents events = mock( ClusterMemberEvents.class ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - + HighAvailabilityMemberStateMachine memberStateMachine = buildMockedStateMachine(); // Then - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( memberStateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); } + @Test public void shouldMoveToToMasterFromPendingOnMasterElectedForItself() throws Throwable { // Given InstanceId me = new InstanceId( 1 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - final Set listener = new HashSet<>(); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events ); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + // When + memberListener.coordinatorIsElected( me ); - } ).when( events ).addClusterMemberListener( Matchers.any() ); + // Then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); + } + + @Test + public void ignoreAnotherMemberNotAvailable() throws Throwable + { + InstanceId me = new InstanceId( 1 ); + InstanceId other = new InstanceId( 2 ); + HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); + + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events ); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); + + // When + memberListener.coordinatorIsElected( me ); + + // Then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); + + // When + memberListener.memberIsUnavailable(HighAvailabilityModeSwitcher.SLAVE, other); + // Then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); + } + + @Test + public void switchToPendingAndForceElectionOnThisMemberNotAvailable() throws Throwable + { + InstanceId me = new InstanceId( 1 ); + InstanceId master = new InstanceId( 2 ); + HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + + HighAvailabilityMemberStateMachine stateMachine = new StateMachineBuilder().withContext( context ) + .withElection( election ).withEvents( events ).build(); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); // When - theListener.coordinatorIsElected( me ); + memberListener.memberIsAvailable( HighAvailabilityModeSwitcher.MASTER, master, URI.create( "2" ), + StoreId.DEFAULT ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_SLAVE ) ); + + // When + memberListener.memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE, me ); + + //then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + verify( election ).performRoleElections(); } @Test - public void shouldRemainToPendingOnMasterElectedForSomeoneElse() throws Throwable + public void whilePendingDoNotForceElectionOnThisMemberNotAvailable() throws Throwable { // Given InstanceId me = new InstanceId( 1 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); + Election election = mock( Election.class ); ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - final Set listener = new HashSet<>(); + HighAvailabilityMemberStateMachine stateMachine = new StateMachineBuilder().withElection( election ) + .withEvents( events ).withContext( context ).build(); + stateMachine.init(); - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + // Then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); - } ).when( events ).addClusterMemberListener( Matchers.any() ); + memberListenerContainer.get().memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE, me ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + //then + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + verifyZeroInteractions( election ); + } + + @Test + public void shouldRemainToPendingOnMasterElectedForSomeoneElse() throws Throwable + { + // Given + InstanceId me = new InstanceId( 1 ); + HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); + + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events ); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); // When - theListener.coordinatorIsElected( new InstanceId( 2 ) ); + memberListener.coordinatorIsElected( new InstanceId( 2 ) ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); } @Test @@ -195,38 +231,21 @@ public void shouldSwitchToToSlaveOnMasterAvailableForSomeoneElse() throws Throwa // Given InstanceId me = new InstanceId( 1 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - final Set listener = new HashSet<>(); - - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } - - } ).when( events ).addClusterMemberListener( Matchers.any() ); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); HAStateChangeListener probe = new HAStateChangeListener(); - toTest.addHighAvailabilityMemberListener( probe ); + stateMachine.addHighAvailabilityMemberListener( probe ); // When - theListener.memberIsAvailable( MASTER, new InstanceId( 2 ), URI.create( "ha://whatever" ), StoreId.DEFAULT ); + memberListener.memberIsAvailable( MASTER, new InstanceId( 2 ), URI.create( "ha://whatever" ), StoreId.DEFAULT ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_SLAVE ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_SLAVE ) ); assertThat( probe.masterIsAvailable, is( true ) ); } @@ -237,54 +256,31 @@ public void whenInMasterStateLosingQuorumShouldPutInPending() throws Throwable InstanceId me = new InstanceId( 1 ); InstanceId other = new InstanceId( 2 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); - ClusterMemberEvents events = mock( ClusterMemberEvents.class ); - List membersList = new LinkedList<>(); - // we cannot set outside of the package the isAlive to return false. So do it with a mock - ClusterMember otherMemberMock = mock( ClusterMember.class ); - when( otherMemberMock.getInstanceId() ).thenReturn( other ); - when( otherMemberMock.isAlive() ).thenReturn( false ); - membersList.add( otherMemberMock ); - - membersList.add( new ClusterMember( me ) ); - when( members.getMembers() ).thenReturn( membersList ); - - final Set listener = new HashSet<>(); + AvailabilityGuard guard = mock( AvailabilityGuard.class ); + ClusterMembers members = mockClusterMembers( me, other ); - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - } ).when( events ).addClusterMemberListener( Matchers.any() ); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events, members, guard ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); HAStateChangeListener probe = new HAStateChangeListener(); - toTest.addHighAvailabilityMemberListener( probe ); + stateMachine.addHighAvailabilityMemberListener( probe ); // Send it to MASTER - theListener.coordinatorIsElected( me ); - theListener.memberIsAvailable( MASTER, me, URI.create( "ha://whatever" ), StoreId.DEFAULT ); + memberListener.coordinatorIsElected( me ); + memberListener.memberIsAvailable( MASTER, me, URI.create( "ha://whatever" ), StoreId.DEFAULT ); - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.MASTER ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.MASTER ) ); // When - theListener.memberIsFailed( new InstanceId( 2 ) ); + memberListener.memberIsFailed( new InstanceId( 2 ) ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); assertThat( probe.instanceStops, is( true ) ); verify( guard, times( 2 ) ).deny( any( AvailabilityGuard.AvailabilityRequirement.class ) ); } @@ -297,53 +293,29 @@ public void whenInSlaveStateLosingQuorumShouldPutInPending() throws Throwable InstanceId other = new InstanceId( 2 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); - ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMembers members = mockClusterMembers( me, other ); - List membersList = new LinkedList<>(); - // we cannot set outside of the package the isAlive to return false. So do it with a mock - ClusterMember otherMemberMock = mock( ClusterMember.class ); - when( otherMemberMock.getInstanceId() ).thenReturn( other ); - when( otherMemberMock.isAlive() ).thenReturn( false ); - membersList.add( otherMemberMock ); - - membersList.add( new ClusterMember( me ) ); - when( members.getMembers() ).thenReturn( membersList ); - - final Set listener = new HashSet<>(); - - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - } ).when( events ).addClusterMemberListener( Matchers.any() ); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events, members, guard ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); HAStateChangeListener probe = new HAStateChangeListener(); - toTest.addHighAvailabilityMemberListener( probe ); + stateMachine.addHighAvailabilityMemberListener( probe ); // Send it to MASTER - theListener.memberIsAvailable( MASTER, other, URI.create( "ha://whatever" ), StoreId.DEFAULT ); - theListener.memberIsAvailable( SLAVE, me, URI.create( "ha://whatever2" ), StoreId.DEFAULT ); + memberListener.memberIsAvailable( MASTER, other, URI.create( "ha://whatever" ), StoreId.DEFAULT ); + memberListener.memberIsAvailable( SLAVE, me, URI.create( "ha://whatever2" ), StoreId.DEFAULT ); - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.SLAVE ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.SLAVE ) ); // When - theListener.memberIsFailed( new InstanceId( 2 ) ); + memberListener.memberIsFailed( new InstanceId( 2 ) ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); assertThat( probe.instanceStops, is( true ) ); verify( guard, times( 2 ) ).deny( any( AvailabilityGuard.AvailabilityRequirement.class ) ); } @@ -356,52 +328,28 @@ public void whenInToMasterStateLosingQuorumShouldPutInPending() throws Throwable InstanceId other = new InstanceId( 2 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); - ClusterMemberEvents events = mock( ClusterMemberEvents.class ); - - List membersList = new LinkedList<>(); - // we cannot set outside of the package the isAlive to return false. So do it with a mock - ClusterMember otherMemberMock = mock( ClusterMember.class ); - when( otherMemberMock.getInstanceId() ).thenReturn( other ); - when( otherMemberMock.isAlive() ).thenReturn( false ); - membersList.add( otherMemberMock ); - - membersList.add( new ClusterMember( me ) ); - when( members.getMembers() ).thenReturn( membersList ); - - final Set listener = new HashSet<>(); + ClusterMembers members = mockClusterMembers( me, other ); - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - } ).when( events ).addClusterMemberListener( Matchers.any() ); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events, members, guard ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); HAStateChangeListener probe = new HAStateChangeListener(); - toTest.addHighAvailabilityMemberListener( probe ); + stateMachine.addHighAvailabilityMemberListener( probe ); // Send it to MASTER - theListener.coordinatorIsElected( me ); + memberListener.coordinatorIsElected( me ); - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_MASTER ) ); // When - theListener.memberIsFailed( new InstanceId( 2 ) ); + memberListener.memberIsFailed( new InstanceId( 2 ) ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); assertThat( probe.instanceStops, is( true ) ); verify( guard, times( 1 ) ).deny( any( AvailabilityGuard.AvailabilityRequirement.class ) ); } @@ -414,52 +362,27 @@ public void whenInToSlaveStateLosingQuorumShouldPutInPending() throws Throwable InstanceId other = new InstanceId( 2 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, false ); AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); - ClusterMemberEvents events = mock( ClusterMemberEvents.class ); - - List membersList = new LinkedList<>(); - // we cannot set outside of the package the isAlive to return false. So do it with a mock - ClusterMember otherMemberMock = mock( ClusterMember.class ); - when( otherMemberMock.getInstanceId() ).thenReturn( other ); - when( otherMemberMock.isAlive() ).thenReturn( false ); - membersList.add( otherMemberMock ); + ClusterMembers members = mockClusterMembers( me, other ); - membersList.add( new ClusterMember( me ) ); - when( members.getMembers() ).thenReturn( membersList ); - - final Set listener = new HashSet<>(); - - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } - - } ).when( events ).addClusterMemberListener( Matchers.any() ); + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - ClusterMemberListener theListener = listener.iterator().next(); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events, members, guard ); + stateMachine.init(); + ClusterMemberListener memberListener = memberListenerContainer.get(); HAStateChangeListener probe = new HAStateChangeListener(); - toTest.addHighAvailabilityMemberListener( probe ); + stateMachine.addHighAvailabilityMemberListener( probe ); // Send it to MASTER - theListener.memberIsAvailable( MASTER, other, URI.create( "ha://whatever" ), StoreId.DEFAULT ); + memberListener.memberIsAvailable( MASTER, other, URI.create( "ha://whatever" ), StoreId.DEFAULT ); - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_SLAVE ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.TO_SLAVE ) ); // When - theListener.memberIsFailed( new InstanceId( 2 ) ); + memberListener.memberIsFailed( new InstanceId( 2 ) ); // Then - assertThat( listener.size(), equalTo( 1 ) ); // Sanity check. - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); assertThat( probe.instanceStops, is( true ) ); verify( guard, times( 1 ) ).deny( any( AvailabilityGuard.AvailabilityRequirement.class ) ); } @@ -470,38 +393,20 @@ public void whenSlaveOnlyIsElectedStayInPending() throws Throwable // Given InstanceId me = new InstanceId( 1 ); HighAvailabilityMemberContext context = new SimpleHighAvailabilityMemberContext( me, true ); - AvailabilityGuard guard = mock( AvailabilityGuard.class ); - ClusterMembers members = mock( ClusterMembers.class ); ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMemberListenerContainer memberListenerContainer = mockAddClusterMemberListener( events ); - final Set listener = new HashSet<>(); + HighAvailabilityMemberStateMachine stateMachine = buildMockedStateMachine( context, events ); - doAnswer( new Answer() - { - @Override - public Object answer( InvocationOnMock invocation ) throws Throwable - { - listener.add( (ClusterMemberListener) invocation.getArguments()[0] ); - return null; - } + stateMachine.init(); - } ).when( events ).addClusterMemberListener( Matchers.any() ); - - Election election = mock( Election.class ); - StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = - new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - - toTest.init(); - - ClusterMemberListener theListener = listener.iterator().next(); + ClusterMemberListener memberListener = memberListenerContainer.get(); // When - theListener.coordinatorIsElected( me ); + memberListener.coordinatorIsElected( me ); // Then - assertThat( toTest.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); - + assertThat( stateMachine.getCurrentState(), equalTo( HighAvailabilityMemberState.PENDING ) ); } @Test @@ -541,15 +446,15 @@ public void whenHAModeSwitcherSwitchesToSlaveTheOtherModeSwitcherDoNotGetTheOldM Election election = mock( Election.class ); StringLogger logger = mock( StringLogger.class ); - HighAvailabilityMemberStateMachine toTest = + HighAvailabilityMemberStateMachine stateMachine = new HighAvailabilityMemberStateMachine( context, guard, members, events, election, logger ); - toTest.init(); - toTest.start(); + stateMachine.init(); + stateMachine.start(); final DelegateInvocationHandler handler = new DelegateInvocationHandler<>( Master.class ); - MasterClientResolver masterClientResolver = mock( MasterClientResolver.class ); + MasterClientResolver masterClientResolver = mock( MasterClientResolver.class ); MasterClient masterClient = mock( MasterClient.class ); when( masterClient.getProtocolVersion() ).thenReturn( MasterClient214.PROTOCOL_VERSION ); when( masterClient.handshake( anyLong(), any( StoreId.class ) ) ).thenReturn( @@ -624,7 +529,7 @@ updatePuller, mock( PullerFactory.class, RETURNS_MOCKS ), mock( ByteCounterMonit haModeSwitcher.start(); haModeSwitcher.listeningAt( URI.create( "http://localhost:12345" ) ); - toTest.addHighAvailabilityMemberListener( haModeSwitcher ); + stateMachine.addHighAvailabilityMemberListener( haModeSwitcher ); final AtomicReference ref = new AtomicReference<>( null ); @@ -661,14 +566,129 @@ protected Object getMasterImpl( LifeSupport life ) // let's test the toString()s since there are too many wrappers of proxies assertEquals( expected.toString(), actual.toString() ); - toTest.stop(); - toTest.shutdown(); + stateMachine.stop(); + stateMachine.shutdown(); haModeSwitcher.stop(); haModeSwitcher.shutdown(); otherModeSwitcher.stop(); otherModeSwitcher.shutdown(); } + private ClusterMembers mockClusterMembers( InstanceId me, InstanceId other ) + { + ClusterMembers members = mock( ClusterMembers.class ); + List membersList = new LinkedList<>(); + // we cannot set outside of the package the isAlive to return false. So do it with a mock + ClusterMember otherMemberMock = mock( ClusterMember.class ); + when( otherMemberMock.getInstanceId() ).thenReturn( other ); + when( otherMemberMock.isAlive() ).thenReturn( false ); + membersList.add( otherMemberMock ); + + membersList.add( new ClusterMember( me ) ); + when( members.getMembers() ).thenReturn( membersList ); + return members; + } + + private ClusterMemberListenerContainer mockAddClusterMemberListener( ClusterMemberEvents events ) + { + final ClusterMemberListenerContainer listenerContainer = new ClusterMemberListenerContainer(); + doAnswer( new Answer() + { + @Override + public Object answer( InvocationOnMock invocation ) throws Throwable + { + listenerContainer.set( (ClusterMemberListener) invocation.getArguments()[0] ); + return null; + } + + } ).when( events ).addClusterMemberListener( Matchers.any() ); + return listenerContainer; + } + + private HighAvailabilityMemberStateMachine buildMockedStateMachine() + { + return new StateMachineBuilder().build(); + } + + private HighAvailabilityMemberStateMachine buildMockedStateMachine ( HighAvailabilityMemberContext context, + ClusterMemberEvents events ) + { + return new StateMachineBuilder().withContext( context ).withEvents( events ).build(); + } + + private HighAvailabilityMemberStateMachine buildMockedStateMachine( HighAvailabilityMemberContext context, + ClusterMemberEvents events, ClusterMembers clusterMembers, AvailabilityGuard guard ) + { + return new StateMachineBuilder().withContext( context ).withEvents( events ).withClusterMembers( + clusterMembers ).withGuard( guard ).build(); + } + + private class StateMachineBuilder + { + HighAvailabilityMemberContext context = mock( HighAvailabilityMemberContext.class ); + ClusterMemberEvents events = mock( ClusterMemberEvents.class ); + ClusterMembers clusterMembers = mock( ClusterMembers.class ); + AvailabilityGuard guard = mock( AvailabilityGuard.class ); + Election election = mock( Election.class ); + StringLogger logger = mock( StringLogger.class ); + + public StateMachineBuilder withContext( HighAvailabilityMemberContext context ) + { + this.context = context; + return this; + } + + public StateMachineBuilder withEvents(ClusterMemberEvents events) + { + this.events = events; + return this; + } + + public StateMachineBuilder withClusterMembers(ClusterMembers clusterMember) + { + this.clusterMembers = clusterMember; + return this; + } + + public StateMachineBuilder withGuard(AvailabilityGuard guard) + { + this.guard = guard; + return this; + } + + public StateMachineBuilder withElection(Election election) + { + this.election = election; + return this; + } + + public HighAvailabilityMemberStateMachine build() + { + return new HighAvailabilityMemberStateMachine( context, guard, clusterMembers, events, election, logger ); + } + } + + private static class ClusterMemberListenerContainer + { + private ClusterMemberListener clusterMemberListener; + + public ClusterMemberListener get() + { + return clusterMemberListener; + } + + public void set( ClusterMemberListener clusterMemberListener ) + { + if ( this.clusterMemberListener != null ) + { + throw new IllegalStateException( "Expected to have only 1 listener, but have more. " + + "Defined listener: " + this.clusterMemberListener + + ". Newly added listener:" + clusterMemberListener ); + } + this.clusterMemberListener = clusterMemberListener; + } + } + private static final class HAStateChangeListener implements HighAvailabilityMemberListener { boolean masterIsElected = false; diff --git a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcherTest.java b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcherTest.java index 12eb1ae2fd448..cf70971dcd516 100644 --- a/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcherTest.java +++ b/enterprise/ha/src/test/java/org/neo4j/kernel/ha/cluster/HighAvailabilityModeSwitcherTest.java @@ -434,7 +434,7 @@ public void shouldTakeNoActionIfSwitchingToSlaveForItselfAsMaster() throws Throw } @Test - public void shouldPerformForcedElections() + public void shouldPostMemberUnavailableEvent() { // Given ClusterMemberAvailability memberAvailability = mock( ClusterMemberAvailability.class ); @@ -445,40 +445,16 @@ public void shouldPerformForcedElections() mock( InstanceId.class ), new DevNullLoggingService() ); // When - modeSwitcher.forceElections(); + modeSwitcher.postMemberUnavailable(); // Then InOrder inOrder = inOrder( memberAvailability, election ); inOrder.verify( memberAvailability ).memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE ); - inOrder.verify( election ).performRoleElections(); inOrder.verifyNoMoreInteractions(); } @Test - public void shouldPerformForcedElectionsOnlyOnce() - { - // Given: HAMS - ClusterMemberAvailability memberAvailability = mock( ClusterMemberAvailability.class ); - Election election = mock( Election.class ); - - HighAvailabilityModeSwitcher modeSwitcher = new HighAvailabilityModeSwitcher( mock( SwitchToSlave.class ), - mock( SwitchToMaster.class ), election, memberAvailability, dependencyResolverMock(), - mock( InstanceId.class ), new DevNullLoggingService() ); - - // When: reelections are forced multiple times - modeSwitcher.forceElections(); - modeSwitcher.forceElections(); - modeSwitcher.forceElections(); - - // Then: instance sens out memberIsUnavailable and asks for elections and does this only once - InOrder inOrder = inOrder( memberAvailability, election ); - inOrder.verify( memberAvailability ).memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE ); - inOrder.verify( election ).performRoleElections(); - inOrder.verifyNoMoreInteractions(); - } - - @Test - public void shouldAllowForcedElectionsAfterModeSwitch() throws Throwable + public void shouldPostMemberNotAvailableEventAfterModeSwitch() throws Throwable { // Given SwitchToSlave switchToSlave = mock( SwitchToSlave.class ); @@ -516,7 +492,7 @@ public Future answer( InvocationOnMock invocation ) throws Throwable modeSwitcher.init(); modeSwitcher.start(); - modeSwitcher.forceElections(); + modeSwitcher.postMemberUnavailable(); reset( memberAvailability, election ); // When @@ -524,12 +500,11 @@ public Future answer( InvocationOnMock invocation ) throws Throwable .class ), URI.create( "http://localhost:9090?serverId=42" ) ) ); modeSwitchHappened.await(); - modeSwitcher.forceElections(); + modeSwitcher.postMemberUnavailable(); // Then InOrder inOrder = inOrder( memberAvailability, election ); inOrder.verify( memberAvailability ).memberIsUnavailable( HighAvailabilityModeSwitcher.SLAVE ); - inOrder.verify( election ).performRoleElections(); inOrder.verifyNoMoreInteractions(); }