diff --git a/genie-web/src/main/java/com/netflix/genie/web/services/ClusterLeaderService.java b/genie-web/src/main/java/com/netflix/genie/web/services/ClusterLeaderService.java new file mode 100644 index 00000000000..6fd0c642dfc --- /dev/null +++ b/genie-web/src/main/java/com/netflix/genie/web/services/ClusterLeaderService.java @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.services; + +/** + * Service interface for the abstracts the details of leadership within nodes in a Genie cluster. + * + * @author mprimi + * @since 4.0.0 + */ +public interface ClusterLeaderService { + + /** + * Stop the service (i.e. renounce leadership and leave the election). + */ + void stop(); + + /** + * Start the service (i.e. join the the election). + */ + void start(); + + /** + * Whether or not this node is participating in the cluster leader election. + * + * @return true if the node is participating in leader election + */ + boolean isRunning(); + + /** + * Whether or not this node is the current cluster leader. + * + * @return true if the node is the current cluster leader + */ + boolean isLeader(); +} diff --git a/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImpl.java b/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImpl.java new file mode 100644 index 00000000000..e5c5b192a15 --- /dev/null +++ b/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImpl.java @@ -0,0 +1,74 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.services.impl; + +import com.netflix.genie.web.services.ClusterLeaderService; +import org.springframework.integration.zookeeper.leader.LeaderInitiator; + +/** + * Implementation of {@link ClusterLeaderService} using Spring's {@link LeaderInitiator} (Zookeeper/Curator based + * leader election mechanism). + * + * @author mprimi + * @since 4.0.0 + */ +public class ClusterLeaderServiceCuratorImpl implements ClusterLeaderService { + + private LeaderInitiator leaderInitiator; + + /** + * Constructor. + * + * @param leaderInitiator the leader initiator component + */ + public ClusterLeaderServiceCuratorImpl(final LeaderInitiator leaderInitiator) { + this.leaderInitiator = leaderInitiator; + } + + /** + * {@inheritDoc} + */ + @Override + public void stop() { + this.leaderInitiator.stop(); + } + + /** + * {@inheritDoc} + */ + @Override + public void start() { + this.leaderInitiator.start(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isRunning() { + return this.leaderInitiator.isRunning(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isLeader() { + return this.leaderInitiator.getContext().isLeader(); + } +} diff --git a/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImpl.java b/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImpl.java new file mode 100644 index 00000000000..ef8277ec53a --- /dev/null +++ b/genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImpl.java @@ -0,0 +1,72 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.services.impl; + +import com.netflix.genie.web.services.ClusterLeaderService; +import com.netflix.genie.web.tasks.leader.LocalLeader; + +/** + * Implementation of {@link ClusterLeaderService} using statically configured {@link LocalLeader} module. + * + * @author mprimi + * @since 4.0.0 + */ +public class ClusterLeaderServiceLocalLeaderImpl implements ClusterLeaderService { + private final LocalLeader localLeader; + + /** + * Constructor. + * + * @param localLeader the local leader module + */ + public ClusterLeaderServiceLocalLeaderImpl(final LocalLeader localLeader) { + this.localLeader = localLeader; + } + + /** + * {@inheritDoc} + */ + @Override + public void stop() { + this.localLeader.stop(); + } + + /** + * {@inheritDoc} + */ + @Override + public void start() { + this.localLeader.start(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isRunning() { + return this.localLeader.isRunning(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isLeader() { + return this.localLeader.isLeader(); + } +} diff --git a/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/LeaderElectionActuator.java b/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/LeaderElectionActuator.java new file mode 100644 index 00000000000..d2b157ba175 --- /dev/null +++ b/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/LeaderElectionActuator.java @@ -0,0 +1,117 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.spring.actuators; + +import com.google.common.collect.ImmutableMap; +import com.netflix.genie.web.services.ClusterLeaderService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.boot.actuate.endpoint.annotation.WriteOperation; + +import java.util.Map; + + +/** + * An actuator endpoint that exposes leadership status and allows stop/start/restart of the leader election service. + * Useful when a specific set of nodes should be given priority to win the leader election (e.g., because they are + * running newer code). + * + * @since 4.0.0 + * @author mprimi + */ +@Endpoint(id = "leaderElection") +@Slf4j +public class LeaderElectionActuator { + + /** + * Operations that this actuator can perform on the leader service. + */ + public enum Action { + /** + * Stop the leader election service. + */ + STOP, + /** + * Start the leader election service. + */ + START, + /** + * Stop then start the leader election service. + */ + RESTART, + + /** + * NOOP action for the purpose of testing unknown actions. + */ + TEST, + } + + private static final String RUNNING = "running"; + private static final String LEADER = "leader"; + private final ClusterLeaderService clusterLeaderService; + + /** + * Constructor. + * + * @param clusterLeaderService the cluster leader service + */ + public LeaderElectionActuator(final ClusterLeaderService clusterLeaderService) { + this.clusterLeaderService = clusterLeaderService; + } + + /** + * Provides the current leader service status: whether the leader service is running and whether the node is leader. + * + * @return a map of attributes + */ + @ReadOperation + public Map getStatus() { + return ImmutableMap.builder() + .put(RUNNING, this.clusterLeaderService.isRunning()) + .put(LEADER, this.clusterLeaderService.isLeader()) + .build(); + } + + /** + * Forces the node to leave the leader election, then re-join it. + * + * @param action the action to perform + */ + @WriteOperation + public void doAction(final Action action) { + switch (action) { + case START: + log.info("Starting leader election service"); + this.clusterLeaderService.start(); + break; + case STOP: + log.info("Stopping leader election service"); + this.clusterLeaderService.stop(); + break; + case RESTART: + log.info("Restarting leader election service"); + this.clusterLeaderService.stop(); + this.clusterLeaderService.start(); + break; + default: + log.error("Unknown action: " + action); + throw new UnsupportedOperationException("Unknown action: " + action.name()); + } + } +} diff --git a/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/package-info.java b/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/package-info.java new file mode 100644 index 00000000000..3a498274ce9 --- /dev/null +++ b/genie-web/src/main/java/com/netflix/genie/web/spring/actuators/package-info.java @@ -0,0 +1,28 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +/** + * Actuator endpoints. + * + * @author mprimi + * @since 4.0.0 + */ +@ParametersAreNonnullByDefault +package com.netflix.genie.web.spring.actuators; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/genie-web/src/main/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfiguration.java b/genie-web/src/main/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfiguration.java index c8543a3c0c0..f5a27782ff4 100644 --- a/genie-web/src/main/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfiguration.java +++ b/genie-web/src/main/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfiguration.java @@ -26,6 +26,10 @@ import com.netflix.genie.web.properties.LeadershipProperties; import com.netflix.genie.web.properties.UserMetricsProperties; import com.netflix.genie.web.properties.ZookeeperLeaderProperties; +import com.netflix.genie.web.services.ClusterLeaderService; +import com.netflix.genie.web.services.impl.ClusterLeaderServiceCuratorImpl; +import com.netflix.genie.web.services.impl.ClusterLeaderServiceLocalLeaderImpl; +import com.netflix.genie.web.spring.actuators.LeaderElectionActuator; import com.netflix.genie.web.spring.autoconfigure.tasks.TasksAutoConfiguration; import com.netflix.genie.web.tasks.leader.AgentJobCleanupTask; import com.netflix.genie.web.tasks.leader.ClusterCheckerTask; @@ -48,6 +52,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; import org.springframework.integration.zookeeper.config.LeaderInitiatorFactoryBean; +import org.springframework.integration.zookeeper.leader.LeaderInitiator; import org.springframework.scheduling.TaskScheduler; import org.springframework.web.client.RestTemplate; @@ -241,4 +246,46 @@ public AgentJobCleanupTask agentJobCleanupTask( registry ); } + + /** + * Create a {@link ClusterLeaderService} based on Zookeeper/Curator if {@link LeaderInitiator} is + * available and the bean does not already exist. + * + * @param leaderInitiator the Spring Zookeeper/Curator based leader election component + * @return a {@link ClusterLeaderService} + */ + @Bean + @ConditionalOnBean(LeaderInitiator.class) + @ConditionalOnMissingBean(ClusterLeaderService.class) + public ClusterLeaderService curatorClusterLeaderService(final LeaderInitiator leaderInitiator) { + return new ClusterLeaderServiceCuratorImpl(leaderInitiator); + } + + /** + * Create a {@link ClusterLeaderService} based on static configuration if {@link LocalLeader} is + * available and the bean does not already exist. + * + * @param localLeader the configuration-based leader election component + * @return a {@link ClusterLeaderService} + */ + @Bean + @ConditionalOnBean(LocalLeader.class) + @ConditionalOnMissingBean(ClusterLeaderService.class) + public ClusterLeaderService localClusterLeaderService(final LocalLeader localLeader) { + return new ClusterLeaderServiceLocalLeaderImpl(localLeader); + } + + /** + * Create a {@link LeaderElectionActuator} bean if one is not already defined and if + * {@link ClusterLeaderService} is available. This bean is an endpoint that gets registered in Spring Actuator. + * + * @param clusterLeaderService the cluster leader service + * @return a {@link LeaderElectionActuator} + */ + @Bean + @ConditionalOnBean(ClusterLeaderService.class) + @ConditionalOnMissingBean(LeaderElectionActuator.class) + public LeaderElectionActuator leaderElectionActuator(final ClusterLeaderService clusterLeaderService) { + return new LeaderElectionActuator(clusterLeaderService); + } } diff --git a/genie-web/src/main/java/com/netflix/genie/web/tasks/leader/LocalLeader.java b/genie-web/src/main/java/com/netflix/genie/web/tasks/leader/LocalLeader.java index e5e8b40d23a..eb5e3177f3a 100644 --- a/genie-web/src/main/java/com/netflix/genie/web/tasks/leader/LocalLeader.java +++ b/genie-web/src/main/java/com/netflix/genie/web/tasks/leader/LocalLeader.java @@ -25,6 +25,9 @@ import org.springframework.integration.leader.event.OnGrantedEvent; import org.springframework.integration.leader.event.OnRevokedEvent; +import javax.annotation.concurrent.ThreadSafe; +import java.util.concurrent.atomic.AtomicBoolean; + /** * A class to control leadership activities when remote leadership isn't enabled and this node has been forcibly * elected as the leader. @@ -33,10 +36,12 @@ * @since 3.0.0 */ @Slf4j +@ThreadSafe public class LocalLeader { private final GenieEventBus genieEventBus; private final boolean isLeader; + private final AtomicBoolean isRunning; /** * Constructor. @@ -48,6 +53,7 @@ public class LocalLeader { public LocalLeader(final GenieEventBus genieEventBus, final boolean isLeader) { this.genieEventBus = genieEventBus; this.isLeader = isLeader; + this.isRunning = new AtomicBoolean(false); if (this.isLeader) { log.info("Constructing LocalLeader. This node IS the leader."); } else { @@ -56,28 +62,64 @@ public LocalLeader(final GenieEventBus genieEventBus, final boolean isLeader) { } /** - * Event listener for when a context is started up. + * Auto-start once application is up and running. * * @param event The Spring Boot application ready event to startup on */ @EventListener public void startLeadership(final ContextRefreshedEvent event) { - if (this.isLeader) { - log.debug("Starting Leadership due to {}", event); - this.genieEventBus.publishSynchronousEvent(new OnGrantedEvent(this, null, "leader")); - } + log.debug("Starting Leadership due to {}", event); + this.start(); } /** - * Before the application shuts down need to turn off leadership activities. + * Auto-stop once application is shutting down. * * @param event The application context closing event */ @EventListener public void stopLeadership(final ContextClosedEvent event) { - if (this.isLeader) { - log.debug("Stopping Leadership due to {}", event); + log.debug("Stopping Leadership due to {}", event); + this.stop(); + } + + /** + * If configured to be leader and previously started, deactivate and send a leadership lost notification. + * NOOP if not running or if this node is not configured to be leader. + */ + public void stop() { + if (this.isRunning.compareAndSet(true, false) && this.isLeader) { + log.info("Stopping Leadership"); this.genieEventBus.publishSynchronousEvent(new OnRevokedEvent(this, null, "leader")); } } + + /** + * If configured to be leader, activate and send a leadership acquired notification. + * NOOP if already running or if this node is not configured to be leader. + */ + public void start() { + if (this.isRunning.compareAndSet(false, true) && this.isLeader) { + log.debug("Starting Leadership"); + this.genieEventBus.publishSynchronousEvent(new OnGrantedEvent(this, null, "leader")); + } + } + + /** + * Whether this module is active. + * + * @return true if {@link LocalLeader#start()} was called. + */ + public boolean isRunning() { + return this.isRunning.get(); + } + + /** + * Whether local node is leader. + * + * @return true if this module is active and the node is configured to be leader + */ + public boolean isLeader() { + return this.isRunning() && this.isLeader; + } } diff --git a/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImplSpec.groovy b/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImplSpec.groovy new file mode 100644 index 00000000000..3f6e63026e0 --- /dev/null +++ b/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceCuratorImplSpec.groovy @@ -0,0 +1,71 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.services.impl + +import com.netflix.genie.web.services.ClusterLeaderService +import org.springframework.integration.leader.Context +import org.springframework.integration.zookeeper.leader.LeaderInitiator +import spock.lang.Specification + +class ClusterLeaderServiceCuratorImplSpec extends Specification { + LeaderInitiator leaderInitiator + ClusterLeaderService service + + void setup() { + this.leaderInitiator = Mock(LeaderInitiator) + this.service = new ClusterLeaderServiceCuratorImpl(leaderInitiator) + } + + def "Start"() { + when: + this.service.start() + + then: + 1 * leaderInitiator.start() + } + + def "Stop"() { + when: + this.service.stop() + + then: + 1 * leaderInitiator.stop() + } + + def "isLeader"() { + Context context = Mock(Context) + when: + boolean isLeader = this.service.isLeader() + + + then: + 1 * leaderInitiator.getContext() >> context + 1 * context.isLeader() >> true + isLeader + } + + + def "isRunning"() { + when: + boolean isRunning = this.service.isRunning() + + then: + 1 * leaderInitiator.isRunning() >> true + isRunning + } +} diff --git a/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImplSpec.groovy b/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImplSpec.groovy new file mode 100644 index 00000000000..b66ece725e1 --- /dev/null +++ b/genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceLocalLeaderImplSpec.groovy @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.services.impl + +import com.netflix.genie.web.services.ClusterLeaderService +import com.netflix.genie.web.tasks.leader.LocalLeader +import spock.lang.Specification + +class ClusterLeaderServiceLocalLeaderImplSpec extends Specification { + LocalLeader localLeader + ClusterLeaderService service + + void setup() { + this.localLeader = Mock(LocalLeader) + this.service = new ClusterLeaderServiceLocalLeaderImpl(localLeader) + } + + def "Start"() { + when: + this.service.start() + + then: + 1 * localLeader.start() + } + + def "Stop"() { + when: + this.service.stop() + + then: + 1 * localLeader.stop() + } + + def "isLeader"() { + when: + boolean isLeader = this.service.isLeader() + + then: + 1 * localLeader.isLeader() >> true + isLeader + } + + + def "isRunning"() { + when: + boolean isRunning = this.service.isRunning() + + then: + 1 * localLeader.isRunning() >> true + isRunning + } +} diff --git a/genie-web/src/test/groovy/com/netflix/genie/web/spring/actuators/LeaderElectionActuatorSpec.groovy b/genie-web/src/test/groovy/com/netflix/genie/web/spring/actuators/LeaderElectionActuatorSpec.groovy new file mode 100644 index 00000000000..93350ad5c6b --- /dev/null +++ b/genie-web/src/test/groovy/com/netflix/genie/web/spring/actuators/LeaderElectionActuatorSpec.groovy @@ -0,0 +1,75 @@ +/* + * + * Copyright 2020 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.genie.web.spring.actuators + +import com.netflix.genie.web.services.ClusterLeaderService +import spock.lang.Specification + +class LeaderElectionActuatorSpec extends Specification { + ClusterLeaderService clusterLeaderService + LeaderElectionActuator actuator + + void setup() { + this.clusterLeaderService = Mock(ClusterLeaderService) + this.actuator = new LeaderElectionActuator(clusterLeaderService) + } + + def "Status"() { + when: + Map status = this.actuator.getStatus() + + then: + 1 * clusterLeaderService.isRunning() >> running + 1 * clusterLeaderService.isLeader() >> leader + status.get(LeaderElectionActuator.RUNNING) == running + status.get(LeaderElectionActuator.LEADER) == leader + + where: + running | leader + true | false + false | true + + } + + def "Actions"() { + when: + this.actuator.doAction(LeaderElectionActuator.Action.START) + + then: + 1 * clusterLeaderService.start() + + when: + this.actuator.doAction(LeaderElectionActuator.Action.STOP) + + then: + 1 * clusterLeaderService.stop() + + when: + this.actuator.doAction(LeaderElectionActuator.Action.RESTART) + + then: + 1 * clusterLeaderService.stop() + 1 * clusterLeaderService.start() + + when: + this.actuator.doAction(LeaderElectionActuator.Action.TEST) + + then: + thrown(UnsupportedOperationException) + } +} diff --git a/genie-web/src/test/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfigurationTest.java b/genie-web/src/test/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfigurationTest.java index 5695dca9397..3cba5c52e03 100644 --- a/genie-web/src/test/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfigurationTest.java +++ b/genie-web/src/test/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfigurationTest.java @@ -35,6 +35,8 @@ import com.netflix.genie.web.properties.LeadershipProperties; import com.netflix.genie.web.properties.UserMetricsProperties; import com.netflix.genie.web.properties.ZookeeperLeaderProperties; +import com.netflix.genie.web.services.ClusterLeaderService; +import com.netflix.genie.web.spring.actuators.LeaderElectionActuator; import com.netflix.genie.web.spring.autoconfigure.tasks.TasksAutoConfiguration; import com.netflix.genie.web.tasks.leader.AgentJobCleanupTask; import com.netflix.genie.web.tasks.leader.ClusterCheckerTask; @@ -92,11 +94,14 @@ void expectedBeansExist() { Assertions.assertThat(context).doesNotHaveBean(LeaderInitiatorFactoryBean.class); Assertions.assertThat(context).hasSingleBean(LocalLeader.class); Assertions.assertThat(context).hasSingleBean(ClusterCheckerTask.class); + Assertions.assertThat(context).hasSingleBean(ClusterLeaderService.class); + Assertions.assertThat(context).hasSingleBean(LeaderElectionActuator.class); // Optional beans Assertions.assertThat(context).doesNotHaveBean(DatabaseCleanupTask.class); Assertions.assertThat(context).doesNotHaveBean(UserMetricsTask.class); Assertions.assertThat(context).doesNotHaveBean(AgentJobCleanupTask.class); + Assertions.assertThat(context).doesNotHaveBean(LeaderInitiatorFactoryBean.class); } ); } @@ -126,10 +131,14 @@ void optionalBeansCreated() { Assertions.assertThat(context).hasSingleBean(LocalLeader.class); Assertions.assertThat(context).hasSingleBean(ClusterCheckerTask.class); + Assertions.assertThat(context).hasSingleBean(ClusterLeaderService.class); + Assertions.assertThat(context).hasSingleBean(LeaderElectionActuator.class); + // Optional beans Assertions.assertThat(context).hasSingleBean(DatabaseCleanupTask.class); Assertions.assertThat(context).hasSingleBean(UserMetricsTask.class); Assertions.assertThat(context).hasSingleBean(AgentJobCleanupTask.class); + Assertions.assertThat(context).doesNotHaveBean(LeaderInitiatorFactoryBean.class); } ); } @@ -155,6 +164,9 @@ void expectedZookeeperBeansExist() { Assertions.assertThat(context).doesNotHaveBean(LocalLeader.class); Assertions.assertThat(context).hasSingleBean(ClusterCheckerTask.class); + Assertions.assertThat(context).hasSingleBean(ClusterLeaderService.class); + Assertions.assertThat(context).hasSingleBean(LeaderElectionActuator.class); + // Optional beans Assertions.assertThat(context).doesNotHaveBean(DatabaseCleanupTask.class); Assertions.assertThat(context).doesNotHaveBean(UserMetricsTask.class); diff --git a/genie-web/src/test/java/com/netflix/genie/web/tasks/leader/LocalLeaderTest.java b/genie-web/src/test/java/com/netflix/genie/web/tasks/leader/LocalLeaderTest.java index 791dc8bf459..162f44d3664 100644 --- a/genie-web/src/test/java/com/netflix/genie/web/tasks/leader/LocalLeaderTest.java +++ b/genie-web/src/test/java/com/netflix/genie/web/tasks/leader/LocalLeaderTest.java @@ -18,6 +18,7 @@ package com.netflix.genie.web.tasks.leader; import com.netflix.genie.web.events.GenieEventBus; +import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,42 +60,69 @@ void tearDown() { } /** - * Make sure the event to grant leadership is fired if the node is the leader. + * Ensure behavior in case of start, stop, start when already running and stop when not running in case the module + * is configured to be leader. */ @Test - void canStartLeadershipIfLeader() { + void startAndStopIfLeader() { this.localLeader = new LocalLeader(this.genieEventBus, true); + Assertions.assertThat(this.localLeader.isRunning()).isFalse(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); + + // Start with application context event this.localLeader.startLeadership(this.contextRefreshedEvent); + Assertions.assertThat(this.localLeader.isRunning()).isTrue(); + Assertions.assertThat(this.localLeader.isLeader()).isTrue(); Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); - } + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); - /** - * Make sure the event to grant leadership is not fired if the node is not the leader. - */ - @Test - void wontStartLeadershipIfNotLeader() { - this.localLeader = new LocalLeader(this.genieEventBus, false); - this.localLeader.startLeadership(this.contextRefreshedEvent); - Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); - } + // Start again + this.localLeader.start(); + Assertions.assertThat(this.localLeader.isRunning()).isTrue(); + Assertions.assertThat(this.localLeader.isLeader()).isTrue(); + Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); - /** - * Make sure the event to revoke leadership is fired if the node is the leader. - */ - @Test - void canStopLeadershipIfLeader() { - this.localLeader = new LocalLeader(this.genieEventBus, true); + // Stop with application context event + this.localLeader.stopLeadership(this.contextClosedEvent); + Assertions.assertThat(this.localLeader.isRunning()).isFalse(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); + Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); + + // Stop again this.localLeader.stopLeadership(this.contextClosedEvent); + Assertions.assertThat(this.localLeader.isRunning()).isFalse(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); Mockito.verify(this.genieEventBus, Mockito.times(1)).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); } /** - * Make sure the event to revoke leadership is not fired if the node is not the leader. + * Ensure behavior in case of start, stop in case the module is not configured to be leader. */ @Test - void wontStopLeadershipIfNotLeader() { + void startAndStopIfNotLeader() { this.localLeader = new LocalLeader(this.genieEventBus, false); + Assertions.assertThat(this.localLeader.isRunning()).isFalse(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); + + // Start + this.localLeader.start(); + Assertions.assertThat(this.localLeader.isRunning()).isTrue(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); + + // Stop this.localLeader.stopLeadership(this.contextClosedEvent); + Assertions.assertThat(this.localLeader.isRunning()).isFalse(); + Assertions.assertThat(this.localLeader.isLeader()).isFalse(); + Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnGrantedEvent.class)); Mockito.verify(this.genieEventBus, Mockito.never()).publishSynchronousEvent(Mockito.any(OnRevokedEvent.class)); } }