Merge 9743896 into 9412a0b

genie-web/src/main/java/com/netflix/genie/web/services/ClusterLeaderService.java

@@ -0,0 +1,51 @@
+/*
+ *
+ *  Copyright 2017 Netflix, Inc.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+package com.netflix.genie.web.services;
+
+/**
+ * Service interface for the abstracts the details of leadership within nodes in a Genie cluster.
+ *
+ * @author mprimi
+ * @since 4.0.0
+ */
+public interface ClusterLeaderService {
+
+    /**
+     * Stop the service (i.e. renounce leadership and leave the election).
+     */
+    void stop();
+
+    /**
+     * Start the service (i.e. join the the election).
+     */
+    void start();
+
+    /**
+     * Whether or not this node is participating in the cluster leader election.
+     *
+     * @return true if the node is participating in leader election
+     */
+    boolean isRunning();
+
+    /**
+     * Whether or not this node is the current cluster leader.
+     *
+     * @return true if the node is the current cluster leader
+     */
+    boolean isLeader();
+}

genie-web/src/main/java/com/netflix/genie/web/services/impl/ClusterLeaderServiceImpl.java

@@ -0,0 +1,73 @@
+/*
+ *
+ *  Copyright 2017 Netflix, Inc.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+package com.netflix.genie.web.services.impl;
+
+import com.netflix.genie.web.services.ClusterLeaderService;
+import org.springframework.integration.zookeeper.leader.LeaderInitiator;
+
+/**
+ * Implementation of {@link ClusterLeaderService} using Spring's {@link LeaderInitiator}.
+ *
+ * @since 4.0.0
+ * @author mprimi
+ */
+public class ClusterLeaderServiceImpl implements ClusterLeaderService {
+
+    private LeaderInitiator leaderInitiator;
+
+    /**
+     * Constructor.
+     *
+     * @param leaderInitiator the leader initiator component
+     */
+    public ClusterLeaderServiceImpl(final LeaderInitiator leaderInitiator) {
+        this.leaderInitiator = leaderInitiator;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void stop() {
+        this.leaderInitiator.stop();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void start() {
+        this.leaderInitiator.start();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public boolean isRunning() {
+        return this.leaderInitiator.isRunning();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public boolean isLeader() {
+        return this.leaderInitiator.getContext().isLeader();
+    }
+}

genie-web/src/main/java/com/netflix/genie/web/spring/actuators/RejoinLeaderElectionActuator.java

@@ -0,0 +1,76 @@
+/*
+ *
+ *  Copyright 2017 Netflix, Inc.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+package com.netflix.genie.web.spring.actuators;
+
+import com.google.common.collect.ImmutableMap;
+import com.netflix.genie.web.services.ClusterLeaderService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
+import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
+import org.springframework.boot.actuate.endpoint.annotation.WriteOperation;
+
+import java.util.Map;
+
+
+/**
+ * An actuator endpoint that exposes leadership status.
+ * Allows leaving and rejoining the election. Useful when a specific set of nodes should be given priority to win the
+ * leader election (e.g., because they are running newer code).
+ *
+ * @since 4.0.0
+ * @author mprimi
+ */
+@Endpoint(id = "rejoinLeaderElection")
+@Slf4j
+public class RejoinLeaderElectionActuator {
+    private static final String RUNNING = "running";
+    private static final String LEADER = "leader";
+    private final ClusterLeaderService clusterLeaderService;
+
+    /**
+     * Constructor.
+     *
+     * @param clusterLeaderService the cluster leader service
+     */
+    public RejoinLeaderElectionActuator(final ClusterLeaderService clusterLeaderService) {
+        this.clusterLeaderService = clusterLeaderService;
+    }
+
+    /**
+     * Provides the current leader service status: whether the leader service is running and whether the node is leader.
+     *
+     * @return a map of attributes
+     */
+    @ReadOperation
+    public Map<String, Object> status() {
+        return ImmutableMap.<String, Object>builder()
+            .put(RUNNING, this.clusterLeaderService.isRunning())
+            .put(LEADER, this.clusterLeaderService.isLeader())
+            .build();
+    }
+
+    /**
+     * Forces the node to leave the leader election, then re-join it.
+     */
+    @WriteOperation
+    public void rejoin() {
+        log.info("Leaving then rejoining cluster leader election");
+        this.clusterLeaderService.stop();
+        this.clusterLeaderService.start();
+    }
+}

genie-web/src/main/java/com/netflix/genie/web/spring/actuators/package-info.java

@@ -0,0 +1,28 @@
+/*
+ *
+ *  Copyright 2020 Netflix, Inc.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+
+/**
+ * Actuator endpoints.
+ *
+ * @author tgianos
+ * @since 4.0.0
+ */
+@ParametersAreNonnullByDefault
+package com.netflix.genie.web.spring.actuators;
+
+import javax.annotation.ParametersAreNonnullByDefault;

genie-web/src/main/java/com/netflix/genie/web/spring/autoconfigure/tasks/leader/LeaderAutoConfiguration.java

@@ -26,6 +26,9 @@
 import com.netflix.genie.web.properties.LeadershipProperties;
 import com.netflix.genie.web.properties.UserMetricsProperties;
 import com.netflix.genie.web.properties.ZookeeperLeaderProperties;
+import com.netflix.genie.web.services.ClusterLeaderService;
+import com.netflix.genie.web.services.impl.ClusterLeaderServiceImpl;
+import com.netflix.genie.web.spring.actuators.RejoinLeaderElectionActuator;
 import com.netflix.genie.web.spring.autoconfigure.tasks.TasksAutoConfiguration;
 import com.netflix.genie.web.tasks.leader.AgentJobCleanupTask;
 import com.netflix.genie.web.tasks.leader.ClusterCheckerTask;
@@ -48,6 +51,7 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.env.Environment;
 import org.springframework.integration.zookeeper.config.LeaderInitiatorFactoryBean;
+import org.springframework.integration.zookeeper.leader.LeaderInitiator;
 import org.springframework.scheduling.TaskScheduler;
 import org.springframework.web.client.RestTemplate;
 
@@ -241,4 +245,32 @@ public AgentJobCleanupTask agentJobCleanupTask(
             registry
         );
     }
+
+    /**
+     * Create a {@link ClusterLeaderService} bean if one is not already defined and if {@link LeaderInitiator} is
+     * available.
+     *
+     * @param leaderInitiator the Spring Zookeeper/Curator based leader election component
+     * @return a {@link ClusterLeaderService}
+     */
+    @Bean
+    @ConditionalOnBean(LeaderInitiator.class)
+    @ConditionalOnMissingBean(ClusterLeaderService.class)
+    public ClusterLeaderService clusterLeaderService(final LeaderInitiator leaderInitiator) {
+        return new ClusterLeaderServiceImpl(leaderInitiator);
+    }
+
+    /**
+     * Create a {@link RejoinLeaderElectionActuator} bean if one is not already defined and if
+     * {@link ClusterLeaderService} is available. This bean is an endpoint that gets registered in Spring Actuator.
+     *
+     * @param clusterLeaderService the cluster leader service
+     * @return a {@link RejoinLeaderElectionActuator}
+     */
+    @Bean
+    @ConditionalOnBean(ClusterLeaderService.class)
+    @ConditionalOnMissingBean(RejoinLeaderElectionActuator.class)
+    public RejoinLeaderElectionActuator leaderElectionActuator(final ClusterLeaderService clusterLeaderService) {
+        return new RejoinLeaderElectionActuator(clusterLeaderService);
+    }
 }

genie-web/src/test/groovy/com/netflix/genie/web/services/impl/ClusterLeaderServiceImplSpec.groovy

@@ -0,0 +1,71 @@
+/*
+ *
+ *  Copyright 2017 Netflix, Inc.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License");
+ *     you may not use this file except in compliance with the License.
+ *     You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS,
+ *     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *     See the License for the specific language governing permissions and
+ *     limitations under the License.
+ *
+ */
+package com.netflix.genie.web.services.impl
+
+import com.netflix.genie.web.services.ClusterLeaderService
+import org.springframework.integration.leader.Context
+import org.springframework.integration.zookeeper.leader.LeaderInitiator
+import spock.lang.Specification
+
+class ClusterLeaderServiceImplSpec extends Specification {
+    LeaderInitiator leaderInitiator
+    ClusterLeaderService service
+
+    void setup() {
+        this.leaderInitiator = Mock(LeaderInitiator)
+        this.service = new ClusterLeaderServiceImpl(leaderInitiator)
+    }
+
+    def "Start"() {
+        when:
+        this.service.start()
+
+        then:
+        1 * leaderInitiator.start()
+    }
+
+    def "Stop"() {
+        when:
+        this.service.stop()
+
+        then:
+        1 * leaderInitiator.stop()
+    }
+
+    def "isLeader"() {
+        Context context = Mock(Context)
+        when:
+        boolean isLeader = this.service.isLeader()
+
+
+        then:
+        1 * leaderInitiator.getContext() >> context
+        1 * context.isLeader() >> true
+        isLeader
+    }
+
+
+    def "isRunning"() {
+        when:
+        boolean isRunning = this.service.isRunning()
+
+        then:
+        1 * leaderInitiator.isRunning() >> true
+        isRunning
+    }
+}