From 43265d2a39b916f98d6fe96335ac8b56a9e3f4fc Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Fri, 30 Jun 2017 17:42:33 +0200 Subject: [PATCH 01/13] CAMEL-11331: Implemented KubernetesClusterService --- components/camel-kubernetes/pom.xml | 6 +- .../AbstractKubernetesEndpoint.java | 53 +--- .../kubernetes/KubernetesConfiguration.java | 15 +- .../kubernetes/KubernetesHelper.java | 98 +++++++ .../ha/KubernetesClusterService.java | 151 +++++++++++ .../kubernetes/ha/KubernetesClusterView.java | 168 ++++++++++++ .../ha/lock/KubernetesClusterEvent.java | 46 ++++ .../lock/KubernetesClusterEventHandler.java | 27 ++ .../ha/lock/KubernetesLeaderMonitor.java | 256 ++++++++++++++++++ .../lock/KubernetesLeadershipController.java | 211 +++++++++++++++ .../ha/lock/KubernetesLockConfiguration.java | 153 +++++++++++ .../ha/lock/KubernetesMembersMonitor.java | 239 ++++++++++++++++ 12 files changed, 1368 insertions(+), 55 deletions(-) create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEvent.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEventHandler.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java diff --git a/components/camel-kubernetes/pom.xml b/components/camel-kubernetes/pom.xml index e5409c8d9cdbf..c44406885cb82 100644 --- a/components/camel-kubernetes/pom.xml +++ b/components/camel-kubernetes/pom.xml @@ -44,12 +44,14 @@ io.fabric8 kubernetes-client - ${kubernetes-client-version} + 2.3-SNAPSHOT + io.fabric8 openshift-client - ${kubernetes-client-version} + 2.3-SNAPSHOT + diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/AbstractKubernetesEndpoint.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/AbstractKubernetesEndpoint.java index f48bf6d1108ad..b7aeb37039b88 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/AbstractKubernetesEndpoint.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/AbstractKubernetesEndpoint.java @@ -18,14 +18,10 @@ import java.util.concurrent.ExecutorService; -import io.fabric8.kubernetes.client.Config; -import io.fabric8.kubernetes.client.ConfigBuilder; -import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; import org.apache.camel.impl.DefaultEndpoint; import org.apache.camel.spi.UriParam; -import org.apache.camel.util.ObjectHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +50,7 @@ public boolean isSingleton() { @Override protected void doStart() throws Exception { super.doStart(); - client = configuration.getKubernetesClient() != null ? configuration.getKubernetesClient() : createKubernetesClient(); + client = KubernetesHelper.getKubernetesClient(configuration); } @Override @@ -80,52 +76,5 @@ public KubernetesConfiguration getKubernetesConfiguration() { return configuration; } - private KubernetesClient createKubernetesClient() { - LOG.debug("Create Kubernetes client with the following Configuration: " + configuration.toString()); - ConfigBuilder builder = new ConfigBuilder(); - builder.withMasterUrl(configuration.getMasterUrl()); - if ((ObjectHelper.isNotEmpty(configuration.getUsername()) - && ObjectHelper.isNotEmpty(configuration.getPassword())) - && ObjectHelper.isEmpty(configuration.getOauthToken())) { - builder.withUsername(configuration.getUsername()); - builder.withPassword(configuration.getPassword()); - } - if (ObjectHelper.isNotEmpty(configuration.getOauthToken())) { - builder.withOauthToken(configuration.getOauthToken()); - } - if (ObjectHelper.isNotEmpty(configuration.getCaCertData())) { - builder.withCaCertData(configuration.getCaCertData()); - } - if (ObjectHelper.isNotEmpty(configuration.getCaCertFile())) { - builder.withCaCertFile(configuration.getCaCertFile()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientCertData())) { - builder.withClientCertData(configuration.getClientCertData()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientCertFile())) { - builder.withClientCertFile(configuration.getClientCertFile()); - } - if (ObjectHelper.isNotEmpty(configuration.getApiVersion())) { - builder.withApiVersion(configuration.getApiVersion()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientKeyAlgo())) { - builder.withClientKeyAlgo(configuration.getClientKeyAlgo()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientKeyData())) { - builder.withClientKeyData(configuration.getClientKeyData()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientKeyFile())) { - builder.withClientKeyFile(configuration.getClientKeyFile()); - } - if (ObjectHelper.isNotEmpty(configuration.getClientKeyPassphrase())) { - builder.withClientKeyPassphrase(configuration.getClientKeyPassphrase()); - } - if (ObjectHelper.isNotEmpty(configuration.getTrustCerts())) { - builder.withTrustCerts(configuration.getTrustCerts()); - } - - Config conf = builder.build(); - return new DefaultKubernetesClient(conf); - } } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java index 89d0d9a3407b4..271ef711a533b 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java @@ -19,13 +19,14 @@ import io.fabric8.kubernetes.client.DefaultKubernetesClient; import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.camel.RuntimeCamelException; import org.apache.camel.spi.Metadata; import org.apache.camel.spi.UriParam; import org.apache.camel.spi.UriParams; import org.apache.camel.spi.UriPath; @UriParams -public class KubernetesConfiguration { +public class KubernetesConfiguration implements Cloneable { @UriPath @Metadata(required = "true") @@ -395,6 +396,18 @@ public void setResourceName(String resourceName) { this.resourceName = resourceName; } + // **************************************** + // Copy + // **************************************** + + public KubernetesConfiguration copy() { + try { + return (KubernetesConfiguration) super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeCamelException(e); + } + } + @Override public String toString() { return "KubernetesConfiguration [masterUrl=" + masterUrl + ", category=" + category + ", kubernetesClient=" diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java new file mode 100644 index 0000000000000..62235adb2ee9a --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes; + +import io.fabric8.kubernetes.client.Config; +import io.fabric8.kubernetes.client.ConfigBuilder; +import io.fabric8.kubernetes.client.DefaultKubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.apache.camel.util.ObjectHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper moethods for Kubernetes resources. + */ +public final class KubernetesHelper { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesHelper.class); + + private KubernetesHelper() { + } + + public static KubernetesClient getKubernetesClient(KubernetesConfiguration configuration) { + if (configuration.getKubernetesClient() != null) { + return configuration.getKubernetesClient(); + } else if (configuration.getMasterUrl() != null) { + return createKubernetesClient(configuration); + } else { + LOG.info("Creating default kubernetes client without applying configuration"); + return new DefaultKubernetesClient(); + } + } + + private static KubernetesClient createKubernetesClient(KubernetesConfiguration configuration) { + LOG.debug("Create Kubernetes client with the following Configuration: " + configuration.toString()); + + ConfigBuilder builder = new ConfigBuilder(); + builder.withMasterUrl(configuration.getMasterUrl()); + if ((ObjectHelper.isNotEmpty(configuration.getUsername()) + && ObjectHelper.isNotEmpty(configuration.getPassword())) + && ObjectHelper.isEmpty(configuration.getOauthToken())) { + builder.withUsername(configuration.getUsername()); + builder.withPassword(configuration.getPassword()); + } + if (ObjectHelper.isNotEmpty(configuration.getOauthToken())) { + builder.withOauthToken(configuration.getOauthToken()); + } + if (ObjectHelper.isNotEmpty(configuration.getCaCertData())) { + builder.withCaCertData(configuration.getCaCertData()); + } + if (ObjectHelper.isNotEmpty(configuration.getCaCertFile())) { + builder.withCaCertFile(configuration.getCaCertFile()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientCertData())) { + builder.withClientCertData(configuration.getClientCertData()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientCertFile())) { + builder.withClientCertFile(configuration.getClientCertFile()); + } + if (ObjectHelper.isNotEmpty(configuration.getApiVersion())) { + builder.withApiVersion(configuration.getApiVersion()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientKeyAlgo())) { + builder.withClientKeyAlgo(configuration.getClientKeyAlgo()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientKeyData())) { + builder.withClientKeyData(configuration.getClientKeyData()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientKeyFile())) { + builder.withClientKeyFile(configuration.getClientKeyFile()); + } + if (ObjectHelper.isNotEmpty(configuration.getClientKeyPassphrase())) { + builder.withClientKeyPassphrase(configuration.getClientKeyPassphrase()); + } + if (ObjectHelper.isNotEmpty(configuration.getTrustCerts())) { + builder.withTrustCerts(configuration.getTrustCerts()); + } + + Config conf = builder.build(); + return new DefaultKubernetesClient(conf); + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java new file mode 100644 index 0000000000000..6d87d48df1271 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha; + +import java.net.InetAddress; +import java.util.Map; + +import org.apache.camel.CamelContext; +import org.apache.camel.RuntimeCamelException; +import org.apache.camel.component.kubernetes.KubernetesConfiguration; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesLockConfiguration; +import org.apache.camel.impl.ha.AbstractCamelClusterService; +import org.apache.camel.util.ObjectHelper; + +/** + * A Kubernetes based cluster service leveraging Kubernetes optimistic locks on resources (specifically ConfigMaps). + */ +public class KubernetesClusterService extends AbstractCamelClusterService { + + public static final String DEFAULT_CONFIGMAP_NAME = "leaders"; + + private KubernetesConfiguration configuration; + + private KubernetesLockConfiguration lockConfiguration; + + public KubernetesClusterService() { + this.configuration = new KubernetesConfiguration(); + this.lockConfiguration = new KubernetesLockConfiguration(); + } + + public KubernetesClusterService(KubernetesConfiguration configuration) { + this.configuration = configuration.copy(); + this.lockConfiguration = new KubernetesLockConfiguration(); + } + + public KubernetesClusterService(CamelContext camelContext, KubernetesConfiguration configuration) { + super(null, camelContext); + this.configuration = configuration.copy(); + this.lockConfiguration = new KubernetesLockConfiguration(); + } + + @Override + protected KubernetesClusterView createView(String namespace) throws Exception { + KubernetesLockConfiguration lockConfig = configWithGroupNameAndDefaults(namespace); + return new KubernetesClusterView(this, configuration, lockConfig); + } + + protected KubernetesLockConfiguration configWithGroupNameAndDefaults(String groupName) { + KubernetesLockConfiguration config = this.lockConfiguration.copy(); + + config.setGroupName(ObjectHelper.notNull(groupName, "groupName")); + + // Check defaults (Namespace and podName can be null) + if (config.getConfigMapName() == null) { + config.setConfigMapName(DEFAULT_CONFIGMAP_NAME); + } + if (config.getPodName() == null) { + config.setPodName(System.getenv("HOSTNAME")); + if (config.getPodName() == null) { + try { + config.setPodName(InetAddress.getLocalHost().getHostName()); + } catch (Exception e) { + throw new RuntimeCamelException("Unable to determine pod name", e); + } + } + } + + return config; + } + + public String getMasterUrl() { + return configuration.getMasterUrl(); + } + + /** + * Set the URL of the Kubernetes master (read from Kubernetes client properties by default). + */ + public void setMasterUrl(String masterUrl) { + configuration.setMasterUrl(masterUrl); + } + + public String getKubernetesNamespace() { + return this.lockConfiguration.getKubernetesResourcesNamespace(); + } + + /** + * Set the name of the Kubernetes namespace containing the pods and the configmap (autodetected by default) + */ + public void setKubernetesNamespace(String kubernetesNamespace) { + this.lockConfiguration.setKubernetesResourcesNamespace(kubernetesNamespace); + } + + public String getConfigMapName() { + return this.lockConfiguration.getConfigMapName(); + } + + /** + * Set the name of the ConfigMap used to do optimistic locking (defaults to 'leaders'). + */ + public void setConfigMapName(String configMapName) { + this.lockConfiguration.setConfigMapName(configMapName); + } + + public String getPodName() { + return this.lockConfiguration.getPodName(); + } + + /** + * Set the name of the current pod (autodetected from container host name by default). + */ + public void setPodName(String podName) { + this.lockConfiguration.setPodName(podName); + } + + public Map getClusterLabels() { + return lockConfiguration.getClusterLabels(); + } + + /** + * Set the labels used to identify the pods composing the cluster. + */ + public void setClusterLabels(Map clusterLabels) { + lockConfiguration.setClusterLabels(clusterLabels); + } + + public Long getWatchRefreshIntervalSeconds() { + return lockConfiguration.getWatchRefreshIntervalSeconds(); + } + + /** + * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. + * Watch recreation can be disabled by putting a negative value (the default will be used in case of null). + */ + public void setWatchRefreshIntervalSeconds(Long watchRefreshIntervalSeconds) { + lockConfiguration.setWatchRefreshIntervalSeconds(watchRefreshIntervalSeconds); + } +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java new file mode 100644 index 0000000000000..9ac6a86afe2a9 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.apache.camel.component.kubernetes.KubernetesConfiguration; +import org.apache.camel.component.kubernetes.KubernetesHelper; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesLeadershipController; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesLockConfiguration; +import org.apache.camel.ha.CamelClusterMember; +import org.apache.camel.impl.ha.AbstractCamelClusterView; +import org.apache.camel.util.ObjectHelper; + +/** + * The cluster view on a specific Camel cluster namespace (not to be confused with Kubernetes namespaces). + * Namespaces are represented as keys in a Kubernetes ConfigMap (values are the current leader pods). + */ +public class KubernetesClusterView extends AbstractCamelClusterView { + + private KubernetesClient kubernetesClient; + + private KubernetesConfiguration configuration; + + private KubernetesLockConfiguration lockConfiguration; + + private KubernetesClusterMember localMember; + + private Map memberCache; + + private volatile Optional currentLeader = Optional.empty(); + + private volatile List currentMembers = Collections.emptyList(); + + private KubernetesLeadershipController controller; + + public KubernetesClusterView(KubernetesClusterService cluster, KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { + super(cluster, lockConfiguration.getGroupName()); + this.configuration = configuration; + this.lockConfiguration = lockConfiguration; + this.localMember = new KubernetesClusterMember(lockConfiguration.getPodName()); + this.memberCache = new HashMap<>(); + } + + @Override + public Optional getMaster() { + return currentLeader; + } + + @Override + public CamelClusterMember getLocalMember() { + return localMember; + } + + @Override + public List getMembers() { + return currentMembers; + } + + @Override + protected void doStart() throws Exception { + if (controller == null) { + this.kubernetesClient = KubernetesHelper.getKubernetesClient(configuration); + + controller = new KubernetesLeadershipController(kubernetesClient, this.lockConfiguration, event -> { + if (event instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { + // New leader + Optional leader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(event).getData(); + currentLeader = leader.map(this::toMember); + if (currentLeader.isPresent()) { + fireLeadershipChangedEvent(currentLeader.get()); + } + } else if (event instanceof KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) { + Set members = KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent.class.cast(event).getData(); + Set oldMembers = currentMembers.stream().map(CamelClusterMember::getId).collect(Collectors.toSet()); + currentMembers = members.stream().map(this::toMember).collect(Collectors.toList()); + + // Computing differences + Set added = new HashSet<>(members); + added.removeAll(oldMembers); + + Set removed = new HashSet<>(oldMembers); + removed.removeAll(members); + + for (String id : added) { + fireMemberAddedEvent(toMember(id)); + } + + for (String id : removed) { + fireMemberRemovedEvent(toMember(id)); + } + } + }); + + controller.start(); + } + } + + @Override + protected void doStop() throws Exception { + if (controller != null) { + controller.stop(); + controller = null; + kubernetesClient.close(); + kubernetesClient = null; + } + } + + protected KubernetesClusterMember toMember(String name) { + if (name.equals(localMember.getId())) { + return localMember; + } + return memberCache.computeIfAbsent(name, KubernetesClusterMember::new); + } + + class KubernetesClusterMember implements CamelClusterMember { + + private String podName; + + public KubernetesClusterMember(String podName) { + this.podName = ObjectHelper.notNull(podName, "podName"); + } + + @Override + public boolean isMaster() { + return currentLeader.isPresent() && currentLeader.get().getId().equals(podName); + } + + @Override + public String getId() { + return podName; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("KubernetesClusterMember{"); + sb.append("podName='").append(podName).append('\''); + sb.append('}'); + return sb.toString(); + } + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEvent.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEvent.java new file mode 100644 index 0000000000000..59f8768e140cb --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEvent.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.Optional; +import java.util.Set; + +/** + * Super interface for events produced by the Kubernetes cluster. + */ +@FunctionalInterface +public interface KubernetesClusterEvent { + + Object getData(); + + /** + * Event signalling that the list of members of the Kubernetes cluster has changed. + */ + interface KubernetesClusterMemberListChangedEvent extends KubernetesClusterEvent { + @Override + Set getData(); + } + + /** + * Event signalling the presence of a new leader. + */ + interface KubernetesClusterLeaderChangedEvent extends KubernetesClusterEvent { + @Override + Optional getData(); + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEventHandler.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEventHandler.java new file mode 100644 index 0000000000000..0962847aa5093 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesClusterEventHandler.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +/** + * Interface for handling Kubernetes cluster events. + */ +@FunctionalInterface +public interface KubernetesClusterEventHandler { + + void onKubernetesClusterEvent(KubernetesClusterEvent event); + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java new file mode 100644 index 0000000000000..5555fe1f2ecd7 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Monitors continuously the configmap to detect changes in leadership. + * It calls the callback eventHandlers only when the leader changes w.r.t. the previous invocation. + */ +class KubernetesLeaderMonitor implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeaderMonitor.class); + + private ScheduledExecutorService serializedExecutor; + + private KubernetesClient kubernetesClient; + + private KubernetesLockConfiguration lockConfiguration; + + private List eventHandlers; + + private Watch watch; + + private boolean terminated; + + private boolean refreshing; + + private ConfigMap latestConfigMap; + + public KubernetesLeaderMonitor(ScheduledExecutorService serializedExecutor, KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration) { + this.serializedExecutor = serializedExecutor; + this.kubernetesClient = kubernetesClient; + this.lockConfiguration = lockConfiguration; + this.eventHandlers = new LinkedList<>(); + } + + public void addClusterEventHandler(KubernetesClusterEventHandler leaderEventHandler) { + this.eventHandlers.add(leaderEventHandler); + } + + @Override + public void start() throws Exception { + this.terminated = false; + serializedExecutor.execute(this::startWatch); + serializedExecutor.execute(() -> doPoll(true)); + + long recreationDelay = lockConfiguration.getWatchRefreshIntervalSecondsOrDefault(); + if (recreationDelay > 0) { + serializedExecutor.scheduleWithFixedDelay(this::refresh, recreationDelay, recreationDelay, TimeUnit.SECONDS); + } + } + + @Override + public void stop() throws Exception { + this.terminated = true; + Watch watch = this.watch; + if (watch != null) { + watch.close(); + } + } + + public void refresh() { + serializedExecutor.execute(() -> { + if (!terminated) { + refreshing = true; + try { + doPoll(false); + + Watch w = this.watch; + if (w != null) { + // It will be recreated + w.close(); + } + } finally { + refreshing = false; + } + } + }); + } + + private void startWatch() { + try { + LOG.debug("Starting ConfigMap watcher for monitoring the current leader"); + this.watch = kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .watch(new Watcher() { + + @Override + public void eventReceived(Action action, ConfigMap configMap) { + switch (action) { + case MODIFIED: + case DELETED: + case ADDED: + LOG.debug("Received update from watch on ConfigMap {}", configMap); + serializedExecutor.execute(() -> checkAndNotify(configMap)); + break; + default: + } + } + + @Override + public void onClose(KubernetesClientException e) { + if (!terminated) { + KubernetesLeaderMonitor.this.watch = null; + if (refreshing) { + LOG.info("Refreshing ConfigMap watcher..."); + serializedExecutor.execute(KubernetesLeaderMonitor.this::startWatch); + } else { + LOG.warn("ConfigMap watcher has been closed unexpectedly. Recreating it in 1 second...", e); + serializedExecutor.schedule(KubernetesLeaderMonitor.this::startWatch, 1, TimeUnit.SECONDS); + } + } + } + }); + } catch (Exception ex) { + LOG.warn("Unable to watch for configmap changes. Retrying in 5 seconds..."); + LOG.debug("Error while trying to watch the configmap", ex); + + this.serializedExecutor.schedule(this::startWatch, 5, TimeUnit.SECONDS); + } + } + + private void doPoll(boolean retry) { + LOG.debug("Starting poll to get configmap {}", this.lockConfiguration.getConfigMapName()); + ConfigMap configMap; + try { + configMap = pollConfigMap(); + } catch (Exception ex) { + if (retry) { + LOG.warn("ConfigMap poll failed. Retrying in 5 seconds...", ex); + this.serializedExecutor.schedule(() -> doPoll(true), 5, TimeUnit.SECONDS); + } else { + LOG.warn("ConfigMap poll failed", ex); + } + return; + } + + checkAndNotify(configMap); + } + + private void checkAndNotify(ConfigMap candidateConfigMap) { + LOG.debug("Checking configMap {}", candidateConfigMap); + ConfigMap newConfigMap = newest(this.latestConfigMap, candidateConfigMap); + Optional leader = extractLeader(newConfigMap); + Optional oldLeader = extractLeader(this.latestConfigMap); + + this.latestConfigMap = newConfigMap; + + LOG.debug("The new leader is {}. Old leader was {}", leader, oldLeader); + if (!leader.equals(oldLeader)) { + LOG.debug("Notifying the new leader to all eventHandlers"); + for (KubernetesClusterEventHandler eventHandler : eventHandlers) { + eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> leader); + } + } else { + LOG.debug("Leader has not changed"); + } + } + + private ConfigMap pollConfigMap() { + return kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .get(); + } + + private Optional extractLeader(ConfigMap configMap) { + Optional leader = Optional.empty(); + if (configMap != null && configMap.getData() != null) { + leader = Optional.ofNullable(configMap.getData().get(this.lockConfiguration.getGroupName())); + } + return leader; + } + + private ConfigMap newest(ConfigMap configMap1, ConfigMap configMap2) { + ConfigMap newest = null; + + if (configMap1 != null && configMap2 == null) { + newest = configMap1; + } else if (configMap1 == null && configMap2 != null) { + newest = configMap2; + } + + if (newest == null) { + String rv1 = extractResourceVersion(configMap1); + String rv2 = extractResourceVersion(configMap2); + newest = newest(configMap1, configMap2, rv1, rv2); + } + + if (newest == null) { + String ct1 = extractCreationTimestamp(configMap1); + String ct2 = extractCreationTimestamp(configMap2); + // timestamps are string-comparable + newest = newest(configMap1, configMap2, ct1, ct2); + } + + return newest; + } + + private > ConfigMap newest(ConfigMap configMap1, ConfigMap configMap2, T cmp1, T cmp2) { + if (cmp1 != null && cmp2 != null) { + int comp = cmp1.compareTo(cmp2); + if (comp > 0) { + return configMap1; + } else { + return configMap2; + } + } + return null; + } + + private String extractResourceVersion(ConfigMap configMap) { + if (configMap != null && configMap.getMetadata() != null) { + return configMap.getMetadata().getResourceVersion(); + } + return null; + } + + private String extractCreationTimestamp(ConfigMap configMap) { + if (configMap != null && configMap.getMetadata() != null) { + return configMap.getMetadata().getCreationTimestamp(); + } + return null; + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java new file mode 100644 index 0000000000000..ad2f9bc726a33 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Start the monitors and participate to leader election when no active leaders are present. + * It communicates changes in leadership and cluster members to the given event handler. + */ +public class KubernetesLeadershipController implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeadershipController.class); + + private KubernetesClient kubernetesClient; + + private KubernetesLockConfiguration lockConfiguration; + + private ScheduledExecutorService executor; + + private KubernetesLeaderMonitor leaderMonitor; + + private KubernetesMembersMonitor membersMonitor; + + private Optional currentLeader; + + private Set currentMembers; + + private KubernetesClusterEventHandler eventHandler; + + public KubernetesLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { + + this.kubernetesClient = kubernetesClient; + this.lockConfiguration = lockConfiguration; + this.eventHandler = eventHandler; + + this.currentLeader = Optional.empty(); + this.currentMembers = Collections.emptySet(); + } + + @Override + public void start() throws Exception { + + if (executor == null) { + executor = Executors.newSingleThreadScheduledExecutor(); // No concurrency + leaderMonitor = new KubernetesLeaderMonitor(this.executor, this.kubernetesClient, this.lockConfiguration); + membersMonitor = new KubernetesMembersMonitor(this.executor, this.kubernetesClient, this.lockConfiguration); + + leaderMonitor.addClusterEventHandler(e -> executor.execute(() -> onLeaderChanged(e))); + if (eventHandler != null) { + leaderMonitor.addClusterEventHandler(eventHandler); + } + + membersMonitor.addClusterEventHandler(e -> executor.execute(() -> onMembersChanged(e))); + if (eventHandler != null) { + membersMonitor.addClusterEventHandler(eventHandler); + } + + // Start all services + leaderMonitor.start(); + membersMonitor.start(); + + // Fire a new election if possible + executor.execute(this::runLeaderElection); + } + + } + + @Override + public void stop() throws Exception { + if (executor != null) { + membersMonitor.stop(); + leaderMonitor.stop(); + executor.shutdown(); + executor.shutdownNow(); + + membersMonitor = null; + leaderMonitor = null; + executor = null; + } + } + + private void onLeaderChanged(KubernetesClusterEvent e) { + Optional newLeader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(e).getData(); + if (!newLeader.isPresent()) { + executor.execute(this::tryLeaderElection); + } + this.currentLeader = newLeader; + } + + private void onMembersChanged(KubernetesClusterEvent e) { + Set newMembers = KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent.class.cast(e).getData(); + if (currentLeader.isPresent()) { + // Check if the current leader is still present in the list + if (!newMembers.contains(currentLeader.get()) && currentMembers.contains(currentLeader.get())) { + executor.execute(this::runLeaderElection); + } + } + this.currentMembers = newMembers; + } + + private void runLeaderElection() { + boolean finished = false; + try { + finished = tryLeaderElection(); + } catch (Exception ex) { + LOG.warn("Exception while trying to acquire the leadership", ex); + } + + if (!finished) { + executor.schedule(this::runLeaderElection, 1, TimeUnit.SECONDS); + } + } + + private boolean tryLeaderElection() { + LOG.debug("Starting leader election"); + if (!currentMembers.contains(this.lockConfiguration.getPodName())) { + LOG.debug("The current pod ({}) is not in the list of participating pods {}. Cannot participate to the election", this.lockConfiguration.getPodName(), currentMembers); + return false; + } + + ConfigMap configMap = kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .get(); + + if (configMap == null) { + // No configmap created so far + LOG.info("Lock configmap is not present in the Kubernetes namespace. A new ConfigMap will be created"); + + ConfigMap newConfigMap = new ConfigMapBuilder(). + withNewMetadata() + .withName(this.lockConfiguration.getConfigMapName()) + .addToLabels("provider", "camel") + .addToLabels("kind", "locks"). + endMetadata() + .addToData(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName()) + .build(); + + try { + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .create(newConfigMap); + } catch (Exception ex) { + // Suppress exception + LOG.warn("Unable to create the ConfigMap, it may have been created by other cluster members concurrently. If the problem persists, check if the service account has the right " + + "permissions to create it"); + LOG.debug("Exception while trying to create the ConfigMap", ex); + return false; + } + return true; + } else { + LOG.info("Lock configmap already present in the Kubernetes namespace. Checking..."); + Map leaders = configMap.getData(); + Optional oldLeader = leaders != null ? Optional.ofNullable(leaders.get(this.lockConfiguration.getGroupName())) : Optional.empty(); + + boolean noLeaderPresent = !oldLeader.isPresent() || !currentMembers.contains(oldLeader.get()); + boolean alreadyLeader = oldLeader.isPresent() && oldLeader.get().equals(this.lockConfiguration.getPodName()); + + if (noLeaderPresent && !alreadyLeader) { + LOG.info("Trying to acquire the lock in configmap={}, key={}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName()); + ConfigMap newConfigMap = new ConfigMapBuilder(configMap) + .addToData(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName()) + .build(); + + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .lockResourceVersion(configMap.getMetadata().getResourceVersion()) + .replace(newConfigMap); + + LOG.info("Lock acquired for configmap={}, key={}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName()); + } else if (!noLeaderPresent) { + LOG.info("A leader is already present for configmap={}, key={}: {}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName(), oldLeader); + } else { + LOG.info("This pod ({}) is already the leader for configmap={}, key={}", this.lockConfiguration.getPodName(), this.lockConfiguration.getConfigMapName(), this.lockConfiguration + .getGroupName()); + } + return true; + } + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java new file mode 100644 index 0000000000000..f203c0ae27c57 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.HashMap; +import java.util.Map; + +import io.fabric8.kubernetes.client.KubernetesClient; + +/** + * Configuration for Kubernetes Lock. + */ +public class KubernetesLockConfiguration implements Cloneable { + + private static final long DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS = 1800; + + /** + * Kubernetes namespace containing the pods and the ConfigMap used for locking. + */ + private String kubernetesResourcesNamespace; + + /** + * Name of the ConfigMap used for locking. + */ + private String configMapName; + + /** + * Name of the lock group (or namespace according to the Camel cluster convention) within the chosen ConfgMap. + */ + private String groupName; + + /** + * Name of the current pod (defaults to host name). + */ + private String podName; + + /** + * Labels used to identify the members of the cluster. + */ + private Map clusterLabels = new HashMap<>(); + + /** + * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. + * Watch recreation can be disabled by putting a negative value (the default will be used in case of null). + */ + private Long watchRefreshIntervalSeconds; + + public KubernetesLockConfiguration() { + } + + public String getKubernetesResourcesNamespaceOrDefault(KubernetesClient kubernetesClient) { + if (kubernetesResourcesNamespace != null) { + return kubernetesResourcesNamespace; + } + return kubernetesClient.getNamespace(); + } + + public String getKubernetesResourcesNamespace() { + return kubernetesResourcesNamespace; + } + + public void setKubernetesResourcesNamespace(String kubernetesResourcesNamespace) { + this.kubernetesResourcesNamespace = kubernetesResourcesNamespace; + } + + public String getConfigMapName() { + return configMapName; + } + + public void setConfigMapName(String configMapName) { + this.configMapName = configMapName; + } + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getPodName() { + return podName; + } + + public void setPodName(String podName) { + this.podName = podName; + } + + public Map getClusterLabels() { + return clusterLabels; + } + + public void addToClusterLabels(String key, String value) { + this.clusterLabels.put(key, value); + } + + public void setClusterLabels(Map clusterLabels) { + this.clusterLabels = clusterLabels; + } + + public Long getWatchRefreshIntervalSeconds() { + return watchRefreshIntervalSeconds; + } + + public long getWatchRefreshIntervalSecondsOrDefault() { + Long interval = watchRefreshIntervalSeconds; + if (interval == null) { + interval = DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS; + } + return interval; + } + + public void setWatchRefreshIntervalSeconds(Long watchRefreshIntervalSeconds) { + this.watchRefreshIntervalSeconds = watchRefreshIntervalSeconds; + } + + public KubernetesLockConfiguration copy() { + try { + KubernetesLockConfiguration copy = (KubernetesLockConfiguration) this.clone(); + return copy; + } catch (CloneNotSupportedException e) { + throw new IllegalStateException("Cannot clone", e); + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("KubernetesLockConfiguration{"); + sb.append("kubernetesResourcesNamespace='").append(kubernetesResourcesNamespace).append('\''); + sb.append(", configMapName='").append(configMapName).append('\''); + sb.append(", groupName='").append(groupName).append('\''); + sb.append(", podName='").append(podName).append('\''); + sb.append(", clusterLabels=").append(clusterLabels); + sb.append(", watchRefreshIntervalSeconds=").append(watchRefreshIntervalSeconds); + sb.append('}'); + return sb.toString(); + } +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java new file mode 100644 index 0000000000000..d9173b26d2c27 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Monitors the list of participants in a leader election and provides the most recently updated list. + * It calls the callback eventHandlers only when the member set changes w.r.t. the previous invocation. + */ +class KubernetesMembersMonitor implements Service { + + private static final long DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS = 1800; + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesMembersMonitor.class); + + private ScheduledExecutorService serializedExecutor; + + private KubernetesClient kubernetesClient; + + private KubernetesLockConfiguration lockConfiguration; + + private List eventHandlers; + + private Watch watch; + + private boolean terminated; + + private boolean refreshing; + + private Set previousMembers = new HashSet<>(); + + private Set basePoll = new HashSet<>(); + private Set deleted = new HashSet<>(); + private Set added = new HashSet<>(); + + public KubernetesMembersMonitor(ScheduledExecutorService serializedExecutor, KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration) { + this.serializedExecutor = serializedExecutor; + this.kubernetesClient = kubernetesClient; + this.lockConfiguration = lockConfiguration; + this.eventHandlers = new LinkedList<>(); + } + + public void addClusterEventHandler(KubernetesClusterEventHandler eventHandler) { + this.eventHandlers.add(eventHandler); + } + + @Override + public void start() throws Exception { + serializedExecutor.execute(() -> doPoll(true)); + serializedExecutor.execute(this::createWatch); + + long recreationDelay = lockConfiguration.getWatchRefreshIntervalSecondsOrDefault(); + if (recreationDelay > 0) { + serializedExecutor.scheduleWithFixedDelay(this::refresh, recreationDelay, recreationDelay, TimeUnit.SECONDS); + } + } + + private void createWatch() { + try { + LOG.debug("Starting cluster members watcher"); + this.watch = kubernetesClient.pods() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withLabels(this.lockConfiguration.getClusterLabels()) + .watch(new Watcher() { + + @Override + public void eventReceived(Action action, Pod pod) { + switch (action) { + case DELETED: + serializedExecutor.execute(() -> deleteAndNotify(podName(pod))); + break; + case ADDED: + serializedExecutor.execute(() -> addAndNotify(podName(pod))); + break; + default: + } + } + + @Override + public void onClose(KubernetesClientException e) { + if (!terminated) { + KubernetesMembersMonitor.this.watch = null; + if (refreshing) { + LOG.info("Refreshing pod list watcher..."); + serializedExecutor.execute(KubernetesMembersMonitor.this::createWatch); + } else { + LOG.warn("Pod list watcher has been closed unexpectedly. Recreating it in 1 second...", e); + serializedExecutor.schedule(KubernetesMembersMonitor.this::createWatch, 1, TimeUnit.SECONDS); + } + } + } + }); + } catch (Exception ex) { + LOG.warn("Unable to watch for pod list changes. Retrying in 5 seconds..."); + LOG.debug("Error while trying to watch the pod list", ex); + + serializedExecutor.schedule(this::createWatch, 5, TimeUnit.SECONDS); + } + } + + @Override + public void stop() throws Exception { + this.terminated = true; + Watch watch = this.watch; + if (watch != null) { + watch.close(); + } + } + + public void refresh() { + serializedExecutor.execute(() -> { + if (!terminated) { + refreshing = true; + try { + doPoll(false); + + Watch w = this.watch; + if (w != null) { + // It will be recreated + w.close(); + } + } finally { + refreshing = false; + } + } + }); + } + + private void doPoll(boolean retry) { + LOG.debug("Starting poll to get all cluster members"); + List pods; + try { + pods = pollPods(); + } catch (Exception ex) { + if (retry) { + LOG.warn("Pods poll failed. Retrying in 5 seconds...", ex); + this.serializedExecutor.schedule(() -> doPoll(true), 5, TimeUnit.SECONDS); + } else { + LOG.warn("Pods poll failed", ex); + } + return; + } + + this.basePoll = pods.stream() + .map(p -> Optional.ofNullable(podName(p))) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet()); + + this.added = new HashSet<>(); + this.deleted = new HashSet<>(); + + LOG.debug("Base list of members is {}", this.basePoll); + + checkAndNotify(); + } + + private List pollPods() { + return kubernetesClient.pods() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withLabels(this.lockConfiguration.getClusterLabels()) + .list().getItems(); + } + + private String podName(Pod pod) { + if (pod != null && pod.getMetadata() != null) { + return pod.getMetadata().getName(); + } + return null; + } + + private void checkAndNotify() { + Set newMembers = new HashSet<>(basePoll); + newMembers.addAll(added); + newMembers.removeAll(deleted); + + LOG.debug("Current list of members is: {}", newMembers); + + if (!newMembers.equals(this.previousMembers)) { + LOG.debug("List of members changed: sending notifications"); + this.previousMembers = newMembers; + + for (KubernetesClusterEventHandler eventHandler : eventHandlers) { + eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) () -> newMembers); + } + } else { + LOG.debug("List of members has not changed"); + } + } + + private void addAndNotify(String member) { + LOG.debug("Adding new member to the list: {}", member); + if (member != null) { + this.added.add(member); + checkAndNotify(); + } + } + + private void deleteAndNotify(String member) { + LOG.debug("Deleting member to the list: {}", member); + if (member != null) { + this.deleted.add(member); + checkAndNotify(); + } + } + +} From f9e5f450e6d7fe0532430b68c3ee1d9d7e2f8728 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Fri, 7 Jul 2017 17:05:31 +0200 Subject: [PATCH 02/13] CAMEL-11331: Lease based implementation of Kubernetes lock --- .../ha/KubernetesClusterService.java | 117 +++++- .../kubernetes/ha/KubernetesClusterView.java | 6 +- .../ha/lock/ConfigMapLockUtils.java | 106 +++++ .../ha/lock/KubernetesLeaderMonitor.java | 256 ------------ .../lock/KubernetesLeadershipController.java | 211 ---------- ...ernetesLeaseBasedLeadershipController.java | 374 ++++++++++++++++++ .../ha/lock/KubernetesLockConfiguration.java | 99 ++++- .../ha/lock/KubernetesMembersMonitor.java | 4 +- .../kubernetes/ha/lock/LeaderInfo.java | 90 +++++ 9 files changed, 767 insertions(+), 496 deletions(-) create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java delete mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java delete mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java index 6d87d48df1271..a868d1641e55d 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java @@ -19,6 +19,8 @@ import java.net.InetAddress; import java.util.Map; +import io.fabric8.kubernetes.client.KubernetesClient; + import org.apache.camel.CamelContext; import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.kubernetes.KubernetesConfiguration; @@ -31,8 +33,6 @@ */ public class KubernetesClusterService extends AbstractCamelClusterService { - public static final String DEFAULT_CONFIGMAP_NAME = "leaders"; - private KubernetesConfiguration configuration; private KubernetesLockConfiguration lockConfiguration; @@ -64,10 +64,7 @@ protected KubernetesLockConfiguration configWithGroupNameAndDefaults(String grou config.setGroupName(ObjectHelper.notNull(groupName, "groupName")); - // Check defaults (Namespace and podName can be null) - if (config.getConfigMapName() == null) { - config.setConfigMapName(DEFAULT_CONFIGMAP_NAME); - } + // Determine the pod name if not provided if (config.getPodName() == null) { config.setPodName(System.getenv("HOSTNAME")); if (config.getPodName() == null) { @@ -79,6 +76,33 @@ protected KubernetesLockConfiguration configWithGroupNameAndDefaults(String grou } } + ObjectHelper.notNull(config.getConfigMapName(), "configMapName"); + ObjectHelper.notNull(config.getClusterLabels(), "clusterLabels"); + + if (config.getJitterFactor() < 1) { + throw new IllegalStateException("jitterFactor must be >= 1 (found: " + config.getJitterFactor() + ")"); + } + if (config.getRetryOnErrorIntervalSeconds() <= 0) { + throw new IllegalStateException("retryOnErrorIntervalSeconds must be > 0 (found: " + config.getRetryOnErrorIntervalSeconds() + ")"); + } + if (config.getRetryPeriodSeconds() <= 0) { + throw new IllegalStateException("retryPeriodSeconds must be > 0 (found: " + config.getRetryPeriodSeconds() + ")"); + } + if (config.getRenewDeadlineSeconds() <= 0) { + throw new IllegalStateException("renewDeadlineSeconds must be > 0 (found: " + config.getRenewDeadlineSeconds() + ")"); + } + if (config.getLeaseDurationSeconds() <= 0) { + throw new IllegalStateException("leaseDurationSeconds must be > 0 (found: " + config.getLeaseDurationSeconds() + ")"); + } + if (config.getLeaseDurationSeconds() <= config.getRenewDeadlineSeconds()) { + throw new IllegalStateException("leaseDurationSeconds must be greater than renewDeadlineSeconds " + + "(" + config.getLeaseDurationSeconds() + " is not greater than " + config.getRenewDeadlineSeconds() + ")"); + } + if (config.getRenewDeadlineSeconds() <= config.getJitterFactor() * config.getRetryPeriodSeconds()) { + throw new IllegalStateException("renewDeadlineSeconds must be greater than jitterFactor*retryPeriodSeconds " + + "(" + config.getRenewDeadlineSeconds() + " is not greater than " + config.getJitterFactor() + "*" + config.getRetryPeriodSeconds() + ")"); + } + return config; } @@ -137,15 +161,88 @@ public void setClusterLabels(Map clusterLabels) { lockConfiguration.setClusterLabels(clusterLabels); } - public Long getWatchRefreshIntervalSeconds() { - return lockConfiguration.getWatchRefreshIntervalSeconds(); + public void addToClusterLabels(String key, String value) { + lockConfiguration.addToClusterLabels(key, value); + } + + public String getKubernetesResourcesNamespace() { + return lockConfiguration.getKubernetesResourcesNamespace(); + } + + /** + * Kubernetes namespace containing the pods and the ConfigMap used for locking. + */ + public void setKubernetesResourcesNamespace(String kubernetesResourcesNamespace) { + lockConfiguration.setKubernetesResourcesNamespace(kubernetesResourcesNamespace); + } + + public long getRetryOnErrorIntervalSeconds() { + return lockConfiguration.getRetryOnErrorIntervalSeconds(); } /** * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. - * Watch recreation can be disabled by putting a negative value (the default will be used in case of null). + * Watch recreation can be disabled by putting value <= 0. + */ + public void setRetryOnErrorIntervalSeconds(long retryOnErrorIntervalSeconds) { + lockConfiguration.setRetryOnErrorIntervalSeconds(retryOnErrorIntervalSeconds); + } + + public double getJitterFactor() { + return lockConfiguration.getJitterFactor(); + } + + /** + * A jitter factor to apply in order to prevent all pods to try to become leaders in the same instant. */ - public void setWatchRefreshIntervalSeconds(Long watchRefreshIntervalSeconds) { + public void setJitterFactor(double jitterFactor) { + lockConfiguration.setJitterFactor(jitterFactor); + } + + public long getLeaseDurationSeconds() { + return lockConfiguration.getLeaseDurationSeconds(); + } + + /** + * The default duration of the lease for the current leader. + */ + public void setLeaseDurationSeconds(long leaseDurationSeconds) { + lockConfiguration.setLeaseDurationSeconds(leaseDurationSeconds); + } + + public long getRenewDeadlineSeconds() { + return lockConfiguration.getRenewDeadlineSeconds(); + } + + /** + * The deadline after which the leader must stop trying to renew its leadership (and yield it). + */ + public void setRenewDeadlineSeconds(long renewDeadlineSeconds) { + lockConfiguration.setRenewDeadlineSeconds(renewDeadlineSeconds); + } + + public long getRetryPeriodSeconds() { + return lockConfiguration.getRetryPeriodSeconds(); + } + + /** + * The time between two subsequent attempts to acquire/renew the leadership (or after the lease expiration). + * It is randomized using the jitter factor in case of new leader election (not renewal). + */ + public void setRetryPeriodSeconds(long retryPeriodSeconds) { + lockConfiguration.setRetryPeriodSeconds(retryPeriodSeconds); + } + + public long getWatchRefreshIntervalSeconds() { + return lockConfiguration.getWatchRefreshIntervalSeconds(); + } + + /** + * Set this to a positive value in order to recreate watchers after a certain amount of time, + * to avoid having stale watchers. + */ + public void setWatchRefreshIntervalSeconds(long watchRefreshIntervalSeconds) { lockConfiguration.setWatchRefreshIntervalSeconds(watchRefreshIntervalSeconds); } + } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java index 9ac6a86afe2a9..e324b3ff4ab28 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java @@ -30,7 +30,7 @@ import org.apache.camel.component.kubernetes.KubernetesConfiguration; import org.apache.camel.component.kubernetes.KubernetesHelper; import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; -import org.apache.camel.component.kubernetes.ha.lock.KubernetesLeadershipController; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesLeaseBasedLeadershipController; import org.apache.camel.component.kubernetes.ha.lock.KubernetesLockConfiguration; import org.apache.camel.ha.CamelClusterMember; import org.apache.camel.impl.ha.AbstractCamelClusterView; @@ -56,7 +56,7 @@ public class KubernetesClusterView extends AbstractCamelClusterView { private volatile List currentMembers = Collections.emptyList(); - private KubernetesLeadershipController controller; + private KubernetesLeaseBasedLeadershipController controller; public KubernetesClusterView(KubernetesClusterService cluster, KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { super(cluster, lockConfiguration.getGroupName()); @@ -86,7 +86,7 @@ protected void doStart() throws Exception { if (controller == null) { this.kubernetesClient = KubernetesHelper.getKubernetesClient(configuration); - controller = new KubernetesLeadershipController(kubernetesClient, this.lockConfiguration, event -> { + controller = new KubernetesLeaseBasedLeadershipController(kubernetesClient, this.lockConfiguration, event -> { if (event instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { // New leader Optional leader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(event).getData(); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java new file mode 100644 index 0000000000000..84718f31ff0bc --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.text.SimpleDateFormat; +import java.util.Date; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +public final class ConfigMapLockUtils { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigMapLockUtils.class); + + private static final String DATE_TIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ssX"; + + private static final String LEADER_PREFIX = "leader.pod."; + + private static final String TIMESTAMP_PREFIX = "leader.timestamp."; + + private ConfigMapLockUtils() { + } + + public static ConfigMap createNewConfigMap(String configMapName, LeaderInfo leaderInfo) { + return new ConfigMapBuilder(). + withNewMetadata() + .withName(configMapName) + .addToLabels("provider", "camel") + .addToLabels("kind", "locks"). + endMetadata() + .addToData(LEADER_PREFIX + leaderInfo.getGroupName(), leaderInfo.getLeader()) + .addToData(TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getTimestamp())) + .build(); + } + + public static ConfigMap getConfigMapWithNewLeader(ConfigMap configMap, LeaderInfo leaderInfo) { + return new ConfigMapBuilder(configMap) + .addToData(LEADER_PREFIX + leaderInfo.getGroupName(), leaderInfo.getLeader()) + .addToData(TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getTimestamp())) + .build(); + } + + public static LeaderInfo getLeaderInfo(ConfigMap configMap, String group) { + return new LeaderInfo(group, getLeader(configMap, group), getTimestamp(configMap, group)); + } + + private static String getLeader(ConfigMap configMap, String group) { + return getConfigMapValue(configMap, LEADER_PREFIX + group); + } + + private static String formatDate(Date date) { + if (date == null) { + return null; + } + try { + return new SimpleDateFormat(DATE_TIME_FORMAT).format(date); + } catch (Exception e) { + LOG.warn("Unable to format date '" + date + "' using format " + DATE_TIME_FORMAT, e); + } + + return null; + } + + private static Date getTimestamp(ConfigMap configMap, String group) { + String timestamp = getConfigMapValue(configMap, TIMESTAMP_PREFIX + group); + if (timestamp == null) { + return null; + } + + try { + return new SimpleDateFormat(DATE_TIME_FORMAT).parse(timestamp); + } catch (Exception e) { + LOG.warn("Unable to parse time string '" + timestamp + "' using format " + DATE_TIME_FORMAT, e); + } + + return null; + } + + private static String getConfigMapValue(ConfigMap configMap, String key) { + if (configMap == null || configMap.getData() == null) { + return null; + } + return configMap.getData().get(key); + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java deleted file mode 100644 index 5555fe1f2ecd7..0000000000000 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaderMonitor.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.component.kubernetes.ha.lock; - -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watch; -import io.fabric8.kubernetes.client.Watcher; - -import org.apache.camel.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Monitors continuously the configmap to detect changes in leadership. - * It calls the callback eventHandlers only when the leader changes w.r.t. the previous invocation. - */ -class KubernetesLeaderMonitor implements Service { - - private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeaderMonitor.class); - - private ScheduledExecutorService serializedExecutor; - - private KubernetesClient kubernetesClient; - - private KubernetesLockConfiguration lockConfiguration; - - private List eventHandlers; - - private Watch watch; - - private boolean terminated; - - private boolean refreshing; - - private ConfigMap latestConfigMap; - - public KubernetesLeaderMonitor(ScheduledExecutorService serializedExecutor, KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration) { - this.serializedExecutor = serializedExecutor; - this.kubernetesClient = kubernetesClient; - this.lockConfiguration = lockConfiguration; - this.eventHandlers = new LinkedList<>(); - } - - public void addClusterEventHandler(KubernetesClusterEventHandler leaderEventHandler) { - this.eventHandlers.add(leaderEventHandler); - } - - @Override - public void start() throws Exception { - this.terminated = false; - serializedExecutor.execute(this::startWatch); - serializedExecutor.execute(() -> doPoll(true)); - - long recreationDelay = lockConfiguration.getWatchRefreshIntervalSecondsOrDefault(); - if (recreationDelay > 0) { - serializedExecutor.scheduleWithFixedDelay(this::refresh, recreationDelay, recreationDelay, TimeUnit.SECONDS); - } - } - - @Override - public void stop() throws Exception { - this.terminated = true; - Watch watch = this.watch; - if (watch != null) { - watch.close(); - } - } - - public void refresh() { - serializedExecutor.execute(() -> { - if (!terminated) { - refreshing = true; - try { - doPoll(false); - - Watch w = this.watch; - if (w != null) { - // It will be recreated - w.close(); - } - } finally { - refreshing = false; - } - } - }); - } - - private void startWatch() { - try { - LOG.debug("Starting ConfigMap watcher for monitoring the current leader"); - this.watch = kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .watch(new Watcher() { - - @Override - public void eventReceived(Action action, ConfigMap configMap) { - switch (action) { - case MODIFIED: - case DELETED: - case ADDED: - LOG.debug("Received update from watch on ConfigMap {}", configMap); - serializedExecutor.execute(() -> checkAndNotify(configMap)); - break; - default: - } - } - - @Override - public void onClose(KubernetesClientException e) { - if (!terminated) { - KubernetesLeaderMonitor.this.watch = null; - if (refreshing) { - LOG.info("Refreshing ConfigMap watcher..."); - serializedExecutor.execute(KubernetesLeaderMonitor.this::startWatch); - } else { - LOG.warn("ConfigMap watcher has been closed unexpectedly. Recreating it in 1 second...", e); - serializedExecutor.schedule(KubernetesLeaderMonitor.this::startWatch, 1, TimeUnit.SECONDS); - } - } - } - }); - } catch (Exception ex) { - LOG.warn("Unable to watch for configmap changes. Retrying in 5 seconds..."); - LOG.debug("Error while trying to watch the configmap", ex); - - this.serializedExecutor.schedule(this::startWatch, 5, TimeUnit.SECONDS); - } - } - - private void doPoll(boolean retry) { - LOG.debug("Starting poll to get configmap {}", this.lockConfiguration.getConfigMapName()); - ConfigMap configMap; - try { - configMap = pollConfigMap(); - } catch (Exception ex) { - if (retry) { - LOG.warn("ConfigMap poll failed. Retrying in 5 seconds...", ex); - this.serializedExecutor.schedule(() -> doPoll(true), 5, TimeUnit.SECONDS); - } else { - LOG.warn("ConfigMap poll failed", ex); - } - return; - } - - checkAndNotify(configMap); - } - - private void checkAndNotify(ConfigMap candidateConfigMap) { - LOG.debug("Checking configMap {}", candidateConfigMap); - ConfigMap newConfigMap = newest(this.latestConfigMap, candidateConfigMap); - Optional leader = extractLeader(newConfigMap); - Optional oldLeader = extractLeader(this.latestConfigMap); - - this.latestConfigMap = newConfigMap; - - LOG.debug("The new leader is {}. Old leader was {}", leader, oldLeader); - if (!leader.equals(oldLeader)) { - LOG.debug("Notifying the new leader to all eventHandlers"); - for (KubernetesClusterEventHandler eventHandler : eventHandlers) { - eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> leader); - } - } else { - LOG.debug("Leader has not changed"); - } - } - - private ConfigMap pollConfigMap() { - return kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .get(); - } - - private Optional extractLeader(ConfigMap configMap) { - Optional leader = Optional.empty(); - if (configMap != null && configMap.getData() != null) { - leader = Optional.ofNullable(configMap.getData().get(this.lockConfiguration.getGroupName())); - } - return leader; - } - - private ConfigMap newest(ConfigMap configMap1, ConfigMap configMap2) { - ConfigMap newest = null; - - if (configMap1 != null && configMap2 == null) { - newest = configMap1; - } else if (configMap1 == null && configMap2 != null) { - newest = configMap2; - } - - if (newest == null) { - String rv1 = extractResourceVersion(configMap1); - String rv2 = extractResourceVersion(configMap2); - newest = newest(configMap1, configMap2, rv1, rv2); - } - - if (newest == null) { - String ct1 = extractCreationTimestamp(configMap1); - String ct2 = extractCreationTimestamp(configMap2); - // timestamps are string-comparable - newest = newest(configMap1, configMap2, ct1, ct2); - } - - return newest; - } - - private > ConfigMap newest(ConfigMap configMap1, ConfigMap configMap2, T cmp1, T cmp2) { - if (cmp1 != null && cmp2 != null) { - int comp = cmp1.compareTo(cmp2); - if (comp > 0) { - return configMap1; - } else { - return configMap2; - } - } - return null; - } - - private String extractResourceVersion(ConfigMap configMap) { - if (configMap != null && configMap.getMetadata() != null) { - return configMap.getMetadata().getResourceVersion(); - } - return null; - } - - private String extractCreationTimestamp(ConfigMap configMap) { - if (configMap != null && configMap.getMetadata() != null) { - return configMap.getMetadata().getCreationTimestamp(); - } - return null; - } - -} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java deleted file mode 100644 index ad2f9bc726a33..0000000000000 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.component.kubernetes.ha.lock; - -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.ConfigMapBuilder; -import io.fabric8.kubernetes.client.KubernetesClient; - -import org.apache.camel.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Start the monitors and participate to leader election when no active leaders are present. - * It communicates changes in leadership and cluster members to the given event handler. - */ -public class KubernetesLeadershipController implements Service { - - private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeadershipController.class); - - private KubernetesClient kubernetesClient; - - private KubernetesLockConfiguration lockConfiguration; - - private ScheduledExecutorService executor; - - private KubernetesLeaderMonitor leaderMonitor; - - private KubernetesMembersMonitor membersMonitor; - - private Optional currentLeader; - - private Set currentMembers; - - private KubernetesClusterEventHandler eventHandler; - - public KubernetesLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { - - this.kubernetesClient = kubernetesClient; - this.lockConfiguration = lockConfiguration; - this.eventHandler = eventHandler; - - this.currentLeader = Optional.empty(); - this.currentMembers = Collections.emptySet(); - } - - @Override - public void start() throws Exception { - - if (executor == null) { - executor = Executors.newSingleThreadScheduledExecutor(); // No concurrency - leaderMonitor = new KubernetesLeaderMonitor(this.executor, this.kubernetesClient, this.lockConfiguration); - membersMonitor = new KubernetesMembersMonitor(this.executor, this.kubernetesClient, this.lockConfiguration); - - leaderMonitor.addClusterEventHandler(e -> executor.execute(() -> onLeaderChanged(e))); - if (eventHandler != null) { - leaderMonitor.addClusterEventHandler(eventHandler); - } - - membersMonitor.addClusterEventHandler(e -> executor.execute(() -> onMembersChanged(e))); - if (eventHandler != null) { - membersMonitor.addClusterEventHandler(eventHandler); - } - - // Start all services - leaderMonitor.start(); - membersMonitor.start(); - - // Fire a new election if possible - executor.execute(this::runLeaderElection); - } - - } - - @Override - public void stop() throws Exception { - if (executor != null) { - membersMonitor.stop(); - leaderMonitor.stop(); - executor.shutdown(); - executor.shutdownNow(); - - membersMonitor = null; - leaderMonitor = null; - executor = null; - } - } - - private void onLeaderChanged(KubernetesClusterEvent e) { - Optional newLeader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(e).getData(); - if (!newLeader.isPresent()) { - executor.execute(this::tryLeaderElection); - } - this.currentLeader = newLeader; - } - - private void onMembersChanged(KubernetesClusterEvent e) { - Set newMembers = KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent.class.cast(e).getData(); - if (currentLeader.isPresent()) { - // Check if the current leader is still present in the list - if (!newMembers.contains(currentLeader.get()) && currentMembers.contains(currentLeader.get())) { - executor.execute(this::runLeaderElection); - } - } - this.currentMembers = newMembers; - } - - private void runLeaderElection() { - boolean finished = false; - try { - finished = tryLeaderElection(); - } catch (Exception ex) { - LOG.warn("Exception while trying to acquire the leadership", ex); - } - - if (!finished) { - executor.schedule(this::runLeaderElection, 1, TimeUnit.SECONDS); - } - } - - private boolean tryLeaderElection() { - LOG.debug("Starting leader election"); - if (!currentMembers.contains(this.lockConfiguration.getPodName())) { - LOG.debug("The current pod ({}) is not in the list of participating pods {}. Cannot participate to the election", this.lockConfiguration.getPodName(), currentMembers); - return false; - } - - ConfigMap configMap = kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .get(); - - if (configMap == null) { - // No configmap created so far - LOG.info("Lock configmap is not present in the Kubernetes namespace. A new ConfigMap will be created"); - - ConfigMap newConfigMap = new ConfigMapBuilder(). - withNewMetadata() - .withName(this.lockConfiguration.getConfigMapName()) - .addToLabels("provider", "camel") - .addToLabels("kind", "locks"). - endMetadata() - .addToData(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName()) - .build(); - - try { - kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .create(newConfigMap); - } catch (Exception ex) { - // Suppress exception - LOG.warn("Unable to create the ConfigMap, it may have been created by other cluster members concurrently. If the problem persists, check if the service account has the right " - + "permissions to create it"); - LOG.debug("Exception while trying to create the ConfigMap", ex); - return false; - } - return true; - } else { - LOG.info("Lock configmap already present in the Kubernetes namespace. Checking..."); - Map leaders = configMap.getData(); - Optional oldLeader = leaders != null ? Optional.ofNullable(leaders.get(this.lockConfiguration.getGroupName())) : Optional.empty(); - - boolean noLeaderPresent = !oldLeader.isPresent() || !currentMembers.contains(oldLeader.get()); - boolean alreadyLeader = oldLeader.isPresent() && oldLeader.get().equals(this.lockConfiguration.getPodName()); - - if (noLeaderPresent && !alreadyLeader) { - LOG.info("Trying to acquire the lock in configmap={}, key={}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName()); - ConfigMap newConfigMap = new ConfigMapBuilder(configMap) - .addToData(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName()) - .build(); - - kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .lockResourceVersion(configMap.getMetadata().getResourceVersion()) - .replace(newConfigMap); - - LOG.info("Lock acquired for configmap={}, key={}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName()); - } else if (!noLeaderPresent) { - LOG.info("A leader is already present for configmap={}, key={}: {}", this.lockConfiguration.getConfigMapName(), this.lockConfiguration.getGroupName(), oldLeader); - } else { - LOG.info("This pod ({}) is already the leader for configmap={}, key={}", this.lockConfiguration.getPodName(), this.lockConfiguration.getConfigMapName(), this.lockConfiguration - .getGroupName()); - } - return true; - } - } - -} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java new file mode 100644 index 0000000000000..b385925dd99a2 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java @@ -0,0 +1,374 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Monitors current status and participate to leader election when no active leaders are present. + * It communicates changes in leadership and cluster members to the given event handler. + */ +public class KubernetesLeaseBasedLeadershipController implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeaseBasedLeadershipController.class); + + private static final long FIXED_ADDITIONAL_DELAY = 100; + + private KubernetesClient kubernetesClient; + + private KubernetesLockConfiguration lockConfiguration; + + private KubernetesClusterEventHandler eventHandler; + + private ScheduledExecutorService serializedExecutor; + private ScheduledExecutorService eventDispatcherExecutor; + + private KubernetesMembersMonitor membersMonitor; + + private Optional currentLeader = Optional.empty(); + + private volatile LeaderInfo latestLeaderInfo; + + public KubernetesLeaseBasedLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { + this.kubernetesClient = kubernetesClient; + this.lockConfiguration = lockConfiguration; + this.eventHandler = eventHandler; + } + + @Override + public void start() throws Exception { + if (serializedExecutor == null) { + LOG.debug("Starting leadership controller..."); + serializedExecutor = Executors.newSingleThreadScheduledExecutor(); + + eventDispatcherExecutor = Executors.newSingleThreadScheduledExecutor(); + + membersMonitor = new KubernetesMembersMonitor(this.serializedExecutor, this.kubernetesClient, this.lockConfiguration); + if (eventHandler != null) { + membersMonitor.addClusterEventHandler(eventHandler); + } + + membersMonitor.start(); + serializedExecutor.execute(this::initialization); + } + } + + @Override + public void stop() throws Exception { + LOG.debug("Stopping leadership controller..."); + if (serializedExecutor != null) { + serializedExecutor.shutdownNow(); + } + if (eventDispatcherExecutor != null) { + eventDispatcherExecutor.shutdown(); + eventDispatcherExecutor.awaitTermination(2, TimeUnit.SECONDS); + eventDispatcherExecutor.shutdownNow(); + } + if (membersMonitor != null) { + membersMonitor.stop(); + } + + membersMonitor = null; + eventDispatcherExecutor = null; + serializedExecutor = null; + } + + /** + * Get the first ConfigMap and setup the initial state. + */ + private void initialization() { + LOG.debug("Reading (with retry) the configmap {} to detect the current leader", this.lockConfiguration.getConfigMapName()); + refreshConfigMapFromCluster(true); + + if (isCurrentPodTheActiveLeader()) { + serializedExecutor.execute(this::onLeadershipAcquired); + } else { + LOG.info("The current pod ({}) is not the leader of the group '{}' in ConfigMap '{}' at this time", this.lockConfiguration.getPodName(), this.lockConfiguration + .getGroupName(), this.lockConfiguration.getConfigMapName()); + serializedExecutor.execute(this::acquireLeadershipCycle); + } + } + + /** + * Signals the acquisition of the leadership and move to the keep-leadership state. + */ + private void onLeadershipAcquired() { + LOG.info("The current pod ({}) is now the leader of the group '{}' in ConfigMap '{}'", this.lockConfiguration.getPodName(), this.lockConfiguration + .getGroupName(), this.lockConfiguration.getConfigMapName()); + + this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + + long nextDelay = computeNextRenewWaitTime(this.latestLeaderInfo.getTimestamp(), this.latestLeaderInfo.getTimestamp()); + serializedExecutor.schedule(this::keepLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + } + + /** + * While in the keep-leadership state, the controller periodically renews the lease. + * If a renewal deadline is met and it was not possible to renew the lease, the leadership is lost. + */ + private void keepLeadershipCycle() { + // renew lease periodically + refreshConfigMapFromCluster(false); // if possible, update + + if (this.latestLeaderInfo.isTimeElapsedSeconds(lockConfiguration.getRenewDeadlineSeconds()) || !this.latestLeaderInfo.isLeader(this.lockConfiguration.getPodName())) { + // Time over for renewal or leadership lost + LOG.debug("The current pod ({}) has lost the leadership", this.lockConfiguration.getPodName()); + serializedExecutor.execute(this::onLeadershipLost); + return; + } + + boolean success = tryAcquireOrRenewLeadership(); + LOG.debug("Attempted to renew the lease. Success={}", success); + + long nextDelay = computeNextRenewWaitTime(this.latestLeaderInfo.getTimestamp(), new Date()); + serializedExecutor.schedule(this::keepLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + } + + /** + * Compute the timestamp of next event while in keep-leadership state. + */ + private long computeNextRenewWaitTime(Date lastRenewal, Date lastRenewalAttempt) { + long timeDeadline = lastRenewal.getTime() + this.lockConfiguration.getRenewDeadlineSeconds() * 1000; + long timeRetry; + long counter = 0; + do { + counter++; + timeRetry = lastRenewal.getTime() + counter * this.lockConfiguration.getRetryPeriodSeconds() * 1000; + } while (timeRetry < lastRenewalAttempt.getTime() && timeRetry < timeDeadline); + + long time = Math.min(timeRetry, timeDeadline); + long delay = Math.max(0, time - System.currentTimeMillis()); + LOG.debug("Next renewal timeout event will be fired in {} seconds", delay / 1000); + return delay; + } + + + /** + * Signals the loss of leadership and move to the acquire-leadership state. + */ + private void onLeadershipLost() { + LOG.info("The local pod ({}) is no longer the leader of the group '{}' in ConfigMap '{}'", this.lockConfiguration.getPodName(), this.lockConfiguration.getGroupName(), + this.lockConfiguration.getConfigMapName()); + + this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + serializedExecutor.execute(this::acquireLeadershipCycle); + } + + /** + * While in the acquire-leadership state, the controller waits for the current lease to expire before trying to acquire the leadership. + */ + private void acquireLeadershipCycle() { + // wait for the current lease to finish then fire an election + refreshConfigMapFromCluster(false); // if possible, update + + // Notify about changes in current leader if any + this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + + if (!this.latestLeaderInfo.isTimeElapsedSeconds(lockConfiguration.getLeaseDurationSeconds())) { + // Wait for the lease to finish before trying leader election + long nextDelay = computeNextElectionWaitTime(this.latestLeaderInfo.getTimestamp()); + serializedExecutor.schedule(this::acquireLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + return; + } + + boolean acquired = tryAcquireOrRenewLeadership(); + if (acquired) { + LOG.debug("Leadership acquired for ConfigMap {}. Notification in progress...", this.lockConfiguration.getConfigMapName()); + serializedExecutor.execute(this::onLeadershipAcquired); + return; + } + + // Notify about changes in current leader if any + this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + + LOG.debug("Cannot acquire the leadership for ConfigMap {}", this.lockConfiguration.getConfigMapName()); + long nextDelay = computeNextElectionWaitTime(this.latestLeaderInfo.getTimestamp()); + serializedExecutor.schedule(this::acquireLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + } + + private long computeNextElectionWaitTime(Date lastRenewal) { + if (lastRenewal == null) { + LOG.debug("Error detected while getting leadership info, next election timeout event will be fired in {} seconds", this.lockConfiguration.getRetryOnErrorIntervalSeconds()); + return this.lockConfiguration.getRetryOnErrorIntervalSeconds(); + } + long time = lastRenewal.getTime() + this.lockConfiguration.getLeaseDurationSeconds() * 1000 + + jitter(this.lockConfiguration.getRetryPeriodSeconds() * 1000, this.lockConfiguration.getJitterFactor()); + + long delay = Math.max(0, time - System.currentTimeMillis()); + LOG.debug("Next election timeout event will be fired in {} seconds", delay / 1000); + return delay; + } + + private long jitter(long num, double factor) { + return (long) (num * (1 + Math.random() * (factor - 1))); + } + + private boolean tryAcquireOrRenewLeadership() { + LOG.debug("Trying to acquire or renew the leadership..."); + + ConfigMap configMap; + try { + configMap = pullConfigMap(); + } catch (Exception e) { + LOG.warn("Unable to retrieve the current ConfigMap " + this.lockConfiguration.getConfigMapName() + " from Kubernetes", e); + return false; + } + + // Info to set in the configmap to become leaders + LeaderInfo newLeaderInfo = new LeaderInfo(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName(), new Date()); + + if (configMap == null) { + // No configmap created so far + LOG.debug("Lock configmap is not present in the Kubernetes namespace. A new ConfigMap will be created"); + ConfigMap newConfigMap = ConfigMapLockUtils.createNewConfigMap(this.lockConfiguration.getConfigMapName(), newLeaderInfo); + + try { + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .create(newConfigMap); + } catch (Exception ex) { + // Suppress exception + LOG.warn("Unable to create the ConfigMap, it may have been created by other cluster members concurrently. If the problem persists, check if the service account has the right " + + "permissions to create it"); + LOG.debug("Exception while trying to create the ConfigMap", ex); + + // Try to get the configMap and return the current leader + refreshConfigMapFromCluster(false); + return isCurrentPodTheActiveLeader(); + } + + LOG.debug("ConfigMap {} successfully created and local pod is leader", this.lockConfiguration.getConfigMapName()); + updateLatestLeaderInfo(newConfigMap); + return true; + } else { + LOG.debug("Lock configmap already present in the Kubernetes namespace. Checking..."); + LeaderInfo leaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); + + boolean weWereLeader = leaderInfo.isLeader(this.lockConfiguration.getPodName()); + boolean leaseExpired = leaderInfo.isTimeElapsedSeconds(this.lockConfiguration.getLeaseDurationSeconds()); + + if (weWereLeader || leaseExpired) { + // Renew the lease or set the new leader + try { + ConfigMap updatedConfigMap = ConfigMapLockUtils.getConfigMapWithNewLeader(configMap, newLeaderInfo); + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .lockResourceVersion(configMap.getMetadata().getResourceVersion()) + .replace(updatedConfigMap); + + LOG.debug("ConfigMap {} successfully updated and local pod is leader", this.lockConfiguration.getConfigMapName()); + updateLatestLeaderInfo(updatedConfigMap); + return true; + } catch (Exception ex) { + LOG.warn("An attempt to become leader has failed. It's possible that the leadership has been taken by another pod"); + LOG.debug("Error received during configmap lock replace", ex); + + // Try to get the configMap and return the current leader + refreshConfigMapFromCluster(false); + return isCurrentPodTheActiveLeader(); + } + } else { + // Another pod is the leader and lease is not expired + LOG.debug("Another pod is the current leader and lease has not expired yet"); + updateLatestLeaderInfo(configMap); + return false; + } + } + } + + + private void refreshConfigMapFromCluster(boolean retry) { + LOG.debug("Retrieving configmap {}", this.lockConfiguration.getConfigMapName()); + try { + updateLatestLeaderInfo(pullConfigMap()); + } catch (Exception ex) { + if (retry) { + LOG.warn("ConfigMap pull failed. Retrying in " + this.lockConfiguration.getRetryOnErrorIntervalSeconds() + " seconds...", ex); + try { + Thread.sleep(this.lockConfiguration.getRetryOnErrorIntervalSeconds() * 1000); + refreshConfigMapFromCluster(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Controller Thread interrupted, shutdown in progress", e); + } + } else { + LOG.warn("Cannot retrieve the ConfigMap: pull failed", ex); + } + } + } + + private boolean isCurrentPodTheActiveLeader() { + return latestLeaderInfo != null + && latestLeaderInfo.isLeader(this.lockConfiguration.getPodName()) + && !latestLeaderInfo.isTimeElapsedSeconds(this.lockConfiguration.getRenewDeadlineSeconds()); + } + + private ConfigMap pullConfigMap() { + return kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .get(); + } + + + private void updateLatestLeaderInfo(ConfigMap configMap) { + LOG.debug("Updating internal status about the current leader"); + this.latestLeaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); + } + + private void checkAndNotifyNewLeader() { + LOG.debug("Checking if the current leader has changed to notify the event handler..."); + LeaderInfo newLeaderInfo = this.latestLeaderInfo; + if (newLeaderInfo == null) { + return; + } + + long leaderInfoDurationSeconds = newLeaderInfo.isLeader(this.lockConfiguration.getPodName()) + ? this.lockConfiguration.getRenewDeadlineSeconds() + : this.lockConfiguration.getLeaseDurationSeconds(); + + Optional newLeader; + if (newLeaderInfo.getLeader() != null && !newLeaderInfo.isTimeElapsedSeconds(leaderInfoDurationSeconds)) { + newLeader = Optional.of(newLeaderInfo.getLeader()); + } else { + newLeader = Optional.empty(); + } + + // Sending notifications in case of leader change + if (!newLeader.equals(this.currentLeader)) { + LOG.debug("Current leader has changed from {} to {}. Sending notifications...", this.currentLeader, newLeader); + this.currentLeader = newLeader; + eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> newLeader); + } else { + LOG.debug("Current leader unchanged: {}", this.currentLeader); + } + } + + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java index f203c0ae27c57..37e02514d9df3 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java @@ -26,7 +26,16 @@ */ public class KubernetesLockConfiguration implements Cloneable { - private static final long DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS = 1800; + public static final String DEFAULT_CONFIGMAP_NAME = "leaders"; + + + public static final double DEFAULT_JITTER_FACTOR = 1.2; + public static final long DEFAULT_LEASE_DURATION_SECONDS = 20; + public static final long DEFAULT_RENEW_DEADLINE_SECONDS = 15; + public static final long DEFAULT_RETRY_PERIOD_SECONDS = 6; + + public static final long DEFAULT_RETRY_ON_ERROR_INTERVAL_SECONDS = 5; + public static final long DEFAULT_WATCH_REFRESH_INTERVAL_SECONDS = 1800; /** * Kubernetes namespace containing the pods and the ConfigMap used for locking. @@ -36,7 +45,7 @@ public class KubernetesLockConfiguration implements Cloneable { /** * Name of the ConfigMap used for locking. */ - private String configMapName; + private String configMapName = DEFAULT_CONFIGMAP_NAME; /** * Name of the lock group (or namespace according to the Camel cluster convention) within the chosen ConfgMap. @@ -55,9 +64,36 @@ public class KubernetesLockConfiguration implements Cloneable { /** * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. - * Watch recreation can be disabled by putting a negative value (the default will be used in case of null). + * Watch recreation can be disabled by putting value <= 0. + */ + private long retryOnErrorIntervalSeconds = DEFAULT_RETRY_ON_ERROR_INTERVAL_SECONDS; + + /** + * A jitter factor to apply in order to prevent all pods to try to become leaders in the same instant. + */ + private double jitterFactor = DEFAULT_JITTER_FACTOR; + + /** + * The default duration of the lease for the current leader. + */ + private long leaseDurationSeconds = DEFAULT_LEASE_DURATION_SECONDS; + + /** + * The deadline after which the leader must stop trying to renew its leadership (and yield it). + */ + private long renewDeadlineSeconds = DEFAULT_RENEW_DEADLINE_SECONDS; + + /** + * The time between two subsequent attempts to acquire/renew the leadership (or after the lease expiration). + * It is randomized using the jitter factor in case of new leader election (not renewal). */ - private Long watchRefreshIntervalSeconds; + private long retryPeriodSeconds = DEFAULT_RETRY_PERIOD_SECONDS; + + /** + * Set this to a positive value in order to recreate watchers after a certain amount of time + * (to prevent them becoming stale). + */ + private long watchRefreshIntervalSeconds = DEFAULT_WATCH_REFRESH_INTERVAL_SECONDS; public KubernetesLockConfiguration() { } @@ -113,19 +149,51 @@ public void setClusterLabels(Map clusterLabels) { this.clusterLabels = clusterLabels; } - public Long getWatchRefreshIntervalSeconds() { - return watchRefreshIntervalSeconds; + public long getRetryOnErrorIntervalSeconds() { + return retryOnErrorIntervalSeconds; } - public long getWatchRefreshIntervalSecondsOrDefault() { - Long interval = watchRefreshIntervalSeconds; - if (interval == null) { - interval = DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS; - } - return interval; + public void setRetryOnErrorIntervalSeconds(long retryOnErrorIntervalSeconds) { + this.retryOnErrorIntervalSeconds = retryOnErrorIntervalSeconds; + } + + public double getJitterFactor() { + return jitterFactor; + } + + public void setJitterFactor(double jitterFactor) { + this.jitterFactor = jitterFactor; + } + + public long getLeaseDurationSeconds() { + return leaseDurationSeconds; + } + + public void setLeaseDurationSeconds(long leaseDurationSeconds) { + this.leaseDurationSeconds = leaseDurationSeconds; + } + + public long getRenewDeadlineSeconds() { + return renewDeadlineSeconds; + } + + public void setRenewDeadlineSeconds(long renewDeadlineSeconds) { + this.renewDeadlineSeconds = renewDeadlineSeconds; + } + + public long getRetryPeriodSeconds() { + return retryPeriodSeconds; + } + + public void setRetryPeriodSeconds(long retryPeriodSeconds) { + this.retryPeriodSeconds = retryPeriodSeconds; + } + + public long getWatchRefreshIntervalSeconds() { + return watchRefreshIntervalSeconds; } - public void setWatchRefreshIntervalSeconds(Long watchRefreshIntervalSeconds) { + public void setWatchRefreshIntervalSeconds(long watchRefreshIntervalSeconds) { this.watchRefreshIntervalSeconds = watchRefreshIntervalSeconds; } @@ -146,6 +214,11 @@ public String toString() { sb.append(", groupName='").append(groupName).append('\''); sb.append(", podName='").append(podName).append('\''); sb.append(", clusterLabels=").append(clusterLabels); + sb.append(", retryOnErrorIntervalSeconds=").append(retryOnErrorIntervalSeconds); + sb.append(", jitterFactor=").append(jitterFactor); + sb.append(", leaseDurationSeconds=").append(leaseDurationSeconds); + sb.append(", renewDeadlineSeconds=").append(renewDeadlineSeconds); + sb.append(", retryPeriodSeconds=").append(retryPeriodSeconds); sb.append(", watchRefreshIntervalSeconds=").append(watchRefreshIntervalSeconds); sb.append('}'); return sb.toString(); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java index d9173b26d2c27..586a11f5ad2e4 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java @@ -41,8 +41,6 @@ */ class KubernetesMembersMonitor implements Service { - private static final long DEFAULT_WATCHER_REFRESH_INTERVAL_SECONDS = 1800; - private static final Logger LOG = LoggerFactory.getLogger(KubernetesMembersMonitor.class); private ScheduledExecutorService serializedExecutor; @@ -81,7 +79,7 @@ public void start() throws Exception { serializedExecutor.execute(() -> doPoll(true)); serializedExecutor.execute(this::createWatch); - long recreationDelay = lockConfiguration.getWatchRefreshIntervalSecondsOrDefault(); + long recreationDelay = lockConfiguration.getWatchRefreshIntervalSeconds(); if (recreationDelay > 0) { serializedExecutor.scheduleWithFixedDelay(this::refresh, recreationDelay, recreationDelay, TimeUnit.SECONDS); } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java new file mode 100644 index 0000000000000..50d16031e3a98 --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.Date; + +import org.apache.camel.util.ObjectHelper; + +/** + * Overview of a leadership status. + */ +public class LeaderInfo { + + private String groupName; + + private String leader; + + private Date timestamp; + + public LeaderInfo() { + } + + public LeaderInfo(String groupName, String leader, Date timestamp) { + this.groupName = groupName; + this.leader = leader; + this.timestamp = timestamp; + } + + public boolean isTimeElapsedSeconds(long timeSeconds) { + if (timestamp == null) { + return true; + } + long now = System.currentTimeMillis(); + return timestamp.getTime() + timeSeconds * 1000 <= now; + } + + public boolean isLeader(String pod) { + ObjectHelper.notNull(pod, "pod"); + return pod.equals(leader); + } + + public String getGroupName() { + return groupName; + } + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + public String getLeader() { + return leader; + } + + public void setLeader(String leader) { + this.leader = leader; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LeaderInfo{"); + sb.append("groupName='").append(groupName).append('\''); + sb.append(", leader='").append(leader).append('\''); + sb.append(", timestamp=").append(timestamp); + sb.append('}'); + return sb.toString(); + } + +} From c8f5fe298aa3aeb61adb1ee8e8dcbc4bbc30b38b Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Fri, 7 Jul 2017 17:18:05 +0200 Subject: [PATCH 03/13] CAMEL-11331: Adding clarifications to the leader interface and fix impl --- .../camel/ha/CamelClusterEventListener.java | 29 +++++++++++++++++-- .../kubernetes/ha/KubernetesClusterView.java | 4 +-- ...ernetesLeaseBasedLeadershipController.java | 2 +- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/camel-core/src/main/java/org/apache/camel/ha/CamelClusterEventListener.java b/camel-core/src/main/java/org/apache/camel/ha/CamelClusterEventListener.java index 5c1997051518a..1a972cab66638 100644 --- a/camel-core/src/main/java/org/apache/camel/ha/CamelClusterEventListener.java +++ b/camel-core/src/main/java/org/apache/camel/ha/CamelClusterEventListener.java @@ -17,16 +17,39 @@ package org.apache.camel.ha; /** - * Marker interface + * Marker interface for cluster events */ public interface CamelClusterEventListener { interface Leadership extends CamelClusterEventListener { + + /** + * Notify a change in the leadership for a particular cluster. + * + * @param view the cluster view + * @param leader the new leader or null (when there are no active leaders) + */ void leadershipChanged(CamelClusterView view, CamelClusterMember leader); + } interface Membership extends CamelClusterEventListener { - void memberAdded(CamelClusterView view, CamelClusterMember leader); - void memberRemoved(CamelClusterView view, CamelClusterMember leader); + + /** + * Notify a change (addition) in the cluster composition. + * + * @param view the cluster view + * @param member the member that has been added + */ + void memberAdded(CamelClusterView view, CamelClusterMember member); + + /** + * Notify a change (removal) in the cluster composition. + * + * @param view the cluster view + * @param member the member that has been removed + */ + void memberRemoved(CamelClusterView view, CamelClusterMember member); + } } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java index e324b3ff4ab28..28f38a54d6701 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java @@ -91,9 +91,7 @@ protected void doStart() throws Exception { // New leader Optional leader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(event).getData(); currentLeader = leader.map(this::toMember); - if (currentLeader.isPresent()) { - fireLeadershipChangedEvent(currentLeader.get()); - } + fireLeadershipChangedEvent(currentLeader.orElse(null)); } else if (event instanceof KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) { Set members = KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent.class.cast(event).getData(); Set oldMembers = currentMembers.stream().map(CamelClusterMember::getId).collect(Collectors.toSet()); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java index b385925dd99a2..8e96a72f5bdd8 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java @@ -362,7 +362,7 @@ private void checkAndNotifyNewLeader() { // Sending notifications in case of leader change if (!newLeader.equals(this.currentLeader)) { - LOG.debug("Current leader has changed from {} to {}. Sending notifications...", this.currentLeader, newLeader); + LOG.info("Current leader has changed from {} to {}. Sending notifications...", this.currentLeader, newLeader); this.currentLeader = newLeader; eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> newLeader); } else { From 4ba197660b0dbcaac72d1b41b3dd6a62b663b2d9 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Fri, 14 Jul 2017 11:21:50 +0200 Subject: [PATCH 04/13] CAMEL-11331: Adding tests and fixing impl --- components/camel-kubernetes/pom.xml | 6 + .../ha/lock/ConfigMapLockUtils.java | 2 +- ...ernetesLeaseBasedLeadershipController.java | 15 +- .../ha/lock/KubernetesLockConfiguration.java | 6 +- .../ha/KubernetesClusterServiceTest.java | 291 ++++++++++++++++++ .../ha/utils/ConfigMapLockSimulator.java | 83 +++++ .../kubernetes/ha/utils/LeaderRecorder.java | 115 +++++++ .../kubernetes/ha/utils/LockTestServer.java | 175 +++++++++++ .../ha/utils/LockTestServerTest.java | 97 ++++++ parent/pom.xml | 1 + 10 files changed, 784 insertions(+), 7 deletions(-) create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/ConfigMapLockSimulator.java create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServerTest.java diff --git a/components/camel-kubernetes/pom.xml b/components/camel-kubernetes/pom.xml index c44406885cb82..38fa0370f3bd8 100644 --- a/components/camel-kubernetes/pom.xml +++ b/components/camel-kubernetes/pom.xml @@ -66,6 +66,12 @@ ${kubernetes-client-version} test + + io.fabric8 + mockwebserver + ${mockwebserver-version} + test + org.apache.camel camel-test-spring diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java index 84718f31ff0bc..70fa860598323 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java @@ -26,7 +26,7 @@ import org.slf4j.LoggerFactory; /** - * + * Utilities for managing ConfigMaps that contain lock information. */ public final class ConfigMapLockUtils { diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java index 8e96a72f5bdd8..42be2e7b854f9 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java @@ -163,8 +163,9 @@ private long computeNextRenewWaitTime(Date lastRenewal, Date lastRenewalAttempt) long time = Math.min(timeRetry, timeDeadline); long delay = Math.max(0, time - System.currentTimeMillis()); - LOG.debug("Next renewal timeout event will be fired in {} seconds", delay / 1000); - return delay; + long delayJittered = jitter(delay, lockConfiguration.getJitterFactor()); + LOG.debug("Next renewal timeout event will be fired in {} seconds", delayJittered / 1000); + return delayJittered; } @@ -340,10 +341,18 @@ private ConfigMap pullConfigMap() { private void updateLatestLeaderInfo(ConfigMap configMap) { LOG.debug("Updating internal status about the current leader"); this.latestLeaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); + + // Notify about changes in current leader if any + this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + if (this.latestLeaderInfo.isLeader(this.lockConfiguration.getPodName())) { + this.eventDispatcherExecutor.schedule(this::checkAndNotifyNewLeader, this.lockConfiguration.getRenewDeadlineSeconds() * 1000 + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + } else if (this.latestLeaderInfo.getLeader() != null) { + this.eventDispatcherExecutor.schedule(this::checkAndNotifyNewLeader, this.lockConfiguration.getLeaseDurationSeconds() * 1000 + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); + } } private void checkAndNotifyNewLeader() { - LOG.debug("Checking if the current leader has changed to notify the event handler..."); + LOG.info("Checking if the current leader has changed to notify the event handler..."); LeaderInfo newLeaderInfo = this.latestLeaderInfo; if (newLeaderInfo == null) { return; diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java index 37e02514d9df3..64617080fe7d3 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java @@ -30,9 +30,9 @@ public class KubernetesLockConfiguration implements Cloneable { public static final double DEFAULT_JITTER_FACTOR = 1.2; - public static final long DEFAULT_LEASE_DURATION_SECONDS = 20; - public static final long DEFAULT_RENEW_DEADLINE_SECONDS = 15; - public static final long DEFAULT_RETRY_PERIOD_SECONDS = 6; + public static final long DEFAULT_LEASE_DURATION_SECONDS = 60; + public static final long DEFAULT_RENEW_DEADLINE_SECONDS = 45; + public static final long DEFAULT_RETRY_PERIOD_SECONDS = 9; public static final long DEFAULT_RETRY_ON_ERROR_INTERVAL_SECONDS = 5; public static final long DEFAULT_WATCH_REFRESH_INTERVAL_SECONDS = 1800; diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java new file mode 100644 index 0000000000000..4baebc6bf8d54 --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; + +import org.apache.camel.component.kubernetes.KubernetesConfiguration; +import org.apache.camel.component.kubernetes.ha.utils.ConfigMapLockSimulator; +import org.apache.camel.component.kubernetes.ha.utils.LeaderRecorder; +import org.apache.camel.component.kubernetes.ha.utils.LockTestServer; +import org.apache.camel.test.junit4.CamelTestSupport; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Test leader election scenarios using a mock server. + */ +public class KubernetesClusterServiceTest extends CamelTestSupport { + + private static final int LEASE_TIME_SECONDS = 5; + + private ConfigMapLockSimulator lockSimulator; + + private Map lockServers; + + @Before + public void prepareLock() { + this.lockSimulator = new ConfigMapLockSimulator("leaders"); + this.lockServers = new HashMap<>(); + } + + @After + public void shutdownLock() { + for (LockTestServer server : this.lockServers.values()) { + try { + server.destroy(); + } catch (Exception e) { + // can happen in case of delay + } + } + } + + @Test + public void testSimpleLeaderElection() throws Exception { + LeaderRecorder mypod1 = addMember("mypod1"); + LeaderRecorder mypod2 = addMember("mypod2"); + context.start(); + + mypod1.waitForAnyLeader(2, TimeUnit.SECONDS); + mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); + + String leader = mypod1.getCurrentLeader(); + assertTrue(leader.startsWith("mypod")); + assertEquals("Leaders should be equals", mypod2.getCurrentLeader(), leader); + } + + @Test + public void testMultipleMembersLeaderElection() throws Exception { + int number = 5; + List members = IntStream.range(0, number).mapToObj(i -> addMember("mypod" + i)).collect(Collectors.toList()); + context.start(); + + for (LeaderRecorder member : members) { + member.waitForAnyLeader(2, TimeUnit.SECONDS); + } + + Set leaders = members.stream().map(LeaderRecorder::getCurrentLeader).collect(Collectors.toSet()); + assertEquals(1, leaders.size()); + String leader = leaders.iterator().next(); + assertTrue(leader.startsWith("mypod")); + } + + @Test + public void testSimpleLeaderElectionWithExistingConfigMap() throws Exception { + lockSimulator.setConfigMap(new ConfigMapBuilder() + .withNewMetadata() + .withName("leaders") + .and().build(), true); + + LeaderRecorder mypod1 = addMember("mypod1"); + LeaderRecorder mypod2 = addMember("mypod2"); + context.start(); + + mypod1.waitForAnyLeader(2, TimeUnit.SECONDS); + mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); + + String leader = mypod1.getCurrentLeader(); + assertTrue(leader.startsWith("mypod")); + assertEquals("Leaders should be equals", mypod2.getCurrentLeader(), leader); + } + + @Test + public void testLeadershipLoss() throws Exception { + LeaderRecorder mypod1 = addMember("mypod1"); + LeaderRecorder mypod2 = addMember("mypod2"); + context.start(); + + mypod1.waitForAnyLeader(2, TimeUnit.SECONDS); + mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); + + String firstLeader = mypod1.getCurrentLeader(); + + LeaderRecorder formerLeaderRecorder = firstLeader.equals("mypod1") ? mypod1 : mypod2; + LeaderRecorder formerLoserRecorder = firstLeader.equals("mypod1") ? mypod2 : mypod1; + + refuseRequestsFromPod(firstLeader); + + formerLeaderRecorder.waitForALeaderChange(7, TimeUnit.SECONDS); + formerLoserRecorder.waitForANewLeader(firstLeader, 7, TimeUnit.SECONDS); + + String secondLeader = formerLoserRecorder.getCurrentLeader(); + assertNotEquals("The firstLeader should be different from the new one", firstLeader, secondLeader); + + Long lossTimestamp = formerLeaderRecorder.getLastTimeOf(l -> l == null); + Long gainTimestamp = formerLoserRecorder.getLastTimeOf(secondLeader::equals); + + assertTrue("At least 2 seconds must elapse from leadership loss and regain (see renewDeadlineSeconds)", gainTimestamp >= lossTimestamp + 2000); + checkLeadershipChangeDistance(LEASE_TIME_SECONDS, TimeUnit.SECONDS, mypod1, mypod2); + } + + @Test + public void testSlowLeaderLosingLeadership() throws Exception { + LeaderRecorder mypod1 = addMember("mypod1"); + LeaderRecorder mypod2 = addMember("mypod2"); + context.start(); + + mypod1.waitForAnyLeader(2, TimeUnit.SECONDS); + mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); + + String firstLeader = mypod1.getCurrentLeader(); + + LeaderRecorder formerLeaderRecorder = firstLeader.equals("mypod1") ? mypod1 : mypod2; + LeaderRecorder formerLoserRecorder = firstLeader.equals("mypod1") ? mypod2 : mypod1; + + delayRequestsFromPod(firstLeader, 10, TimeUnit.SECONDS); + + formerLeaderRecorder.waitForALeaderChange(7, TimeUnit.SECONDS); + formerLoserRecorder.waitForANewLeader(firstLeader, 7, TimeUnit.SECONDS); + + String secondLeader = formerLoserRecorder.getCurrentLeader(); + assertNotEquals("The firstLeader should be different from the new one", firstLeader, secondLeader); + + Long lossTimestamp = formerLeaderRecorder.getLastTimeOf(l -> l == null); + Long gainTimestamp = formerLoserRecorder.getLastTimeOf(secondLeader::equals); + + assertTrue("At least 2 seconds must elapse from leadership loss and regain (see renewDeadlineSeconds)", gainTimestamp >= lossTimestamp + 2000); + checkLeadershipChangeDistance(LEASE_TIME_SECONDS, TimeUnit.SECONDS, mypod1, mypod2); + } + + @Test + public void testRecoveryAfterFailure() throws Exception { + LeaderRecorder mypod1 = addMember("mypod1"); + LeaderRecorder mypod2 = addMember("mypod2"); + context.start(); + + mypod1.waitForAnyLeader(2, TimeUnit.SECONDS); + mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); + + String firstLeader = mypod1.getCurrentLeader(); + + for (int i = 0; i < 3; i++) { + refuseRequestsFromPod(firstLeader); + Thread.sleep(1000); + allowRequestsFromPod(firstLeader); + Thread.sleep(2000); + } + + assertEquals(firstLeader, mypod1.getCurrentLeader()); + assertEquals(firstLeader, mypod2.getCurrentLeader()); + } + + @Test + public void testSharedConfigMap() throws Exception { + LeaderRecorder a1 = addMember("a1"); + LeaderRecorder a2 = addMember("a2"); + LeaderRecorder b1 = addMember("b1", "app2"); + LeaderRecorder b2 = addMember("b2", "app2"); + context.start(); + + a1.waitForAnyLeader(2, TimeUnit.SECONDS); + a2.waitForAnyLeader(2, TimeUnit.SECONDS); + b1.waitForAnyLeader(2, TimeUnit.SECONDS); + b1.waitForAnyLeader(2, TimeUnit.SECONDS); + + assertNotNull(a1.getCurrentLeader()); + assertTrue(a1.getCurrentLeader().startsWith("a")); + assertEquals(a1.getCurrentLeader(), a2.getCurrentLeader()); + assertNotNull(b1.getCurrentLeader()); + assertTrue(b1.getCurrentLeader().startsWith("b")); + assertEquals(b1.getCurrentLeader(), b2.getCurrentLeader()); + + assertNotEquals(a1.getCurrentLeader(), b2.getCurrentLeader()); + } + + private void delayRequestsFromPod(String pod, long delay, TimeUnit unit) { + this.lockServers.get(pod).setDelayRequests(TimeUnit.MILLISECONDS.convert(delay, unit)); + } + + private void refuseRequestsFromPod(String pod) { + this.lockServers.get(pod).setRefuseRequests(true); + } + + private void allowRequestsFromPod(String pod) { + this.lockServers.get(pod).setRefuseRequests(false); + } + + private void checkLeadershipChangeDistance(long minimum, TimeUnit unit, LeaderRecorder... recorders) { + List infos = Arrays.stream(recorders) + .flatMap(lr -> lr.getLeadershipInfo().stream()) + .sorted((li1, li2) -> Long.compare(li1.getChangeTimestamp(), li2.getChangeTimestamp())) + .collect(Collectors.toList()); + + LeaderRecorder.LeadershipInfo currentLeaderLastSeen = null; + for (LeaderRecorder.LeadershipInfo info : infos) { + if (currentLeaderLastSeen == null || currentLeaderLastSeen.getLeader() == null) { + currentLeaderLastSeen = info; + } else { + if (Objects.equals(info.getLeader(), currentLeaderLastSeen.getLeader())) { + currentLeaderLastSeen = info; + } else if (info.getLeader() != null && !info.getLeader().equals(currentLeaderLastSeen.getLeader())) { + // switch + long delay = info.getChangeTimestamp() - currentLeaderLastSeen.getChangeTimestamp(); + assertTrue("Lease time not elapsed between switch", delay >= TimeUnit.MILLISECONDS.convert(minimum, unit)); + currentLeaderLastSeen = info; + } + } + } + } + + private LeaderRecorder addMember(String name) { + return addMember(name, "app"); + } + + private LeaderRecorder addMember(String name, String namespace) { + assertNull(this.lockServers.get(name)); + + LockTestServer lockServer = new LockTestServer(lockSimulator); + this.lockServers.put(name, lockServer); + + KubernetesConfiguration configuration = new KubernetesConfiguration(); + configuration.setKubernetesClient(lockServer.createClient()); + + KubernetesClusterService member = new KubernetesClusterService(configuration); + member.setKubernetesNamespace("test"); + member.setPodName(name); + member.setLeaseDurationSeconds(LEASE_TIME_SECONDS); + member.setRenewDeadlineSeconds(3); // 5-3 = at least 2 seconds for switching on leadership loss + member.setRetryPeriodSeconds(1); + member.setRetryOnErrorIntervalSeconds(1); + member.setJitterFactor(1.2); + + LeaderRecorder recorder = new LeaderRecorder(); + try { + member.getView(namespace).addEventListener(recorder); + context().addService(member); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + return recorder; + } + + @Override + public boolean isUseRouteBuilder() { + return false; + } +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/ConfigMapLockSimulator.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/ConfigMapLockSimulator.java new file mode 100644 index 0000000000000..1c3d7d09d15d1 --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/ConfigMapLockSimulator.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.utils; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Central lock for testing leader election. + */ +public class ConfigMapLockSimulator { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigMapLockSimulator.class); + + private String configMapName; + + private ConfigMap currentMap; + + private long versionCounter = 1000000; + + public ConfigMapLockSimulator(String configMapName) { + this.configMapName = configMapName; + } + + public String getConfigMapName() { + return configMapName; + } + + public synchronized boolean setConfigMap(ConfigMap map, boolean insert) { + // Insert + if (insert && currentMap != null) { + LOG.error("Current map should have been null"); + return false; + } + + // Update + if (!insert && currentMap == null) { + LOG.error("Current map should not have been null"); + return false; + } + String version = map.getMetadata() != null ? map.getMetadata().getResourceVersion() : null; + if (version != null) { + long versionLong = Long.parseLong(version); + if (versionLong != versionCounter) { + LOG.warn("Current resource version is {} while the update is related to version {}", versionCounter, versionLong); + return false; + } + } + + this.currentMap = new ConfigMapBuilder(map) + .editOrNewMetadata() + .withResourceVersion(String.valueOf(++versionCounter)) + .endMetadata() + .build(); + return true; + } + + public synchronized ConfigMap getConfigMap() { + if (currentMap == null) { + return null; + } + + return new ConfigMapBuilder(currentMap).build(); + } + +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java new file mode 100644 index 0000000000000..6670f375a8d49 --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.utils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import org.apache.camel.ha.CamelClusterEventListener; +import org.apache.camel.ha.CamelClusterMember; +import org.apache.camel.ha.CamelClusterView; +import org.junit.Assert; + +/** + * Records leadership changes and allow to do assertions. + */ +public class LeaderRecorder implements CamelClusterEventListener.Leadership { + + private List leaderships = new CopyOnWriteArrayList<>(); + + @Override + public void leadershipChanged(CamelClusterView view, CamelClusterMember leader) { + this.leaderships.add(new LeadershipInfo(leader != null ? leader.getId() : null, System.currentTimeMillis())); + } + + public List getLeadershipInfo() { + return leaderships; + } + + public void waitForAnyLeader(long time, TimeUnit unit) { + waitForLeader(leader -> leader != null, time, unit); + } + + public void waitForALeaderChange(long time, TimeUnit unit) { + String current = getCurrentLeader(); + waitForLeader(leader -> !Objects.equals(current, leader), time, unit); + } + + public void waitForANewLeader(String current, long time, TimeUnit unit) { + waitForLeader(leader -> leader != null && !Objects.equals(current, leader), time, unit); + } + + public void waitForLeader(Predicate as, long time, TimeUnit unit) { + long start = System.currentTimeMillis(); + while (!as.test(getCurrentLeader())) { + if (System.currentTimeMillis() - start > TimeUnit.MILLISECONDS.convert(time, unit)) { + Assert.fail("Timeout while waiting for condition"); + } + doWait(50); + } + } + + private void doWait(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + public String getCurrentLeader() { + if (leaderships.size() > 0) { + return leaderships.get(leaderships.size() - 1).getLeader(); + } + return null; + } + + public Long getLastTimeOf(Predicate p) { + List lst = new ArrayList<>(leaderships); + Collections.reverse(lst); + for (LeadershipInfo info : lst) { + if (p.test(info.getLeader())) { + return info.getChangeTimestamp(); + } + } + return null; + } + + public static class LeadershipInfo { + private String leader; + private long changeTimestamp; + + public LeadershipInfo(String leader, long changeTimestamp) { + this.leader = leader; + this.changeTimestamp = changeTimestamp; + } + + public String getLeader() { + return leader; + } + + public long getChangeTimestamp() { + return changeTimestamp; + } + } + +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java new file mode 100644 index 0000000000000..6422e353b5e01 --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.utils; + +import java.io.IOException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.PodListBuilder; +import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer; +import io.fabric8.mockwebserver.utils.ResponseProvider; + +import okhttp3.mockwebserver.RecordedRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Test server to interact with Kubernetes for locking on a ConfigMap. + */ +public class LockTestServer extends KubernetesMockServer { + + private static final Logger LOG = LoggerFactory.getLogger(LockTestServer.class); + + private boolean refuseRequests; + + private Long delayRequests; + + public LockTestServer(ConfigMapLockSimulator lockSimulator) { + + expect().get().withPath("/api/v1/namespaces/test/configmaps/" + lockSimulator.getConfigMapName()).andReply(new ResponseProvider() { + ThreadLocal responseCode = new ThreadLocal<>(); + + @Override + public int getStatusCode() { + return responseCode.get(); + } + + @Override + public Object getBody(RecordedRequest recordedRequest) { + delayIfNecessary(); + if (refuseRequests) { + responseCode.set(500); + return ""; + } + + ConfigMap map = lockSimulator.getConfigMap(); + if (map != null) { + responseCode.set(200); + return map; + } else { + responseCode.set(404); + return ""; + } + } + }).always(); + + expect().post().withPath("/api/v1/namespaces/test/configmaps").andReply(new ResponseProvider() { + ThreadLocal responseCode = new ThreadLocal<>(); + + @Override + public int getStatusCode() { + return responseCode.get(); + } + + @Override + public Object getBody(RecordedRequest recordedRequest) { + delayIfNecessary(); + if (refuseRequests) { + responseCode.set(500); + return ""; + } + + ConfigMap map = convert(recordedRequest); + if (map == null || map.getMetadata() == null || !lockSimulator.getConfigMapName().equals(map.getMetadata().getName())) { + throw new IllegalArgumentException("Illegal configMap received"); + } + + boolean done = lockSimulator.setConfigMap(map, true); + if (done) { + responseCode.set(201); + return lockSimulator.getConfigMap(); + } else { + responseCode.set(500); + return ""; + } + } + }).always(); + + expect().put().withPath("/api/v1/namespaces/test/configmaps/" + lockSimulator.getConfigMapName()).andReply(new ResponseProvider() { + ThreadLocal responseCode = new ThreadLocal<>(); + + @Override + public int getStatusCode() { + return responseCode.get(); + } + + @Override + public Object getBody(RecordedRequest recordedRequest) { + delayIfNecessary(); + if (refuseRequests) { + responseCode.set(500); + return ""; + } + + ConfigMap map = convert(recordedRequest); + + boolean done = lockSimulator.setConfigMap(map, false); + if (done) { + responseCode.set(200); + return lockSimulator.getConfigMap(); + } else { + responseCode.set(409); + return ""; + } + } + }).always(); + + // Other resources + expect().get().withPath("/api/v1/namespaces/test/pods").andReturn(200, new PodListBuilder().withNewMetadata().withResourceVersion("1").and().build()).always(); + expect().get().withPath("/api/v1/namespaces/test/pods?resourceVersion=1&watch=true").andUpgradeToWebSocket().open().done().always(); + } + + + public boolean isRefuseRequests() { + return refuseRequests; + } + + public void setRefuseRequests(boolean refuseRequests) { + this.refuseRequests = refuseRequests; + } + + public Long getDelayRequests() { + return delayRequests; + } + + public void setDelayRequests(Long delayRequests) { + this.delayRequests = delayRequests; + } + + private void delayIfNecessary() { + if (delayRequests != null) { + try { + Thread.sleep(delayRequests); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + private ConfigMap convert(RecordedRequest request) { + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(request.getBody().readByteArray(), ConfigMap.class); + } catch (IOException e) { + throw new IllegalArgumentException("Erroneous data", e); + } + } + +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServerTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServerTest.java new file mode 100644 index 0000000000000..282b83f27626b --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServerTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.utils; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Basic tests on the lock test server. + */ +public class LockTestServerTest { + + @Test + public void test() { + ConfigMapLockSimulator lock = new ConfigMapLockSimulator("xxx"); + LockTestServer server = new LockTestServer(lock); + KubernetesClient client = server.createClient(); + + assertNull(client.configMaps().withName("xxx").get()); + + client.configMaps().withName("xxx").createNew() + .withNewMetadata() + .withName("xxx") + .and().done(); + + try { + client.configMaps().withName("xxx").createNew() + .withNewMetadata() + .withName("xxx") + .and().done(); + Assert.fail("Should have failed for duplicate insert"); + } catch (Exception e) { + } + + client.configMaps().withName("xxx") + .createOrReplaceWithNew() + .editOrNewMetadata() + .withName("xxx") + .addToLabels("a", "b") + .and().done(); + + ConfigMap map = client.configMaps().withName("xxx").get(); + assertEquals("b", map.getMetadata().getLabels().get("a")); + + + client.configMaps().withName("xxx") + .lockResourceVersion(map.getMetadata().getResourceVersion()) + .replace(new ConfigMapBuilder(map) + .editOrNewMetadata() + .withName("xxx") + .addToLabels("c", "d") + .and() + .build()); + + ConfigMap newMap = client.configMaps().withName("xxx").get(); + assertEquals("d", newMap.getMetadata().getLabels().get("c")); + + try { + client.configMaps().withName("xxx") + .lockResourceVersion(map.getMetadata().getResourceVersion()) + .replace(new ConfigMapBuilder(map) + .editOrNewMetadata() + .withName("xxx") + .addToLabels("e", "f") + .and() + .build()); + Assert.fail("Should have failed for wrong version"); + } catch (Exception ex) { + } + + ConfigMap newMap2 = client.configMaps().withName("xxx").get(); + assertNull(newMap2.getMetadata().getLabels().get("e")); + + } + +} diff --git a/parent/pom.xml b/parent/pom.xml index 8030a2a1bb7dd..516033f0745a7 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -491,6 +491,7 @@ 2.0.16 0.9.4 1.9 + 0.0.13 1.10.19 3.5.0 3.2.2 From caa1b7c11e1ddfa244137ae76bf14f6cc1689dab Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Wed, 19 Jul 2017 15:24:48 +0200 Subject: [PATCH 05/13] CAMEL-11331: Adjusted log level --- .../ha/lock/KubernetesLeaseBasedLeadershipController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java index 42be2e7b854f9..1366ee1f87b3c 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java @@ -352,7 +352,7 @@ private void updateLatestLeaderInfo(ConfigMap configMap) { } private void checkAndNotifyNewLeader() { - LOG.info("Checking if the current leader has changed to notify the event handler..."); + LOG.debug("Checking if the current leader has changed to notify the event handler..."); LeaderInfo newLeaderInfo = this.latestLeaderInfo; if (newLeaderInfo == null) { return; @@ -371,7 +371,7 @@ private void checkAndNotifyNewLeader() { // Sending notifications in case of leader change if (!newLeader.equals(this.currentLeader)) { - LOG.info("Current leader has changed from {} to {}. Sending notifications...", this.currentLeader, newLeader); + LOG.info("Current leader has changed from {} to {}. Sending notification...", this.currentLeader, newLeader); this.currentLeader = newLeader; eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> newLeader); } else { From 21d16f76ac8cbefb9e958b9b770f6877ee04f6e4 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Wed, 19 Jul 2017 16:34:51 +0200 Subject: [PATCH 06/13] CAMEL-11331: Removed unnecessary change in leadership for followers --- .../lock/KubernetesLeaseBasedLeadershipController.java | 9 +++++---- .../kubernetes/ha/KubernetesClusterServiceTest.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java index 1366ee1f87b3c..76e91bf062c8c 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java @@ -265,6 +265,7 @@ private boolean tryAcquireOrRenewLeadership() { LOG.debug("ConfigMap {} successfully created and local pod is leader", this.lockConfiguration.getConfigMapName()); updateLatestLeaderInfo(newConfigMap); + scheduleCheckForPossibleLeadershipLoss(); return true; } else { LOG.debug("Lock configmap already present in the Kubernetes namespace. Checking..."); @@ -285,6 +286,7 @@ private boolean tryAcquireOrRenewLeadership() { LOG.debug("ConfigMap {} successfully updated and local pod is leader", this.lockConfiguration.getConfigMapName()); updateLatestLeaderInfo(updatedConfigMap); + scheduleCheckForPossibleLeadershipLoss(); return true; } catch (Exception ex) { LOG.warn("An attempt to become leader has failed. It's possible that the leadership has been taken by another pod"); @@ -341,13 +343,12 @@ private ConfigMap pullConfigMap() { private void updateLatestLeaderInfo(ConfigMap configMap) { LOG.debug("Updating internal status about the current leader"); this.latestLeaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); + } - // Notify about changes in current leader if any - this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); + private void scheduleCheckForPossibleLeadershipLoss() { + // Adding check for the case of main thread busy on http calls if (this.latestLeaderInfo.isLeader(this.lockConfiguration.getPodName())) { this.eventDispatcherExecutor.schedule(this::checkAndNotifyNewLeader, this.lockConfiguration.getRenewDeadlineSeconds() * 1000 + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - } else if (this.latestLeaderInfo.getLeader() != null) { - this.eventDispatcherExecutor.schedule(this::checkAndNotifyNewLeader, this.lockConfiguration.getLeaseDurationSeconds() * 1000 + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); } } diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java index 4baebc6bf8d54..3bdffbd79e7df 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java @@ -205,7 +205,7 @@ public void testSharedConfigMap() throws Exception { a1.waitForAnyLeader(2, TimeUnit.SECONDS); a2.waitForAnyLeader(2, TimeUnit.SECONDS); b1.waitForAnyLeader(2, TimeUnit.SECONDS); - b1.waitForAnyLeader(2, TimeUnit.SECONDS); + b2.waitForAnyLeader(2, TimeUnit.SECONDS); assertNotNull(a1.getCurrentLeader()); assertTrue(a1.getCurrentLeader().startsWith("a")); From 930fcee08f8996051166b9ee013fca9fef3d7d01 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Mon, 31 Jul 2017 17:17:34 +0200 Subject: [PATCH 07/13] CAMEL-11331: Clock-drift-free version of the protocol --- .../ha/KubernetesClusterService.java | 84 ++-- .../kubernetes/ha/KubernetesClusterView.java | 6 +- .../ha/lock/ConfigMapLockUtils.java | 15 +- .../lock/KubernetesLeadershipController.java | 352 ++++++++++++++++ ...ernetesLeaseBasedLeadershipController.java | 384 ------------------ .../ha/lock/KubernetesLockConfiguration.java | 84 ++-- .../ha/lock/KubernetesMembersMonitor.java | 237 ----------- .../kubernetes/ha/lock/LeaderInfo.java | 46 ++- .../ha/lock/TimedLeaderNotifier.java | 179 ++++++++ .../ha/KubernetesClusterServiceTest.java | 59 +-- .../ha/TimedLeaderNotifierTest.java | 117 ++++++ .../kubernetes/ha/utils/LeaderRecorder.java | 5 + .../kubernetes/ha/utils/LockTestServer.java | 31 +- 13 files changed, 813 insertions(+), 786 deletions(-) create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java delete mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java delete mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java create mode 100644 components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java create mode 100644 components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java index a868d1641e55d..08ebb70ea062a 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java @@ -82,25 +82,22 @@ protected KubernetesLockConfiguration configWithGroupNameAndDefaults(String grou if (config.getJitterFactor() < 1) { throw new IllegalStateException("jitterFactor must be >= 1 (found: " + config.getJitterFactor() + ")"); } - if (config.getRetryOnErrorIntervalSeconds() <= 0) { - throw new IllegalStateException("retryOnErrorIntervalSeconds must be > 0 (found: " + config.getRetryOnErrorIntervalSeconds() + ")"); + if (config.getRetryPeriodMillis() <= 0) { + throw new IllegalStateException("retryPeriodMillis must be > 0 (found: " + config.getRetryPeriodMillis() + ")"); } - if (config.getRetryPeriodSeconds() <= 0) { - throw new IllegalStateException("retryPeriodSeconds must be > 0 (found: " + config.getRetryPeriodSeconds() + ")"); + if (config.getRenewDeadlineMillis() <= 0) { + throw new IllegalStateException("renewDeadlineMillis must be > 0 (found: " + config.getRenewDeadlineMillis() + ")"); } - if (config.getRenewDeadlineSeconds() <= 0) { - throw new IllegalStateException("renewDeadlineSeconds must be > 0 (found: " + config.getRenewDeadlineSeconds() + ")"); + if (config.getLeaseDurationMillis() <= 0) { + throw new IllegalStateException("leaseDurationMillis must be > 0 (found: " + config.getLeaseDurationMillis() + ")"); } - if (config.getLeaseDurationSeconds() <= 0) { - throw new IllegalStateException("leaseDurationSeconds must be > 0 (found: " + config.getLeaseDurationSeconds() + ")"); + if (config.getLeaseDurationMillis() <= config.getRenewDeadlineMillis()) { + throw new IllegalStateException("leaseDurationMillis must be greater than renewDeadlineMillis " + + "(" + config.getLeaseDurationMillis() + " is not greater than " + config.getRenewDeadlineMillis() + ")"); } - if (config.getLeaseDurationSeconds() <= config.getRenewDeadlineSeconds()) { - throw new IllegalStateException("leaseDurationSeconds must be greater than renewDeadlineSeconds " - + "(" + config.getLeaseDurationSeconds() + " is not greater than " + config.getRenewDeadlineSeconds() + ")"); - } - if (config.getRenewDeadlineSeconds() <= config.getJitterFactor() * config.getRetryPeriodSeconds()) { - throw new IllegalStateException("renewDeadlineSeconds must be greater than jitterFactor*retryPeriodSeconds " - + "(" + config.getRenewDeadlineSeconds() + " is not greater than " + config.getJitterFactor() + "*" + config.getRetryPeriodSeconds() + ")"); + if (config.getRenewDeadlineMillis() <= config.getJitterFactor() * config.getRetryPeriodMillis()) { + throw new IllegalStateException("renewDeadlineMillis must be greater than jitterFactor*retryPeriodMillis " + + "(" + config.getRenewDeadlineMillis() + " is not greater than " + config.getJitterFactor() + "*" + config.getRetryPeriodMillis() + ")"); } return config; @@ -176,16 +173,8 @@ public void setKubernetesResourcesNamespace(String kubernetesResourcesNamespace) lockConfiguration.setKubernetesResourcesNamespace(kubernetesResourcesNamespace); } - public long getRetryOnErrorIntervalSeconds() { - return lockConfiguration.getRetryOnErrorIntervalSeconds(); - } - - /** - * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. - * Watch recreation can be disabled by putting value <= 0. - */ - public void setRetryOnErrorIntervalSeconds(long retryOnErrorIntervalSeconds) { - lockConfiguration.setRetryOnErrorIntervalSeconds(retryOnErrorIntervalSeconds); + public String getKubernetesResourcesNamespaceOrDefault(KubernetesClient kubernetesClient) { + return lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient); } public double getJitterFactor() { @@ -193,56 +182,43 @@ public double getJitterFactor() { } /** - * A jitter factor to apply in order to prevent all pods to try to become leaders in the same instant. + * A jitter factor to apply in order to prevent all pods to call Kubernetes APIs in the same instant. */ public void setJitterFactor(double jitterFactor) { lockConfiguration.setJitterFactor(jitterFactor); } - public long getLeaseDurationSeconds() { - return lockConfiguration.getLeaseDurationSeconds(); + public long getLeaseDurationMillis() { + return lockConfiguration.getLeaseDurationMillis(); } /** * The default duration of the lease for the current leader. */ - public void setLeaseDurationSeconds(long leaseDurationSeconds) { - lockConfiguration.setLeaseDurationSeconds(leaseDurationSeconds); - } - - public long getRenewDeadlineSeconds() { - return lockConfiguration.getRenewDeadlineSeconds(); - } - - /** - * The deadline after which the leader must stop trying to renew its leadership (and yield it). - */ - public void setRenewDeadlineSeconds(long renewDeadlineSeconds) { - lockConfiguration.setRenewDeadlineSeconds(renewDeadlineSeconds); + public void setLeaseDurationMillis(long leaseDurationMillis) { + lockConfiguration.setLeaseDurationMillis(leaseDurationMillis); } - public long getRetryPeriodSeconds() { - return lockConfiguration.getRetryPeriodSeconds(); + public long getRenewDeadlineMillis() { + return lockConfiguration.getRenewDeadlineMillis(); } /** - * The time between two subsequent attempts to acquire/renew the leadership (or after the lease expiration). - * It is randomized using the jitter factor in case of new leader election (not renewal). + * The deadline after which the leader must stop its services because it may have lost the leadership. */ - public void setRetryPeriodSeconds(long retryPeriodSeconds) { - lockConfiguration.setRetryPeriodSeconds(retryPeriodSeconds); + public void setRenewDeadlineMillis(long renewDeadlineMillis) { + lockConfiguration.setRenewDeadlineMillis(renewDeadlineMillis); } - public long getWatchRefreshIntervalSeconds() { - return lockConfiguration.getWatchRefreshIntervalSeconds(); + public long getRetryPeriodMillis() { + return lockConfiguration.getRetryPeriodMillis(); } /** - * Set this to a positive value in order to recreate watchers after a certain amount of time, - * to avoid having stale watchers. + * The time between two subsequent attempts to check and acquire the leadership. + * It is randomized using the jitter factor. */ - public void setWatchRefreshIntervalSeconds(long watchRefreshIntervalSeconds) { - lockConfiguration.setWatchRefreshIntervalSeconds(watchRefreshIntervalSeconds); + public void setRetryPeriodMillis(long retryPeriodMillis) { + lockConfiguration.setRetryPeriodMillis(retryPeriodMillis); } - } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java index 28f38a54d6701..ddda67529e4c8 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java @@ -30,7 +30,7 @@ import org.apache.camel.component.kubernetes.KubernetesConfiguration; import org.apache.camel.component.kubernetes.KubernetesHelper; import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; -import org.apache.camel.component.kubernetes.ha.lock.KubernetesLeaseBasedLeadershipController; +import org.apache.camel.component.kubernetes.ha.lock.KubernetesLeadershipController; import org.apache.camel.component.kubernetes.ha.lock.KubernetesLockConfiguration; import org.apache.camel.ha.CamelClusterMember; import org.apache.camel.impl.ha.AbstractCamelClusterView; @@ -56,7 +56,7 @@ public class KubernetesClusterView extends AbstractCamelClusterView { private volatile List currentMembers = Collections.emptyList(); - private KubernetesLeaseBasedLeadershipController controller; + private KubernetesLeadershipController controller; public KubernetesClusterView(KubernetesClusterService cluster, KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { super(cluster, lockConfiguration.getGroupName()); @@ -86,7 +86,7 @@ protected void doStart() throws Exception { if (controller == null) { this.kubernetesClient = KubernetesHelper.getKubernetesClient(configuration); - controller = new KubernetesLeaseBasedLeadershipController(kubernetesClient, this.lockConfiguration, event -> { + controller = new KubernetesLeadershipController(kubernetesClient, this.lockConfiguration, event -> { if (event instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { // New leader Optional leader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(event).getData(); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java index 70fa860598323..feea1c6f26eec 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/ConfigMapLockUtils.java @@ -18,6 +18,7 @@ import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Set; import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ConfigMapBuilder; @@ -36,7 +37,7 @@ public final class ConfigMapLockUtils { private static final String LEADER_PREFIX = "leader.pod."; - private static final String TIMESTAMP_PREFIX = "leader.timestamp."; + private static final String LOCAL_TIMESTAMP_PREFIX = "leader.local.timestamp."; private ConfigMapLockUtils() { } @@ -49,19 +50,19 @@ public static ConfigMap createNewConfigMap(String configMapName, LeaderInfo lead .addToLabels("kind", "locks"). endMetadata() .addToData(LEADER_PREFIX + leaderInfo.getGroupName(), leaderInfo.getLeader()) - .addToData(TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getTimestamp())) + .addToData(LOCAL_TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getLocalTimestamp())) .build(); } public static ConfigMap getConfigMapWithNewLeader(ConfigMap configMap, LeaderInfo leaderInfo) { return new ConfigMapBuilder(configMap) .addToData(LEADER_PREFIX + leaderInfo.getGroupName(), leaderInfo.getLeader()) - .addToData(TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getTimestamp())) + .addToData(LOCAL_TIMESTAMP_PREFIX + leaderInfo.getGroupName(), formatDate(leaderInfo.getLocalTimestamp())) .build(); } - public static LeaderInfo getLeaderInfo(ConfigMap configMap, String group) { - return new LeaderInfo(group, getLeader(configMap, group), getTimestamp(configMap, group)); + public static LeaderInfo getLeaderInfo(ConfigMap configMap, Set members, String group) { + return new LeaderInfo(group, getLeader(configMap, group), getLocalTimestamp(configMap, group), members); } private static String getLeader(ConfigMap configMap, String group) { @@ -81,8 +82,8 @@ private static String formatDate(Date date) { return null; } - private static Date getTimestamp(ConfigMap configMap, String group) { - String timestamp = getConfigMapValue(configMap, TIMESTAMP_PREFIX + group); + private static Date getLocalTimestamp(ConfigMap configMap, String group) { + String timestamp = getConfigMapValue(configMap, LOCAL_TIMESTAMP_PREFIX + group); if (timestamp == null) { return null; } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java new file mode 100644 index 0000000000000..f5277798d01fd --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java @@ -0,0 +1,352 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.math.BigDecimal; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.client.KubernetesClient; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Monitors current status and participate to leader election when no active leaders are present. + * It communicates changes in leadership and cluster members to the given event handler. + */ +public class KubernetesLeadershipController implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeadershipController.class); + + private enum State { + NOT_LEADER, + BECOMING_LEADER, + LEADER + } + + private KubernetesClient kubernetesClient; + + private KubernetesLockConfiguration lockConfiguration; + + private KubernetesClusterEventHandler eventHandler; + + private State currentState = State.NOT_LEADER; + + private ScheduledExecutorService serializedExecutor; + + private TimedLeaderNotifier leaderNotifier; + + private volatile LeaderInfo latestLeaderInfo; + private volatile ConfigMap latestConfigMap; + private volatile Set latestMembers; + + public KubernetesLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { + this.kubernetesClient = kubernetesClient; + this.lockConfiguration = lockConfiguration; + this.eventHandler = eventHandler; + } + + @Override + public void start() throws Exception { + if (serializedExecutor == null) { + LOG.debug("{} Starting leadership controller...", logPrefix()); + serializedExecutor = Executors.newSingleThreadScheduledExecutor(); + leaderNotifier = new TimedLeaderNotifier(this.eventHandler); + + leaderNotifier.start(); + serializedExecutor.execute(this::refreshStatus); + } + } + + @Override + public void stop() throws Exception { + LOG.debug("{} Stopping leadership controller...", logPrefix()); + if (serializedExecutor != null) { + serializedExecutor.shutdownNow(); + } + serializedExecutor = null; + + if (leaderNotifier != null) { + leaderNotifier.stop(); + } + leaderNotifier = null; + } + + private void refreshStatus() { + switch (currentState) { + case NOT_LEADER: + refreshStatusNotLeader(); + break; + case BECOMING_LEADER: + refreshStatusBecomingLeader(); + break; + case LEADER: + refreshStatusLeader(); + break; + default: + throw new RuntimeException("Unsupported state " + currentState); + } + } + + /** + * This pod is currently not leader. It should monitor the leader configuration and try + * to acquire the leadership if possible. + */ + private void refreshStatusNotLeader() { + LOG.debug("{} Pod is not leader, pulling new data from the cluster", logPrefix()); + boolean pulled = lookupNewLeaderInfo(); + if (!pulled) { + rescheduleAfterDelay(); + return; + } + + if (this.latestLeaderInfo.hasEmptyLeader()) { + // There is no previous leader + LOG.info("{} The cluster has no leaders. Trying to acquire the leadership...", logPrefix()); + boolean acquired = tryAcquireLeadership(); + if (acquired) { + LOG.info("{} Leadership acquired by current pod ({}) with immediate effect", logPrefix(), this.lockConfiguration.getPodName()); + this.currentState = State.LEADER; + this.serializedExecutor.execute(this::refreshStatus); + return; + } else { + LOG.info("{} Unable to acquire the leadership, it may have been acquired by another pod", logPrefix()); + } + } else if (!this.latestLeaderInfo.hasValidLeader()) { + // There's a previous leader and it's invalid + LOG.info("{} Leadership has been lost by old owner. Trying to acquire the leadership...", logPrefix()); + boolean acquired = tryAcquireLeadership(); + if (acquired) { + LOG.info("{} Leadership acquired by current pod ({})", logPrefix(), this.lockConfiguration.getPodName()); + this.currentState = State.BECOMING_LEADER; + this.serializedExecutor.execute(this::refreshStatus); + return; + } else { + LOG.info("{} Unable to acquire the leadership, it may have been acquired by another pod", logPrefix()); + } + } else if (this.latestLeaderInfo.isValidLeader(this.lockConfiguration.getPodName())) { + // We are leaders for some reason (e.g. pod restart on failure) + LOG.info("{} Leadership is already owned by current pod ({})", logPrefix(), this.lockConfiguration.getPodName()); + this.currentState = State.BECOMING_LEADER; + this.serializedExecutor.execute(this::refreshStatus); + return; + } + + this.leaderNotifier.refreshLeadership(Optional.ofNullable(this.latestLeaderInfo.getLeader()), + System.currentTimeMillis(), + this.lockConfiguration.getLeaseDurationMillis(), + this.latestLeaderInfo.getMembers()); + rescheduleAfterDelay(); + } + + /** + * This pod has acquired the leadership but it should wait for the old leader + * to tear down resources before starting the local services. + */ + private void refreshStatusBecomingLeader() { + // Wait always the same amount of time before becoming the leader + // Even if the current pod is already leader, we should let a possible old version of the pod to shut down + long delay = this.lockConfiguration.getLeaseDurationMillis(); + LOG.info("{} Current pod ({}) owns the leadership, but it will be effective in {} seconds...", logPrefix(), this.lockConfiguration.getPodName(), new BigDecimal(delay).divide(BigDecimal + .valueOf(1000), 2, BigDecimal.ROUND_HALF_UP)); + + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + LOG.warn("Thread interrupted", e); + } + + LOG.info("{} Current pod ({}) is becoming the new leader now...", logPrefix(), this.lockConfiguration.getPodName()); + this.currentState = State.LEADER; + this.serializedExecutor.execute(this::refreshStatus); + } + + private void refreshStatusLeader() { + LOG.debug("{} Pod should be the leader, pulling new data from the cluster", logPrefix()); + long timeBeforePulling = System.currentTimeMillis(); + boolean pulled = lookupNewLeaderInfo(); + if (!pulled) { + rescheduleAfterDelay(); + return; + } + + if (this.latestLeaderInfo.isValidLeader(this.lockConfiguration.getPodName())) { + LOG.debug("{} Current Pod ({}) is still the leader", logPrefix(), this.lockConfiguration.getPodName()); + this.leaderNotifier.refreshLeadership(Optional.of(this.lockConfiguration.getPodName()), + timeBeforePulling, + this.lockConfiguration.getRenewDeadlineMillis(), + this.latestLeaderInfo.getMembers()); + rescheduleAfterDelay(); + return; + } else { + LOG.debug("{} Current Pod ({}) has lost the leadership", logPrefix(), this.lockConfiguration.getPodName()); + this.currentState = State.NOT_LEADER; + // set a empty leader to signal leadership loss + this.leaderNotifier.refreshLeadership(Optional.empty(), + System.currentTimeMillis(), + lockConfiguration.getLeaseDurationMillis(), + this.latestLeaderInfo.getMembers()); + + // wait a lease time and restart + this.serializedExecutor.schedule(this::refreshStatus, this.lockConfiguration.getLeaseDurationMillis(), TimeUnit.MILLISECONDS); + } + } + + private void rescheduleAfterDelay() { + this.serializedExecutor.schedule(this::refreshStatus, jitter(this.lockConfiguration.getRetryPeriodMillis(), this.lockConfiguration.getJitterFactor()), TimeUnit.MILLISECONDS); + } + + private boolean lookupNewLeaderInfo() { + LOG.debug("{} Looking up leadership information...", logPrefix()); + + ConfigMap configMap; + try { + configMap = pullConfigMap(); + } catch (Throwable e) { + LOG.warn(logPrefix() + " Unable to retrieve the current ConfigMap " + this.lockConfiguration.getConfigMapName() + " from Kubernetes"); + LOG.debug(logPrefix() + " Exception thrown during ConfigMap lookup", e); + return false; + } + + Set members; + try { + members = Objects.requireNonNull(pullClusterMembers(), "Retrieved a null set of members"); + } catch (Throwable e) { + LOG.warn(logPrefix() + " Unable to retrieve the list of cluster members from Kubernetes"); + LOG.debug(logPrefix() + " Exception thrown during Pod list lookup", e); + return false; + } + + updateLatestLeaderInfo(configMap, members); + return true; + } + + private boolean tryAcquireLeadership() { + LOG.debug("{} Trying to acquire the leadership...", logPrefix()); + + ConfigMap configMap = this.latestConfigMap; + Set members = this.latestMembers; + LeaderInfo latestLeaderInfo = this.latestLeaderInfo; + + if (latestLeaderInfo == null || members == null) { + LOG.warn(logPrefix() + " Unexpected condition. Latest leader info or list of members is empty."); + return false; + } else if (!members.contains(this.lockConfiguration.getPodName())) { + LOG.warn(logPrefix() + " The list of cluster members " + latestLeaderInfo.getMembers() + " does not contain the current pod (" + this.lockConfiguration.getPodName() + "). Cannot acquire" + + " leadership."); + return false; + } + + // Info we would set set in the configmap to become leaders + LeaderInfo newLeaderInfo = new LeaderInfo(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName(), new Date(), members); + + if (configMap == null) { + // No ConfigMap created so far + LOG.debug("{} Lock configmap is not present in the Kubernetes namespace. A new ConfigMap will be created", logPrefix()); + ConfigMap newConfigMap = ConfigMapLockUtils.createNewConfigMap(this.lockConfiguration.getConfigMapName(), newLeaderInfo); + + try { + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .create(newConfigMap); + + LOG.debug("{} ConfigMap {} successfully created", logPrefix(), this.lockConfiguration.getConfigMapName()); + updateLatestLeaderInfo(newConfigMap, members); + return true; + } catch (Exception ex) { + // Suppress exception + LOG.warn(logPrefix() + " Unable to create the ConfigMap, it may have been created by other cluster members concurrently. If the problem persists, check if the service account has " + + "the right " + + "permissions to create it"); + LOG.debug(logPrefix() + " Exception while trying to create the ConfigMap", ex); + return false; + } + } else { + LOG.debug("{} Lock configmap already present in the Kubernetes namespace. Checking...", logPrefix()); + LeaderInfo leaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, members, this.lockConfiguration.getGroupName()); + + boolean canAcquire = !leaderInfo.hasValidLeader(); + if (canAcquire) { + // Try to be the new leader + try { + ConfigMap updatedConfigMap = ConfigMapLockUtils.getConfigMapWithNewLeader(configMap, newLeaderInfo); + kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .lockResourceVersion(configMap.getMetadata().getResourceVersion()) + .replace(updatedConfigMap); + + LOG.debug("{} ConfigMap {} successfully updated", logPrefix(), this.lockConfiguration.getConfigMapName()); + updateLatestLeaderInfo(updatedConfigMap, members); + return true; + } catch (Exception ex) { + LOG.warn(logPrefix() + " Unable to update the lock ConfigMap to set leadership information"); + LOG.debug(logPrefix() + " Error received during configmap lock replace", ex); + return false; + } + } else { + // Another pod is the leader and it's still active + LOG.debug("{} Another pod ({}) is the current leader and it is still active", logPrefix(), this.latestLeaderInfo.getLeader()); + return false; + } + } + } + + private void updateLatestLeaderInfo(ConfigMap configMap, Set members) { + LOG.debug("{} Updating internal status about the current leader", logPrefix()); + this.latestConfigMap = configMap; + this.latestMembers = members; + this.latestLeaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, members, this.lockConfiguration.getGroupName()); + LOG.debug("{} Current leader info: {}", logPrefix(), this.latestLeaderInfo); + } + + private ConfigMap pullConfigMap() { + return kubernetesClient.configMaps() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withName(this.lockConfiguration.getConfigMapName()) + .get(); + } + + private Set pullClusterMembers() { + List pods = kubernetesClient.pods() + .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) + .withLabels(this.lockConfiguration.getClusterLabels()) + .list().getItems(); + + return pods.stream().map(pod -> pod.getMetadata().getName()).collect(Collectors.toSet()); + } + + private long jitter(long num, double factor) { + return (long) (num * (1 + Math.random() * (factor - 1))); + } + + private String logPrefix() { + return "Leadership Controller [" + this.lockConfiguration.getPodName() + "]"; + } + +} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java deleted file mode 100644 index 76e91bf062c8c..0000000000000 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeaseBasedLeadershipController.java +++ /dev/null @@ -1,384 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.component.kubernetes.ha.lock; - -import java.util.Date; -import java.util.Optional; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.client.KubernetesClient; - -import org.apache.camel.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Monitors current status and participate to leader election when no active leaders are present. - * It communicates changes in leadership and cluster members to the given event handler. - */ -public class KubernetesLeaseBasedLeadershipController implements Service { - - private static final Logger LOG = LoggerFactory.getLogger(KubernetesLeaseBasedLeadershipController.class); - - private static final long FIXED_ADDITIONAL_DELAY = 100; - - private KubernetesClient kubernetesClient; - - private KubernetesLockConfiguration lockConfiguration; - - private KubernetesClusterEventHandler eventHandler; - - private ScheduledExecutorService serializedExecutor; - private ScheduledExecutorService eventDispatcherExecutor; - - private KubernetesMembersMonitor membersMonitor; - - private Optional currentLeader = Optional.empty(); - - private volatile LeaderInfo latestLeaderInfo; - - public KubernetesLeaseBasedLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { - this.kubernetesClient = kubernetesClient; - this.lockConfiguration = lockConfiguration; - this.eventHandler = eventHandler; - } - - @Override - public void start() throws Exception { - if (serializedExecutor == null) { - LOG.debug("Starting leadership controller..."); - serializedExecutor = Executors.newSingleThreadScheduledExecutor(); - - eventDispatcherExecutor = Executors.newSingleThreadScheduledExecutor(); - - membersMonitor = new KubernetesMembersMonitor(this.serializedExecutor, this.kubernetesClient, this.lockConfiguration); - if (eventHandler != null) { - membersMonitor.addClusterEventHandler(eventHandler); - } - - membersMonitor.start(); - serializedExecutor.execute(this::initialization); - } - } - - @Override - public void stop() throws Exception { - LOG.debug("Stopping leadership controller..."); - if (serializedExecutor != null) { - serializedExecutor.shutdownNow(); - } - if (eventDispatcherExecutor != null) { - eventDispatcherExecutor.shutdown(); - eventDispatcherExecutor.awaitTermination(2, TimeUnit.SECONDS); - eventDispatcherExecutor.shutdownNow(); - } - if (membersMonitor != null) { - membersMonitor.stop(); - } - - membersMonitor = null; - eventDispatcherExecutor = null; - serializedExecutor = null; - } - - /** - * Get the first ConfigMap and setup the initial state. - */ - private void initialization() { - LOG.debug("Reading (with retry) the configmap {} to detect the current leader", this.lockConfiguration.getConfigMapName()); - refreshConfigMapFromCluster(true); - - if (isCurrentPodTheActiveLeader()) { - serializedExecutor.execute(this::onLeadershipAcquired); - } else { - LOG.info("The current pod ({}) is not the leader of the group '{}' in ConfigMap '{}' at this time", this.lockConfiguration.getPodName(), this.lockConfiguration - .getGroupName(), this.lockConfiguration.getConfigMapName()); - serializedExecutor.execute(this::acquireLeadershipCycle); - } - } - - /** - * Signals the acquisition of the leadership and move to the keep-leadership state. - */ - private void onLeadershipAcquired() { - LOG.info("The current pod ({}) is now the leader of the group '{}' in ConfigMap '{}'", this.lockConfiguration.getPodName(), this.lockConfiguration - .getGroupName(), this.lockConfiguration.getConfigMapName()); - - this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); - - long nextDelay = computeNextRenewWaitTime(this.latestLeaderInfo.getTimestamp(), this.latestLeaderInfo.getTimestamp()); - serializedExecutor.schedule(this::keepLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - } - - /** - * While in the keep-leadership state, the controller periodically renews the lease. - * If a renewal deadline is met and it was not possible to renew the lease, the leadership is lost. - */ - private void keepLeadershipCycle() { - // renew lease periodically - refreshConfigMapFromCluster(false); // if possible, update - - if (this.latestLeaderInfo.isTimeElapsedSeconds(lockConfiguration.getRenewDeadlineSeconds()) || !this.latestLeaderInfo.isLeader(this.lockConfiguration.getPodName())) { - // Time over for renewal or leadership lost - LOG.debug("The current pod ({}) has lost the leadership", this.lockConfiguration.getPodName()); - serializedExecutor.execute(this::onLeadershipLost); - return; - } - - boolean success = tryAcquireOrRenewLeadership(); - LOG.debug("Attempted to renew the lease. Success={}", success); - - long nextDelay = computeNextRenewWaitTime(this.latestLeaderInfo.getTimestamp(), new Date()); - serializedExecutor.schedule(this::keepLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - } - - /** - * Compute the timestamp of next event while in keep-leadership state. - */ - private long computeNextRenewWaitTime(Date lastRenewal, Date lastRenewalAttempt) { - long timeDeadline = lastRenewal.getTime() + this.lockConfiguration.getRenewDeadlineSeconds() * 1000; - long timeRetry; - long counter = 0; - do { - counter++; - timeRetry = lastRenewal.getTime() + counter * this.lockConfiguration.getRetryPeriodSeconds() * 1000; - } while (timeRetry < lastRenewalAttempt.getTime() && timeRetry < timeDeadline); - - long time = Math.min(timeRetry, timeDeadline); - long delay = Math.max(0, time - System.currentTimeMillis()); - long delayJittered = jitter(delay, lockConfiguration.getJitterFactor()); - LOG.debug("Next renewal timeout event will be fired in {} seconds", delayJittered / 1000); - return delayJittered; - } - - - /** - * Signals the loss of leadership and move to the acquire-leadership state. - */ - private void onLeadershipLost() { - LOG.info("The local pod ({}) is no longer the leader of the group '{}' in ConfigMap '{}'", this.lockConfiguration.getPodName(), this.lockConfiguration.getGroupName(), - this.lockConfiguration.getConfigMapName()); - - this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); - serializedExecutor.execute(this::acquireLeadershipCycle); - } - - /** - * While in the acquire-leadership state, the controller waits for the current lease to expire before trying to acquire the leadership. - */ - private void acquireLeadershipCycle() { - // wait for the current lease to finish then fire an election - refreshConfigMapFromCluster(false); // if possible, update - - // Notify about changes in current leader if any - this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); - - if (!this.latestLeaderInfo.isTimeElapsedSeconds(lockConfiguration.getLeaseDurationSeconds())) { - // Wait for the lease to finish before trying leader election - long nextDelay = computeNextElectionWaitTime(this.latestLeaderInfo.getTimestamp()); - serializedExecutor.schedule(this::acquireLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - return; - } - - boolean acquired = tryAcquireOrRenewLeadership(); - if (acquired) { - LOG.debug("Leadership acquired for ConfigMap {}. Notification in progress...", this.lockConfiguration.getConfigMapName()); - serializedExecutor.execute(this::onLeadershipAcquired); - return; - } - - // Notify about changes in current leader if any - this.eventDispatcherExecutor.execute(this::checkAndNotifyNewLeader); - - LOG.debug("Cannot acquire the leadership for ConfigMap {}", this.lockConfiguration.getConfigMapName()); - long nextDelay = computeNextElectionWaitTime(this.latestLeaderInfo.getTimestamp()); - serializedExecutor.schedule(this::acquireLeadershipCycle, nextDelay + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - } - - private long computeNextElectionWaitTime(Date lastRenewal) { - if (lastRenewal == null) { - LOG.debug("Error detected while getting leadership info, next election timeout event will be fired in {} seconds", this.lockConfiguration.getRetryOnErrorIntervalSeconds()); - return this.lockConfiguration.getRetryOnErrorIntervalSeconds(); - } - long time = lastRenewal.getTime() + this.lockConfiguration.getLeaseDurationSeconds() * 1000 - + jitter(this.lockConfiguration.getRetryPeriodSeconds() * 1000, this.lockConfiguration.getJitterFactor()); - - long delay = Math.max(0, time - System.currentTimeMillis()); - LOG.debug("Next election timeout event will be fired in {} seconds", delay / 1000); - return delay; - } - - private long jitter(long num, double factor) { - return (long) (num * (1 + Math.random() * (factor - 1))); - } - - private boolean tryAcquireOrRenewLeadership() { - LOG.debug("Trying to acquire or renew the leadership..."); - - ConfigMap configMap; - try { - configMap = pullConfigMap(); - } catch (Exception e) { - LOG.warn("Unable to retrieve the current ConfigMap " + this.lockConfiguration.getConfigMapName() + " from Kubernetes", e); - return false; - } - - // Info to set in the configmap to become leaders - LeaderInfo newLeaderInfo = new LeaderInfo(this.lockConfiguration.getGroupName(), this.lockConfiguration.getPodName(), new Date()); - - if (configMap == null) { - // No configmap created so far - LOG.debug("Lock configmap is not present in the Kubernetes namespace. A new ConfigMap will be created"); - ConfigMap newConfigMap = ConfigMapLockUtils.createNewConfigMap(this.lockConfiguration.getConfigMapName(), newLeaderInfo); - - try { - kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .create(newConfigMap); - } catch (Exception ex) { - // Suppress exception - LOG.warn("Unable to create the ConfigMap, it may have been created by other cluster members concurrently. If the problem persists, check if the service account has the right " - + "permissions to create it"); - LOG.debug("Exception while trying to create the ConfigMap", ex); - - // Try to get the configMap and return the current leader - refreshConfigMapFromCluster(false); - return isCurrentPodTheActiveLeader(); - } - - LOG.debug("ConfigMap {} successfully created and local pod is leader", this.lockConfiguration.getConfigMapName()); - updateLatestLeaderInfo(newConfigMap); - scheduleCheckForPossibleLeadershipLoss(); - return true; - } else { - LOG.debug("Lock configmap already present in the Kubernetes namespace. Checking..."); - LeaderInfo leaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); - - boolean weWereLeader = leaderInfo.isLeader(this.lockConfiguration.getPodName()); - boolean leaseExpired = leaderInfo.isTimeElapsedSeconds(this.lockConfiguration.getLeaseDurationSeconds()); - - if (weWereLeader || leaseExpired) { - // Renew the lease or set the new leader - try { - ConfigMap updatedConfigMap = ConfigMapLockUtils.getConfigMapWithNewLeader(configMap, newLeaderInfo); - kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .lockResourceVersion(configMap.getMetadata().getResourceVersion()) - .replace(updatedConfigMap); - - LOG.debug("ConfigMap {} successfully updated and local pod is leader", this.lockConfiguration.getConfigMapName()); - updateLatestLeaderInfo(updatedConfigMap); - scheduleCheckForPossibleLeadershipLoss(); - return true; - } catch (Exception ex) { - LOG.warn("An attempt to become leader has failed. It's possible that the leadership has been taken by another pod"); - LOG.debug("Error received during configmap lock replace", ex); - - // Try to get the configMap and return the current leader - refreshConfigMapFromCluster(false); - return isCurrentPodTheActiveLeader(); - } - } else { - // Another pod is the leader and lease is not expired - LOG.debug("Another pod is the current leader and lease has not expired yet"); - updateLatestLeaderInfo(configMap); - return false; - } - } - } - - - private void refreshConfigMapFromCluster(boolean retry) { - LOG.debug("Retrieving configmap {}", this.lockConfiguration.getConfigMapName()); - try { - updateLatestLeaderInfo(pullConfigMap()); - } catch (Exception ex) { - if (retry) { - LOG.warn("ConfigMap pull failed. Retrying in " + this.lockConfiguration.getRetryOnErrorIntervalSeconds() + " seconds...", ex); - try { - Thread.sleep(this.lockConfiguration.getRetryOnErrorIntervalSeconds() * 1000); - refreshConfigMapFromCluster(true); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Controller Thread interrupted, shutdown in progress", e); - } - } else { - LOG.warn("Cannot retrieve the ConfigMap: pull failed", ex); - } - } - } - - private boolean isCurrentPodTheActiveLeader() { - return latestLeaderInfo != null - && latestLeaderInfo.isLeader(this.lockConfiguration.getPodName()) - && !latestLeaderInfo.isTimeElapsedSeconds(this.lockConfiguration.getRenewDeadlineSeconds()); - } - - private ConfigMap pullConfigMap() { - return kubernetesClient.configMaps() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withName(this.lockConfiguration.getConfigMapName()) - .get(); - } - - - private void updateLatestLeaderInfo(ConfigMap configMap) { - LOG.debug("Updating internal status about the current leader"); - this.latestLeaderInfo = ConfigMapLockUtils.getLeaderInfo(configMap, this.lockConfiguration.getGroupName()); - } - - private void scheduleCheckForPossibleLeadershipLoss() { - // Adding check for the case of main thread busy on http calls - if (this.latestLeaderInfo.isLeader(this.lockConfiguration.getPodName())) { - this.eventDispatcherExecutor.schedule(this::checkAndNotifyNewLeader, this.lockConfiguration.getRenewDeadlineSeconds() * 1000 + FIXED_ADDITIONAL_DELAY, TimeUnit.MILLISECONDS); - } - } - - private void checkAndNotifyNewLeader() { - LOG.debug("Checking if the current leader has changed to notify the event handler..."); - LeaderInfo newLeaderInfo = this.latestLeaderInfo; - if (newLeaderInfo == null) { - return; - } - - long leaderInfoDurationSeconds = newLeaderInfo.isLeader(this.lockConfiguration.getPodName()) - ? this.lockConfiguration.getRenewDeadlineSeconds() - : this.lockConfiguration.getLeaseDurationSeconds(); - - Optional newLeader; - if (newLeaderInfo.getLeader() != null && !newLeaderInfo.isTimeElapsedSeconds(leaderInfoDurationSeconds)) { - newLeader = Optional.of(newLeaderInfo.getLeader()); - } else { - newLeader = Optional.empty(); - } - - // Sending notifications in case of leader change - if (!newLeader.equals(this.currentLeader)) { - LOG.info("Current leader has changed from {} to {}. Sending notification...", this.currentLeader, newLeader); - this.currentLeader = newLeader; - eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) () -> newLeader); - } else { - LOG.debug("Current leader unchanged: {}", this.currentLeader); - } - } - - -} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java index 64617080fe7d3..be0b42407ea2b 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java @@ -28,14 +28,10 @@ public class KubernetesLockConfiguration implements Cloneable { public static final String DEFAULT_CONFIGMAP_NAME = "leaders"; - public static final double DEFAULT_JITTER_FACTOR = 1.2; - public static final long DEFAULT_LEASE_DURATION_SECONDS = 60; - public static final long DEFAULT_RENEW_DEADLINE_SECONDS = 45; - public static final long DEFAULT_RETRY_PERIOD_SECONDS = 9; - - public static final long DEFAULT_RETRY_ON_ERROR_INTERVAL_SECONDS = 5; - public static final long DEFAULT_WATCH_REFRESH_INTERVAL_SECONDS = 1800; + public static final long DEFAULT_LEASE_DURATION_MILLIS = 60000; + public static final long DEFAULT_RENEW_DEADLINE_MILLIS = 45000; + public static final long DEFAULT_RETRY_PERIOD_MILLIS = 9000; /** * Kubernetes namespace containing the pods and the ConfigMap used for locking. @@ -63,37 +59,25 @@ public class KubernetesLockConfiguration implements Cloneable { private Map clusterLabels = new HashMap<>(); /** - * Indicates the maximum amount of time a Kubernetes watch should be kept active, before being recreated. - * Watch recreation can be disabled by putting value <= 0. - */ - private long retryOnErrorIntervalSeconds = DEFAULT_RETRY_ON_ERROR_INTERVAL_SECONDS; - - /** - * A jitter factor to apply in order to prevent all pods to try to become leaders in the same instant. + * A jitter factor to apply in order to prevent all pods to call Kubernetes APIs in the same instant. */ private double jitterFactor = DEFAULT_JITTER_FACTOR; /** * The default duration of the lease for the current leader. */ - private long leaseDurationSeconds = DEFAULT_LEASE_DURATION_SECONDS; - - /** - * The deadline after which the leader must stop trying to renew its leadership (and yield it). - */ - private long renewDeadlineSeconds = DEFAULT_RENEW_DEADLINE_SECONDS; + private long leaseDurationMillis = DEFAULT_LEASE_DURATION_MILLIS; /** - * The time between two subsequent attempts to acquire/renew the leadership (or after the lease expiration). - * It is randomized using the jitter factor in case of new leader election (not renewal). + * The deadline after which the leader must stop its services because it may have lost the leadership. */ - private long retryPeriodSeconds = DEFAULT_RETRY_PERIOD_SECONDS; + private long renewDeadlineMillis = DEFAULT_RENEW_DEADLINE_MILLIS; /** - * Set this to a positive value in order to recreate watchers after a certain amount of time - * (to prevent them becoming stale). + * The time between two subsequent attempts to check and acquire the leadership. + * It is randomized using the jitter factor. */ - private long watchRefreshIntervalSeconds = DEFAULT_WATCH_REFRESH_INTERVAL_SECONDS; + private long retryPeriodMillis = DEFAULT_RETRY_PERIOD_MILLIS; public KubernetesLockConfiguration() { } @@ -149,14 +133,6 @@ public void setClusterLabels(Map clusterLabels) { this.clusterLabels = clusterLabels; } - public long getRetryOnErrorIntervalSeconds() { - return retryOnErrorIntervalSeconds; - } - - public void setRetryOnErrorIntervalSeconds(long retryOnErrorIntervalSeconds) { - this.retryOnErrorIntervalSeconds = retryOnErrorIntervalSeconds; - } - public double getJitterFactor() { return jitterFactor; } @@ -165,36 +141,28 @@ public void setJitterFactor(double jitterFactor) { this.jitterFactor = jitterFactor; } - public long getLeaseDurationSeconds() { - return leaseDurationSeconds; - } - - public void setLeaseDurationSeconds(long leaseDurationSeconds) { - this.leaseDurationSeconds = leaseDurationSeconds; - } - - public long getRenewDeadlineSeconds() { - return renewDeadlineSeconds; + public long getLeaseDurationMillis() { + return leaseDurationMillis; } - public void setRenewDeadlineSeconds(long renewDeadlineSeconds) { - this.renewDeadlineSeconds = renewDeadlineSeconds; + public void setLeaseDurationMillis(long leaseDurationMillis) { + this.leaseDurationMillis = leaseDurationMillis; } - public long getRetryPeriodSeconds() { - return retryPeriodSeconds; + public long getRenewDeadlineMillis() { + return renewDeadlineMillis; } - public void setRetryPeriodSeconds(long retryPeriodSeconds) { - this.retryPeriodSeconds = retryPeriodSeconds; + public void setRenewDeadlineMillis(long renewDeadlineMillis) { + this.renewDeadlineMillis = renewDeadlineMillis; } - public long getWatchRefreshIntervalSeconds() { - return watchRefreshIntervalSeconds; + public long getRetryPeriodMillis() { + return retryPeriodMillis; } - public void setWatchRefreshIntervalSeconds(long watchRefreshIntervalSeconds) { - this.watchRefreshIntervalSeconds = watchRefreshIntervalSeconds; + public void setRetryPeriodMillis(long retryPeriodMillis) { + this.retryPeriodMillis = retryPeriodMillis; } public KubernetesLockConfiguration copy() { @@ -214,12 +182,10 @@ public String toString() { sb.append(", groupName='").append(groupName).append('\''); sb.append(", podName='").append(podName).append('\''); sb.append(", clusterLabels=").append(clusterLabels); - sb.append(", retryOnErrorIntervalSeconds=").append(retryOnErrorIntervalSeconds); sb.append(", jitterFactor=").append(jitterFactor); - sb.append(", leaseDurationSeconds=").append(leaseDurationSeconds); - sb.append(", renewDeadlineSeconds=").append(renewDeadlineSeconds); - sb.append(", retryPeriodSeconds=").append(retryPeriodSeconds); - sb.append(", watchRefreshIntervalSeconds=").append(watchRefreshIntervalSeconds); + sb.append(", leaseDurationMillis=").append(leaseDurationMillis); + sb.append(", renewDeadlineMillis=").append(renewDeadlineMillis); + sb.append(", retryPeriodMillis=").append(retryPeriodMillis); sb.append('}'); return sb.toString(); } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java deleted file mode 100644 index 586a11f5ad2e4..0000000000000 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesMembersMonitor.java +++ /dev/null @@ -1,237 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.component.kubernetes.ha.lock; - -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.client.KubernetesClient; -import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.Watch; -import io.fabric8.kubernetes.client.Watcher; - -import org.apache.camel.Service; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Monitors the list of participants in a leader election and provides the most recently updated list. - * It calls the callback eventHandlers only when the member set changes w.r.t. the previous invocation. - */ -class KubernetesMembersMonitor implements Service { - - private static final Logger LOG = LoggerFactory.getLogger(KubernetesMembersMonitor.class); - - private ScheduledExecutorService serializedExecutor; - - private KubernetesClient kubernetesClient; - - private KubernetesLockConfiguration lockConfiguration; - - private List eventHandlers; - - private Watch watch; - - private boolean terminated; - - private boolean refreshing; - - private Set previousMembers = new HashSet<>(); - - private Set basePoll = new HashSet<>(); - private Set deleted = new HashSet<>(); - private Set added = new HashSet<>(); - - public KubernetesMembersMonitor(ScheduledExecutorService serializedExecutor, KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration) { - this.serializedExecutor = serializedExecutor; - this.kubernetesClient = kubernetesClient; - this.lockConfiguration = lockConfiguration; - this.eventHandlers = new LinkedList<>(); - } - - public void addClusterEventHandler(KubernetesClusterEventHandler eventHandler) { - this.eventHandlers.add(eventHandler); - } - - @Override - public void start() throws Exception { - serializedExecutor.execute(() -> doPoll(true)); - serializedExecutor.execute(this::createWatch); - - long recreationDelay = lockConfiguration.getWatchRefreshIntervalSeconds(); - if (recreationDelay > 0) { - serializedExecutor.scheduleWithFixedDelay(this::refresh, recreationDelay, recreationDelay, TimeUnit.SECONDS); - } - } - - private void createWatch() { - try { - LOG.debug("Starting cluster members watcher"); - this.watch = kubernetesClient.pods() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withLabels(this.lockConfiguration.getClusterLabels()) - .watch(new Watcher() { - - @Override - public void eventReceived(Action action, Pod pod) { - switch (action) { - case DELETED: - serializedExecutor.execute(() -> deleteAndNotify(podName(pod))); - break; - case ADDED: - serializedExecutor.execute(() -> addAndNotify(podName(pod))); - break; - default: - } - } - - @Override - public void onClose(KubernetesClientException e) { - if (!terminated) { - KubernetesMembersMonitor.this.watch = null; - if (refreshing) { - LOG.info("Refreshing pod list watcher..."); - serializedExecutor.execute(KubernetesMembersMonitor.this::createWatch); - } else { - LOG.warn("Pod list watcher has been closed unexpectedly. Recreating it in 1 second...", e); - serializedExecutor.schedule(KubernetesMembersMonitor.this::createWatch, 1, TimeUnit.SECONDS); - } - } - } - }); - } catch (Exception ex) { - LOG.warn("Unable to watch for pod list changes. Retrying in 5 seconds..."); - LOG.debug("Error while trying to watch the pod list", ex); - - serializedExecutor.schedule(this::createWatch, 5, TimeUnit.SECONDS); - } - } - - @Override - public void stop() throws Exception { - this.terminated = true; - Watch watch = this.watch; - if (watch != null) { - watch.close(); - } - } - - public void refresh() { - serializedExecutor.execute(() -> { - if (!terminated) { - refreshing = true; - try { - doPoll(false); - - Watch w = this.watch; - if (w != null) { - // It will be recreated - w.close(); - } - } finally { - refreshing = false; - } - } - }); - } - - private void doPoll(boolean retry) { - LOG.debug("Starting poll to get all cluster members"); - List pods; - try { - pods = pollPods(); - } catch (Exception ex) { - if (retry) { - LOG.warn("Pods poll failed. Retrying in 5 seconds...", ex); - this.serializedExecutor.schedule(() -> doPoll(true), 5, TimeUnit.SECONDS); - } else { - LOG.warn("Pods poll failed", ex); - } - return; - } - - this.basePoll = pods.stream() - .map(p -> Optional.ofNullable(podName(p))) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toSet()); - - this.added = new HashSet<>(); - this.deleted = new HashSet<>(); - - LOG.debug("Base list of members is {}", this.basePoll); - - checkAndNotify(); - } - - private List pollPods() { - return kubernetesClient.pods() - .inNamespace(this.lockConfiguration.getKubernetesResourcesNamespaceOrDefault(kubernetesClient)) - .withLabels(this.lockConfiguration.getClusterLabels()) - .list().getItems(); - } - - private String podName(Pod pod) { - if (pod != null && pod.getMetadata() != null) { - return pod.getMetadata().getName(); - } - return null; - } - - private void checkAndNotify() { - Set newMembers = new HashSet<>(basePoll); - newMembers.addAll(added); - newMembers.removeAll(deleted); - - LOG.debug("Current list of members is: {}", newMembers); - - if (!newMembers.equals(this.previousMembers)) { - LOG.debug("List of members changed: sending notifications"); - this.previousMembers = newMembers; - - for (KubernetesClusterEventHandler eventHandler : eventHandlers) { - eventHandler.onKubernetesClusterEvent((KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) () -> newMembers); - } - } else { - LOG.debug("List of members has not changed"); - } - } - - private void addAndNotify(String member) { - LOG.debug("Adding new member to the list: {}", member); - if (member != null) { - this.added.add(member); - checkAndNotify(); - } - } - - private void deleteAndNotify(String member) { - LOG.debug("Deleting member to the list: {}", member); - if (member != null) { - this.deleted.add(member); - checkAndNotify(); - } - } - -} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java index 50d16031e3a98..d06194541d653 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/LeaderInfo.java @@ -17,6 +17,7 @@ package org.apache.camel.component.kubernetes.ha.lock; import java.util.Date; +import java.util.Set; import org.apache.camel.util.ObjectHelper; @@ -29,28 +30,31 @@ public class LeaderInfo { private String leader; - private Date timestamp; + private Date localTimestamp; + + private Set members; public LeaderInfo() { } - public LeaderInfo(String groupName, String leader, Date timestamp) { + public LeaderInfo(String groupName, String leader, Date timestamp, Set members) { this.groupName = groupName; this.leader = leader; - this.timestamp = timestamp; + this.localTimestamp = timestamp; + this.members = members; + } + + public boolean hasEmptyLeader() { + return this.leader == null; } - public boolean isTimeElapsedSeconds(long timeSeconds) { - if (timestamp == null) { - return true; - } - long now = System.currentTimeMillis(); - return timestamp.getTime() + timeSeconds * 1000 <= now; + public boolean hasValidLeader() { + return this.leader != null && this.members.contains(this.leader); } - public boolean isLeader(String pod) { + public boolean isValidLeader(String pod) { ObjectHelper.notNull(pod, "pod"); - return pod.equals(leader); + return hasValidLeader() && pod.equals(leader); } public String getGroupName() { @@ -69,12 +73,20 @@ public void setLeader(String leader) { this.leader = leader; } - public Date getTimestamp() { - return timestamp; + public Date getLocalTimestamp() { + return localTimestamp; + } + + public void setLocalTimestamp(Date localTimestamp) { + this.localTimestamp = localTimestamp; + } + + public Set getMembers() { + return members; } - public void setTimestamp(Date timestamp) { - this.timestamp = timestamp; + public void setMembers(Set members) { + this.members = members; } @Override @@ -82,9 +94,9 @@ public String toString() { final StringBuilder sb = new StringBuilder("LeaderInfo{"); sb.append("groupName='").append(groupName).append('\''); sb.append(", leader='").append(leader).append('\''); - sb.append(", timestamp=").append(timestamp); + sb.append(", localTimestamp=").append(localTimestamp); + sb.append(", members=").append(members); sb.append('}'); return sb.toString(); } - } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java new file mode 100644 index 0000000000000..6ada830b04b1a --- /dev/null +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha.lock; + +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.camel.Service; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Receives information about the current leader and handles expiration automatically. + */ +public class TimedLeaderNotifier implements Service { + + private static final Logger LOG = LoggerFactory.getLogger(TimedLeaderNotifier.class); + + private static final long FIXED_DELAY = 10; + + private KubernetesClusterEventHandler handler; + + private Lock lock = new ReentrantLock(); + + private ScheduledExecutorService executor; + + private Optional lastCommunicatedLeader = Optional.empty(); + private Set lastCommunicatedMembers = Collections.emptySet(); + + private Optional currentLeader = Optional.empty(); + + private Set currentMembers; + + private Long timestamp; + + private Long lease; + + private long changeCounter; + + public TimedLeaderNotifier(KubernetesClusterEventHandler handler) { + this.handler = Objects.requireNonNull(handler, "Handler must be present"); + } + + public void refreshLeadership(Optional leader, Long timestamp, Long lease, Set members) { + Objects.requireNonNull(leader, "leader must be non null (use Optional.empty)"); + Objects.requireNonNull(members, "members must be non null (use empty set)"); + long version; + try { + lock.lock(); + + this.currentLeader = leader; + this.currentMembers = members; + this.timestamp = timestamp; + this.lease = lease; + version = ++changeCounter; + } finally { + lock.unlock(); + } + + LOG.debug("Updated leader to {} at version version {}", leader, version); + this.executor.execute(() -> checkAndNotify(version)); + if (leader.isPresent()) { + long time = System.currentTimeMillis(); + long delay = Math.max(timestamp + lease + FIXED_DELAY - time, FIXED_DELAY); + LOG.debug("Setting expiration in {} millis for version {}", delay, version); + this.executor.schedule(() -> expiration(version), delay, TimeUnit.MILLISECONDS); + } + } + + @Override + public void start() throws Exception { + if (this.executor == null) { + this.executor = Executors.newSingleThreadScheduledExecutor(); + } + } + + @Override + public void stop() throws Exception { + if (this.executor != null) { + ScheduledExecutorService executor = this.executor; + this.executor = null; + + executor.shutdownNow(); + executor.awaitTermination(1, TimeUnit.SECONDS); + } + } + + private void expiration(long version) { + try { + lock.lock(); + + if (version != this.changeCounter) { + return; + } + + long time = System.currentTimeMillis(); + if (time < this.timestamp + this.lease) { + long delay = this.timestamp + this.lease - time; + LOG.debug("Delaying expiration by {} millis at version version {}", delay + FIXED_DELAY, version); + this.executor.schedule(() -> expiration(version), delay + FIXED_DELAY, TimeUnit.MILLISECONDS); + return; + } + } finally { + lock.unlock(); + } + + checkAndNotify(version); + } + + private void checkAndNotify(long version) { + Optional leader; + Set members; + try { + lock.lock(); + + if (version != this.changeCounter) { + return; + } + + leader = this.currentLeader; + members = this.currentMembers; + + if (leader.isPresent()) { + long time = System.currentTimeMillis(); + if (time >= this.timestamp + this.lease) { + leader = Optional.empty(); + } + } + + } finally { + lock.unlock(); + } + + final Optional newLeader = leader; + if (!newLeader.equals(lastCommunicatedLeader)) { + lastCommunicatedLeader = newLeader; + handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent() { + @Override + public Optional getData() { + return newLeader; + } + }); + } + + final Set newMembers = members; + if (!newMembers.equals(lastCommunicatedMembers)) { + lastCommunicatedMembers = newMembers; + handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent() { + @Override + public Set getData() { + return newMembers; + } + }); + } + + } + +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java index 3bdffbd79e7df..4a2a11e5d95d5 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java @@ -42,7 +42,10 @@ */ public class KubernetesClusterServiceTest extends CamelTestSupport { - private static final int LEASE_TIME_SECONDS = 5; + private static final int LEASE_TIME_MILLIS = 2000; + private static final int RENEW_DEADLINE_MILLIS = 1000; + private static final int RETRY_PERIOD_MILLIS = 200; + private static final double JITTER_FACTOR = 1.1; private ConfigMapLockSimulator lockSimulator; @@ -75,6 +78,7 @@ public void testSimpleLeaderElection() throws Exception { mypod2.waitForAnyLeader(2, TimeUnit.SECONDS); String leader = mypod1.getCurrentLeader(); + assertNotNull(leader); assertTrue(leader.startsWith("mypod")); assertEquals("Leaders should be equals", mypod2.getCurrentLeader(), leader); } @@ -129,6 +133,7 @@ public void testLeadershipLoss() throws Exception { LeaderRecorder formerLoserRecorder = firstLeader.equals("mypod1") ? mypod2 : mypod1; refuseRequestsFromPod(firstLeader); + disconnectPod(firstLeader); formerLeaderRecorder.waitForALeaderChange(7, TimeUnit.SECONDS); formerLoserRecorder.waitForANewLeader(firstLeader, 7, TimeUnit.SECONDS); @@ -139,12 +144,12 @@ public void testLeadershipLoss() throws Exception { Long lossTimestamp = formerLeaderRecorder.getLastTimeOf(l -> l == null); Long gainTimestamp = formerLoserRecorder.getLastTimeOf(secondLeader::equals); - assertTrue("At least 2 seconds must elapse from leadership loss and regain (see renewDeadlineSeconds)", gainTimestamp >= lossTimestamp + 2000); - checkLeadershipChangeDistance(LEASE_TIME_SECONDS, TimeUnit.SECONDS, mypod1, mypod2); + assertTrue("At least half distance must elapse from leadership loss and regain (see renewDeadlineSeconds)", gainTimestamp >= lossTimestamp + (LEASE_TIME_MILLIS - RENEW_DEADLINE_MILLIS) / 2); + checkLeadershipChangeDistance((LEASE_TIME_MILLIS - RENEW_DEADLINE_MILLIS) / 2, TimeUnit.MILLISECONDS, mypod1, mypod2); } @Test - public void testSlowLeaderLosingLeadership() throws Exception { + public void testSlowLeaderLosingLeadershipOnlyInternally() throws Exception { LeaderRecorder mypod1 = addMember("mypod1"); LeaderRecorder mypod2 = addMember("mypod2"); context.start(); @@ -159,17 +164,9 @@ public void testSlowLeaderLosingLeadership() throws Exception { delayRequestsFromPod(firstLeader, 10, TimeUnit.SECONDS); - formerLeaderRecorder.waitForALeaderChange(7, TimeUnit.SECONDS); - formerLoserRecorder.waitForANewLeader(firstLeader, 7, TimeUnit.SECONDS); - - String secondLeader = formerLoserRecorder.getCurrentLeader(); - assertNotEquals("The firstLeader should be different from the new one", firstLeader, secondLeader); - - Long lossTimestamp = formerLeaderRecorder.getLastTimeOf(l -> l == null); - Long gainTimestamp = formerLoserRecorder.getLastTimeOf(secondLeader::equals); - - assertTrue("At least 2 seconds must elapse from leadership loss and regain (see renewDeadlineSeconds)", gainTimestamp >= lossTimestamp + 2000); - checkLeadershipChangeDistance(LEASE_TIME_SECONDS, TimeUnit.SECONDS, mypod1, mypod2); + Thread.sleep(LEASE_TIME_MILLIS); + assertNull(formerLeaderRecorder.getCurrentLeader()); + assertEquals(firstLeader, formerLoserRecorder.getCurrentLeader()); } @Test @@ -185,9 +182,9 @@ public void testRecoveryAfterFailure() throws Exception { for (int i = 0; i < 3; i++) { refuseRequestsFromPod(firstLeader); - Thread.sleep(1000); + Thread.sleep(RENEW_DEADLINE_MILLIS); allowRequestsFromPod(firstLeader); - Thread.sleep(2000); + Thread.sleep(LEASE_TIME_MILLIS); } assertEquals(firstLeader, mypod1.getCurrentLeader()); @@ -229,6 +226,18 @@ private void allowRequestsFromPod(String pod) { this.lockServers.get(pod).setRefuseRequests(false); } + private void disconnectPod(String pod) { + for (LockTestServer server : this.lockServers.values()) { + server.removePod(pod); + } + } + + private void connectPod(String pod) { + for (LockTestServer server : this.lockServers.values()) { + server.addPod(pod); + } + } + private void checkLeadershipChangeDistance(long minimum, TimeUnit unit, LeaderRecorder... recorders) { List infos = Arrays.stream(recorders) .flatMap(lr -> lr.getLeadershipInfo().stream()) @@ -245,7 +254,8 @@ private void checkLeadershipChangeDistance(long minimum, TimeUnit unit, LeaderRe } else if (info.getLeader() != null && !info.getLeader().equals(currentLeaderLastSeen.getLeader())) { // switch long delay = info.getChangeTimestamp() - currentLeaderLastSeen.getChangeTimestamp(); - assertTrue("Lease time not elapsed between switch", delay >= TimeUnit.MILLISECONDS.convert(minimum, unit)); + assertTrue("Lease time not elapsed between switch, minimum=" + TimeUnit.MILLISECONDS.convert(minimum, unit) + ", found=" + delay, delay >= TimeUnit.MILLISECONDS.convert(minimum, + unit)); currentLeaderLastSeen = info; } } @@ -268,11 +278,10 @@ private LeaderRecorder addMember(String name, String namespace) { KubernetesClusterService member = new KubernetesClusterService(configuration); member.setKubernetesNamespace("test"); member.setPodName(name); - member.setLeaseDurationSeconds(LEASE_TIME_SECONDS); - member.setRenewDeadlineSeconds(3); // 5-3 = at least 2 seconds for switching on leadership loss - member.setRetryPeriodSeconds(1); - member.setRetryOnErrorIntervalSeconds(1); - member.setJitterFactor(1.2); + member.setLeaseDurationMillis(LEASE_TIME_MILLIS); + member.setRenewDeadlineMillis(RENEW_DEADLINE_MILLIS); + member.setRetryPeriodMillis(RETRY_PERIOD_MILLIS); + member.setJitterFactor(JITTER_FACTOR); LeaderRecorder recorder = new LeaderRecorder(); try { @@ -281,6 +290,10 @@ private LeaderRecorder addMember(String name, String namespace) { } catch (Exception ex) { throw new RuntimeException(ex); } + + for (String pod : this.lockServers.keySet()) { + connectPod(pod); + } return recorder; } diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java new file mode 100644 index 0000000000000..8380147da357f --- /dev/null +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.kubernetes.ha; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; +import org.apache.camel.component.kubernetes.ha.lock.TimedLeaderNotifier; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Test the behavior of the timed notifier. + */ +public class TimedLeaderNotifierTest { + + private TimedLeaderNotifier notifier; + + private volatile Optional currentLeader; + + private volatile Set currentMembers; + + @Before + public void init() throws Exception { + this.notifier = new TimedLeaderNotifier(e -> { + if (e instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { + currentLeader = ((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) e).getData(); + } else if (e instanceof KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) { + currentMembers = ((KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) e).getData(); + } + }); + this.notifier.start(); + } + + @After + public void destroy() throws Exception { + this.notifier.stop(); + } + + @Test + public void testMultipleCalls() throws Exception { + Set members = new TreeSet<>(Arrays.asList("one", "two", "three")); + notifier.refreshLeadership(Optional.of("one"), System.currentTimeMillis(), 50L, members); + notifier.refreshLeadership(Optional.of("two"), System.currentTimeMillis(), 50L, members); + notifier.refreshLeadership(Optional.of("three"), System.currentTimeMillis(), 5000L, members); + Thread.sleep(80); + assertEquals(Optional.of("three"), currentLeader); + assertEquals(members, currentMembers); + } + + @Test + public void testExpiration() throws Exception { + Set members = new TreeSet<>(Arrays.asList("one", "two", "three")); + notifier.refreshLeadership(Optional.of("one"), System.currentTimeMillis(), 50L, members); + notifier.refreshLeadership(Optional.of("two"), System.currentTimeMillis(), 50L, members); + Thread.sleep(160); + assertEquals(Optional.empty(), currentLeader); + assertEquals(members, currentMembers); + notifier.refreshLeadership(Optional.of("three"), System.currentTimeMillis(), 5000L, members); + Thread.sleep(80); + assertEquals(Optional.of("three"), currentLeader); + assertEquals(members, currentMembers); + } + + @Test + public void testMemberChanging() throws Exception { + Set members1 = Collections.singleton("one"); + Set members2 = new TreeSet<>(Arrays.asList("one", "two")); + notifier.refreshLeadership(Optional.of("one"), System.currentTimeMillis(), 50L, members1); + notifier.refreshLeadership(Optional.of("two"), System.currentTimeMillis(), 5000L, members2); + Thread.sleep(80); + assertEquals(Optional.of("two"), currentLeader); + assertEquals(members2, currentMembers); + } + + @Test + public void testOldData() throws Exception { + Set members = new TreeSet<>(Arrays.asList("one", "two", "three")); + notifier.refreshLeadership(Optional.of("one"), System.currentTimeMillis(), 1000L, members); + Thread.sleep(80); + notifier.refreshLeadership(Optional.of("two"), System.currentTimeMillis() - 1000, 900L, members); + Thread.sleep(80); + assertEquals(Optional.empty(), currentLeader); + } + + @Test + public void testNewLeaderEmpty() throws Exception { + Set members = new TreeSet<>(Arrays.asList("one", "two", "three")); + notifier.refreshLeadership(Optional.of("one"), System.currentTimeMillis(), 1000L, members); + Thread.sleep(80); + notifier.refreshLeadership(Optional.empty(), null, null, members); + Thread.sleep(80); + assertEquals(Optional.empty(), currentLeader); + } + +} diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java index 6670f375a8d49..7d7147b6cbca3 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LeaderRecorder.java @@ -28,16 +28,21 @@ import org.apache.camel.ha.CamelClusterMember; import org.apache.camel.ha.CamelClusterView; import org.junit.Assert; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Records leadership changes and allow to do assertions. */ public class LeaderRecorder implements CamelClusterEventListener.Leadership { + private static final Logger LOG = LoggerFactory.getLogger(LeaderRecorder.class); + private List leaderships = new CopyOnWriteArrayList<>(); @Override public void leadershipChanged(CamelClusterView view, CamelClusterMember leader) { + LOG.info("Cluster view {} - leader changed to: {}", view.getLocalMember(), leader); this.leaderships.add(new LeadershipInfo(leader != null ? leader.getId() : null, System.currentTimeMillis())); } diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java index 6422e353b5e01..3dc242315964b 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/utils/LockTestServer.java @@ -17,10 +17,16 @@ package org.apache.camel.component.kubernetes.ha.utils; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.PodListBuilder; import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer; import io.fabric8.mockwebserver.utils.ResponseProvider; @@ -41,7 +47,15 @@ public class LockTestServer extends KubernetesMockServer { private Long delayRequests; + private Set pods; + public LockTestServer(ConfigMapLockSimulator lockSimulator) { + this(lockSimulator, Collections.emptySet()); + } + + public LockTestServer(ConfigMapLockSimulator lockSimulator, Collection initialPods) { + + this.pods = new TreeSet<>(initialPods); expect().get().withPath("/api/v1/namespaces/test/configmaps/" + lockSimulator.getConfigMapName()).andReply(new ResponseProvider() { ThreadLocal responseCode = new ThreadLocal<>(); @@ -132,8 +146,9 @@ public Object getBody(RecordedRequest recordedRequest) { }).always(); // Other resources - expect().get().withPath("/api/v1/namespaces/test/pods").andReturn(200, new PodListBuilder().withNewMetadata().withResourceVersion("1").and().build()).always(); - expect().get().withPath("/api/v1/namespaces/test/pods?resourceVersion=1&watch=true").andUpgradeToWebSocket().open().done().always(); + expect().get().withPath("/api/v1/namespaces/test/pods").andReply(200, request -> new PodListBuilder().withNewMetadata().withResourceVersion("1").and().withItems( + getCurrentPods().stream().map(name -> new PodBuilder().withNewMetadata().withName(name).and().build()).collect(Collectors.toList()) + ).build()).always(); } @@ -145,6 +160,18 @@ public void setRefuseRequests(boolean refuseRequests) { this.refuseRequests = refuseRequests; } + public synchronized Collection getCurrentPods() { + return new TreeSet<>(this.pods); + } + + public synchronized void removePod(String pod) { + this.pods.remove(pod); + } + + public synchronized void addPod(String pod) { + this.pods.add(pod); + } + public Long getDelayRequests() { return delayRequests; } From cc19aa24b9c992fc690114f39ea2ad564c1170fa Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Mon, 31 Jul 2017 17:40:53 +0200 Subject: [PATCH 08/13] CAMEL-11331: Speeding up default config --- .../kubernetes/ha/lock/KubernetesLeadershipController.java | 2 +- .../kubernetes/ha/lock/KubernetesLockConfiguration.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java index f5277798d01fd..b0c2110d8d8fb 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java @@ -346,7 +346,7 @@ private long jitter(long num, double factor) { } private String logPrefix() { - return "Leadership Controller [" + this.lockConfiguration.getPodName() + "]"; + return "Pod[" + this.lockConfiguration.getPodName() + "]"; } } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java index be0b42407ea2b..69c54d96386cd 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLockConfiguration.java @@ -29,9 +29,9 @@ public class KubernetesLockConfiguration implements Cloneable { public static final String DEFAULT_CONFIGMAP_NAME = "leaders"; public static final double DEFAULT_JITTER_FACTOR = 1.2; - public static final long DEFAULT_LEASE_DURATION_MILLIS = 60000; - public static final long DEFAULT_RENEW_DEADLINE_MILLIS = 45000; - public static final long DEFAULT_RETRY_PERIOD_MILLIS = 9000; + public static final long DEFAULT_LEASE_DURATION_MILLIS = 30000; + public static final long DEFAULT_RENEW_DEADLINE_MILLIS = 20000; + public static final long DEFAULT_RETRY_PERIOD_MILLIS = 5000; /** * Kubernetes namespace containing the pods and the ConfigMap used for locking. From 90f622d3b72e56d8342063d26f269e630860da04 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Mon, 31 Jul 2017 17:56:08 +0200 Subject: [PATCH 09/13] CAMEL-11331: Fixed logging and avoid unnecessary wait --- .../lock/KubernetesLeadershipController.java | 22 +++++++++---------- .../ha/lock/TimedLeaderNotifier.java | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java index b0c2110d8d8fb..2f79bd7a5eb33 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java @@ -130,7 +130,7 @@ private void refreshStatusNotLeader() { LOG.info("{} The cluster has no leaders. Trying to acquire the leadership...", logPrefix()); boolean acquired = tryAcquireLeadership(); if (acquired) { - LOG.info("{} Leadership acquired by current pod ({}) with immediate effect", logPrefix(), this.lockConfiguration.getPodName()); + LOG.info("{} Leadership acquired by current pod with immediate effect", logPrefix()); this.currentState = State.LEADER; this.serializedExecutor.execute(this::refreshStatus); return; @@ -142,7 +142,7 @@ private void refreshStatusNotLeader() { LOG.info("{} Leadership has been lost by old owner. Trying to acquire the leadership...", logPrefix()); boolean acquired = tryAcquireLeadership(); if (acquired) { - LOG.info("{} Leadership acquired by current pod ({})", logPrefix(), this.lockConfiguration.getPodName()); + LOG.info("{} Leadership acquired by current pod", logPrefix()); this.currentState = State.BECOMING_LEADER; this.serializedExecutor.execute(this::refreshStatus); return; @@ -151,7 +151,7 @@ private void refreshStatusNotLeader() { } } else if (this.latestLeaderInfo.isValidLeader(this.lockConfiguration.getPodName())) { // We are leaders for some reason (e.g. pod restart on failure) - LOG.info("{} Leadership is already owned by current pod ({})", logPrefix(), this.lockConfiguration.getPodName()); + LOG.info("{} Leadership is already owned by current pod", logPrefix()); this.currentState = State.BECOMING_LEADER; this.serializedExecutor.execute(this::refreshStatus); return; @@ -172,7 +172,7 @@ private void refreshStatusBecomingLeader() { // Wait always the same amount of time before becoming the leader // Even if the current pod is already leader, we should let a possible old version of the pod to shut down long delay = this.lockConfiguration.getLeaseDurationMillis(); - LOG.info("{} Current pod ({}) owns the leadership, but it will be effective in {} seconds...", logPrefix(), this.lockConfiguration.getPodName(), new BigDecimal(delay).divide(BigDecimal + LOG.info("{} Current pod owns the leadership, but it will be effective in {} seconds...", logPrefix(), new BigDecimal(delay).divide(BigDecimal .valueOf(1000), 2, BigDecimal.ROUND_HALF_UP)); try { @@ -181,7 +181,7 @@ private void refreshStatusBecomingLeader() { LOG.warn("Thread interrupted", e); } - LOG.info("{} Current pod ({}) is becoming the new leader now...", logPrefix(), this.lockConfiguration.getPodName()); + LOG.info("{} Current pod is becoming the new leader now...", logPrefix()); this.currentState = State.LEADER; this.serializedExecutor.execute(this::refreshStatus); } @@ -196,7 +196,7 @@ private void refreshStatusLeader() { } if (this.latestLeaderInfo.isValidLeader(this.lockConfiguration.getPodName())) { - LOG.debug("{} Current Pod ({}) is still the leader", logPrefix(), this.lockConfiguration.getPodName()); + LOG.debug("{} Current Pod is still the leader", logPrefix()); this.leaderNotifier.refreshLeadership(Optional.of(this.lockConfiguration.getPodName()), timeBeforePulling, this.lockConfiguration.getRenewDeadlineMillis(), @@ -204,7 +204,7 @@ private void refreshStatusLeader() { rescheduleAfterDelay(); return; } else { - LOG.debug("{} Current Pod ({}) has lost the leadership", logPrefix(), this.lockConfiguration.getPodName()); + LOG.debug("{} Current Pod has lost the leadership", logPrefix()); this.currentState = State.NOT_LEADER; // set a empty leader to signal leadership loss this.leaderNotifier.refreshLeadership(Optional.empty(), @@ -212,8 +212,8 @@ private void refreshStatusLeader() { lockConfiguration.getLeaseDurationMillis(), this.latestLeaderInfo.getMembers()); - // wait a lease time and restart - this.serializedExecutor.schedule(this::refreshStatus, this.lockConfiguration.getLeaseDurationMillis(), TimeUnit.MILLISECONDS); + // restart from scratch to acquire leadership + this.serializedExecutor.execute(this::refreshStatus); } } @@ -257,7 +257,7 @@ private boolean tryAcquireLeadership() { LOG.warn(logPrefix() + " Unexpected condition. Latest leader info or list of members is empty."); return false; } else if (!members.contains(this.lockConfiguration.getPodName())) { - LOG.warn(logPrefix() + " The list of cluster members " + latestLeaderInfo.getMembers() + " does not contain the current pod (" + this.lockConfiguration.getPodName() + "). Cannot acquire" + LOG.warn(logPrefix() + " The list of cluster members " + latestLeaderInfo.getMembers() + " does not contain the current Pod. Cannot acquire" + " leadership."); return false; } @@ -311,7 +311,7 @@ private boolean tryAcquireLeadership() { } } else { // Another pod is the leader and it's still active - LOG.debug("{} Another pod ({}) is the current leader and it is still active", logPrefix(), this.latestLeaderInfo.getLeader()); + LOG.debug("{} Another Pod ({}) is the current leader and it is still active", logPrefix(), this.latestLeaderInfo.getLeader()); return false; } } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java index 6ada830b04b1a..6c512267092cc 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java @@ -155,6 +155,7 @@ private void checkAndNotify(long version) { final Optional newLeader = leader; if (!newLeader.equals(lastCommunicatedLeader)) { lastCommunicatedLeader = newLeader; + LOG.debug("Communicating new leader: {}" + newLeader); handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent() { @Override public Optional getData() { @@ -166,6 +167,7 @@ public Optional getData() { final Set newMembers = members; if (!newMembers.equals(lastCommunicatedMembers)) { lastCommunicatedMembers = newMembers; + LOG.debug("Communicating new cluster members: {}" + newMembers); handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent() { @Override public Set getData() { From 101dadbc0f1a59181615fd2b7dd4fd76992d7037 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Tue, 1 Aug 2017 10:30:26 +0200 Subject: [PATCH 10/13] CAMEL-11331: Added better logging and upgrade library --- components/camel-kubernetes/pom.xml | 6 ++-- .../ha/lock/TimedLeaderNotifier.java | 36 +++++++++++-------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/components/camel-kubernetes/pom.xml b/components/camel-kubernetes/pom.xml index 38fa0370f3bd8..0d7b72bbcb614 100644 --- a/components/camel-kubernetes/pom.xml +++ b/components/camel-kubernetes/pom.xml @@ -44,14 +44,12 @@ io.fabric8 kubernetes-client - 2.3-SNAPSHOT - + ${kubernetes-client-version} io.fabric8 openshift-client - 2.3-SNAPSHOT - + ${kubernetes-client-version} diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java index 6c512267092cc..c95b51711538d 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java @@ -155,25 +155,33 @@ private void checkAndNotify(long version) { final Optional newLeader = leader; if (!newLeader.equals(lastCommunicatedLeader)) { lastCommunicatedLeader = newLeader; - LOG.debug("Communicating new leader: {}" + newLeader); - handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent() { - @Override - public Optional getData() { - return newLeader; - } - }); + LOG.info("The cluster has a new leader: {}", newLeader); + try { + handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent() { + @Override + public Optional getData() { + return newLeader; + } + }); + } catch (Throwable t) { + LOG.warn("Error while communicating the new leader to the handler", t); + } } final Set newMembers = members; if (!newMembers.equals(lastCommunicatedMembers)) { lastCommunicatedMembers = newMembers; - LOG.debug("Communicating new cluster members: {}" + newMembers); - handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent() { - @Override - public Set getData() { - return newMembers; - } - }); + LOG.info("The list of cluster members has changed: {}", newMembers); + try { + handler.onKubernetesClusterEvent(new KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent() { + @Override + public Set getData() { + return newMembers; + } + }); + } catch (Throwable t) { + LOG.warn("Error while communicating the cluster members to the handler", t); + } } } From 5e2f1d579dfd6d23eeff2e8aeb4790b8edc2a881 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Tue, 8 Aug 2017 12:05:59 +0200 Subject: [PATCH 11/13] CAMEL-11331: Adding connection timeout as parameter (auto-computed) --- .../kubernetes-build-configs-component.adoc | 3 +- .../docs/kubernetes-builds-component.adoc | 3 +- .../src/main/docs/kubernetes-component.adoc | 3 +- .../kubernetes-config-maps-component.adoc | 3 +- .../kubernetes-deployments-component.adoc | 3 +- .../docs/kubernetes-namespaces-component.adoc | 3 +- .../main/docs/kubernetes-nodes-component.adoc | 3 +- ...s-persistent-volumes-claims-component.adoc | 3 +- ...bernetes-persistent-volumes-component.adoc | 3 +- .../main/docs/kubernetes-pods-component.adoc | 3 +- ...tes-replication-controllers-component.adoc | 3 +- .../kubernetes-resources-quota-component.adoc | 3 +- .../docs/kubernetes-secrets-component.adoc | 3 +- .../docs/kubernetes-services-component.adoc | 3 +- .../kubernetes/KubernetesConfiguration.java | 16 +++++++++- .../kubernetes/KubernetesHelper.java | 3 ++ .../ha/KubernetesClusterService.java | 30 +++++++++++++++++-- 17 files changed, 73 insertions(+), 18 deletions(-) diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc index d0029aad173ae..5acd8c3323e17 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc @@ -30,7 +30,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -40,6 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc index 61714a75f313d..e5c73ed594ff7 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc @@ -30,7 +30,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -40,6 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc index 8c6551b3e3b49..91a7ad17b3eaa 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc @@ -78,7 +78,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (27 parameters): +#### Query Parameters (28 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -97,6 +97,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc index f1feefa5e89dd..d424b5951b7cd 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc @@ -30,7 +30,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -40,6 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc index 938635aca823d..8e9a0c1458df4 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -49,6 +49,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc index 8c2546f65002c..1d15a94c53ab6 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc @@ -32,7 +32,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -50,6 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc index 0e76719c0ca4c..04b938cb76474 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc @@ -32,7 +32,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -50,6 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc index cc3f73ca71ea9..46b4ecf2133dc 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -41,6 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc index e60f116106c6a..a2e38267b42b8 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -41,6 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc index 45d3407700e46..988e3da204bb0 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -49,6 +49,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc index 937f49a61ed34..184ba3fae3143 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc @@ -32,7 +32,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -50,6 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc index c89027e2560e5..400c0863684df 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -41,6 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc index 7e0dc792604cf..ac734c4c91dba 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc @@ -31,7 +31,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (18 parameters): +#### Query Parameters (19 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -41,6 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc index 8970059b77bbc..4713ab01934f1 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc @@ -32,7 +32,7 @@ with the following path and query parameters: | **masterUrl** | *Required* Kubernetes Master url | | String |======================================================================= -#### Query Parameters (26 parameters): +#### Query Parameters (27 parameters): [width="100%",cols="2,5,^1,2",options="header"] |======================================================================= @@ -50,6 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java index 271ef711a533b..51bd8394a04b8 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesConfiguration.java @@ -115,6 +115,9 @@ public class KubernetesConfiguration implements Cloneable { @UriParam(label = "consumer", defaultValue = "1") private int poolSize = 1; + @UriParam(label = "advanced") + private Integer connectionTimeout; + /** * Kubernetes Master url */ @@ -396,6 +399,17 @@ public void setResourceName(String resourceName) { this.resourceName = resourceName; } + public Integer getConnectionTimeout() { + return connectionTimeout; + } + + /** + * Connection timeout in milliseconds to use when making requests to the Kubernetes API server. + */ + public void setConnectionTimeout(Integer connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + // **************************************** // Copy // **************************************** @@ -418,7 +432,7 @@ public String toString() { + ", clientKeyPassphrase=" + clientKeyPassphrase + ", oauthToken=" + oauthToken + ", trustCerts=" + trustCerts + ", namespace=" + namespace + ", labelKey=" + labelKey + ", labelValue=" + labelValue + ", resourceName=" + resourceName + ", portName=" + portName + ", dnsDomain=" + dnsDomain - + ", poolSize=" + poolSize + "]"; + + ", poolSize=" + poolSize + ", connectionTimeout=" + connectionTimeout + "]"; } } diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java index 62235adb2ee9a..213fd5c472c03 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/KubernetesHelper.java @@ -90,6 +90,9 @@ private static KubernetesClient createKubernetesClient(KubernetesConfiguration c if (ObjectHelper.isNotEmpty(configuration.getTrustCerts())) { builder.withTrustCerts(configuration.getTrustCerts()); } + if (ObjectHelper.isNotEmpty(configuration.getConnectionTimeout())) { + builder.withConnectionTimeout(configuration.getConnectionTimeout()); + } Config conf = builder.build(); return new DefaultKubernetesClient(conf); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java index 08ebb70ea062a..1c95b2263a156 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java @@ -55,11 +55,24 @@ public KubernetesClusterService(CamelContext camelContext, KubernetesConfigurati @Override protected KubernetesClusterView createView(String namespace) throws Exception { - KubernetesLockConfiguration lockConfig = configWithGroupNameAndDefaults(namespace); - return new KubernetesClusterView(this, configuration, lockConfig); + KubernetesLockConfiguration lockConfig = lockConfigWithGroupNameAndDefaults(namespace); + KubernetesConfiguration config = setConfigDefaults(this.configuration.copy(), lockConfig); + return new KubernetesClusterView(this, config, lockConfig); + } + + protected KubernetesConfiguration setConfigDefaults(KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { + if (configuration.getConnectionTimeout() == null) { + // Set the connection timeout to be much lower than the renewal deadline, + // to avoid losing the leadership in case of stale connections + int timeout = (int) (lockConfiguration.getRenewDeadlineMillis() / 3); + timeout = Math.max(timeout, 3000); + timeout = Math.min(timeout, 30000); + configuration.setConnectionTimeout(timeout); + } + return configuration; } - protected KubernetesLockConfiguration configWithGroupNameAndDefaults(String groupName) { + protected KubernetesLockConfiguration lockConfigWithGroupNameAndDefaults(String groupName) { KubernetesLockConfiguration config = this.lockConfiguration.copy(); config.setGroupName(ObjectHelper.notNull(groupName, "groupName")); @@ -114,6 +127,17 @@ public void setMasterUrl(String masterUrl) { configuration.setMasterUrl(masterUrl); } + public Integer getConnectionTimeout() { + return configuration.getConnectionTimeout(); + } + + /** + * Connection timeout in milliseconds to use when making requests to the Kubernetes API server. + */ + public void setConnectionTimeout(Integer connectionTimeout) { + configuration.setConnectionTimeout(connectionTimeout); + } + public String getKubernetesNamespace() { return this.lockConfiguration.getKubernetesResourcesNamespace(); } From 9b829f29c8265a5561a4b373a8199d5d1a0c48d1 Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Tue, 8 Aug 2017 12:40:24 +0200 Subject: [PATCH 12/13] CAMEL-11331: Fix doc --- .../src/main/docs/kubernetes-build-configs-component.adoc | 2 +- .../src/main/docs/kubernetes-builds-component.adoc | 2 +- .../camel-kubernetes/src/main/docs/kubernetes-component.adoc | 2 +- .../src/main/docs/kubernetes-config-maps-component.adoc | 2 +- .../src/main/docs/kubernetes-deployments-component.adoc | 2 +- .../src/main/docs/kubernetes-namespaces-component.adoc | 2 +- .../src/main/docs/kubernetes-nodes-component.adoc | 2 +- .../docs/kubernetes-persistent-volumes-claims-component.adoc | 2 +- .../src/main/docs/kubernetes-persistent-volumes-component.adoc | 2 +- .../src/main/docs/kubernetes-pods-component.adoc | 2 +- .../main/docs/kubernetes-replication-controllers-component.adoc | 2 +- .../src/main/docs/kubernetes-resources-quota-component.adoc | 2 +- .../src/main/docs/kubernetes-secrets-component.adoc | 2 +- .../src/main/docs/kubernetes-services-component.adoc | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc index 5acd8c3323e17..b9ba15bf66a05 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-build-configs-component.adoc @@ -40,7 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc index e5c73ed594ff7..6273f2361a55e 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-builds-component.adoc @@ -40,7 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc index 91a7ad17b3eaa..b9ecb5bebaf39 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-component.adoc @@ -97,7 +97,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc index d424b5951b7cd..cdcaf945b6ece 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-config-maps-component.adoc @@ -40,7 +40,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc index 8e9a0c1458df4..b3a2349b6591c 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-deployments-component.adoc @@ -49,7 +49,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc index 1d15a94c53ab6..ba4893f269bf0 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-namespaces-component.adoc @@ -50,7 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc index 04b938cb76474..cfc9ecc36ff7d 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-nodes-component.adoc @@ -50,7 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc index 46b4ecf2133dc..37343db0732b3 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-claims-component.adoc @@ -41,7 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc index a2e38267b42b8..fbc18b7213d47 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-persistent-volumes-component.adoc @@ -41,7 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc index 988e3da204bb0..0a720fba18304 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-pods-component.adoc @@ -49,7 +49,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc index 184ba3fae3143..05c725c52617c 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-replication-controllers-component.adoc @@ -50,7 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc index 400c0863684df..f0c5dde42cc1f 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-resources-quota-component.adoc @@ -41,7 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc index ac734c4c91dba..ed30fa2331723 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-secrets-component.adoc @@ -41,7 +41,7 @@ with the following path and query parameters: | **kubernetesClient** (producer) | Default KubernetesClient to use if provided | | KubernetesClient | **operation** (producer) | Producer operation to do on Kubernetes | | String | **portName** (producer) | The port name used for ServiceCall EIP | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String diff --git a/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc b/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc index 4713ab01934f1..ec622072b94e4 100644 --- a/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc +++ b/components/camel-kubernetes/src/main/docs/kubernetes-services-component.adoc @@ -50,7 +50,7 @@ with the following path and query parameters: | **exceptionHandler** (consumer) | To let the consumer use a custom ExceptionHandler. Notice if the option bridgeErrorHandler is enabled then this options is not in use. By default the consumer will deal with exceptions that will be logged at WARN or ERROR level and ignored. | | ExceptionHandler | **exchangePattern** (consumer) | Sets the exchange pattern when the consumer creates an exchange. | | ExchangePattern | **operation** (producer) | Producer operation to do on Kubernetes | | String -| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the kubernetes API. | | Integer +| **connectionTimeout** (advanced) | Connection timeout in milliseconds to use when making requests to the Kubernetes API server. | | Integer | **synchronous** (advanced) | Sets whether synchronous processing should be strictly used or Camel is allowed to use asynchronous processing (if supported). | false | boolean | **caCertData** (security) | The CA Cert Data | | String | **caCertFile** (security) | The CA Cert File | | String From 7806c175051b62b3dc17ca9a1b624b3b8a3980ea Mon Sep 17 00:00:00 2001 From: Nicola Ferraro Date: Tue, 8 Aug 2017 15:39:22 +0200 Subject: [PATCH 13/13] CAMEL-11331: Using Camel thread pools --- .../kubernetes/ha/KubernetesClusterService.java | 2 +- .../kubernetes/ha/KubernetesClusterView.java | 12 ++++++++---- .../ha/lock/KubernetesLeadershipController.java | 11 +++++++---- .../kubernetes/ha/lock/TimedLeaderNotifier.java | 9 ++++++--- .../kubernetes/ha/KubernetesClusterServiceTest.java | 2 +- .../kubernetes/ha/TimedLeaderNotifierTest.java | 10 +++++++++- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java index 1c95b2263a156..00cb04d71b145 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterService.java @@ -57,7 +57,7 @@ public KubernetesClusterService(CamelContext camelContext, KubernetesConfigurati protected KubernetesClusterView createView(String namespace) throws Exception { KubernetesLockConfiguration lockConfig = lockConfigWithGroupNameAndDefaults(namespace); KubernetesConfiguration config = setConfigDefaults(this.configuration.copy(), lockConfig); - return new KubernetesClusterView(this, config, lockConfig); + return new KubernetesClusterView(getCamelContext(), this, config, lockConfig); } protected KubernetesConfiguration setConfigDefaults(KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java index ddda67529e4c8..a67b6623033fc 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterView.java @@ -27,6 +27,7 @@ import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.camel.CamelContext; import org.apache.camel.component.kubernetes.KubernetesConfiguration; import org.apache.camel.component.kubernetes.KubernetesHelper; import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; @@ -42,6 +43,8 @@ */ public class KubernetesClusterView extends AbstractCamelClusterView { + private CamelContext camelContext; + private KubernetesClient kubernetesClient; private KubernetesConfiguration configuration; @@ -58,10 +61,11 @@ public class KubernetesClusterView extends AbstractCamelClusterView { private KubernetesLeadershipController controller; - public KubernetesClusterView(KubernetesClusterService cluster, KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { + public KubernetesClusterView(CamelContext camelContext, KubernetesClusterService cluster, KubernetesConfiguration configuration, KubernetesLockConfiguration lockConfiguration) { super(cluster, lockConfiguration.getGroupName()); - this.configuration = configuration; - this.lockConfiguration = lockConfiguration; + this.camelContext = ObjectHelper.notNull(camelContext, "camelContext"); + this.configuration = ObjectHelper.notNull(configuration, "configuration"); + this.lockConfiguration = ObjectHelper.notNull(lockConfiguration, "lockConfiguration"); this.localMember = new KubernetesClusterMember(lockConfiguration.getPodName()); this.memberCache = new HashMap<>(); } @@ -86,7 +90,7 @@ protected void doStart() throws Exception { if (controller == null) { this.kubernetesClient = KubernetesHelper.getKubernetesClient(configuration); - controller = new KubernetesLeadershipController(kubernetesClient, this.lockConfiguration, event -> { + controller = new KubernetesLeadershipController(camelContext, kubernetesClient, this.lockConfiguration, event -> { if (event instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { // New leader Optional leader = KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent.class.cast(event).getData(); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java index 2f79bd7a5eb33..25a09f82254e8 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/KubernetesLeadershipController.java @@ -22,7 +22,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -31,6 +30,7 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.camel.CamelContext; import org.apache.camel.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +49,8 @@ private enum State { LEADER } + private CamelContext camelContext; + private KubernetesClient kubernetesClient; private KubernetesLockConfiguration lockConfiguration; @@ -65,7 +67,8 @@ private enum State { private volatile ConfigMap latestConfigMap; private volatile Set latestMembers; - public KubernetesLeadershipController(KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { + public KubernetesLeadershipController(CamelContext camelContext, KubernetesClient kubernetesClient, KubernetesLockConfiguration lockConfiguration, KubernetesClusterEventHandler eventHandler) { + this.camelContext = camelContext; this.kubernetesClient = kubernetesClient; this.lockConfiguration = lockConfiguration; this.eventHandler = eventHandler; @@ -75,8 +78,8 @@ public KubernetesLeadershipController(KubernetesClient kubernetesClient, Kuberne public void start() throws Exception { if (serializedExecutor == null) { LOG.debug("{} Starting leadership controller...", logPrefix()); - serializedExecutor = Executors.newSingleThreadScheduledExecutor(); - leaderNotifier = new TimedLeaderNotifier(this.eventHandler); + serializedExecutor = camelContext.getExecutorServiceManager().newSingleThreadScheduledExecutor(this, "CamelKubernetesLeadershipController"); + leaderNotifier = new TimedLeaderNotifier(this.camelContext, this.eventHandler); leaderNotifier.start(); serializedExecutor.execute(this::refreshStatus); diff --git a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java index c95b51711538d..f8055360d73b0 100644 --- a/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java +++ b/components/camel-kubernetes/src/main/java/org/apache/camel/component/kubernetes/ha/lock/TimedLeaderNotifier.java @@ -20,12 +20,12 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.camel.CamelContext; import org.apache.camel.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,6 +39,8 @@ public class TimedLeaderNotifier implements Service { private static final long FIXED_DELAY = 10; + private CamelContext camelContext; + private KubernetesClusterEventHandler handler; private Lock lock = new ReentrantLock(); @@ -58,7 +60,8 @@ public class TimedLeaderNotifier implements Service { private long changeCounter; - public TimedLeaderNotifier(KubernetesClusterEventHandler handler) { + public TimedLeaderNotifier(CamelContext camelContext, KubernetesClusterEventHandler handler) { + this.camelContext = Objects.requireNonNull(camelContext, "Camel context must be present"); this.handler = Objects.requireNonNull(handler, "Handler must be present"); } @@ -91,7 +94,7 @@ public void refreshLeadership(Optional leader, Long timestamp, Long leas @Override public void start() throws Exception { if (this.executor == null) { - this.executor = Executors.newSingleThreadScheduledExecutor(); + this.executor = camelContext.getExecutorServiceManager().newSingleThreadScheduledExecutor(this, "CamelKubernetesLeaderNotifier"); } } diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java index 4a2a11e5d95d5..62174c23dfd06 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/KubernetesClusterServiceTest.java @@ -285,8 +285,8 @@ private LeaderRecorder addMember(String name, String namespace) { LeaderRecorder recorder = new LeaderRecorder(); try { - member.getView(namespace).addEventListener(recorder); context().addService(member); + member.getView(namespace).addEventListener(recorder); } catch (Exception ex) { throw new RuntimeException(ex); } diff --git a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java index 8380147da357f..164912d60e95d 100644 --- a/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java +++ b/components/camel-kubernetes/src/test/java/org/apache/camel/component/kubernetes/ha/TimedLeaderNotifierTest.java @@ -22,8 +22,10 @@ import java.util.Set; import java.util.TreeSet; +import org.apache.camel.CamelContext; import org.apache.camel.component.kubernetes.ha.lock.KubernetesClusterEvent; import org.apache.camel.component.kubernetes.ha.lock.TimedLeaderNotifier; +import org.apache.camel.impl.DefaultCamelContext; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -35,6 +37,8 @@ */ public class TimedLeaderNotifierTest { + private CamelContext context; + private TimedLeaderNotifier notifier; private volatile Optional currentLeader; @@ -43,7 +47,10 @@ public class TimedLeaderNotifierTest { @Before public void init() throws Exception { - this.notifier = new TimedLeaderNotifier(e -> { + this.context = new DefaultCamelContext(); + this.context.start(); + + this.notifier = new TimedLeaderNotifier(context, e -> { if (e instanceof KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) { currentLeader = ((KubernetesClusterEvent.KubernetesClusterLeaderChangedEvent) e).getData(); } else if (e instanceof KubernetesClusterEvent.KubernetesClusterMemberListChangedEvent) { @@ -56,6 +63,7 @@ public void init() throws Exception { @After public void destroy() throws Exception { this.notifier.stop(); + this.context.stop(); } @Test