forked from openshift/cluster-etcd-operator
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Replace etcd endpoint representation with configmap
Kube service design asserts `endpoint` resources cannot exist without a corresponding `service` resource, and Kube will actively delete the endpoint when the service is deleted or if Kube detects the endpoint is a "stray". The operator needs to: 1. Manage etcd endpoint state atomically. 2. Maintain exclusive ownership of the etcd endpoint state resource. Altogether this makes the `endpoint` resource inappropriate for the task. The competition between the operator and the Kube endpoints controller to manage the endpoint has led to instability. To resolve the problems, persist etcd endpoint state in a `configmap`. Maintain compatibility by continuing to write the `endpoint`, and update consuming components to prefer the `configmap` over the `endpoint`. Also requires: openshift/cluster-kube-apiserver-operator#859 openshift/cluster-openshift-apiserver-operator#364
- Loading branch information
1 parent
ec550f4
commit c52f9e7
Showing
6 changed files
with
163 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: etcd-endpoints | ||
| namespace: openshift-etcd | ||
| annotations: | ||
| alpha.installer.openshift.io/etcd-bootstrap: {{ .BootstrapIP }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
126 changes: 126 additions & 0 deletions
126
pkg/operator/etcdendpointscontroller/etcdendpointscontroller.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| package etcdendpointscontroller | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/base64" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| operatorv1 "github.com/openshift/api/operator/v1" | ||
| "github.com/openshift/library-go/pkg/controller/factory" | ||
| "github.com/openshift/library-go/pkg/operator/events" | ||
| "github.com/openshift/library-go/pkg/operator/resource/resourceapply" | ||
| "github.com/openshift/library-go/pkg/operator/v1helpers" | ||
| operatorv1helpers "github.com/openshift/library-go/pkg/operator/v1helpers" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/labels" | ||
| "k8s.io/client-go/kubernetes" | ||
| corev1client "k8s.io/client-go/kubernetes/typed/core/v1" | ||
| corev1listers "k8s.io/client-go/listers/core/v1" | ||
|
|
||
| "github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient" | ||
| ) | ||
|
|
||
| // EtcdEndpointsController maintains a configmap resource with | ||
| // IP addresses for etcd. It should never depend on DNS directly or transitively. | ||
| type EtcdEndpointsController struct { | ||
| operatorClient v1helpers.OperatorClient | ||
| nodeLister corev1listers.NodeLister | ||
| configmapLister corev1listers.ConfigMapLister | ||
| configmapClient corev1client.ConfigMapsGetter | ||
| } | ||
|
|
||
| func NewEtcdEndpointsController( | ||
| operatorClient v1helpers.OperatorClient, | ||
| eventRecorder events.Recorder, | ||
| kubeClient kubernetes.Interface, | ||
| kubeInformers operatorv1helpers.KubeInformersForNamespaces, | ||
| ) factory.Controller { | ||
| kubeInformersForTargetNamespace := kubeInformers.InformersFor(operatorclient.TargetNamespace) | ||
| configmapsInformer := kubeInformersForTargetNamespace.Core().V1().ConfigMaps() | ||
| kubeInformersForCluster := kubeInformers.InformersFor("") | ||
| nodeInformer := kubeInformersForCluster.Core().V1().Nodes() | ||
|
|
||
| c := &EtcdEndpointsController{ | ||
| operatorClient: operatorClient, | ||
| nodeLister: nodeInformer.Lister(), | ||
| configmapLister: configmapsInformer.Lister(), | ||
| configmapClient: kubeClient.CoreV1(), | ||
| } | ||
| return factory.New().ResyncEvery(time.Minute).WithInformers( | ||
| operatorClient.Informer(), | ||
| configmapsInformer.Informer(), | ||
| nodeInformer.Informer(), | ||
| ).WithSync(c.sync).ToController("EtcdEndpointsController", eventRecorder.WithComponentSuffix("etcd-endpoints-controller")) | ||
| } | ||
|
|
||
| func (c *EtcdEndpointsController) sync(ctx context.Context, syncCtx factory.SyncContext) error { | ||
| err := c.syncConfigMap(ctx, syncCtx.Recorder()) | ||
|
|
||
| if err != nil { | ||
| _, _, updateErr := v1helpers.UpdateStatus(c.operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ | ||
| Type: "EtcdEndpointsDegraded", | ||
| Status: operatorv1.ConditionTrue, | ||
| Reason: "ErrorUpdatingEtcdEndpoints", | ||
| Message: err.Error(), | ||
| })) | ||
| if updateErr != nil { | ||
| syncCtx.Recorder().Warning("EtcdEndpointsErrorUpdatingStatus", updateErr.Error()) | ||
| } | ||
| return err | ||
| } | ||
|
|
||
| _, _, updateErr := v1helpers.UpdateStatus(c.operatorClient, v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{ | ||
| Type: "EtcdEndpointsDegraded", | ||
| Status: operatorv1.ConditionFalse, | ||
| Reason: "EtcdEndpointsUpdated", | ||
| })) | ||
| if updateErr != nil { | ||
| syncCtx.Recorder().Warning("EtcdEndpointsErrorUpdatingStatus", updateErr.Error()) | ||
| return updateErr | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (c *EtcdEndpointsController) syncConfigMap(ctx context.Context, recorder events.Recorder) error { | ||
| required := configMapAsset() | ||
|
|
||
| // create endpoint addresses for each node | ||
| nodes, err := c.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector()) | ||
| if err != nil { | ||
| return fmt.Errorf("unable to list expected etcd member nodes: %v", err) | ||
| } | ||
| endpointAddresses := map[string]string{} | ||
| for _, node := range nodes { | ||
| var nodeInternalIP string | ||
| for _, nodeAddress := range node.Status.Addresses { | ||
| if nodeAddress.Type == corev1.NodeInternalIP { | ||
| nodeInternalIP = nodeAddress.Address | ||
| break | ||
| } | ||
| } | ||
| if len(nodeInternalIP) == 0 { | ||
| return fmt.Errorf("unable to determine internal ip address for node %s", node.Name) | ||
| } | ||
| endpointAddresses[base64.StdEncoding.WithPadding(base64.NoPadding).EncodeToString([]byte(nodeInternalIP))] = nodeInternalIP | ||
| } | ||
|
|
||
| if len(endpointAddresses) == 0 { | ||
| return fmt.Errorf("no master nodes are present") | ||
| } | ||
|
|
||
| required.Data = endpointAddresses | ||
|
|
||
| _, _, err = resourceapply.ApplyConfigMap(c.configmapClient, recorder, required) | ||
| return err | ||
| } | ||
|
|
||
| func configMapAsset() *corev1.ConfigMap { | ||
| return &corev1.ConfigMap{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: "etcd-endpoints", | ||
| Namespace: operatorclient.TargetNamespace, | ||
| }, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters