From d6e430c4e1de6c22cc132f722452463ab8b5c502 Mon Sep 17 00:00:00 2001 From: Sergey Shevchenko Date: Wed, 27 Mar 2024 20:25:34 +0200 Subject: [PATCH] feat: Refactor resource generation (#84) Closes #55 --- .../controller/etcdcluster_controller_test.go | 2 +- internal/controller/factory/builders.go | 42 ++++++- internal/controller/factory/configMap.go | 55 ++++++++- internal/controller/factory/configmap_test.go | 90 ++++++++++++++ internal/controller/factory/statefulset.go | 2 +- .../controller/factory/statefulset_test.go | 18 +++ internal/controller/factory/svc.go | 110 ++++++++++++++++++ internal/controller/factory/svc_test.go | 100 ++++++++++++++++ 8 files changed, 415 insertions(+), 4 deletions(-) create mode 100644 internal/controller/factory/configmap_test.go create mode 100644 internal/controller/factory/svc.go create mode 100644 internal/controller/factory/svc_test.go diff --git a/internal/controller/etcdcluster_controller_test.go b/internal/controller/etcdcluster_controller_test.go index be4af94e..386e5908 100644 --- a/internal/controller/etcdcluster_controller_test.go +++ b/internal/controller/etcdcluster_controller_test.go @@ -119,7 +119,7 @@ var _ = Describe("EtcdCluster Controller", func() { svc = &v1.Service{} clientSvcName := types.NamespacedName{ Namespace: typeNamespacedName.Namespace, - Name: controllerReconciler.getClientServiceName(etcdcluster), + Name: factory.GetClientServiceName(etcdcluster), } err = k8sClient.Get(ctx, clientSvcName, svc) Expect(err).NotTo(HaveOccurred(), "cluster client Service should exist") diff --git a/internal/controller/factory/builders.go b/internal/controller/factory/builders.go index 8887bb0f..f6724515 100644 --- a/internal/controller/factory/builders.go +++ b/internal/controller/factory/builders.go @@ -21,6 +21,7 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -28,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) -func reconcileSTS(ctx context.Context, rclient client.Client, crdName string, sts *appsv1.StatefulSet) error { +func reconcileStatefulSet(ctx context.Context, rclient client.Client, crdName string, sts *appsv1.StatefulSet) error { logger := log.FromContext(ctx) currentSts := &appsv1.StatefulSet{} @@ -47,3 +48,42 @@ func reconcileSTS(ctx context.Context, rclient client.Client, crdName string, st sts.Status = currentSts.Status return rclient.Update(ctx, sts) } + +func reconcileConfigMap(ctx context.Context, rclient client.Client, crdName string, configMap *corev1.ConfigMap) error { + logger := log.FromContext(ctx) + + currentConfigMap := &corev1.ConfigMap{} + err := rclient.Get(ctx, types.NamespacedName{Namespace: configMap.Namespace, Name: configMap.Name}, currentConfigMap) + if err != nil { + if errors.IsNotFound(err) { + logger.V(2).Info("creating new configMap", "cm_name", configMap.Name, "crd_object", crdName) + return rclient.Create(ctx, configMap) + } + return fmt.Errorf("cannot get existing configMap: %s, for crd_object: %s, err: %w", configMap.Name, crdName, err) + } + configMap.Annotations = labels.Merge(currentConfigMap.Annotations, configMap.Annotations) + if configMap.ResourceVersion != "" { + configMap.ResourceVersion = currentConfigMap.ResourceVersion + } + return rclient.Update(ctx, configMap) +} + +func reconcileService(ctx context.Context, rclient client.Client, crdName string, svc *corev1.Service) error { + logger := log.FromContext(ctx) + + currentSvc := &corev1.Service{} + err := rclient.Get(ctx, types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}, currentSvc) + if err != nil { + if errors.IsNotFound(err) { + logger.V(2).Info("creating new service", "svc_name", svc.Name, "crd_object", crdName) + return rclient.Create(ctx, svc) + } + return fmt.Errorf("cannot get existing service: %s, for crd_object: %s, err: %w", svc.Name, crdName, err) + } + svc.Annotations = labels.Merge(currentSvc.Annotations, svc.Annotations) + if svc.ResourceVersion != "" { + svc.ResourceVersion = currentSvc.ResourceVersion + } + svc.Status = currentSvc.Status + return rclient.Update(ctx, svc) +} diff --git a/internal/controller/factory/configMap.go b/internal/controller/factory/configMap.go index c1356333..1f6b0104 100644 --- a/internal/controller/factory/configMap.go +++ b/internal/controller/factory/configMap.go @@ -16,8 +16,61 @@ limitations under the License. package factory -import etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" +) func GetClusterStateConfigMapName(cluster *etcdaenixiov1alpha1.EtcdCluster) string { return cluster.Name + "-cluster-state" } + +func CreateOrUpdateClusterStateConfigMap( + ctx context.Context, + cluster *etcdaenixiov1alpha1.EtcdCluster, + isClusterReady bool, + rclient client.Client, + rscheme *runtime.Scheme, +) error { + initialCluster := "" + for i := int32(0); i < *cluster.Spec.Replicas; i++ { + if i > 0 { + initialCluster += "," + } + initialCluster += fmt.Sprintf("%s-%d=https://%s-%d.%s.%s.svc:2380", + cluster.Name, i, + cluster.Name, i, cluster.Name, cluster.Namespace, + ) + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: GetClusterStateConfigMapName(cluster), + }, + Data: map[string]string{ + "ETCD_INITIAL_CLUSTER_STATE": "new", + "ETCD_INITIAL_CLUSTER": initialCluster, + "ETCD_INITIAL_CLUSTER_TOKEN": cluster.Name + "-" + cluster.Namespace, + }, + } + + if isClusterReady { + // update cluster state to existing + configMap.Data["ETCD_INITIAL_CLUSTER_STATE"] = "existing" + } + + if err := ctrl.SetControllerReference(cluster, configMap, rscheme); err != nil { + return fmt.Errorf("cannot set controller reference: %w", err) + } + + return reconcileConfigMap(ctx, rclient, cluster.Name, configMap) +} diff --git a/internal/controller/factory/configmap_test.go b/internal/controller/factory/configmap_test.go new file mode 100644 index 00000000..f6fcda3a --- /dev/null +++ b/internal/controller/factory/configmap_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package factory + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" +) + +var _ = Describe("CreateOrUpdateClusterStateConfigMap handlers", func() { + Context("When ensuring a configMap", func() { + const resourceName = "test-resource" + + ctx := context.Background() + + etcdcluster := &etcdaenixiov1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + UID: "test-uid", + }, + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Replicas: ptr.To(int32(3)), + }, + } + typeNamespacedName := types.NamespacedName{ + Name: GetClusterStateConfigMapName(etcdcluster), + Namespace: "default", + } + + It("should successfully ensure the configmap", func() { + cm := &corev1.ConfigMap{} + + By("creating the configmap for initial cluster") + err := CreateOrUpdateClusterStateConfigMap(ctx, etcdcluster, false, k8sClient, k8sClient.Scheme()) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, cm) + cmUid := cm.UID + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["ETCD_INITIAL_CLUSTER_STATE"]).To(Equal("new")) + + By("updating the configmap for initialized cluster") + err = CreateOrUpdateClusterStateConfigMap(ctx, etcdcluster, true, k8sClient, k8sClient.Scheme()) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, cm) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["ETCD_INITIAL_CLUSTER_STATE"]).To(Equal("existing")) + // Check that we are updating the same configmap + Expect(cm.UID).To(Equal(cmUid)) + + By("deleting the configmap") + + Expect(k8sClient.Delete(ctx, cm)).To(Succeed()) + }) + + It("should fail on creating the configMap with invalid owner reference", func() { + etcdcluster := etcdcluster.DeepCopy() + emptyScheme := runtime.NewScheme() + + err := CreateOrUpdateClusterStateConfigMap(ctx, etcdcluster, false, k8sClient, emptyScheme) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/internal/controller/factory/statefulset.go b/internal/controller/factory/statefulset.go index 413dbe24..4762d855 100644 --- a/internal/controller/factory/statefulset.go +++ b/internal/controller/factory/statefulset.go @@ -198,7 +198,7 @@ func CreateOrUpdateStatefulSet( return fmt.Errorf("cannot set controller reference: %w", err) } - return reconcileSTS(ctx, rclient, cluster.Name, statefulSet) + return reconcileStatefulSet(ctx, rclient, cluster.Name, statefulSet) } func generateEtcdCommand(cluster *etcdaenixiov1alpha1.EtcdCluster) []string { diff --git a/internal/controller/factory/statefulset_test.go b/internal/controller/factory/statefulset_test.go index 88f904b8..49f3ece9 100644 --- a/internal/controller/factory/statefulset_test.go +++ b/internal/controller/factory/statefulset_test.go @@ -69,6 +69,24 @@ var _ = Describe("CreateOrUpdateStatefulSet handler", func() { It("should successfully create the statefulset with filled spec", func() { By("Creating the statefulset") etcdcluster := etcdcluster.DeepCopy() + etcdcluster.Spec.Storage = etcdaenixiov1alpha1.StorageSpec{ + VolumeClaimTemplate: etcdaenixiov1alpha1.EmbeddedPersistentVolumeClaim{ + EmbeddedObjectMetadata: etcdaenixiov1alpha1.EmbeddedObjectMetadata{ + Name: "etcd-data", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{ + v1.ReadWriteOnce, + }, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, + Status: v1.PersistentVolumeClaimStatus{}, + }, + } etcdcluster.Spec.PodSpec = etcdaenixiov1alpha1.PodSpec{ Resources: v1.ResourceRequirements{ Requests: v1.ResourceList{ diff --git a/internal/controller/factory/svc.go b/internal/controller/factory/svc.go new file mode 100644 index 00000000..41e657d3 --- /dev/null +++ b/internal/controller/factory/svc.go @@ -0,0 +1,110 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package factory + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" +) + +func GetClientServiceName(cluster *etcdaenixiov1alpha1.EtcdCluster) string { + return fmt.Sprintf("%s-client", cluster.Name) +} + +func CreateOrUpdateClusterService( + ctx context.Context, + cluster *etcdaenixiov1alpha1.EtcdCluster, + rclient client.Client, + rscheme *runtime.Scheme, +) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluster.Name, + Namespace: cluster.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "etcd", + "app.kubernetes.io/instance": cluster.Name, + "app.kubernetes.io/managed-by": "etcd-operator", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + {Name: "peer", TargetPort: intstr.FromInt32(2380), Port: 2380, Protocol: corev1.ProtocolTCP}, + {Name: "client", TargetPort: intstr.FromInt32(2379), Port: 2379, Protocol: corev1.ProtocolTCP}, + }, + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "None", + Selector: map[string]string{ + "app.kubernetes.io/name": "etcd", + "app.kubernetes.io/instance": cluster.Name, + "app.kubernetes.io/managed-by": "etcd-operator", + }, + PublishNotReadyAddresses: true, + }, + } + + if err := ctrl.SetControllerReference(cluster, svc, rscheme); err != nil { + return fmt.Errorf("cannot set controller reference: %w", err) + } + + return reconcileService(ctx, rclient, cluster.Name, svc) +} + +func CreateOrUpdateClientService( + ctx context.Context, + cluster *etcdaenixiov1alpha1.EtcdCluster, + rclient client.Client, + rscheme *runtime.Scheme, +) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: GetClientServiceName(cluster), + Namespace: cluster.Namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "etcd", + "app.kubernetes.io/instance": cluster.Name, + "app.kubernetes.io/managed-by": "etcd-operator", + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + {Name: "client", TargetPort: intstr.FromInt32(2379), Port: 2379, Protocol: corev1.ProtocolTCP}, + }, + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app.kubernetes.io/name": "etcd", + "app.kubernetes.io/instance": cluster.Name, + "app.kubernetes.io/managed-by": "etcd-operator", + }, + }, + } + + if err := ctrl.SetControllerReference(cluster, svc, rscheme); err != nil { + return fmt.Errorf("cannot set controller reference: %w", err) + } + + return reconcileService(ctx, rclient, cluster.Name, svc) +} diff --git a/internal/controller/factory/svc_test.go b/internal/controller/factory/svc_test.go new file mode 100644 index 00000000..d8579b75 --- /dev/null +++ b/internal/controller/factory/svc_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2024 The etcd-operator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package factory + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1" +) + +var _ = Describe("CreateOrUpdateService handlers", func() { + Context("When ensuring a cluster services", func() { + const resourceName = "test-resource" + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: "default", + } + etcdcluster := &etcdaenixiov1alpha1.EtcdCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: "default", + UID: "test-uid", + }, + Spec: etcdaenixiov1alpha1.EtcdClusterSpec{ + Replicas: ptr.To(int32(3)), + }, + } + + It("should successfully create the cluster service", func() { + svc := &corev1.Service{} + err := CreateOrUpdateClusterService(ctx, etcdcluster, k8sClient, k8sClient.Scheme()) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, typeNamespacedName, svc) + Expect(err).NotTo(HaveOccurred()) + Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(svc.Spec.ClusterIP).To(Equal("None")) + + Expect(k8sClient.Delete(ctx, svc)).To(Succeed()) + }) + + It("should fail on creating the cluster service with invalid owner reference", func() { + etcdcluster := etcdcluster.DeepCopy() + emptyScheme := runtime.NewScheme() + + err := CreateOrUpdateClusterService(ctx, etcdcluster, k8sClient, emptyScheme) + Expect(err).To(HaveOccurred()) + }) + + It("should successfully create the client service", func() { + svc := &corev1.Service{} + err := CreateOrUpdateClientService(ctx, etcdcluster, k8sClient, k8sClient.Scheme()) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: GetClientServiceName(etcdcluster), + Namespace: "default", + }, svc) + Expect(err).NotTo(HaveOccurred()) + Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(svc.Spec.ClusterIP).To(Not(Equal("None"))) + + Expect(k8sClient.Delete(ctx, svc)).To(Succeed()) + }) + + It("should fail on creating the client service with invalid owner reference", func() { + etcdcluster := etcdcluster.DeepCopy() + emptyScheme := runtime.NewScheme() + + err := CreateOrUpdateClientService(ctx, etcdcluster, k8sClient, emptyScheme) + Expect(err).To(HaveOccurred()) + }) + }) +})