Skip to content

Commit

Permalink
improve EtcdCluster conditions processing (#92)
Browse files Browse the repository at this point in the history
Fixes: #79

---------

Signed-off-by: Artem Bortnikov <artem.bortnikov@telekom.com>
  • Loading branch information
aobort committed Mar 28, 2024
1 parent fee8e09 commit 2f64d81
Show file tree
Hide file tree
Showing 7 changed files with 230 additions and 67 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ RUN go mod download
# Copy the go source
COPY cmd/main.go ./cmd/
COPY api/ ./api/
COPY internal/controller/ ./internal/controller/
COPY internal/ ./internal/

# Build
# the GOARCH has not a default value to allow the binary be built according to the host where the command
Expand Down
8 changes: 8 additions & 0 deletions api/v1alpha1/etcdcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ const (
)

type EtcdCondType string
type EtcdCondMessage string

const (
EtcdCondTypeInitStarted EtcdCondType = "InitializationStarted"
Expand All @@ -49,6 +50,13 @@ const (
EtcdCondTypeStatefulSetNotReady EtcdCondType = "StatefulSetNotReady"
)

const (
EtcdInitCondNegMessage EtcdCondMessage = "Cluster initialization started"
EtcdInitCondPosMessage EtcdCondMessage = "Cluster managed resources created"
EtcdReadyCondNegMessage EtcdCondMessage = "Cluster StatefulSet is not Ready"
EtcdReadyCondPosMessage EtcdCondMessage = "Cluster StatefulSet is Ready"
)

// EtcdClusterStatus defines the observed state of EtcdCluster
type EtcdClusterStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`
Expand Down
92 changes: 31 additions & 61 deletions internal/controller/etcdcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,13 @@ import (
"context"
goerrors "errors"
"fmt"
"slices"

"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/predicate"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -71,67 +69,45 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return reconcile.Result{}, nil
}

// 3. mark CR as initialized
// fill conditions
if len(instance.Status.Conditions) == 0 {
instance.Status.Conditions = append(instance.Status.Conditions, metav1.Condition{
Type: etcdaenixiov1alpha1.EtcdConditionInitialized,
Status: metav1.ConditionFalse,
ObservedGeneration: instance.Generation,
LastTransitionTime: metav1.Now(),
Reason: string(etcdaenixiov1alpha1.EtcdCondTypeInitStarted),
Message: "Cluster initialization has started",
})
factory.FillConditions(instance)
}

// check sts condition
isClusterReady := false
sts := &appsv1.StatefulSet{}
err = r.Get(ctx, client.ObjectKey{
Namespace: instance.Namespace,
Name: instance.Name,
}, sts)
if err == nil {
isClusterReady = sts.Status.ReadyReplicas == *sts.Spec.Replicas
}

if err := r.ensureClusterObjects(ctx, instance, isClusterReady); err != nil {
// ensure managed resources
if err := r.ensureClusterObjects(ctx, instance); err != nil {
logger.Error(err, "cannot create Cluster auxiliary objects")
return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot create Cluster auxiliary objects: %w", err))
}

r.updateClusterState(instance, metav1.Condition{
Type: etcdaenixiov1alpha1.EtcdConditionInitialized,
Status: metav1.ConditionTrue,
LastTransitionTime: metav1.Now(),
Reason: string(etcdaenixiov1alpha1.EtcdCondTypeInitComplete),
Message: "Cluster initialization is complete",
})
if isClusterReady {
r.updateClusterState(instance, metav1.Condition{
Type: etcdaenixiov1alpha1.EtcdConditionReady,
Status: metav1.ConditionTrue,
LastTransitionTime: metav1.Now(),
Reason: string(etcdaenixiov1alpha1.EtcdCondTypeStatefulSetReady),
Message: "Cluster StatefulSet is Ready",
})
} else {
r.updateClusterState(instance, metav1.Condition{
Type: etcdaenixiov1alpha1.EtcdConditionReady,
Status: metav1.ConditionFalse,
LastTransitionTime: metav1.Now(),
Reason: string(etcdaenixiov1alpha1.EtcdCondTypeStatefulSetNotReady),
Message: "Cluster StatefulSet is not Ready",
})
// set cluster initialization condition
factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized).
WithStatus(true).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeInitComplete)).
WithMessage(string(etcdaenixiov1alpha1.EtcdInitCondPosMessage)).
Complete())

// check sts condition
clusterReady, err := r.isStatefulSetReady(ctx, instance)
if err != nil {
logger.Error(err, "failed to check etcd cluster state")
return r.updateStatusOnErr(ctx, instance, fmt.Errorf("cannot check Cluster readiness: %w", err))
}

// set cluster readiness condition
factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionReady).
WithStatus(clusterReady).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeStatefulSetReady)).
WithMessage(string(etcdaenixiov1alpha1.EtcdReadyCondPosMessage)).
Complete())
return r.updateStatus(ctx, instance)
}

// ensureClusterObjects creates or updates all objects owned by cluster CR
func (r *EtcdClusterReconciler) ensureClusterObjects(
ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster, isClusterReady bool) error {
ctx context.Context, cluster *etcdaenixiov1alpha1.EtcdCluster) error {
// 1. create or update configmap <name>-cluster-state
if err := factory.CreateOrUpdateClusterStateConfigMap(ctx, cluster, isClusterReady, r.Client, r.Scheme); err != nil {
if err := factory.CreateOrUpdateClusterStateConfigMap(ctx, cluster, r.Client, r.Scheme); err != nil {
return err
}
if err := factory.CreateOrUpdateClusterService(ctx, cluster, r.Client, r.Scheme); err != nil {
Expand Down Expand Up @@ -171,20 +147,14 @@ func (r *EtcdClusterReconciler) updateStatus(ctx context.Context, cluster *etcda
return ctrl.Result{}, nil
}

// updateClusterState patches status condition in cluster using merge by Type
func (r *EtcdClusterReconciler) updateClusterState(cluster *etcdaenixiov1alpha1.EtcdCluster, state metav1.Condition) {
if initIdx := slices.IndexFunc(cluster.Status.Conditions, func(condition metav1.Condition) bool {
return condition.Type == state.Type
}); initIdx != -1 {
cluster.Status.Conditions[initIdx].Status = state.Status
cluster.Status.Conditions[initIdx].LastTransitionTime = state.LastTransitionTime
cluster.Status.Conditions[initIdx].ObservedGeneration = cluster.Generation
cluster.Status.Conditions[initIdx].Reason = state.Reason
cluster.Status.Conditions[initIdx].Message = state.Message
} else {
state.ObservedGeneration = cluster.Generation
cluster.Status.Conditions = append(cluster.Status.Conditions, state)
// isStatefulSetReady gets managed StatefulSet and checks its readiness.
func (r *EtcdClusterReconciler) isStatefulSetReady(ctx context.Context, c *etcdaenixiov1alpha1.EtcdCluster) (bool, error) {
sts := &appsv1.StatefulSet{}
err := r.Get(ctx, client.ObjectKeyFromObject(c), sts)
if err == nil {
return sts.Status.ReadyReplicas == *sts.Spec.Replicas, nil
}
return false, client.IgnoreNotFound(err)
}

// SetupWithManager sets up the controller with the Manager.
Expand Down
98 changes: 98 additions & 0 deletions internal/controller/factory/condition.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
Copyright 2024 The etcd-operator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
"slices"

etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Condition is a wrapper over metav1.Condition type.
type Condition struct {
metav1.Condition
}

// NewCondition returns Condition object with provided condition type.
func NewCondition(conditionType etcdaenixiov1alpha1.EtcdCondType) Condition {
return Condition{metav1.Condition{Type: string(conditionType)}}
}

// WithStatus converts boolean value passed as an argument to metav1.ConditionStatus type and sets condition status.
func (c Condition) WithStatus(status bool) Condition {
c.Status = metav1.ConditionFalse
if status {
c.Status = metav1.ConditionTrue
}
return c
}

// WithReason sets condition reason.
func (c Condition) WithReason(reason string) Condition {
c.Reason = reason
return c
}

// WithMessage sets condition message.
func (c Condition) WithMessage(message string) Condition {
c.Message = message
return c
}

// Complete finalizes condition building by setting transition timestamp and returns wrapped metav1.Condition object.
func (c Condition) Complete() metav1.Condition {
c.LastTransitionTime = metav1.Now()
return c.Condition
}

// FillConditions fills EtcdCluster .status.Conditions list with all available conditions in "False" status.
func FillConditions(cluster *etcdaenixiov1alpha1.EtcdCluster) {
SetCondition(cluster, NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized).
WithStatus(false).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeInitStarted)).
WithMessage(string(etcdaenixiov1alpha1.EtcdInitCondNegMessage)).
Complete())
SetCondition(cluster, NewCondition(etcdaenixiov1alpha1.EtcdConditionReady).
WithStatus(false).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeStatefulSetNotReady)).
WithMessage(string(etcdaenixiov1alpha1.EtcdReadyCondNegMessage)).
Complete())
}

// SetCondition sets either replaces corresponding existing condition in the .status.Conditions list or appends
// one passed as an argument. In case operation will not result into condition status change, return.
func SetCondition(
cluster *etcdaenixiov1alpha1.EtcdCluster,
condition metav1.Condition,
) {
condition.ObservedGeneration = cluster.GetGeneration()
idx := slices.IndexFunc(cluster.Status.Conditions, func(c metav1.Condition) bool {
return c.Type == condition.Type
})

if idx == -1 {
cluster.Status.Conditions = append(cluster.Status.Conditions, condition)
return
}
statusNotChanged := cluster.Status.Conditions[idx].Status == condition.Status
reasonNotChanged := cluster.Status.Conditions[idx].Reason == condition.Reason
if statusNotChanged && reasonNotChanged {
return
}
cluster.Status.Conditions[idx] = condition
}
74 changes: 74 additions & 0 deletions internal/controller/factory/condition_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
Copyright 2024 The etcd-operator Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
"slices"

etcdaenixiov1alpha1 "github.com/aenix-io/etcd-operator/api/v1alpha1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("Condition builder", func() {
Context("When updating condition", func() {
etcdCluster := &etcdaenixiov1alpha1.EtcdCluster{}

It("should fill conditions", func() {
FillConditions(etcdCluster)
Expect(etcdCluster.Status.Conditions).NotTo(BeEmpty())
for _, c := range etcdCluster.Status.Conditions {
Expect(c.Status).To(Equal(metav1.ConditionFalse))
Expect(c.LastTransitionTime).NotTo(BeZero())
}
})

It("should update existing condition", func() {
conditionsLength := len(etcdCluster.Status.Conditions)
SetCondition(etcdCluster, NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized).
WithStatus(false).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeInitStarted)).
WithMessage("test").
Complete())
Expect(len(etcdCluster.Status.Conditions)).To(Equal(conditionsLength))
})

It("should keep last transition timestamp without status change, otherwise update", func() {
idx := slices.IndexFunc(etcdCluster.Status.Conditions, func(condition metav1.Condition) bool {
return condition.Type == etcdaenixiov1alpha1.EtcdConditionInitialized
})
timestamp := etcdCluster.Status.Conditions[idx].LastTransitionTime

By("setting condition without status change")
SetCondition(etcdCluster, NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized).
WithStatus(false).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeInitStarted)).
WithMessage("test").
Complete())
Expect(etcdCluster.Status.Conditions[idx].LastTransitionTime).To(Equal(timestamp))

By("setting condition with status changed")
SetCondition(etcdCluster, NewCondition(etcdaenixiov1alpha1.EtcdConditionInitialized).
WithStatus(true).
WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeInitStarted)).
WithMessage("test").
Complete())
Expect(etcdCluster.Status.Conditions[idx].LastTransitionTime).NotTo(Equal(timestamp))
})
})
})
15 changes: 13 additions & 2 deletions internal/controller/factory/configMap.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package factory
import (
"context"
"fmt"
"slices"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -36,7 +37,6 @@ func GetClusterStateConfigMapName(cluster *etcdaenixiov1alpha1.EtcdCluster) stri
func CreateOrUpdateClusterStateConfigMap(
ctx context.Context,
cluster *etcdaenixiov1alpha1.EtcdCluster,
isClusterReady bool,
rclient client.Client,
rscheme *runtime.Scheme,
) error {
Expand All @@ -63,7 +63,7 @@ func CreateOrUpdateClusterStateConfigMap(
},
}

if isClusterReady {
if isEtcdClusterReady(cluster) {
// update cluster state to existing
configMap.Data["ETCD_INITIAL_CLUSTER_STATE"] = "existing"
}
Expand All @@ -74,3 +74,14 @@ func CreateOrUpdateClusterStateConfigMap(

return reconcileConfigMap(ctx, rclient, cluster.Name, configMap)
}

// isEtcdClusterReady returns true if condition "Ready" has status equal to "True", otherwise false.
func isEtcdClusterReady(cluster *etcdaenixiov1alpha1.EtcdCluster) bool {
idx := slices.IndexFunc(cluster.Status.Conditions, func(condition metav1.Condition) bool {
return condition.Type == etcdaenixiov1alpha1.EtcdConditionReady
})
if idx == -1 {
return false
}
return cluster.Status.Conditions[idx].Status == metav1.ConditionTrue
}
Loading

0 comments on commit 2f64d81

Please sign in to comment.