Skip to content

Commit

Permalink
Implement Ready condition for Instance type
Browse files Browse the repository at this point in the history
Signed-off-by: Alena Varkockova <varkockova.a@gmail.com>
  • Loading branch information
alenkacz committed Oct 8, 2020
1 parent 2f7c6c2 commit 960385b
Show file tree
Hide file tree
Showing 19 changed files with 591 additions and 5 deletions.
43 changes: 43 additions & 0 deletions config/crds/kudo.dev_instances.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,49 @@ spec:
status:
description: InstanceStatus defines the observed state of Instance
properties:
conditions:
items:
description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
properties:
lastTransitionTime:
description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: message is a human readable message indicating details about the transition. This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
planStatus:
additionalProperties:
description: "PlanStatus is representing status of a plan \n These are valid states and transitions \n | Never executed | | v | Error |<------>| Pending | ^ | | v | +-------+--------+ | +-------+--------+ | | v v | Fatal error | | Complete |"
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/kudo/v1beta1/instance_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type PlanExecution struct {
type InstanceStatus struct {
// slice would be enough here but we cannot use slice because order of sequence in yaml is considered significant while here it's not
PlanStatus map[string]PlanStatus `json:"planStatus,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty"`
}

// PlanStatus is representing status of a plan
Expand Down
28 changes: 28 additions & 0 deletions pkg/apis/kudo/v1beta1/instance_types_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"fmt"
"log"

"k8s.io/apimachinery/pkg/api/meta"

"github.com/thoas/go-funk"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
Expand Down Expand Up @@ -70,6 +72,7 @@ func (i *Instance) NoPlanEverExecuted() bool {
}

// UpdateInstanceStatus updates `Status.PlanStatus` and `Status.AggregatedStatus` property based on the given plan
// also updates Ready condition for finished plans
func (i *Instance) UpdateInstanceStatus(ps *PlanStatus, updatedTimestamp *metav1.Time) {
for k, v := range i.Status.PlanStatus {
if v.Name == ps.Name {
Expand All @@ -78,6 +81,9 @@ func (i *Instance) UpdateInstanceStatus(ps *PlanStatus, updatedTimestamp *metav1
i.Spec.PlanExecution.Status = ps.Status
}
}
if i.Spec.PlanExecution.Status.IsFinished() {
i.SetReadinessTrue("")
}
}

// ResetPlanStatus method resets a PlanStatus for a passed plan name and instance. Plan/phase/step statuses
Expand Down Expand Up @@ -196,6 +202,28 @@ func (i *Instance) IsTopLevelInstance() bool {
return !i.IsChildInstance()
}

const (
planInProgressReason = "PlanInProgress"
resourceNotReadyReason = "ResourceNotReady"
readyConditionType = "Ready"
)

func (i *Instance) SetReadinessTrue(msg string) {
condition := metav1.Condition{Type: readyConditionType, Status: metav1.ConditionTrue, Message: msg}
meta.SetStatusCondition(&i.Status.Conditions, condition)
}

func (i *Instance) SetReadinessFalse(msg string) {
condition := metav1.Condition{Type: readyConditionType, Status: metav1.ConditionFalse, Message: msg, Reason: resourceNotReadyReason}
meta.SetStatusCondition(&i.Status.Conditions, condition)
}

// SetReadinessUnknown sets an unknown state for Status.Conditions.Ready
func (i *Instance) SetReadinessUnknown() {
condition := metav1.Condition{Type: readyConditionType, Status: metav1.ConditionUnknown, Reason: planInProgressReason}
meta.SetStatusCondition(&i.Status.Conditions, condition)
}

// wasRunAfter returns true if p1 was run after p2
func wasRunAfter(p1 PlanStatus, p2 PlanStatus) bool {
if p1.Status == ExecutionNeverRun || p2.Status == ExecutionNeverRun || p1.LastUpdatedTimestamp == nil || p2.LastUpdatedTimestamp == nil {
Expand Down
12 changes: 10 additions & 2 deletions pkg/apis/kudo/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 32 additions & 2 deletions pkg/controller/instance/instance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ import (
"github.com/kudobuilder/kudo/pkg/engine/renderer"
"github.com/kudobuilder/kudo/pkg/engine/task"
"github.com/kudobuilder/kudo/pkg/engine/workflow"
"github.com/kudobuilder/kudo/pkg/kubernetes/status"
"github.com/kudobuilder/kudo/pkg/kudoctl/resources/dependencies"
"github.com/kudobuilder/kudo/pkg/util/convert"
)
Expand Down Expand Up @@ -157,6 +158,12 @@ func isForPipePod(e event.DeleteEvent) bool {
// | Update instance with new |
// | state of the execution |
// +-------------------------------+
// |
// v
// +-------------------------------+
// | Update readiness even if |
// | no plan running |
// +-------------------------------+
//
// Automatically generate RBAC rules to allow the Controller to read and write Deployments
func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
Expand Down Expand Up @@ -187,8 +194,14 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
// get the scheduled plan
plan, uid := scheduledPlan(instance, ov)
if plan == "" {
log.Printf("InstanceController: Nothing to do, no plan scheduled for instance %s/%s", instance.Namespace, instance.Name)
return reconcile.Result{}, nil
// no plan is running, we still need to make sure the readiness property is up to date
err := setReadinessOnInstance(instance, r.Client)
if err != nil {
log.Printf("InstanceController: Error when computing readiness for %s/%s: %v", instance.Namespace, instance.Name, err)
return reconcile.Result{}, err
}
err = updateInstance(instance, oldInstance, r.Client)
return reconcile.Result{}, err
}

ensurePlanStatusInitialized(instance, ov)
Expand Down Expand Up @@ -258,6 +271,20 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
return computeTheReconcileResult(instance, time.Now), nil
}

func setReadinessOnInstance(instance *kudoapi.Instance, c client.Client) error {
ready, msg, err := status.IsReady(*instance, c)
log.Printf("Updatinginstance %s/%s readiness to: : %t", instance.Namespace, instance.Name, ready)
if err != nil {
return err
}
if ready {
instance.SetReadinessTrue(msg)
} else {
instance.SetReadinessFalse(msg)
}
return nil
}

// computeTheReconcileResult decides whether retry reconciliation or not
// if plan was finished, reconciliation is not retried
// for others it uses LastUpdatedTimestamp of a current plan
Expand Down Expand Up @@ -536,5 +563,8 @@ func scheduledPlan(i *kudoapi.Instance, ov *kudoapi.OperatorVersion) (string, ty
i.Spec.PlanExecution.Status = kudoapi.ExecutionNeverRun
}

if i.Spec.PlanExecution.PlanName == kudoapi.DeployPlanName || i.Spec.PlanExecution.PlanName == kudoapi.UpgradePlanName || i.Spec.PlanExecution.PlanName == kudoapi.UpdatePlanName {
i.SetReadinessUnknown()
}
return i.Spec.PlanExecution.PlanName, i.Spec.PlanExecution.UID
}
100 changes: 100 additions & 0 deletions pkg/kubernetes/status/readiness.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package status

import (
"context"
"fmt"
"log"

kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
label "github.com/kudobuilder/kudo/pkg/util/kudo"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
)

// IsReady computes instance readiness based on current state of underlying resources
// currently readiness examines the following types: Pods, StatefulSets, Deployments, ReplicaSets and DaemonSets
// Instance is considered ready if all the resources linked to this instance are also ready (healthy)
func IsReady(i kudoapi.Instance, c client.Client) (ready bool, msg string, err error) {
resources, err := healthResources(c, i.Name, i.Namespace)
if err != nil {
return false, "", err
}
ready = true
readinessMessage := ""
for _, res := range resources {
healthy, _, err := IsHealthy(res)
if !healthy {
if readinessMessage == "" {
readinessMessage = fmt.Sprintf("%v", err)
} else {
readinessMessage += fmt.Sprintf(", %v", err)
}
ready = false
}
}

return ready, readinessMessage, nil
}

func healthResources(c client.Client, instanceName, instanceNamespace string) ([]runtime.Object, error) {
instanceLabels, err := labels.Parse(fmt.Sprintf("%s=%s,%s=%s", label.InstanceLabel, instanceName, label.HeritageLabel, "kudo"))
if err != nil {
return nil, fmt.Errorf("unable to create list of labels to define health: %v", err)
}

dList := &appsv1.DeploymentList{}
err = c.List(context.TODO(), dList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type Deployment: %v", err)
}

ssList := &appsv1.StatefulSetList{}
err = c.List(context.TODO(), ssList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type StatefulSet: %v", err)
}

rsList := &appsv1.ReplicaSetList{}
err = c.List(context.TODO(), rsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type ReplicaSet: %v", err)
}

dsList := &appsv1.DaemonSetList{}
err = c.List(context.TODO(), dsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type DaemonSet: %v", err)
}

podsList := &corev1.PodList{}
err = c.List(context.TODO(), podsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type Pod: %v", err)
}

var result []runtime.Object
for i := range dList.Items {
result = append(result, &dList.Items[i])
}
for i := range ssList.Items {
result = append(result, &ssList.Items[i])
}
for i := range rsList.Items {
result = append(result, &rsList.Items[i])
}
for i := range dsList.Items {
result = append(result, &dsList.Items[i])
}
for i := range podsList.Items {
result = append(result, &podsList.Items[i])
}

log.Printf("Computing health out of %d Deployments, %d ReplicaSets, %d StatefulSets, %d DaemonSets, %d Pods", len(dList.Items), len(rsList.Items), len(ssList.Items), len(dsList.Items), len(podsList.Items))

return result, nil
}
Loading

0 comments on commit 960385b

Please sign in to comment.