Skip to content

Commit

Permalink
Implement Ready condition for Instance type (#1706)
Browse files Browse the repository at this point in the history
This is an implementation of KEP-34. It adds Status.Conditions.Ready to an Instance while this property is being computed from the owned resources of certain types.

Signed-off-by: Alena Varkockova <varkockova.a@gmail.com>

Co-authored-by: Jan Schlicht <jan@d2iq.com>
Co-authored-by: Andreas Neumann <aneumann@mesosphere.com>
  • Loading branch information
3 people committed Oct 9, 2020
1 parent 853c9b3 commit 7eefce1
Show file tree
Hide file tree
Showing 19 changed files with 618 additions and 10 deletions.
43 changes: 43 additions & 0 deletions config/crds/kudo.dev_instances.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,49 @@ spec:
status:
description: InstanceStatus defines the observed state of Instance
properties:
conditions:
items:
description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }"
properties:
lastTransitionTime:
description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
format: date-time
type: string
message:
description: message is a human readable message indicating details about the transition. This may be an empty string.
maxLength: 32768
type: string
observedGeneration:
description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.
format: int64
minimum: 0
type: integer
reason:
description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.
maxLength: 1024
minLength: 1
pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
type: string
status:
description: status of the condition, one of True, False, Unknown.
enum:
- "True"
- "False"
- Unknown
type: string
type:
description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
maxLength: 316
pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
type: string
required:
- lastTransitionTime
- message
- reason
- status
- type
type: object
type: array
planStatus:
additionalProperties:
description: "PlanStatus is representing status of a plan \n These are valid states and transitions \n | Never executed | | v | Error |<------>| Pending | ^ | | v | +-------+--------+ | +-------+--------+ | | v v | Fatal error | | Complete |"
Expand Down
1 change: 1 addition & 0 deletions pkg/apis/kudo/v1beta1/instance_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type PlanExecution struct {
type InstanceStatus struct {
// slice would be enough here but we cannot use slice because order of sequence in yaml is considered significant while here it's not
PlanStatus map[string]PlanStatus `json:"planStatus,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty"`
}

// PlanStatus is representing status of a plan
Expand Down
35 changes: 35 additions & 0 deletions pkg/apis/kudo/v1beta1/instance_types_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"log"

"github.com/thoas/go-funk"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -70,6 +71,7 @@ func (i *Instance) NoPlanEverExecuted() bool {
}

// UpdateInstanceStatus updates `Status.PlanStatus` and `Status.AggregatedStatus` property based on the given plan
// also updates Ready condition for finished plans
func (i *Instance) UpdateInstanceStatus(ps *PlanStatus, updatedTimestamp *metav1.Time) {
for k, v := range i.Status.PlanStatus {
if v.Name == ps.Name {
Expand All @@ -78,6 +80,12 @@ func (i *Instance) UpdateInstanceStatus(ps *PlanStatus, updatedTimestamp *metav1
i.Spec.PlanExecution.Status = ps.Status
}
}
if i.Spec.PlanExecution.Status.IsFinished() {
i.SetReadiness(ReadinessResourcesReady, "")
}
if i.Spec.PlanExecution.Status == ExecutionFatalError {
i.SetReadiness(ReadinessPlanInFatalError, "")
}
}

// ResetPlanStatus method resets a PlanStatus for a passed plan name and instance. Plan/phase/step statuses
Expand Down Expand Up @@ -196,6 +204,33 @@ func (i *Instance) IsTopLevelInstance() bool {
return !i.IsChildInstance()
}

type ReadinessType string

const (
ReadinessPlanInProgress ReadinessType = "PlanInProgress"
ReadinessPlanInFatalError ReadinessType = "PlanInFatalError"
ReadinessResourceNotReady ReadinessType = "ResourceNotReady"
ReadinessResourcesReady ReadinessType = "ResourcesReady"

readyConditionType = "Ready"
)

func (i *Instance) SetReadiness(reason ReadinessType, msg string) {
var status metav1.ConditionStatus

switch reason {
case ReadinessResourcesReady:
status = metav1.ConditionTrue
case ReadinessResourceNotReady:
status = metav1.ConditionFalse
case ReadinessPlanInFatalError, ReadinessPlanInProgress:
status = metav1.ConditionUnknown
}

condition := metav1.Condition{Type: readyConditionType, Status: status, Message: msg, Reason: string(reason)}
meta.SetStatusCondition(&i.Status.Conditions, condition)
}

// wasRunAfter returns true if p1 was run after p2
func wasRunAfter(p1 PlanStatus, p2 PlanStatus) bool {
if p1.Status == ExecutionNeverRun || p2.Status == ExecutionNeverRun || p1.LastUpdatedTimestamp == nil || p2.LastUpdatedTimestamp == nil {
Expand Down
12 changes: 10 additions & 2 deletions pkg/apis/kudo/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

60 changes: 53 additions & 7 deletions pkg/controller/instance/instance_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,18 @@ import (
"reflect"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"

"github.com/thoas/go-funk"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/discovery"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -50,6 +50,7 @@ import (
"github.com/kudobuilder/kudo/pkg/engine/renderer"
"github.com/kudobuilder/kudo/pkg/engine/task"
"github.com/kudobuilder/kudo/pkg/engine/workflow"
"github.com/kudobuilder/kudo/pkg/kubernetes/status"
"github.com/kudobuilder/kudo/pkg/kudoctl/resources/dependencies"
"github.com/kudobuilder/kudo/pkg/util/convert"
)
Expand Down Expand Up @@ -157,6 +158,12 @@ func isForPipePod(e event.DeleteEvent) bool {
// | Update instance with new |
// | state of the execution |
// +-------------------------------+
// |
// v
// +-------------------------------+
// | Update readiness even if |
// | no plan running |
// +-------------------------------+
//
// Automatically generate RBAC rules to allow the Controller to read and write Deployments
func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
Expand Down Expand Up @@ -187,10 +194,21 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
// get the scheduled plan
plan, uid := scheduledPlan(instance, ov)
if plan == "" {
log.Printf("InstanceController: Nothing to do, no plan scheduled for instance %s/%s", instance.Namespace, instance.Name)
return reconcile.Result{}, nil
// no plan is running, we still need to make sure the readiness property is up to date
err := setReadinessOnInstance(instance, r.Client)
if err != nil {
log.Printf("InstanceController: Error when computing readiness for %s/%s: %v", instance.Namespace, instance.Name, err)
return reconcile.Result{}, err
}
if readinessChanged(oldInstance, instance) {
err = updateInstance(instance, oldInstance, r.Client)
} else {
log.Printf("InstanceController: Readiness did not change for %s/%s. Not updating.", instance.Namespace, instance.Name)
}
return reconcile.Result{}, err
}

ensureReadinessInitialized(instance)
ensurePlanStatusInitialized(instance, ov)

// reset its status if the plan is new and log/record it
Expand Down Expand Up @@ -258,6 +276,34 @@ func (r *Reconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
return computeTheReconcileResult(instance, time.Now), nil
}

func ensureReadinessInitialized(i *kudoapi.Instance) {
if i.Spec.PlanExecution.PlanName == kudoapi.DeployPlanName || i.Spec.PlanExecution.PlanName == kudoapi.UpgradePlanName || i.Spec.PlanExecution.PlanName == kudoapi.UpdatePlanName {
i.SetReadiness(kudoapi.ReadinessPlanInProgress, "")
}
// For any other plan we keep the existing Readiness. As the deploy plan is always the first plan to run, the Readiness is always initialized.
}

func readinessChanged(instance *kudoapi.Instance, instance2 *kudoapi.Instance) bool {
ready := meta.FindStatusCondition(instance.Status.Conditions, "Ready")
ready2 := meta.FindStatusCondition(instance2.Status.Conditions, "Ready")

return !reflect.DeepEqual(ready, ready2)
}

func setReadinessOnInstance(instance *kudoapi.Instance, c client.Client) error {
ready, msg, err := status.IsReady(*instance, c)
log.Printf("Updating instance %s/%s readiness to: %t", instance.Namespace, instance.Name, ready)
if err != nil {
return err
}
if ready {
instance.SetReadiness(kudoapi.ReadinessResourcesReady, msg)
} else {
instance.SetReadiness(kudoapi.ReadinessResourceNotReady, msg)
}
return nil
}

// computeTheReconcileResult decides whether retry reconciliation or not
// if plan was finished, reconciliation is not retried
// for others it uses LastUpdatedTimestamp of a current plan
Expand Down
102 changes: 102 additions & 0 deletions pkg/kubernetes/status/readiness.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package status

import (
"context"
"fmt"
"log"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

kudoapi "github.com/kudobuilder/kudo/pkg/apis/kudo/v1beta1"
label "github.com/kudobuilder/kudo/pkg/util/kudo"
)

// IsReady computes instance readiness based on current state of underlying resources
// currently readiness examines the following types: Pods, StatefulSets, Deployments, ReplicaSets and DaemonSets
// Instance is considered ready if all the resources linked to this instance are also ready (healthy)
func IsReady(i kudoapi.Instance, c client.Client) (bool, string, error) {
resources, err := healthResources(c, i.Name, i.Namespace)
if err != nil {
return false, "", err
}
ready := true
readinessMessage := ""
for _, res := range resources {
healthy, msg, err := IsHealthy(res)
if err != nil {
return false, "", err
}
if !healthy {
if readinessMessage == "" {
readinessMessage = msg
} else {
readinessMessage += fmt.Sprintf(", %s", msg)
}
ready = false
}
}

return ready, readinessMessage, nil
}

func healthResources(c client.Client, instanceName, instanceNamespace string) ([]runtime.Object, error) {
instanceLabels, err := labels.Parse(fmt.Sprintf("%s=%s,%s=%s", label.InstanceLabel, instanceName, label.HeritageLabel, "kudo"))
if err != nil {
return nil, fmt.Errorf("unable to create list of labels to define health: %v", err)
}

dList := &appsv1.DeploymentList{}
err = c.List(context.TODO(), dList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type Deployment: %v", err)
}

ssList := &appsv1.StatefulSetList{}
err = c.List(context.TODO(), ssList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type StatefulSet: %v", err)
}

rsList := &appsv1.ReplicaSetList{}
err = c.List(context.TODO(), rsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type ReplicaSet: %v", err)
}

dsList := &appsv1.DaemonSetList{}
err = c.List(context.TODO(), dsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type DaemonSet: %v", err)
}

podsList := &corev1.PodList{}
err = c.List(context.TODO(), podsList, &client.ListOptions{Namespace: instanceNamespace, LabelSelector: instanceLabels})
if err != nil {
return nil, fmt.Errorf("unable to pull resources of type Pod: %v", err)
}

var result []runtime.Object
for i := range dList.Items {
result = append(result, &dList.Items[i])
}
for i := range ssList.Items {
result = append(result, &ssList.Items[i])
}
for i := range rsList.Items {
result = append(result, &rsList.Items[i])
}
for i := range dsList.Items {
result = append(result, &dsList.Items[i])
}
for i := range podsList.Items {
result = append(result, &podsList.Items[i])
}

log.Printf("Computing health out of %d Deployments, %d ReplicaSets, %d StatefulSets, %d DaemonSets, %d Pods", len(dList.Items), len(rsList.Items), len(ssList.Items), len(dsList.Items), len(podsList.Items))

return result, nil
}
Loading

0 comments on commit 7eefce1

Please sign in to comment.