report template controller's status in ControllerConfig

cmd/machine-config-operator/start.go

@@ -84,6 +84,7 @@ func startControllers(ctx *common.ControllerContext) error {
 		startOpts.imagesFile,
 		ctx.NamespacedInformerFactory.Machineconfiguration().V1().MCOConfigs(),
 		ctx.NamespacedInformerFactory.Machineconfiguration().V1().MachineConfigPools(),
+		ctx.NamespacedInformerFactory.Machineconfiguration().V1().ControllerConfigs(),
 		ctx.NamespacedInformerFactory.Machineconfiguration().V1().MachineConfigs(),
 		ctx.NamespacedInformerFactory.Machineconfiguration().V1().ControllerConfigs(),
 		ctx.KubeNamespacedInformerFactory.Core().V1().ServiceAccounts(),

manifests/controllerconfig.crd.yaml

@@ -15,6 +15,8 @@ spec:
       storage: true
   # either Namespaced or Cluster
   scope: Cluster
+  subresources:
+    status: {}
   names:
     # plural name to be used in the URL: /apis/<group>/<version>/<plural>
     plural: controllerconfigs

pkg/apis/machineconfiguration.openshift.io/v1/helpers.go

@@ -69,17 +69,17 @@ func SetMachineConfigPoolCondition(status *MachineConfigPoolStatus, condition Ma
 	if currentCond != nil && currentCond.Status == condition.Status {
 		condition.LastTransitionTime = currentCond.LastTransitionTime
 	}
-	newConditions := filterOutCondition(status.Conditions, condition.Type)
+	newConditions := filterOutMachineConfigPoolCondition(status.Conditions, condition.Type)
 	status.Conditions = append(newConditions, condition)
 }
 
 // RemoveMachineConfigPoolCondition removes the MachineConfigPool condition with the provided type.
 func RemoveMachineConfigPoolCondition(status *MachineConfigPoolStatus, condType MachineConfigPoolConditionType) {
-	status.Conditions = filterOutCondition(status.Conditions, condType)
+	status.Conditions = filterOutMachineConfigPoolCondition(status.Conditions, condType)
 }
 
 // filterOutCondition returns a new slice of MachineConfigPool conditions without conditions with the provided type.
-func filterOutCondition(conditions []MachineConfigPoolCondition, condType MachineConfigPoolConditionType) []MachineConfigPoolCondition {
+func filterOutMachineConfigPoolCondition(conditions []MachineConfigPoolCondition, condType MachineConfigPoolConditionType) []MachineConfigPoolCondition {
 	var newConditions []MachineConfigPoolCondition
 	for _, c := range conditions {
 		if c.Type == condType {
@@ -119,3 +119,77 @@ func NewKubeletConfigCondition(condType KubeletConfigStatusConditionType, status
 		Message:            message,
 	}
 }
+
+// NewControllerConfigStatusCondition creates a new ControllerConfigStatus condition.
+func NewControllerConfigStatusCondition(condType ControllerConfigStatusConditionType, status corev1.ConditionStatus, reason, message string) *ControllerConfigStatusCondition {
+	return &ControllerConfigStatusCondition{
+		Type:               condType,
+		Status:             status,
+		LastTransitionTime: metav1.Now(),
+		Reason:             reason,
+		Message:            message,
+	}
+}
+
+// GetControllerConfigStatusCondition returns the condition with the provided type.
+func GetControllerConfigStatusCondition(status ControllerConfigStatus, condType ControllerConfigStatusConditionType) *ControllerConfigStatusCondition {
+	for i := range status.Conditions {
+		c := status.Conditions[i]
+		if c.Type == condType {
+			return &c
+		}
+	}
+	return nil
+}
+
+// SetControllerConfigStatusCondition updates the ControllerConfigStatus to include the provided condition. If the condition that
+// we are about to add already exists and has the same status and reason then we are not going to update.
+func SetControllerConfigStatusCondition(status *ControllerConfigStatus, condition ControllerConfigStatusCondition) {
+	currentCond := GetControllerConfigStatusCondition(*status, condition.Type)
+	if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason {
+		return
+	}
+	// Do not update lastTransitionTime if the status of the condition doesn't change.
+	if currentCond != nil && currentCond.Status == condition.Status {
+		condition.LastTransitionTime = currentCond.LastTransitionTime
+	}
+	newConditions := filterOutControllerConfigStatusCondition(status.Conditions, condition.Type)
+	status.Conditions = append(newConditions, condition)
+}
+
+// RemoveControllerConfigStatusCondition removes the ControllerConfigStatus condition with the provided type.
+func RemoveControllerConfigStatusCondition(status *ControllerConfigStatus, condType ControllerConfigStatusConditionType) {
+	status.Conditions = filterOutControllerConfigStatusCondition(status.Conditions, condType)
+}
+
+// filterOutCondition returns a new slice of ControllerConfigStatus conditions without conditions with the provided type.
+func filterOutControllerConfigStatusCondition(conditions []ControllerConfigStatusCondition, condType ControllerConfigStatusConditionType) []ControllerConfigStatusCondition {
+	var newConditions []ControllerConfigStatusCondition
+	for _, c := range conditions {
+		if c.Type == condType {
+			continue
+		}
+		newConditions = append(newConditions, c)
+	}
+	return newConditions
+}
+
+// IsControllerConfigStatusConditionTrue returns true when the conditionType is present and set to `ConditionTrue`
+func IsControllerConfigStatusConditionTrue(conditions []ControllerConfigStatusCondition, conditionType ControllerConfigStatusConditionType) bool {
+	return IsControllerConfigStatusConditionPresentAndEqual(conditions, conditionType, corev1.ConditionTrue)
+}
+
+// IsControllerConfigStatusConditionFalse returns true when the conditionType is present and set to `ConditionFalse`
+func IsControllerConfigStatusConditionFalse(conditions []ControllerConfigStatusCondition, conditionType ControllerConfigStatusConditionType) bool {
+	return IsControllerConfigStatusConditionPresentAndEqual(conditions, conditionType, corev1.ConditionFalse)
+}
+
+// IsControllerConfigStatusConditionPresentAndEqual returns true when conditionType is present and equal to status.
+func IsControllerConfigStatusConditionPresentAndEqual(conditions []ControllerConfigStatusCondition, conditionType ControllerConfigStatusConditionType, status corev1.ConditionStatus) bool {
+	for _, condition := range conditions {
+		if condition.Type == conditionType {
+			return condition.Status == status
+		}
+	}
+	return false
+}

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -103,16 +103,17 @@ type MCOConfigList struct {
 //
 
 // +genclient
-// +genclient:noStatus
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // ControllerConfig describes configuration for MachineConfigController.
+// This is currently only used to drive the machineconfigs generated by the TemplateController.
 type ControllerConfig struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	Spec ControllerConfigSpec `json:"spec"`
+	Spec   ControllerConfigSpec   `json:"spec"`
+	Status ControllerConfigStatus `json:"status"`
 }
 
 // ControllerConfigSpec is the spec for ControllerConfig resource.
@@ -138,9 +139,46 @@ type ControllerConfigSpec struct {
 	SSHKey string `json:"sshKey"`
 
 	// Sourced from configmap/machine-config-osimageurl
-	OSImageURL string `json:"osImageURL`
+	OSImageURL string `json:"osImageURL"`
+}
+
+// ControllerConfigStatus is the status for ControllerConfig
+type ControllerConfigStatus struct {
+	// The generation observed by the controller.
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Represents the latest available observations of current state.
+	Conditions []ControllerConfigStatusCondition `json:"conditions"`
+}
+
+// ControllerConfigStatusCondition contains condition information for ControllerConfigStatus
+type ControllerConfigStatusCondition struct {
+	// type specifies the state of the operator's reconciliation functionality.
+	Type ControllerConfigStatusConditionType `json:"type"`
+
+	// status of the condition, one of True, False, Unknown.
+	Status corev1.ConditionStatus `json:"status"`
+
+	// lastTransitionTime is the time of the last update to the current status object.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime"`
+
+	// reason is the reason for the condition's last transition.  Reasons are CamelCase
+	Reason string `json:"reason,omitempty"`
+
+	// message provides additional information about the current condition.
+	// This is only to be consumed by humans.
+	Message string `json:"message,omitempty"`
 }
 
+// ControllerConfigStatusConditionType valid conditions of a machineconfigpool
+type ControllerConfigStatusConditionType string
+
+const (
+	TemplateContollerRunning   ControllerConfigStatusConditionType = "TemplateContollerRunning"
+	TemplateContollerCompleted ControllerConfigStatusConditionType = "TemplateContollerCompleted"
+	TemplateContollerFailing   ControllerConfigStatusConditionType = "TemplateContollerFailing"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // ControllerConfigList is a list of ControllerConfig resources

pkg/controller/template/status.go

@@ -0,0 +1,100 @@
+package template
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/client-go/util/retry"
+
+	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
+	mcfgclientv1 "github.com/openshift/machine-config-operator/pkg/generated/clientset/versioned/typed/machineconfiguration.openshift.io/v1"
+	"github.com/openshift/machine-config-operator/pkg/version"
+)
+
+// - sets `running` condition to `true`.
+// - reset the `available` condition to `false` when we are syncing to new generation.
+// - does not modify `failing` condition.
+func (ctrl *Controller) syncRunningStatus(ctrlconfig *mcfgv1.ControllerConfig) error {
+	updateFunc := func(cfg *mcfgv1.ControllerConfig) error {
+		reason := fmt.Sprintf("syncing towards (%d) generation using controller version %s", cfg.GetGeneration(), version.Version)
+		rcond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerRunning, corev1.ConditionTrue, reason, "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *rcond)
+		if cfg.GetGeneration() != cfg.Status.ObservedGeneration && mcfgv1.IsControllerConfigStatusConditionPresentAndEqual(cfg.Status.Conditions, mcfgv1.TemplateContollerCompleted, corev1.ConditionTrue) {
+			acond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerCompleted, corev1.ConditionFalse, fmt.Sprintf("%s due to change in Generation", reason), "")
+			mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *acond)
+		}
+		cfg.Status.ObservedGeneration = ctrlconfig.GetGeneration()
+		return nil
+	}
+	return updateControllerConfigStatus(ctrlconfig.GetName(), ctrl.ccLister.Get, ctrl.client.MachineconfigurationV1().ControllerConfigs(), updateFunc)
+}
+
+// - resets `running` condition to `false`
+// - resets `completed` condition to `false`
+// - sets the `failing` condition to `true` using the `oerr`
+func (ctrl *Controller) syncFailingStatus(ctrlconfig *mcfgv1.ControllerConfig, oerr error) error {
+	if oerr == nil {
+		return oerr
+	}
+	updateFunc := func(cfg *mcfgv1.ControllerConfig) error {
+		reason := oerr.Error()
+		message := fmt.Sprintf("failed to syncing towards (%d) generation using controller version %s: %v", cfg.GetGeneration(), version.Version, oerr)
+		fcond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerFailing, corev1.ConditionTrue, reason, message)
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *fcond)
+		acond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerCompleted, corev1.ConditionFalse, "", "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *acond)
+		rcond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerRunning, corev1.ConditionFalse, "", "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *rcond)
+		cfg.Status.ObservedGeneration = ctrlconfig.GetGeneration()
+		return nil
+	}
+	if err := updateControllerConfigStatus(ctrlconfig.GetName(), ctrl.ccLister.Get, ctrl.client.MachineconfigurationV1().ControllerConfigs(), updateFunc); err != nil {
+		return fmt.Errorf("failed to sync status for %v", oerr)
+	}
+	return oerr
+}
+
+// - resets `running` condition to `false`
+// - resets `failing` condition to `false`
+// - sets the `completed` condition to `true`
+func (ctrl *Controller) syncCompletedStatus(ctrlconfig *mcfgv1.ControllerConfig) error {
+	updateFunc := func(cfg *mcfgv1.ControllerConfig) error {
+		reason := fmt.Sprintf("sync completed towards (%d) generation using controller version %s", cfg.GetGeneration(), version.Version)
+		acond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerCompleted, corev1.ConditionTrue, reason, "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *acond)
+		rcond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerRunning, corev1.ConditionFalse, "", "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *rcond)
+		fcond := mcfgv1.NewControllerConfigStatusCondition(mcfgv1.TemplateContollerFailing, corev1.ConditionFalse, "", "")
+		mcfgv1.SetControllerConfigStatusCondition(&cfg.Status, *fcond)
+		cfg.Status.ObservedGeneration = ctrlconfig.GetGeneration()
+		return nil
+	}
+	return updateControllerConfigStatus(ctrlconfig.GetName(), ctrl.ccLister.Get, ctrl.client.MachineconfigurationV1().ControllerConfigs(), updateFunc)
+}
+
+type updateControllerConfigStatusFunc func(*mcfgv1.ControllerConfig) error
+
+func updateControllerConfigStatus(name string,
+	controllerConfigGetter func(name string) (*mcfgv1.ControllerConfig, error),
+	client mcfgclientv1.ControllerConfigInterface,
+	updateFuncs ...updateControllerConfigStatusFunc) error {
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		old, err := controllerConfigGetter(name)
+		if err != nil {
+			return err
+		}
+		new := old.DeepCopy()
+		for _, update := range updateFuncs {
+			if err := update(new); err != nil {
+				return err
+			}
+		}
+
+		if equality.Semantic.DeepEqual(old, new) {
+			return nil
+		}
+		_, err = client.UpdateStatus(new)
+		return err
+	})
+}

pkg/controller/template/template_controller.go

@@ -332,34 +332,40 @@ func (ctrl *Controller) syncControllerConfig(key string) error {
 	// TODO: Deep-copy only when needed.
 	cfg := controllerconfig.DeepCopy()
 
+	if cfg.GetGeneration() != cfg.Status.ObservedGeneration {
+		if err := ctrl.syncRunningStatus(cfg); err != nil {
+			return err
+		}
+	}
+
 	var pullSecretRaw []byte
 	if cfg.Spec.PullSecret != nil {
 		secret, err := ctrl.kubeClient.CoreV1().Secrets(cfg.Spec.PullSecret.Namespace).Get(cfg.Spec.PullSecret.Name, metav1.GetOptions{})
 		if err != nil {
-			return err
+			return ctrl.syncFailingStatus(cfg, err)
 		}
 
 		if secret.Type != corev1.SecretTypeDockerConfigJson {
-			return fmt.Errorf("expected secret type %s found %s", corev1.SecretTypeDockerConfigJson, secret.Type)
+			return ctrl.syncFailingStatus(cfg, fmt.Errorf("expected secret type %s found %s", corev1.SecretTypeDockerConfigJson, secret.Type))
 		}
 		pullSecretRaw = secret.Data[corev1.DockerConfigJsonKey]
 	}
 	mcs, err := getMachineConfigsForControllerConfig(ctrl.templatesDir, cfg, pullSecretRaw)
 	if err != nil {
-		return err
+		return ctrl.syncFailingStatus(cfg, err)
 	}
 
 	for idx := range mcs {
 		_, updated, err := resourceapply.ApplyMachineConfig(ctrl.client.MachineconfigurationV1(), mcs[idx])
 		if err != nil {
-			return err
+			return ctrl.syncFailingStatus(cfg, err)
 		}
 		if updated {
 			glog.V(4).Infof("Machineconfig %s was updated", mcs[idx].Name)
 		}
 	}
 
-	return nil
+	return ctrl.syncCompletedStatus(cfg)
 }
 
 func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte) ([]*mcfgv1.MachineConfig, error) {