OCPBUGS-24416: Sync MCN with node creation and deletion

the MCN object was not reacting to node deletions. the MCO was also creating these objects too early in their lifecycle. We should only
create an MCN  once the node is out of the Provisioning and Pending Status

Signed-off-by: Charlie Doern <cdoern@redhat.com>

install/0000_80_machine-config-operator_00_clusterreader_clusterrole.yaml

@@ -33,6 +33,17 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - machineconfiguration.openshift.io
+    resources:
+      - machineconfignodes
+      - machineconfignodes/status
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
+      - create
   - apiGroups:
       - config.openshift.io
     resources: 

pkg/operator/sync.go

@@ -48,6 +48,7 @@ import (
 	templatectrl "github.com/openshift/machine-config-operator/pkg/controller/template"
 	daemonconsts "github.com/openshift/machine-config-operator/pkg/daemon/constants"
 	"github.com/openshift/machine-config-operator/pkg/server"
+	"github.com/openshift/machine-config-operator/pkg/upgrademonitor"
 	"github.com/openshift/machine-config-operator/pkg/version"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 )
@@ -710,7 +711,14 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig) error {
 	if err != nil {
 		return err
 	}
+	mcns, err := optr.mcNodeLister.List(labels.Everything())
+	if err != nil {
+		return err
+	}
 	for _, node := range nodes {
+		if node.Status.Phase == corev1.NodePending || node.Status.Phase == corev1.NodePhase("Provisioning") {
+			continue
+		}
 		var pool string
 		var ok bool
 		if _, ok = node.Labels["node-role.kubernetes.io/worker"]; ok {
@@ -744,10 +752,31 @@ func (optr *Operator) syncMachineConfigNodes(_ *renderConfig) error {
 			return err
 		}
 		p := mcoResourceRead.ReadMachineConfigNodeV1OrDie(mcsBytes)
-		_, _, err = mcoResourceApply.ApplyMachineConfigNode(optr.client.MachineconfigurationV1alpha1(), p)
+		mcn, _, err := mcoResourceApply.ApplyMachineConfigNode(optr.client.MachineconfigurationV1alpha1(), p)
 		if err != nil {
 			return err
 		}
+		// if this is the first time we are applying the MCN and the node is ready, set the config version probably
+		if mcn.Spec.ConfigVersion.Desired == "NotYetSet" {
+			err = upgrademonitor.GenerateAndApplyMachineConfigNodeSpec(optr.fgAccessor, pool, node, optr.client)
+			if err != nil {
+				klog.Errorf("Error making MCN spec for Update Compatible: %w", err)
+			}
+		}
+
+	}
+	for _, mcn := range mcns {
+		found := false
+		for _, node := range nodes {
+			if node.Name == mcn.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			klog.Infof("Node %s has been removed, deleting associated MCN", mcn.Name)
+			optr.client.MachineconfigurationV1alpha1().MachineConfigNodes().Delete(context.TODO(), mcn.Name, metav1.DeleteOptions{})
+		}
 	}
 	return nil
 }

pkg/upgrademonitor/upgrade_monitor.go

@@ -40,6 +40,14 @@ func GenerateAndApplyMachineConfigNodes(parentCondition, childCondition *Conditi
 		return nil
 	}
 
+	var pool string
+	var ok bool
+	if _, ok = node.Labels["node-role.kubernetes.io/worker"]; ok {
+		pool = "worker"
+	} else if _, ok = node.Labels["node-role.kubernetes.io/master"]; ok {
+		pool = "master"
+	}
+
 	// get the existing MCN, or if it DNE create one below
 	mcNode, needNewMCNode := createOrGetMachineConfigNode(mcfgClient, node)
 	newMCNode := mcNode.DeepCopy()
@@ -162,9 +170,15 @@ func GenerateAndApplyMachineConfigNodes(parentCondition, childCondition *Conditi
 	}
 
 	// for now, keep spec and status aligned
-	newMCNode.Status.ConfigVersion = mcfgalphav1.MachineConfigNodeStatusMachineConfigVersion{
-		Desired: newMCNode.Status.ConfigVersion.Desired,
-		Current: node.Annotations["machineconfiguration.openshift.io/currentConfig"],
+	if node.Annotations["machineconfiguration.openshift.io/currentConfig"] != "" {
+		newMCNode.Status.ConfigVersion = mcfgalphav1.MachineConfigNodeStatusMachineConfigVersion{
+			Desired: newMCNode.Status.ConfigVersion.Desired,
+			Current: node.Annotations["machineconfiguration.openshift.io/currentConfig"],
+		}
+	} else {
+		newMCNode.Status.ConfigVersion = mcfgalphav1.MachineConfigNodeStatusMachineConfigVersion{
+			Desired: newMCNode.Status.ConfigVersion.Desired,
+		}
 	}
 	// if the update is compatible, we can set the desired to the one being used in the update
 	// this happens either if we get prepared == true OR literally any other parent condition, since if we get past prepared, then the desiredConfig is correct.
@@ -176,24 +190,19 @@ func GenerateAndApplyMachineConfigNodes(parentCondition, childCondition *Conditi
 
 	// if we do not need a new MCN, generate the apply configurations for this object
 	if !needNewMCNode {
-		statusconfigVersionApplyConfig := machineconfigurationalphav1.MachineConfigNodeStatusMachineConfigVersion().WithCurrent(newMCNode.Status.ConfigVersion.Current).WithDesired(newMCNode.Status.ConfigVersion.Desired)
+		statusconfigVersionApplyConfig := machineconfigurationalphav1.MachineConfigNodeStatusMachineConfigVersion().WithDesired(newMCNode.Status.ConfigVersion.Desired)
+		if node.Annotations["machineconfiguration.openshift.io/currentConfig"] != "" {
+			statusconfigVersionApplyConfig = statusconfigVersionApplyConfig.WithCurrent(newMCNode.Status.ConfigVersion.Current)
+		}
 		statusApplyConfig := machineconfigurationalphav1.MachineConfigNodeStatus().WithConditions(newMCNode.Status.Conditions...).WithObservedGeneration(newMCNode.Generation + 1).WithConfigVersion(statusconfigVersionApplyConfig)
 		mcnodeApplyConfig := machineconfigurationalphav1.MachineConfigNode(newMCNode.Name).WithStatus(statusApplyConfig)
 		_, err := mcfgClient.MachineconfigurationV1alpha1().MachineConfigNodes().ApplyStatus(context.TODO(), mcnodeApplyConfig, metav1.ApplyOptions{FieldManager: "machine-config-operator", Force: true})
 		if err != nil {
 			klog.Errorf("Error applying MCN status: %w", err)
 			return err
 		}
-	} else {
+	} else if node.Status.Phase != corev1.NodePending && node.Status.Phase != corev1.NodePhase("Provisioning") {
 		// there are cases where we get here before the MCO has settled and applied all of the MCnodes.
-		var pool string
-		var ok bool
-		if _, ok = node.Labels["node-role.kubernetes.io/worker"]; ok {
-			pool = "worker"
-		} else if _, ok = node.Labels["node-role.kubernetes.io/master"]; ok {
-			pool = "master"
-		}
-
 		newMCNode.Spec.ConfigVersion = mcfgalphav1.MachineConfigNodeSpecMachineConfigVersion{
 			Desired: node.Annotations["machineconfiguration.openshift.io/desiredConfig"],
 		}
@@ -209,6 +218,13 @@ func GenerateAndApplyMachineConfigNodes(parentCondition, childCondition *Conditi
 			return err
 		}
 	}
+	// if this is the first time we are applying the MCN and the node is ready, set the config version probably
+	if node.Status.Phase != corev1.NodePending && node.Status.Phase != corev1.NodePhase("Provisioning") && newMCNode.Spec.ConfigVersion.Desired == "NotYetSet" {
+		err = GenerateAndApplyMachineConfigNodeSpec(fgAccessor, pool, node, mcfgClient)
+		if err != nil {
+			klog.Errorf("Error making MCN spec for Update Compatible: %w", err)
+		}
+	}
 	return nil
 }