Migrate NMState to use kubernetes-nmstate-operator if available (#815)

This patch make CNAO use the kubernetes-nmstate-operator if it is
running and available. The only prereq for this is to have the
operator pod running. If that's detected via the
app=kubernetes-nmstate-operator label on a Deployment, then CNAO
will only render an nmstates.nmstate.io CR to kick off the handler
and webhook from the operator.

If the operator is added, it also has upgrade logic to move to using
the operator instead of installing kubernetes-nmstate directly. It
will remove the old instance.

Signed-off-by: Brad P. Crochet <brad@redhat.com>

.gitignore

@@ -29,3 +29,5 @@ whitespace-check
 
 # Goland IDE folder
 .idea/
+# VS Code IDE folder
+.vscode/

data/nmstate/operator/nmstate.io_v1beta1_nmstate_cr.yaml

@@ -0,0 +1,4 @@
+apiVersion: nmstate.io/v1beta1
+kind: NMState
+metadata:
+  name: nmstate

hack/components/bump-nmstate.sh

@@ -46,11 +46,14 @@ echo 'Configure nmstate-webhook and nmstate-handler templates and save the rende
 
 echo 'Copy kubernetes-nmstate manifests'
 rm -rf data/nmstate/*
-cp $NMSTATE_PATH/config/cnao/handler/* data/nmstate/
-cp $NMSTATE_PATH/deploy/crds/nmstate.io_nodenetwork*.yaml data/nmstate/
-cp $NMSTATE_PATH/deploy/openshift/scc.yaml data/nmstate/scc.yaml
-sed -i "s/---/{{ if .EnableSCC }}\n---/" data/nmstate/scc.yaml
-echo "{{ end }}" >> data/nmstate/scc.yaml
+mkdir -p data/nmstate/{operator,operand}/
+cp $NMSTATE_PATH/config/cnao/handler/* data/nmstate/operand/
+cp $NMSTATE_PATH/deploy/crds/nmstate.io_nodenetwork*.yaml data/nmstate/operand/
+cp $NMSTATE_PATH/deploy/openshift/scc.yaml data/nmstate/operand/scc.yaml
+sed -i "s/---/{{ if .EnableSCC }}\n---/" data/nmstate/operand/scc.yaml
+echo "{{ end }}" >> data/nmstate/operand/scc.yaml
+
+cp $NMSTATE_PATH/deploy/crds/nmstate.io_v1beta1_nmstate_cr.yaml data/nmstate/operator/
 
 echo 'Apply custom CNAO patches on kubernetes-nmstate manifests'
-sed -i -z 's#kind: Secret\nmetadata:#kind: Secret\nmetadata:\n  annotations:\n    networkaddonsoperator.network.kubevirt.io\/rejectOwner: ""#' data/nmstate/operator.yaml
+sed -i -z 's#kind: Secret\nmetadata:#kind: Secret\nmetadata:\n  annotations:\n    networkaddonsoperator.network.kubevirt.io\/rejectOwner: ""#' data/nmstate/operand/operator.yaml

pkg/controller/networkaddonsconfig/networkaddonsconfig_controller.go

@@ -191,9 +191,19 @@ func (r *ReconcileNetworkAddonsConfig) Reconcile(request reconcile.Request) (rec
 		return reconcile.Result{}, nil
 	}
 
+	// Check for NMState Operator
+	nmstateOperator, err := isRunningKubernetesNMStateOperator(r.client)
+	if err != nil {
+		return reconcile.Result{}, errors.Wrap(err, "failed to check whether running Kubernetes NMState Operator")
+	}
+	if nmstateOperator {
+		log.Printf("Kubernetes NMState Operator is running")
+	}
+	r.clusterInfo.NmstateOperator = nmstateOperator
+
 	// Fetch the NetworkAddonsConfig instance
 	networkAddonsConfigStorageVersion := &cnaov1.NetworkAddonsConfig{}
-	err := r.client.Get(context.TODO(), request.NamespacedName, networkAddonsConfigStorageVersion)
+	err = r.client.Get(context.TODO(), request.NamespacedName, networkAddonsConfigStorageVersion)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
 			// Request object not found, could have been deleted after reconcile request.
@@ -316,7 +326,7 @@ func (r *ReconcileNetworkAddonsConfig) renderObjectsV1(networkAddonsConfig *cnao
 
 	// Perform any special object changes that are impossible to do with regular Apply. e.g. Remove outdated objects
 	// and objects that cannot be modified by Apply method due to incompatible changes.
-	if err := network.SpecialCleanUp(&networkAddonsConfig.Spec, r.client); err != nil {
+	if err := network.SpecialCleanUp(&networkAddonsConfig.Spec, r.client, r.clusterInfo); err != nil {
 		log.Printf("failed to Clean Up outdated objects: %v", err)
 		return objs, err
 	}
@@ -529,6 +539,21 @@ func isSCCAvailable(c kubernetes.Interface) (bool, error) {
 	return isResourceAvailable(c, "securitycontextconstraints", "security.openshift.io", "v1")
 }
 
+func isRunningKubernetesNMStateOperator(c k8sclient.Client) (bool, error) {
+	deployments := &appsv1.DeploymentList{}
+	err := c.List(context.TODO(), deployments, k8sclient.MatchingLabels{"app": "kubernetes-nmstate-operator"})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	if len(deployments.Items) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
 func isResourceAvailable(kubeClient kubernetes.Interface, name string, group string, version string) (bool, error) {
 	result := kubeClient.ExtensionsV1beta1().RESTClient().Get().RequestURI("/apis/" + group + "/" + version + "/" + name).Do(context.TODO())
 	if result.Error() != nil {

pkg/network/cluster-info.go

@@ -1,6 +1,7 @@
 package network
 
 type ClusterInfo struct {
-	SCCAvailable bool
-	OpenShift4   bool
+	SCCAvailable    bool
+	OpenShift4      bool
+	NmstateOperator bool
 }

pkg/network/network.go

@@ -57,12 +57,12 @@ func FillDefaults(conf, previous *cnao.NetworkAddonsConfigSpec) error {
 }
 
 // specialCleanUp checks if there are any specific outdated objects or ones that are no longer compatible and deletes them.
-func SpecialCleanUp(conf *cnao.NetworkAddonsConfigSpec, client k8sclient.Client) error {
+func SpecialCleanUp(conf *cnao.NetworkAddonsConfigSpec, client k8sclient.Client, clusterInfo *ClusterInfo) error {
 	errs := []error{}
 	ctx := context.TODO()
 
 	errs = append(errs, cleanUpMultus(conf, ctx, client)...)
-	errs = append(errs, cleanUpNMState(conf, ctx, client)...)
+	errs = append(errs, cleanUpNMState(conf, ctx, client, clusterInfo)...)
 
 	if len(errs) > 0 {
 		return errors.Errorf("invalid configuration:\n%v", errorListToMultiLineString(errs))

pkg/network/nmstate.go

@@ -38,14 +38,44 @@ func renderNMState(conf *cnao.NetworkAddonsConfigSpec, manifestDir string, clust
 	data.Data["CertOverlapInterval"] = conf.SelfSignConfiguration.CertOverlapInterval
 	data.Data["PlacementConfiguration"] = conf.PlacementConfiguration
 
-	objs, err := render.RenderDir(filepath.Join(manifestDir, "nmstate"), &data)
+	log.Printf("NMStateOperator == %t", clusterInfo.NmstateOperator)
+	fullManifestDir := filepath.Join(manifestDir, "nmstate", "operand")
+	if clusterInfo.NmstateOperator {
+		fullManifestDir = filepath.Join(manifestDir, "nmstate", "operator")
+	}
+	log.Printf("Rendering NMState directory: %s", fullManifestDir)
+
+	objs, err := render.RenderDir(fullManifestDir, &data)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to render nmstate state handler manifests")
 	}
 
 	return objs, nil
 }
 
+func removeAppsV1Resource(ctx context.Context, client k8sclient.Client, name, namespace, kind string) []error {
+	// Get existing
+	existing := &unstructured.Unstructured{}
+	gvk := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: kind}
+	existing.SetGroupVersionKind(gvk)
+
+	err := client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, existing)
+
+	// if we found the object
+	if err == nil {
+		objDesc := fmt.Sprintf("(%s) %s/%s", gvk.String(), namespace, name)
+		log.Printf("Cleanup up %s Object", objDesc)
+
+		// Delete the object
+		err = client.Delete(ctx, existing)
+		if err != nil {
+			log.Printf("Failed Cleaning up %s Object", objDesc)
+			return []error{err}
+		}
+	}
+	return []error{}
+}
+
 func removeDaemonSetHandlerWorker(ctx context.Context, client k8sclient.Client) []error {
 	// Get existing
 	existing := &unstructured.Unstructured{}
@@ -74,13 +104,42 @@ func removeDaemonSetHandlerWorker(ctx context.Context, client k8sclient.Client)
 	return []error{}
 }
 
-func cleanUpNMState(conf *cnao.NetworkAddonsConfigSpec, ctx context.Context, client k8sclient.Client) []error {
+func removeStandaloneHandler(ctx context.Context, client k8sclient.Client) []error {
+	namespace := os.Getenv("OPERAND_NAMESPACE")
+	name := "nmstate-handler"
+	kind := "DaemonSet"
+
+	return removeAppsV1Resource(ctx, client, name, namespace, kind)
+}
+
+func removeStandaloneWebhook(ctx context.Context, client k8sclient.Client) []error {
+	namespace := os.Getenv("OPERAND_NAMESPACE")
+	name := "nmstate-webhook"
+	kind := "Deployment"
+
+	return removeAppsV1Resource(ctx, client, name, namespace, kind)
+}
+
+func removeStandaloneCertManager(ctx context.Context, client k8sclient.Client) []error {
+	namespace := os.Getenv("OPERAND_NAMESPACE")
+	name := "nmstate-cert-manager"
+	kind := "Deployment"
+
+	return removeAppsV1Resource(ctx, client, name, namespace, kind)
+}
+
+func cleanUpNMState(conf *cnao.NetworkAddonsConfigSpec, ctx context.Context, client k8sclient.Client, clusterInfo *ClusterInfo) []error {
 	if conf.NMState == nil {
 		return []error{}
 	}
 
 	errList := []error{}
 	errList = append(errList, removeDaemonSetHandlerWorker(ctx, client)...)
+	if clusterInfo.NmstateOperator {
+		errList = append(errList, removeStandaloneHandler(ctx, client)...)
+		errList = append(errList, removeStandaloneWebhook(ctx, client)...)
+		errList = append(errList, removeStandaloneCertManager(ctx, client)...)
+	}
 
 	return errList
 }

test/check/check.go

@@ -183,6 +183,17 @@ func CheckOperatorIsReady(timeout time.Duration) {
 	}
 }
 
+func CheckNMStateOperatorIsReady(timeout time.Duration) {
+	By("Checking that the operator is up and running")
+	if timeout != CheckImmediately {
+		Eventually(func() error {
+			return checkForGenericDeployment("nmstate-operator", "nmstate", false)
+		}, timeout, time.Second).ShouldNot(HaveOccurred(), fmt.Sprintf("Timed out waiting for the operator to become ready"))
+	} else {
+		Expect(checkForGenericDeployment("nmstate-operator", "nmstate", false)).ShouldNot(HaveOccurred(), "Operator is not ready")
+	}
+}
+
 func CheckForLeftoverObjects(currentVersion string) {
 	listOptions := client.ListOptions{}
 	key := cnaov1.SchemeGroupVersion.Group + "/version"
@@ -358,17 +369,23 @@ func checkForSecurityContextConstraints(name string) error {
 }
 
 func checkForDeployment(name string) error {
+	return checkForGenericDeployment(name, components.Namespace, true)
+}
+
+func checkForGenericDeployment(name, namespace string, checkLabels bool) error {
 	deployment := appsv1.Deployment{}
 
-	err := framework.Global.Client.Get(context.Background(), types.NamespacedName{Name: name, Namespace: components.Namespace}, &deployment)
+	err := framework.Global.Client.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &deployment)
 	if err != nil {
 		return err
 	}
 
-	labels := deployment.GetLabels()
-	if labels != nil {
-		if _, operatorLabelSet := labels[cnaov1.SchemeGroupVersion.Group+"/version"]; !operatorLabelSet {
-			return fmt.Errorf("Deployment %s/%s is missing operator label", components.Namespace, name)
+	if checkLabels {
+		labels := deployment.GetLabels()
+		if labels != nil {
+			if _, operatorLabelSet := labels[cnaov1.SchemeGroupVersion.Group+"/version"]; !operatorLabelSet {
+				return fmt.Errorf("Deployment %s/%s is missing operator label", components.Namespace, name)
+			}
 		}
 	}
 
@@ -377,7 +394,7 @@ func checkForDeployment(name string) error {
 		if err != nil {
 			panic(err)
 		}
-		return fmt.Errorf("Deployment %s/%s is not ready, current state:\n%v\ncluster Info:\n%v", components.Namespace, name, string(manifest), gatherClusterInfo())
+		return fmt.Errorf("Deployment %s/%s is not ready, current state:\n%v\ncluster Info:\n%v", namespace, name, string(manifest), gatherClusterInfo())
 	}
 
 	return nil

test/check/components.go

@@ -1,5 +1,13 @@
 package check
 
+import (
+	"fmt"
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
 type Component struct {
 	ComponentName                string
 	ClusterRole                  string
@@ -77,3 +85,57 @@ var (
 		MacvtapComponent,
 	}
 )
+
+type ComponentUpdatePolicy string
+
+const (
+	Tagged ComponentUpdatePolicy = "tagged"
+	Static ComponentUpdatePolicy = "static"
+	Latest ComponentUpdatePolicy = "latest"
+)
+
+type ComponentsConfig struct {
+	Components map[string]ComponentSource `yaml:"components"`
+}
+
+type ComponentSource struct {
+	Url          string                `yaml:"url"`
+	Commit       string                `yaml:"commit"`
+	Branch       string                `yaml:"branch"`
+	UpdatePolicy ComponentUpdatePolicy `yaml:"update-policy"`
+	Metadata     string                `yaml:"metadata"`
+}
+
+func GetComponentSource(component string) (ComponentSource, error) {
+	componentsConfig, err := parseComponentsYaml("components.yaml")
+	if err != nil {
+		return ComponentSource{}, errors.Wrapf(err, "Failed to get components config")
+	}
+
+	componentSource, ok := componentsConfig.Components[component]
+	if !ok {
+		return ComponentSource{}, errors.Wrapf(err, "Failed to get component %s", component)
+	}
+
+	return componentSource, nil
+}
+
+func parseComponentsYaml(componentsConfigPath string) (ComponentsConfig, error) {
+	config := ComponentsConfig{}
+
+	componentsData, err := ioutil.ReadFile(componentsConfigPath)
+	if err != nil {
+		return ComponentsConfig{}, errors.Wrapf(err, "Failed to open file %s", componentsConfigPath)
+	}
+
+	err = yaml.Unmarshal(componentsData, &config)
+	if err != nil {
+		return ComponentsConfig{}, errors.Wrapf(err, "Failed to Unmarshal %s", componentsConfigPath)
+	}
+
+	if len(config.Components) == 0 {
+		return ComponentsConfig{}, fmt.Errorf("Failed to Unmarshal %s. Output is empty", componentsConfigPath)
+	}
+
+	return config, nil
+}