From b838f0a078d5bc1e59b15bd664550b4813dff99d Mon Sep 17 00:00:00 2001 From: Harshal Patil Date: Tue, 1 Dec 2020 11:35:23 +0530 Subject: [PATCH] Prevent multile KataConfig CRs Signed-off-by: Harshal Patil --- api/v1/kataconfig_types.go | 2 + ...onfiguration.openshift.io_kataconfigs.yaml | 439 +++++++++--------- controllers/openshift_controller.go | 68 +++ controllers/openshift_controller_test.go | 57 +++ controllers/suite_test.go | 29 +- 5 files changed, 375 insertions(+), 220 deletions(-) create mode 100644 controllers/openshift_controller_test.go diff --git a/api/v1/kataconfig_types.go b/api/v1/kataconfig_types.go index e263afdc..f04c487c 100644 --- a/api/v1/kataconfig_types.go +++ b/api/v1/kataconfig_types.go @@ -26,6 +26,7 @@ type KataConfigSpec struct { // KataConfigPoolSelector is used to filer the worker nodes // if not specified, all worker nodes are selected // +optional + // +nullable KataConfigPoolSelector *metav1.LabelSelector `json:"kataConfigPoolSelector"` // +optional @@ -68,6 +69,7 @@ type KataConfig struct { metav1.ObjectMeta `json:"metadata,omitempty"` // +optional + // +nullable Spec KataConfigSpec `json:"spec,omitempty"` Status KataConfigStatus `json:"status,omitempty"` } diff --git a/config/crd/bases/kataconfiguration.openshift.io_kataconfigs.yaml b/config/crd/bases/kataconfiguration.openshift.io_kataconfigs.yaml index 674d1861..4767bc64 100644 --- a/config/crd/bases/kataconfiguration.openshift.io_kataconfigs.yaml +++ b/config/crd/bases/kataconfiguration.openshift.io_kataconfigs.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.4.0 creationTimestamp: null name: kataconfigs.kataconfiguration.openshift.io spec: @@ -15,229 +15,232 @@ spec: plural: kataconfigs singular: kataconfig scope: Cluster - subresources: - status: {} - validation: - openAPIV3Schema: - description: KataConfig is the Schema for the kataconfigs API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: KataConfigSpec defines the desired state of KataConfig - properties: - config: - description: KataInstallConfig is a placeholder struct - properties: - sourceImage: - description: SourceImage is the name of the kata-deploy image - type: string - required: - - sourceImage - type: object - kataConfigPoolSelector: - description: KataConfigPoolSelector is used to filer the worker nodes - if not specified, all worker nodes are selected - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + versions: + - name: v1 + schema: + openAPIV3Schema: + description: KataConfig is the Schema for the kataconfigs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KataConfigSpec defines the desired state of KataConfig + nullable: true + properties: + config: + description: KataInstallConfig is a placeholder struct + properties: + sourceImage: + description: SourceImage is the name of the kata-deploy image + type: string + required: + - sourceImage + type: object + kataConfigPoolSelector: + description: KataConfigPoolSelector is used to filer the worker nodes + if not specified, all worker nodes are selected + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + type: object + status: + description: KataConfigStatus defines the observed state of KataConfig + properties: + installationStatus: + description: InstallationStatus reflects the status of the ongoing + kata installation + properties: + completed: + description: Completed reflects the status of nodes that have + completed kata installation properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to a - set of values. Valid operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator - is In or NotIn, the values array must be non-empty. If the - operator is Exists or DoesNotExist, the values array must - be empty. This array is replaced during a strategic merge - patch. + completedNodesCount: + description: CompletedNodesCount reflects the number of nodes + that have completed kata operation + type: integer + completedNodesList: + description: CompletedNodesList reflects the list of nodes + that have completed kata operation items: type: string type: array - required: - - key - - operator type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator is - "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - type: object - status: - description: KataConfigStatus defines the observed state of KataConfig - properties: - installationStatus: - description: InstallationStatus reflects the status of the ongoing kata - installation - properties: - completed: - description: Completed reflects the status of nodes that have completed - kata installation - properties: - completedNodesCount: - description: CompletedNodesCount reflects the number of nodes - that have completed kata operation - type: integer - completedNodesList: - description: CompletedNodesList reflects the list of nodes that - have completed kata operation - items: - type: string - type: array - type: object - failed: - description: Failed reflects the status of nodes that have failed - kata installation - properties: - failedNodesCount: - description: FailedNodesCount reflects the number of nodes that - have failed kata operation - type: integer - failedNodesList: - description: FailedNodesList reflects the list of nodes that - have failed kata operation - items: - description: FailedNodeStatus holds the name and the error - message of the failed node - properties: - error: - description: Error message of the failed node reported - by the installation daemon - type: string - name: - description: Name of the failed node - type: string - required: - - error - - name - type: object - type: array - type: object - inProgress: - description: InProgress reflects the status of nodes that are in - the process of kata installation - properties: - binariesInstallNodesList: - items: - type: string - type: array - inProgressNodesCount: - description: InProgressNodesCount reflects the number of nodes - that are in the process of kata installation - type: integer - type: object - type: object - kataImage: - description: KataImage is the image used for delivering kata binaries - type: string - runtimeClass: - description: RuntimeClass is the name of the runtime class used in CRIO - configuration - type: string - totalNodesCount: - description: TotalNodesCounts is the total number of worker nodes targeted - by this CR - type: integer - unInstallationStatus: - description: UnInstallationStatus reflects the status of the ongoing - kata uninstallation - properties: - completed: - description: Completed reflects the status of nodes that have completed - kata uninstallation - properties: - completedNodesCount: - description: CompletedNodesCount reflects the number of nodes - that have completed kata operation - type: integer - completedNodesList: - description: CompletedNodesList reflects the list of nodes that - have completed kata operation - items: - type: string - type: array - type: object - failed: - description: Failed reflects the status of nodes that have failed - kata uninstallation - properties: - failedNodesCount: - description: FailedNodesCount reflects the number of nodes that - have failed kata operation - type: integer - failedNodesList: - description: FailedNodesList reflects the list of nodes that - have failed kata operation - items: - description: FailedNodeStatus holds the name and the error - message of the failed node - properties: - error: - description: Error message of the failed node reported - by the installation daemon - type: string - name: - description: Name of the failed node - type: string - required: - - error - - name - type: object - type: array - type: object - inProgress: - description: InProgress reflects the status of nodes that are in - the process of kata uninstallation - properties: - binariesUninstallNodesList: - items: - type: string - type: array - inProgressNodesCount: - type: integer - type: object - type: object - upgradeStatus: - description: Upgradestatus reflects the status of the ongoing kata upgrade - type: object - required: - - kataImage - - runtimeClass - - totalNodesCount - type: object - type: object - version: v1 - versions: - - name: v1 + failed: + description: Failed reflects the status of nodes that have failed + kata installation + properties: + failedNodesCount: + description: FailedNodesCount reflects the number of nodes + that have failed kata operation + type: integer + failedNodesList: + description: FailedNodesList reflects the list of nodes that + have failed kata operation + items: + description: FailedNodeStatus holds the name and the error + message of the failed node + properties: + error: + description: Error message of the failed node reported + by the installation daemon + type: string + name: + description: Name of the failed node + type: string + required: + - error + - name + type: object + type: array + type: object + inProgress: + description: InProgress reflects the status of nodes that are + in the process of kata installation + properties: + binariesInstallNodesList: + items: + type: string + type: array + inProgressNodesCount: + description: InProgressNodesCount reflects the number of nodes + that are in the process of kata installation + type: integer + type: object + type: object + kataImage: + description: KataImage is the image used for delivering kata binaries + type: string + runtimeClass: + description: RuntimeClass is the name of the runtime class used in + CRIO configuration + type: string + totalNodesCount: + description: TotalNodesCounts is the total number of worker nodes + targeted by this CR + type: integer + unInstallationStatus: + description: UnInstallationStatus reflects the status of the ongoing + kata uninstallation + properties: + completed: + description: Completed reflects the status of nodes that have + completed kata uninstallation + properties: + completedNodesCount: + description: CompletedNodesCount reflects the number of nodes + that have completed kata operation + type: integer + completedNodesList: + description: CompletedNodesList reflects the list of nodes + that have completed kata operation + items: + type: string + type: array + type: object + failed: + description: Failed reflects the status of nodes that have failed + kata uninstallation + properties: + failedNodesCount: + description: FailedNodesCount reflects the number of nodes + that have failed kata operation + type: integer + failedNodesList: + description: FailedNodesList reflects the list of nodes that + have failed kata operation + items: + description: FailedNodeStatus holds the name and the error + message of the failed node + properties: + error: + description: Error message of the failed node reported + by the installation daemon + type: string + name: + description: Name of the failed node + type: string + required: + - error + - name + type: object + type: array + type: object + inProgress: + description: InProgress reflects the status of nodes that are + in the process of kata uninstallation + properties: + binariesUninstallNodesList: + items: + type: string + type: array + inProgressNodesCount: + type: integer + type: object + type: object + upgradeStatus: + description: Upgradestatus reflects the status of the ongoing kata + upgrade + type: object + required: + - kataImage + - runtimeClass + - totalNodesCount + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/controllers/openshift_controller.go b/controllers/openshift_controller.go index fd0fbca6..5cdfdffd 100644 --- a/controllers/openshift_controller.go +++ b/controllers/openshift_controller.go @@ -40,6 +40,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // blank assignment to verify that KataConfigOpenShiftReconciler implements reconcile.Reconciler @@ -83,6 +84,13 @@ func (r *KataConfigOpenShiftReconciler) Reconcile(req ctrl.Request) (ctrl.Result } return func() (ctrl.Result, error) { + oldest, err := r.isOldestCR() + if !oldest && err != nil { + return reconcile.Result{Requeue: true}, err + } else if !oldest && err == nil { + return reconcile.Result{}, nil + } + // Check if the KataConfig instance is marked to be deleted, which is // indicated by the deletion timestamp being set. if r.kataConfig.GetDeletionTimestamp() != nil { @@ -735,3 +743,63 @@ func (r *KataConfigOpenShiftReconciler) SetupWithManager(mgr ctrl.Manager) error For(&kataconfigurationv1.KataConfig{}). Complete(r) } + +func (r *KataConfigOpenShiftReconciler) isOldestCR() (bool, error) { + kataConfigList := &kataconfigurationv1.KataConfigList{} + listOpts := []client.ListOption{ + client.InNamespace(corev1.NamespaceAll), + } + if err := r.Client.List(context.TODO(), kataConfigList, listOpts...); err != nil { + return false, fmt.Errorf("Failed to list KataConfig custom resources: %v", err) + } + + if len(kataConfigList.Items) == 1 { + return true, nil + } + + // Creation time of the CR of the current reconciliation request + tkccd := r.kataConfig.GetCreationTimestamp() + + // holds the oldest CR found so far + var oldestCR *kataconfigurationv1.KataConfig + + for index := range kataConfigList.Items { + if kataConfigList.Items[index].Name == r.kataConfig.Name { + continue + } + + // Creation time of this instance of CR in the loop + ckccd := kataConfigList.Items[index].GetCreationTimestamp() + + if oldestCR == nil { + oldestCR = &kataConfigList.Items[index] + } else { + oldestCreationDateSoFar := oldestCR.GetCreationTimestamp() + if !oldestCreationDateSoFar.Before(&ckccd) { + oldestCR = &kataConfigList.Items[index] + } + } + } + + oldestCRCreationDate := oldestCR.GetCreationTimestamp() + if !tkccd.Before(&oldestCRCreationDate) { + if r.kataConfig.Status.InstallationStatus.Failed.FailedNodesCount != -1 { + r.kataConfig.Status.InstallationStatus.Failed.FailedNodesCount = -1 + r.kataConfig.Status.InstallationStatus.Failed.FailedNodesList = []kataconfigurationv1.FailedNodeStatus{ + { + Name: "", + Error: fmt.Sprintf("Multiple KataConfig CRs are not supported, %s already exists", oldestCR.Name), + }, + } + + err := r.Client.Status().Update(context.TODO(), r.kataConfig) + if err != nil { + return false, err + } + + return false, nil + } + } + + return true, nil +} diff --git a/controllers/openshift_controller_test.go b/controllers/openshift_controller_test.go new file mode 100644 index 00000000..36a7e1d0 --- /dev/null +++ b/controllers/openshift_controller_test.go @@ -0,0 +1,57 @@ +package controllers + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + kataconfigurationv1 "github.com/openshift/kata-operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +var _ = Describe("OpenShift KataConfig Controller", func() { + Context("KataConfig create", func() { + It("Should not support multiple KataConfig CRs", func() { + + const ( + name = "example-kataconfig" + ) + + kataconfig := &kataconfigurationv1.KataConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kataconfiguration.openshift.io/v1", + Kind: "KataConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + + By("Creating the first KataConfig CR successfully") + Expect(k8sClient.Create(context.Background(), kataconfig)).Should(Succeed()) + time.Sleep(time.Second * 5) + + kataconfig2 := &kataconfigurationv1.KataConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kataconfiguration.openshift.io/v1", + Kind: "KataConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name + "2", + }, + } + + Expect(k8sClient.Create(context.Background(), kataconfig2)).Should(Succeed()) + time.Sleep(time.Second * 5) + + kataConfig2Key := types.NamespacedName{Name: kataconfig2.Name} + Expect(k8sClient.Get(context.Background(), kataConfig2Key, kataconfig2)).Should(Succeed()) + + By("Creating marking the second KataConfig CR correctly") + Expect(kataconfig2.Status.InstallationStatus.Failed.FailedNodesCount).Should(Equal(-1)) + }) + }) + +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 5a765fbf..9abfda76 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -22,8 +22,11 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -40,6 +43,7 @@ import ( var cfg *rest.Config var k8sClient client.Client var testEnv *envtest.Environment +var k8sManager ctrl.Manager func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) @@ -62,13 +66,34 @@ var _ = BeforeSuite(func(done Done) { Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - err = kataconfigurationv1.AddToScheme(scheme.Scheme) + s := scheme.Scheme + + s.AddKnownTypes(appsv1.SchemeGroupVersion, &appsv1.DaemonSet{}) + s.AddKnownTypes(corev1.SchemeGroupVersion, &corev1.NodeList{}) + + err = kataconfigurationv1.AddToScheme(s) + Expect(err).NotTo(HaveOccurred()) // +kubebuilder:scaffold:scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + }) Expect(err).ToNot(HaveOccurred()) + + err = (&KataConfigOpenShiftReconciler{ + Client: k8sManager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("KataConfig"), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + + go func() { + err = k8sManager.Start(ctrl.SetupSignalHandler()) + Expect(err).ToNot(HaveOccurred()) + }() + + k8sClient = k8sManager.GetClient() Expect(k8sClient).ToNot(BeNil()) close(done)