add attachedcluater e2e test (kurator-dev#594)

* add attachedcluater e2e test

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* change kurator clusters exist wait time

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* add install cert-manager and fluxcd

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* use go code to write kurator e2e test

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* use commit ID as the VERSION

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* fix create kurator-host cluster timeout

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* Added script to create two clusters as e2e test environments

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* Scoping for go test in pkg and cmd

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* fix Fixed kurator-member.config file path error

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

* add fleet status check and delete resource when finish e2e test

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

---------

Signed-off-by: LiZhenCheng9527 <lizhencheng6@huawei.com>

.github/workflows/e2e.yaml

@@ -0,0 +1,33 @@
+name: E2E
+on:
+  pull_request:
+jobs:
+  e2e-test:
+    runs-on: ubuntu-22.04
+    name: E2E Test
+    timeout-minutes: 40
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.20.x
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Build Clusters
+        uses: helm/kind-action@v1.8.0
+        with:
+          version: v0.20.0
+          install_only: true
+      - name: Set Up Clusters
+        run: |
+          hack/e2e-test/build-clusters.sh
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.10.1
+      - name: Init kurator cluster
+        run: |
+          hack/e2e-test/install-kurator.sh
+      - name: fleet-clusters e2e test
+        run: |
+          hack/e2e-test/run-e2e.sh

Makefile

@@ -150,7 +150,8 @@ gen-chart: sync-crds
 
 .PHONY: test
 test: clean tidy
-	go test ./...
+	go test ./pkg/...
+	go test ./cmd/...
 
 .PHONY: clean
 clean:

e2e/attachedcluster_test.go

@@ -0,0 +1,115 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"kurator.dev/kurator/e2e/resources"
+	clusterv1a1 "kurator.dev/kurator/pkg/apis/cluster/v1alpha1"
+	fleetv1a1 "kurator.dev/kurator/pkg/apis/fleet/v1alpha1"
+)
+
+var _ = ginkgo.Describe("[AttachedClusters] AttachedClusters testing", func() {
+	var (
+		namespace         string
+		fleetname         string
+		memberClusterName string
+		kubeconfigPath    string
+		secret            *corev1.Secret
+		attachedcluster   *clusterv1a1.AttachedCluster
+	)
+
+	ginkgo.BeforeEach(func() {
+		namespace = "e2e-test"
+		fleetname = "e2etest"
+		memberClusterName = "kurator-member"
+		homeDir, err := os.UserHomeDir()
+		gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
+		kubeconfigPath = filepath.Join(homeDir, ".kube/kurator-member.config")
+
+		// create namespace for e2e test
+		e2eNamespace := resources.NewNamespace(namespace)
+		createNSErr := resources.CreateNamespace(kubeClient, e2eNamespace)
+		gomega.Expect(createNSErr).ShouldNot(gomega.HaveOccurred())
+		time.Sleep(3 * time.Second)
+
+		// build secrets use member cluster kubeconfig
+		kubeconfig, readfileErr := os.ReadFile(kubeconfigPath)
+		gomega.Expect(readfileErr).ShouldNot(gomega.HaveOccurred())
+		data := make(map[string][]byte)
+		data[memberClusterName] = kubeconfig
+		secret = resources.NewSecret(namespace, memberClusterName, data)
+
+		// build two attachedclusters
+		secretKeyRef := clusterv1a1.SecretKeyRef{
+			Name: memberClusterName,
+			Key:  memberClusterName,
+		}
+		attachedcluster = resources.NewAttachedCluster(namespace, memberClusterName, secretKeyRef)
+	})
+
+	ginkgo.AfterEach(func() {
+		fleerRemoveErr := resources.RemoveFleet(kuratorClient, namespace, fleetname)
+		gomega.Expect(fleerRemoveErr).ShouldNot(gomega.HaveOccurred())
+
+		attachedclusterRemoveErr := resources.RemoveAttachedCluster(kuratorClient, namespace, memberClusterName)
+		gomega.Expect(attachedclusterRemoveErr).ShouldNot(gomega.HaveOccurred())
+
+		secretRemoveErr := resources.RemoveSecret(kubeClient, namespace, memberClusterName)
+		gomega.Expect(secretRemoveErr).ShouldNot(gomega.HaveOccurred())
+
+		namespaceRemoveErr := resources.RemoveNamespace(kubeClient, namespace)
+		gomega.Expect(namespaceRemoveErr).ShouldNot(gomega.HaveOccurred())
+	})
+
+	ginkgo.It("Create Fleet", func() {
+		// step 1.create secrets
+		secretCreateErr := resources.CreateSecret(kubeClient, secret)
+		gomega.Expect(secretCreateErr).ShouldNot(gomega.HaveOccurred())
+
+		// step 2.create attachedclusters
+		attachedCreateErr := resources.CreateAttachedCluster(kuratorClient, attachedcluster)
+		gomega.Expect(attachedCreateErr).ShouldNot(gomega.HaveOccurred())
+
+		time.Sleep(3 * time.Second)
+		// step 3.create fleet
+		clusters := []*corev1.ObjectReference{
+			{
+				Name: memberClusterName,
+				Kind: "AttachedCluster",
+			},
+		}
+		fleet := resources.NewFleet(namespace, fleetname, clusters)
+		fleetCreateErr := resources.CreateFleet(kuratorClient, fleet)
+		gomega.Expect(fleetCreateErr).ShouldNot(gomega.HaveOccurred())
+		time.Sleep(3 * time.Second)
+
+		// step 4.check fleet status
+		fleetPresentOnCluster, fleetGetErr := kuratorClient.FleetV1alpha1().Fleets(namespace).Get(context.TODO(), fleetname, metav1.GetOptions{})
+		gomega.Expect(fleetGetErr).ShouldNot(gomega.HaveOccurred())
+		gomega.Expect(fleetPresentOnCluster.Status.Phase).Should(gomega.Equal(fleetv1a1.ReadyPhase))
+	})
+})

e2e/framework/cluster.go

@@ -0,0 +1,44 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+)
+
+// LoadRESTClientConfig creates a rest.Config using the passed kubeconfig. If context is empty, current context in kubeconfig will be used.
+func LoadRESTClientConfig(kubeconfig string, context string) (*rest.Config, error) {
+	loader := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}
+	loadedConfig, err := loader.Load()
+	if err != nil {
+		return nil, err
+	}
+
+	if context == "" {
+		context = loadedConfig.CurrentContext
+	}
+	klog.Infof("Use context %v", context)
+
+	return clientcmd.NewNonInteractiveClientConfig(
+		*loadedConfig,
+		context,
+		&clientcmd.ConfigOverrides{},
+		loader,
+	).ClientConfig()
+}

e2e/resources/attachedcluster.go

@@ -0,0 +1,77 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resources
+
+import (
+	"context"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	clusterv1a1 "kurator.dev/kurator/pkg/apis/cluster/v1alpha1"
+	kurator "kurator.dev/kurator/pkg/client-go/generated/clientset/versioned"
+)
+
+func NewAttachedCluster(namespace string, name string, config clusterv1a1.SecretKeyRef) *clusterv1a1.AttachedCluster {
+	return &clusterv1a1.AttachedCluster{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "cluster.kurator.dev/v1alpha1",
+			Kind:       "AttachedCluster",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+		Spec: clusterv1a1.AttachedClusterSpec{
+			Kubeconfig: config,
+		},
+	}
+}
+
+// CreateAttachedCluster create AttachedCluster.
+func CreateAttachedCluster(client kurator.Interface, attachedCluster *clusterv1a1.AttachedCluster) error {
+	_, err := client.ClusterV1alpha1().AttachedClusters(attachedCluster.Namespace).Create(context.TODO(), attachedCluster, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			return UpdateAttachedCluster(client, attachedCluster)
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
+// UpdateAttachedCluster update AttachedCluster
+func UpdateAttachedCluster(client kurator.Interface, attachedCluster *clusterv1a1.AttachedCluster) error {
+	_, err := client.ClusterV1alpha1().AttachedClusters(attachedCluster.Namespace).Update(context.TODO(), attachedCluster, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func RemoveAttachedCluster(client kurator.Interface, namespace, name string) error {
+	err := client.ClusterV1alpha1().AttachedClusters(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		} else {
+			return err
+		}
+	}
+	return nil
+}

e2e/resources/fleet.go

@@ -0,0 +1,79 @@
+/*
+Copyright Kurator Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resources
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fleetv1a1 "kurator.dev/kurator/pkg/apis/fleet/v1alpha1"
+	kurator "kurator.dev/kurator/pkg/client-go/generated/clientset/versioned"
+)
+
+// NewFleet will build a Fleet object.
+func NewFleet(namespace string, name string, clusters []*corev1.ObjectReference) *fleetv1a1.Fleet {
+	return &fleetv1a1.Fleet{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "fleet.kurator.dev/v1alpha1",
+			Kind:       "Fleet",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+		Spec: fleetv1a1.FleetSpec{
+			Clusters: clusters,
+		},
+	}
+}
+
+// CreateAttachedCluster create AttachedCluster.
+func CreateFleet(client kurator.Interface, fleet *fleetv1a1.Fleet) error {
+	_, err := client.FleetV1alpha1().Fleets(fleet.Namespace).Create(context.TODO(), fleet, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			return UpdateFleet(client, fleet)
+		} else {
+			return err
+		}
+	}
+	return nil
+}
+
+// UpdateAttachedCluster update AttachedCluster
+func UpdateFleet(client kurator.Interface, fleet *fleetv1a1.Fleet) error {
+	_, err := client.FleetV1alpha1().Fleets(fleet.Namespace).Update(context.TODO(), fleet, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func RemoveFleet(client kurator.Interface, namespace, name string) error {
+	err := client.FleetV1alpha1().Fleets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		} else {
+			return err
+		}
+	}
+	return nil
+}