add E2E test script files

kubernetes-sigs · Oct 25, 2022 · e1573fd · e1573fd
1 parent 994b31b
commit e1573fd
Show file tree

Hide file tree

Showing 4 changed files with 340 additions and 25 deletions.
diff --git a/hack/post-run-e2e.sh b/hack/post-run-e2e.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubectl delete -n kube-system --ignore-not-found=true daemonset huawei-cloud-controller-manager
+kubectl delete -n kube-system --ignore-not-found=true serviceaccount cloud-controller-manager
+kubectl delete clusterrole --ignore-not-found=true system:cloud-controller-manager
+kubectl delete clusterrolebinding --ignore-not-found=true system:cloud-controller-manager
diff --git a/hack/pre-run-e2e.sh b/hack/pre-run-e2e.sh
@@ -0,0 +1,266 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${REGISTRY_USER_NAME}" = "" ]] || [[ "${REGISTRY_PASSWORD}" = "" ]]; then
+  echo "REGISTRY_USER_NAME and REGISTRY_PASSWORD can not be empty, please set it"
+  exit 1
+fi
+
+export REGISTRY_SERVER_ADDRESS=swr.ap-southeast-1.myhuaweicloud.com
+export REGISTRY=swr.ap-southeast-1.myhuaweicloud.com/dev-registry
+export VERSION=latest
+
+echo -e "\nBuild images"
+make upload-images
+
+echo -e "\nCheck cloud-config secret"
+count=`kubectl get -n kube-system secret cloud-config | grep cloud-config | wc -l`
+if [[ "$count" -ne 1 ]]; then
+  echo "Please create a secret with the name: cloud-config."
+  exit 1
+fi
+
+kubectl delete -n kube-system daemonset --ignore-not-found=true huawei-cloud-controller-manager
+
+echo -e "\nDeploy huawei-cloud-controller-manager"
+cat << EOF | kubectl apply -f -
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+rules:
+  - resources:
+      - tokenreviews
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+    apiGroups:
+      - authentication.k8s.io
+  - resources:
+      - configmaps
+      - endpoints
+      - pods
+      - services
+      - secrets
+      - serviceaccounts
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+    apiGroups:
+      - ''
+  - resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
+      - patch
+      - update
+    apiGroups:
+      - ''
+  - resources:
+      - services/status
+      - pods/status
+    verbs:
+      - update
+      - patch
+    apiGroups:
+      - ''
+  - resources:
+      - nodes/status
+    verbs:
+      - patch
+      - update
+    apiGroups:
+      - ''
+  - resources:
+      - events
+      - endpoints
+    verbs:
+      - create
+      - patch
+      - update
+    apiGroups:
+      - ''
+  - resources:
+      - leases
+    verbs:
+      - get
+      - update
+      - create
+      - delete
+    apiGroups:
+      - coordination.k8s.io
+  - resources:
+      - customresourcedefinitions
+    verbs:
+      - get
+      - update
+      - create
+      - delete
+    apiGroups:
+      - apiextensions.k8s.io
+  - resources:
+      - ingresses
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - create
+      - patch
+      - delete
+    apiGroups:
+      - networking.k8s.io
+  - resources:
+      - ingresses/status
+    verbs:
+      - update
+      - patch
+    apiGroups:
+      - networking.k8s.io
+  - resources:
+      - endpointslices
+    verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - discovery.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: huawei-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: huawei-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: huawei-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: huawei-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsUser: 1001
+      tolerations:
+        - key: node.cloudprovider.kubernetes.io/uninitialized
+          value: "true"
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: huawei-cloud-controller-manager
+          image: swr.ap-southeast-1.myhuaweicloud.com/dev-registry/huawei-cloud-controller-manager:latest
+          args:
+            - /bin/huawei-cloud-controller-manager
+            - --v=5
+            - --cloud-config=/etc/config/cloud.conf
+            - --cloud-provider=huaweicloud
+            - --use-service-account-credentials=true
+            - --bind-address=127.0.0.1
+          volumeMounts:
+            - mountPath: /etc/kubernetes
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/ssl/certs
+              name: ca-certs
+              readOnly: true
+            - mountPath: /etc/config
+              name: cloud-config-volume
+              readOnly: true
+            - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+              name: flexvolume-dir
+          resources:
+            requests:
+              cpu: 200m
+      hostNetwork: true
+      volumes:
+      - hostPath:
+          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+          type: DirectoryOrCreate
+        name: flexvolume-dir
+      - hostPath:
+          path: /etc/kubernetes
+          type: DirectoryOrCreate
+        name: k8s-certs
+      - hostPath:
+          path: /etc/ssl/certs
+          type: DirectoryOrCreate
+        name: ca-certs
+      - name: cloud-config-volume
+        secret:
+          secretName: cloud-config
+EOF
+
+testRes="false"
+set +o errexit
+for (( i=0; i<10; i++));
+do
+    lines=`kubectl get pod -n kube-system | grep huawei-cloud-controller-manager | grep Running | wc -l`
+    if [ "$lines" = "1" ]; then
+        testRes="true"
+        break
+    fi
+    sleep 3
+done
+
+if [ "$testRes" = "true" ]; then
+    echo -e "Deploy huawei-cloud-controller-manager success\n"
+else
+    echo -e "FAIL: deploy huawei-cloud-controller-manager failed\n"
+    exit 1
+fi
diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+kubectl version
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${REPO_ROOT}/e2e-logs"}
+mkdir -p "${ARTIFACTS_PATH}"
+
+# Install ginkgo
+export GO111MODULE="on"
+go get github.com/onsi/ginkgo/ginkgo@v1.16.4
+ginkgo version
+
+# Pre run e2e for extra components
+echo -e "\nRun pre run e2e"
+"${REPO_ROOT}"/hack/pre-run-e2e.sh
+
+# Run e2e
+echo -e "\nRun e2e"
+ginkgo -v -race -failFast -p -randomizeSuites ./test/e2e/
+TESTING_RESULT=$?
+
+# Collect logs
+kubectl logs daemonset/huawei-cloud-controller-manager -n kube-system > ${ARTIFACTS_PATH}/huawei-cloud-controller-manager.log
+echo -e "\nCollected logs at ${ARTIFACTS_PATH}:"
+
+# Post run e2e for delete extra components
+echo -e "\nRun post run e2e"
+"${REPO_ROOT}"/hack/post-run-e2e.sh
+
+exit $TESTING_RESULT
diff --git a/test/e2e/loadbalancerservice_test.go b/test/e2e/loadbalancerservice_test.go
@@ -19,7 +19,6 @@ package e2e
 import (
 	"context"
 	"fmt"
-	"os"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -37,21 +36,15 @@ import (
 var _ = ginkgo.Describe("loadbalancer service testing", func() {
 	var deployment *appsv1.Deployment
 	var service *corev1.Service
-	var secret *corev1.Secret
 
 	ginkgo.BeforeEach(func() {
 		deploymentName := deploymentNamePrefix + rand.String(RandomStrLength)
 		deployment = helper2.NewDeployment(testNamespace, deploymentName)
 		framework.CreateDeployment(kubeClient, deployment)
-
-		// todo(chengxiangdong): This is a temporary solution, I will optimize it later.
-		secret = newSecret(testNamespace)
-		framework.CreateSecret(kubeClient, secret)
 	})
 
 	ginkgo.AfterEach(func() {
 		framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
-		framework.RemoveSecret(kubeClient, secret.Namespace, secret.Name)
 		if service != nil {
 			framework.RemoveService(kubeClient, service.Namespace, service.Name)
 			ginkgo.By(fmt.Sprintf("Wait for the Service(%s/%s) to be deleted", testNamespace, service.Name), func() {
@@ -136,21 +129,3 @@ func newLoadbalancerAutoService(namespace, name, sessionAffinity string) *corev1
 		},
 	}
 }
-
-func newSecret(namespace string) *corev1.Secret {
-	return &corev1.Secret{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "v1",
-			Kind:       "Secret",
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: namespace,
-			Name:      "example.secret",
-		},
-		Type: corev1.SecretTypeOpaque,
-		StringData: map[string]string{
-			"access": os.Getenv("HC_ACCESS_KEY"),
-			"secret": os.Getenv("HC_SECRET_KEY"),
-		},
-	}
-}