diff --git a/hack/post-run-e2e.sh b/hack/post-run-e2e.sh new file mode 100644 index 000000000..9d92b9254 --- /dev/null +++ b/hack/post-run-e2e.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +kubectl delete -n kube-system --ignore-not-found=true daemonset huawei-cloud-controller-manager +kubectl delete -n kube-system --ignore-not-found=true serviceaccount cloud-controller-manager +kubectl delete clusterrole --ignore-not-found=true system:cloud-controller-manager +kubectl delete clusterrolebinding --ignore-not-found=true system:cloud-controller-manager diff --git a/hack/pre-run-e2e.sh b/hack/pre-run-e2e.sh new file mode 100644 index 000000000..16d18f97e --- /dev/null +++ b/hack/pre-run-e2e.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +if [[ "${REGISTRY_USER_NAME}" = "" ]] || [[ "${REGISTRY_PASSWORD}" = "" ]]; then + echo "REGISTRY_USER_NAME and REGISTRY_PASSWORD can not be empty, please set it" + exit 1 +fi + +export REGISTRY_SERVER_ADDRESS=swr.ap-southeast-1.myhuaweicloud.com +export REGISTRY=swr.ap-southeast-1.myhuaweicloud.com/dev-registry +export VERSION=latest + +echo -e "\nBuild images" +make upload-images + +echo -e "\nCheck cloud-config secret" +count=`kubectl get -n kube-system secret cloud-config | grep cloud-config | wc -l` +if [[ "$count" -ne 1 ]]; then + echo "Please create a secret with the name: cloud-config." + exit 1 +fi + +kubectl delete -n kube-system daemonset --ignore-not-found=true huawei-cloud-controller-manager + +echo -e "\nDeploy huawei-cloud-controller-manager" +cat << EOF | kubectl apply -f - +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:cloud-controller-manager +rules: + - resources: + - tokenreviews + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - authentication.k8s.io + - resources: + - configmaps + - endpoints + - pods + - services + - secrets + - serviceaccounts + verbs: + - get + - list + - watch + - create + - update + - patch + apiGroups: + - '' + - resources: + - nodes + verbs: + - get + - list + - watch + - delete + - patch + - update + apiGroups: + - '' + - resources: + - services/status + - pods/status + verbs: + - update + - patch + apiGroups: + - '' + - resources: + - nodes/status + verbs: + - patch + - update + apiGroups: + - '' + - resources: + - events + - endpoints + verbs: + - create + - patch + - update + apiGroups: + - '' + - resources: + - leases + verbs: + - get + - update + - create + - delete + apiGroups: + - coordination.k8s.io + - resources: + - customresourcedefinitions + verbs: + - get + - update + - create + - delete + apiGroups: + - apiextensions.k8s.io + - resources: + - ingresses + verbs: + - get + - list + - watch + - update + - create + - patch + - delete + apiGroups: + - networking.k8s.io + - resources: + - ingresses/status + verbs: + - update + - patch + apiGroups: + - networking.k8s.io + - resources: + - endpointslices + verbs: + - get + - list + - watch + apiGroups: + - discovery.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: huawei-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: huawei-cloud-controller-manager +spec: + selector: + matchLabels: + k8s-app: huawei-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: huawei-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + securityContext: + runAsUser: 1001 + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: cloud-controller-manager + containers: + - name: huawei-cloud-controller-manager + image: swr.ap-southeast-1.myhuaweicloud.com/dev-registry/huawei-cloud-controller-manager:latest + args: + - /bin/huawei-cloud-controller-manager + - --v=5 + - --cloud-config=/etc/config/cloud.conf + - --cloud-provider=huaweicloud + - --use-service-account-credentials=true + - --bind-address=127.0.0.1 + volumeMounts: + - mountPath: /etc/kubernetes + name: k8s-certs + readOnly: true + - mountPath: /etc/ssl/certs + name: ca-certs + readOnly: true + - mountPath: /etc/config + name: cloud-config-volume + readOnly: true + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: flexvolume-dir + resources: + requests: + cpu: 200m + hostNetwork: true + volumes: + - hostPath: + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + type: DirectoryOrCreate + name: flexvolume-dir + - hostPath: + path: /etc/kubernetes + type: DirectoryOrCreate + name: k8s-certs + - hostPath: + path: /etc/ssl/certs + type: DirectoryOrCreate + name: ca-certs + - name: cloud-config-volume + secret: + secretName: cloud-config +EOF + +testRes="false" +set +o errexit +for (( i=0; i<10; i++)); +do + lines=`kubectl get pod -n kube-system | grep huawei-cloud-controller-manager | grep Running | wc -l` + if [ "$lines" = "1" ]; then + testRes="true" + break + fi + sleep 3 +done + +if [ "$testRes" = "true" ]; then + echo -e "Deploy huawei-cloud-controller-manager success\n" +else + echo -e "FAIL: deploy huawei-cloud-controller-manager failed\n" + exit 1 +fi diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh new file mode 100644 index 000000000..5d006fc4f --- /dev/null +++ b/hack/run-e2e.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +kubectl version + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. + +ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${REPO_ROOT}/e2e-logs"} +mkdir -p "${ARTIFACTS_PATH}" + +# Install ginkgo +export GO111MODULE="on" +go get github.com/onsi/ginkgo/ginkgo@v1.16.4 +ginkgo version + +# Pre run e2e for extra components +echo -e "\nRun pre run e2e" +"${REPO_ROOT}"/hack/pre-run-e2e.sh + +# Run e2e +echo -e "\nRun e2e" +ginkgo -v -race -failFast -p -randomizeSuites ./test/e2e/ +TESTING_RESULT=$? + +# Collect logs +kubectl logs daemonset/huawei-cloud-controller-manager -n kube-system > ${ARTIFACTS_PATH}/huawei-cloud-controller-manager.log +echo -e "\nCollected logs at ${ARTIFACTS_PATH}:" + +# Post run e2e for delete extra components +echo -e "\nRun post run e2e" +"${REPO_ROOT}"/hack/post-run-e2e.sh + +exit $TESTING_RESULT diff --git a/test/e2e/loadbalancerservice_test.go b/test/e2e/loadbalancerservice_test.go index 4f341d7f7..fb9271994 100644 --- a/test/e2e/loadbalancerservice_test.go +++ b/test/e2e/loadbalancerservice_test.go @@ -19,7 +19,6 @@ package e2e import ( "context" "fmt" - "os" "github.com/onsi/ginkgo" "github.com/onsi/gomega" @@ -37,21 +36,15 @@ import ( var _ = ginkgo.Describe("loadbalancer service testing", func() { var deployment *appsv1.Deployment var service *corev1.Service - var secret *corev1.Secret ginkgo.BeforeEach(func() { deploymentName := deploymentNamePrefix + rand.String(RandomStrLength) deployment = helper2.NewDeployment(testNamespace, deploymentName) framework.CreateDeployment(kubeClient, deployment) - - // todo(chengxiangdong): This is a temporary solution, I will optimize it later. - secret = newSecret(testNamespace) - framework.CreateSecret(kubeClient, secret) }) ginkgo.AfterEach(func() { framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name) - framework.RemoveSecret(kubeClient, secret.Namespace, secret.Name) if service != nil { framework.RemoveService(kubeClient, service.Namespace, service.Name) ginkgo.By(fmt.Sprintf("Wait for the Service(%s/%s) to be deleted", testNamespace, service.Name), func() { @@ -136,21 +129,3 @@ func newLoadbalancerAutoService(namespace, name, sessionAffinity string) *corev1 }, } } - -func newSecret(namespace string) *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "example.secret", - }, - Type: corev1.SecretTypeOpaque, - StringData: map[string]string{ - "access": os.Getenv("HC_ACCESS_KEY"), - "secret": os.Getenv("HC_SECRET_KEY"), - }, - } -}