From 28b36400ba34b988deacf25000e7ed045044a800 Mon Sep 17 00:00:00 2001 From: Marlin Cremers Date: Sat, 9 Nov 2019 13:28:27 +0100 Subject: [PATCH] Run E2E tests --- .github/workflows/go-test-e2e.yml | 115 ++++++++++++++++++++++++ example/rbac/cluster-role-template.yaml | 2 +- hack/ci/get_dep | 14 +-- hack/ci/run_e2e | 1 - hack/test | 5 +- test/e2e/basic_test.go | 8 +- test/e2e/e2eslow/backup_restore_test.go | 2 +- test/e2e/e2eutil/tls.go | 8 +- test/e2e/framework/framework.go | 28 +++--- test/e2e/recovery_test.go | 4 +- test/e2e/resize_test.go | 8 +- test/e2e/tls_test.go | 4 +- test/pod/run-test-pod | 7 +- test/pod/test-pod-templ.yaml | 6 +- 14 files changed, 166 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/go-test-e2e.yml diff --git a/.github/workflows/go-test-e2e.yml b/.github/workflows/go-test-e2e.yml new file mode 100644 index 000000000..4189caf8c --- /dev/null +++ b/.github/workflows/go-test-e2e.yml @@ -0,0 +1,115 @@ +name: Testing E2E +on: + push: + schedule: + - cron: '0 1 * * *' +jobs: + prepare: + name: Prepare + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.13 + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Docker login + run: docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" + env: + GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} + + - name: Prepare e2e image + run: | + go get -v -t -d ./... + go mod vendor + hack/build/e2e/docker_push + env: + TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}} + + - name: Prepare operator image + run: | + hack/build/operator/build + hack/build/backup-operator/build + hack/build/restore-operator/build + IMAGE=${OPERATOR_IMAGE} hack/build/docker_push + env: + OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}} + test-e2e: + name: E2E + runs-on: ubuntu-latest + needs: prepare + steps: + - name: Set up Go 1.13 + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: KinD (Kubernetes in Docker) Action + uses: engineerd/setup-kind@v0.1.0 + + - name: Test + run: | + docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" + docker pull $TEST_IMAGE + docker pull $OPERATOR_IMAGE + export KUBECONFIG="$(kind get kubeconfig-path)" + kind load docker-image $TEST_IMAGE + kind load docker-image $OPERATOR_IMAGE + hack/ci/run_e2e + cat _output/logs/* + env: + GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} + OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}} + TEST_AWS_SECRET: na + TEST_S3_BUCKET: na + TEST_NAMESPACE: default + BUILD_IMAGE: false + BUILD_E2E: false + TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}} + PASSES: e2e +# test-e2eslow: +# name: E2E slow +# runs-on: ubuntu-latest +# needs: prepare +# steps: +# - name: Set up Go 1.13 +# uses: actions/setup-go@v1 +# with: +# go-version: 1.13 +# id: go +# +# - name: Check out code into the Go module directory +# uses: actions/checkout@v1 +# +# - name: KinD (Kubernetes in Docker) Action +# uses: engineerd/setup-kind@v0.1.0 +# +# - name: Test +# run: | +# docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}" +# docker pull $TEST_IMAGE +# docker pull $OPERATOR_IMAGE +# export KUBECONFIG="$(kind get kubeconfig-path)" +# kind load docker-image $TEST_IMAGE +# kind load docker-image $OPERATOR_IMAGE +# kubectl create secret docker-registry --docker-server=docker.pkg.github.com --docker-username=marlinc --docker-password=${GITHUB_PACKAGE_REGISTRY_TOKEN} github +# kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "github"}]}' +# hack/ci/run_e2e +# env: +# GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }} +# OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}} +# TEST_AWS_SECRET: na +# TEST_S3_BUCKET: na +# TEST_NAMESPACE: default +# BUILD_IMAGE: false +# BUILD_E2E: false +# TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}} +# PASSES: e2eslow \ No newline at end of file diff --git a/example/rbac/cluster-role-template.yaml b/example/rbac/cluster-role-template.yaml index 5d1cae4c4..fa7a41e75 100644 --- a/example/rbac/cluster-role-template.yaml +++ b/example/rbac/cluster-role-template.yaml @@ -39,4 +39,4 @@ rules: resources: - secrets verbs: - - get + - "*" diff --git a/hack/ci/get_dep b/hack/ci/get_dep index 5dd31865f..47b9f4f93 100755 --- a/hack/ci/get_dep +++ b/hack/ci/get_dep @@ -2,10 +2,12 @@ set -o errexit -DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator" +#DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator" +# +#docker run --rm \ +# -v "$PWD":"$DOCKER_REPO_ROOT" \ +# -w "$DOCKER_REPO_ROOT" \ +# gcr.io/coreos-k8s-scale-testing/etcd-operator-builder:0.4.1-2 \ +# hack/update_vendor.sh -docker run --rm \ - -v "$PWD":"$DOCKER_REPO_ROOT" \ - -w "$DOCKER_REPO_ROOT" \ - gcr.io/coreos-k8s-scale-testing/etcd-operator-builder:0.4.1-2 \ - hack/update_vendor.sh +go get -v -t -d ./... diff --git a/hack/ci/run_e2e b/hack/ci/run_e2e index 0d063d8f0..7e5d176c2 100755 --- a/hack/ci/run_e2e +++ b/hack/ci/run_e2e @@ -36,7 +36,6 @@ fi BUILD_E2E=${BUILD_E2E:-true} if [[ ${BUILD_E2E} == "true" ]]; then TEST_IMAGE=${TEST_IMAGE:-"gcr.io/coreos-k8s-scale-testing/etcd-operator-e2e:${GIT_VERSION}"} - gcloud docker -a TEST_IMAGE=${TEST_IMAGE} hack/build/e2e/docker_push fi diff --git a/hack/test b/hack/test index 11987e56a..dba381655 100755 --- a/hack/test +++ b/hack/test @@ -36,7 +36,7 @@ function e2e_pass { build_flags=("-i") # cache package compilation data for faster repeated builds for i in {1..2}; do go test -failfast -parallel=4 "./test/e2e/" ${build_flags[@]} -run "$E2E_TEST_SELECTOR" -timeout 30m --race \ - --kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} + --kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} --retry-attempts 24 build_flags=("") done } @@ -46,7 +46,7 @@ function e2eslow_pass { build_flags=("-i") # cache package compilation data for faster repeated builds for i in {1..2}; do go test -failfast "./test/e2e/e2eslow" ${build_flags[@]} -run "$E2E_TEST_SELECTOR" -timeout 30m --race \ - --kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} + --kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} --retry-attempts 24 build_flags=("") done } @@ -56,6 +56,7 @@ function upgrade_pass { UPGRADE_TEST_SELECTOR=${UPGRADE_TEST_SELECTOR:-.*} go test -failfast ./test/e2e/upgradetest/ -run "$UPGRADE_TEST_SELECTOR" --race -timeout 30m \ --kubeconfig=$KUBECONFIG --kube-ns=$TEST_NAMESPACE \ + --retry-attempts 24 \ --old-image=$UPGRADE_FROM \ --new-image=$UPGRADE_TO } diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go index 463c82c83..232dfdfeb 100644 --- a/test/e2e/basic_test.go +++ b/test/e2e/basic_test.go @@ -40,7 +40,7 @@ func TestCreateCluster(t *testing.T) { } }() - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } } @@ -63,7 +63,7 @@ func TestPauseControl(t *testing.T) { } }() - names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd) + names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd) if err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } @@ -96,7 +96,7 @@ func TestPauseControl(t *testing.T) { t.Fatalf("failed to resume control: %v", err) } - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to resize to 3 members etcd cluster: %v", err) } } @@ -120,7 +120,7 @@ func TestEtcdUpgrade(t *testing.T) { } }() - err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, 6, testEtcd) + err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, f.RetryAttempts, testEtcd) if err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } diff --git a/test/e2e/e2eslow/backup_restore_test.go b/test/e2e/e2eslow/backup_restore_test.go index b387cc7af..7151b512c 100644 --- a/test/e2e/e2eslow/backup_restore_test.go +++ b/test/e2e/e2eslow/backup_restore_test.go @@ -57,7 +57,7 @@ func TestBackupAndRestore(t *testing.T) { memberPeerTLSSecret := "etcd-peer-tls-" + suffix memberClientTLSSecret := "etcd-server-tls-" + suffix operatorClientTLSSecret := "etcd-client-tls-" + suffix - err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret) + err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret) if err != nil { t.Fatal(err) } diff --git a/test/e2e/e2eutil/tls.go b/test/e2e/e2eutil/tls.go index 7fcbc45d2..7274ef51d 100644 --- a/test/e2e/e2eutil/tls.go +++ b/test/e2e/e2eutil/tls.go @@ -28,8 +28,8 @@ import ( ) // PrepareTLS creates all the required tls certs for a given clusterName. -func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error { - err := PreparePeerTLSSecret(clusterName, namespace, memberPeerTLSSecret) +func PrepareTLS(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error { + err := PreparePeerTLSSecret(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret) if err != nil { return fmt.Errorf("failed to prepare peer TLS secret: %v", err) } @@ -45,7 +45,7 @@ func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecr return nil } -func PreparePeerTLSSecret(clusterName, ns, secretName string) error { +func PreparePeerTLSSecret(clusterName, ns, kubeClusterDomain, secretName string) error { dir, err := ioutil.TempDir("", "etcd-operator-tls-") if err != nil { return err @@ -59,7 +59,7 @@ func PreparePeerTLSSecret(clusterName, ns, secretName string) error { fmt.Sprintf("*.%s.%s.svc", clusterName, ns), // Due to issue https://github.com/coreos/etcd/issues/8797, // we need to provide FQDN in certs at the moment. - fmt.Sprintf("*.%s.%s.svc.cluster.local", clusterName, ns), + fmt.Sprintf("*.%s.%s.svc.%s", clusterName, ns, kubeClusterDomain), } err = prepareTLSCerts(certPath, keyPath, caPath, hosts) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index 8909b99d8..d3ba71011 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -48,17 +48,21 @@ const ( ) type Framework struct { - opImage string - KubeClient kubernetes.Interface - CRClient versioned.Interface - Namespace string + opImage string + KubeClient kubernetes.Interface + KubeClusterDomain string + CRClient versioned.Interface + Namespace string + RetryAttempts int } // Setup setups a test framework and points "Global" to it. func setup() error { kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config") + kubeClusterDomain := flag.String("kube-cluster-domain", "cluster.local", "kube cluster domain") opImage := flag.String("operator-image", "", "operator image, e.g. gcr.io/coreos-k8s-scale-testing/etcd-operator") ns := flag.String("namespace", "default", "e2e test namespace") + retryAttemtps := flag.Int("retry-attempts", 6, "e2e cluster create retries (10 seconds per retry)") flag.Parse() config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig) @@ -71,10 +75,12 @@ func setup() error { } Global = &Framework{ - KubeClient: cli, - CRClient: client.MustNew(config), - Namespace: *ns, - opImage: *opImage, + KubeClient: cli, + KubeClusterDomain: *kubeClusterDomain, + CRClient: client.MustNew(config), + Namespace: *ns, + opImage: *opImage, + RetryAttempts: *retryAttemtps, } // Skip the etcd-operator deployment setup if the operator image was not specified @@ -125,7 +131,7 @@ func (f *Framework) SetupEtcdOperator() error { Containers: []v1.Container{{ Name: "etcd-operator", Image: f.opImage, - ImagePullPolicy: v1.PullAlways, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{"/usr/local/bin/etcd-operator"}, Env: []v1.EnvVar{ { @@ -151,7 +157,7 @@ func (f *Framework) SetupEtcdOperator() error { }, { Name: etcdBackupOperatorName, Image: f.opImage, - ImagePullPolicy: v1.PullAlways, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{"/usr/local/bin/etcd-backup-operator"}, Env: []v1.EnvVar{ { @@ -166,7 +172,7 @@ func (f *Framework) SetupEtcdOperator() error { }, { Name: etcdRestoreOperatorName, Image: f.opImage, - ImagePullPolicy: v1.PullAlways, + ImagePullPolicy: v1.PullIfNotPresent, Command: []string{"/usr/local/bin/etcd-restore-operator"}, Env: []v1.EnvVar{ { diff --git a/test/e2e/recovery_test.go b/test/e2e/recovery_test.go index 62a9cd48c..f4ddf98ef 100644 --- a/test/e2e/recovery_test.go +++ b/test/e2e/recovery_test.go @@ -39,7 +39,7 @@ func TestOneMemberRecovery(t *testing.T) { } }() - names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd) + names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd) if err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } @@ -49,7 +49,7 @@ func TestOneMemberRecovery(t *testing.T) { if err := e2eutil.KillMembers(f.KubeClient, f.Namespace, names[2]); err != nil { t.Fatal(err) } - if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to resize to 3 members etcd cluster: %v", err) } } diff --git a/test/e2e/resize_test.go b/test/e2e/resize_test.go index 30da5ed59..2773a7b2e 100644 --- a/test/e2e/resize_test.go +++ b/test/e2e/resize_test.go @@ -39,7 +39,7 @@ func TestResizeCluster3to5(t *testing.T) { } }() - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } t.Log("reached to 3 members cluster") @@ -51,7 +51,7 @@ func TestResizeCluster3to5(t *testing.T) { t.Fatal(err) } - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to resize to 5 members etcd cluster: %v", err) } } @@ -72,7 +72,7 @@ func TestResizeCluster5to3(t *testing.T) { } }() - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 9, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to create 5 members etcd cluster: %v", err) } t.Log("reached to 5 members cluster") @@ -84,7 +84,7 @@ func TestResizeCluster5to3(t *testing.T) { t.Fatal(err) } - if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil { + if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil { t.Fatalf("failed to resize to 3 members etcd cluster: %v", err) } } diff --git a/test/e2e/tls_test.go b/test/e2e/tls_test.go index cb9a2e188..6fd7660f9 100644 --- a/test/e2e/tls_test.go +++ b/test/e2e/tls_test.go @@ -40,7 +40,7 @@ func TestTLS(t *testing.T) { memberClientTLSSecret := "etcd-server-tls" + suffix operatorClientTLSSecret := "etcd-client-tls" + suffix - err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret) + err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret) if err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestTLS(t *testing.T) { } }() - _, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, c) + _, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, c) if err != nil { t.Fatalf("failed to create 3 members etcd cluster: %v", err) } diff --git a/test/pod/run-test-pod b/test/pod/run-test-pod index 51e7757a3..600c3df34 100755 --- a/test/pod/run-test-pod +++ b/test/pod/run-test-pod @@ -24,7 +24,6 @@ else exit 1 fi - kubectl -n ${TEST_NAMESPACE} create -f ${TEST_POD_SPEC} PHASE_RUNNING="Running" @@ -57,11 +56,7 @@ DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator" mkdir -p _output/logs/ cp $KUBECONFIG ./kubeconfig -docker run --rm \ - -v "$PWD":"$DOCKER_REPO_ROOT" \ - -w "$DOCKER_REPO_ROOT" \ - gcr.io/coreos-k8s-scale-testing/logcollector \ - logcollector --kubeconfig=${DOCKER_REPO_ROOT}/kubeconfig --e2e-podname=${POD_NAME} \ +go run test/logcollector/main.go --kubeconfig=$KUBECONFIG --e2e-podname=${POD_NAME} \ --namespace=${TEST_NAMESPACE} --logs-dir="_output/logs/" # Check for pod success or failure diff --git a/test/pod/test-pod-templ.yaml b/test/pod/test-pod-templ.yaml index ef118bed1..1a3f99b23 100644 --- a/test/pod/test-pod-templ.yaml +++ b/test/pod/test-pod-templ.yaml @@ -7,11 +7,13 @@ spec: containers: - name: image: - imagePullPolicy: Always + imagePullPolicy: IfNotPresent command: ["hack/test"] resources: requests: - cpu: 2 + cpu: 250m + limits: + cpu: 1 env: - name: TEST_NAMESPACE valueFrom: