Run E2E tests

.github/workflows/go-test-e2e.yml

@@ -0,0 +1,115 @@
+name: Testing E2E
+on:
+  push:
+  schedule:
+    - cron: '0 1 * * *'
+jobs:
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: Docker login
+        run: docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+        env:
+          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+
+      - name: Prepare e2e image
+        run: |
+          go get -v -t -d ./...
+          go mod vendor
+          hack/build/e2e/docker_push
+        env:
+          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+
+      - name: Prepare operator image
+        run: |
+          hack/build/operator/build
+          hack/build/backup-operator/build
+          hack/build/restore-operator/build
+          IMAGE=${OPERATOR_IMAGE} hack/build/docker_push
+        env:
+          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+  test-e2e:
+    name: E2E
+    runs-on: ubuntu-latest
+    needs: prepare
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: KinD (Kubernetes in Docker) Action
+        uses: engineerd/setup-kind@v0.1.0
+
+      - name: Test
+        run: |
+          docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+          docker pull $TEST_IMAGE
+          docker pull $OPERATOR_IMAGE
+          export KUBECONFIG="$(kind get kubeconfig-path)"
+          kind load docker-image $TEST_IMAGE
+          kind load docker-image $OPERATOR_IMAGE
+          hack/ci/run_e2e
+          cat _output/logs/*
+        env:
+          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+          TEST_AWS_SECRET: na
+          TEST_S3_BUCKET: na
+          TEST_NAMESPACE: default
+          BUILD_IMAGE: false
+          BUILD_E2E: false
+          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+          PASSES: e2e
+#  test-e2eslow:
+#    name: E2E slow
+#    runs-on: ubuntu-latest
+#    needs: prepare
+#    steps:
+#      - name: Set up Go 1.13
+#        uses: actions/setup-go@v1
+#        with:
+#          go-version: 1.13
+#        id: go
+#
+#      - name: Check out code into the Go module directory
+#        uses: actions/checkout@v1
+#
+#      - name: KinD (Kubernetes in Docker) Action
+#        uses: engineerd/setup-kind@v0.1.0
+#
+#      - name: Test
+#        run: |
+#          docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+#          docker pull $TEST_IMAGE
+#          docker pull $OPERATOR_IMAGE
+#          export KUBECONFIG="$(kind get kubeconfig-path)"
+#          kind load docker-image $TEST_IMAGE
+#          kind load docker-image $OPERATOR_IMAGE
+#          kubectl create secret docker-registry --docker-server=docker.pkg.github.com --docker-username=marlinc --docker-password=${GITHUB_PACKAGE_REGISTRY_TOKEN} github
+#          kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "github"}]}'
+#          hack/ci/run_e2e
+#        env:
+#          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+#          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+#          TEST_AWS_SECRET: na
+#          TEST_S3_BUCKET: na
+#          TEST_NAMESPACE: default
+#          BUILD_IMAGE: false
+#          BUILD_E2E: false
+#          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+#          PASSES: e2eslow

example/rbac/cluster-role-template.yaml

@@ -39,4 +39,4 @@ rules:
   resources:
   - secrets
   verbs:
-  - get
+  - "*"

hack/ci/get_dep

@@ -2,10 +2,4 @@
 
 set -o errexit
 
-DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator"
-
-docker run --rm \
-	-v "$PWD":"$DOCKER_REPO_ROOT" \
-	-w "$DOCKER_REPO_ROOT" \
-	gcr.io/coreos-k8s-scale-testing/etcd-operator-builder:0.4.1-2 \
-	hack/update_vendor.sh
+go get -v -t -d ./...

hack/ci/run_e2e

@@ -36,7 +36,6 @@ fi
 BUILD_E2E=${BUILD_E2E:-true}
 if [[ ${BUILD_E2E} == "true" ]]; then
   TEST_IMAGE=${TEST_IMAGE:-"gcr.io/coreos-k8s-scale-testing/etcd-operator-e2e:${GIT_VERSION}"}
-  gcloud docker -a
   TEST_IMAGE=${TEST_IMAGE} hack/build/e2e/docker_push
 fi
 

hack/test

@@ -36,7 +36,7 @@ function e2e_pass {
 	build_flags=("-i") # cache package compilation data for faster repeated builds
 	for i in {1..2}; do
 		go test -failfast -parallel=4 "./test/e2e/" ${build_flags[@]} -run "$E2E_TEST_SELECTOR" -timeout 30m --race \
-			--kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE}
+			--kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} --retry-attempts 24
 		build_flags=("")
 	done
 }
@@ -46,7 +46,7 @@ function e2eslow_pass {
 	build_flags=("-i") # cache package compilation data for faster repeated builds
 	for i in {1..2}; do
 		go test -failfast "./test/e2e/e2eslow" ${build_flags[@]} -run "$E2E_TEST_SELECTOR" -timeout 30m --race \
-			--kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE}
+			--kubeconfig=$KUBECONFIG --operator-image=$OPERATOR_IMAGE --namespace=${TEST_NAMESPACE} --retry-attempts 24
 		build_flags=("")
 	done
 }
@@ -56,6 +56,7 @@ function upgrade_pass {
 	UPGRADE_TEST_SELECTOR=${UPGRADE_TEST_SELECTOR:-.*}
 	go test -failfast ./test/e2e/upgradetest/ -run "$UPGRADE_TEST_SELECTOR" --race -timeout 30m \
 		--kubeconfig=$KUBECONFIG --kube-ns=$TEST_NAMESPACE \
+		--retry-attempts 24 \
 		--old-image=$UPGRADE_FROM \
 		--new-image=$UPGRADE_TO
 }

test/e2e/basic_test.go

@@ -40,7 +40,7 @@ func TestCreateCluster(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
 }
@@ -63,7 +63,7 @@ func TestPauseControl(t *testing.T) {
 		}
 	}()
 
-	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd)
+	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
@@ -96,7 +96,7 @@ func TestPauseControl(t *testing.T) {
 		t.Fatalf("failed to resume control: %v", err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }
@@ -120,7 +120,7 @@ func TestEtcdUpgrade(t *testing.T) {
 		}
 	}()
 
-	err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, 6, testEtcd)
+	err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}

test/e2e/e2eslow/backup_restore_test.go

@@ -57,7 +57,7 @@ func TestBackupAndRestore(t *testing.T) {
 	memberPeerTLSSecret := "etcd-peer-tls-" + suffix
 	memberClientTLSSecret := "etcd-server-tls-" + suffix
 	operatorClientTLSSecret := "etcd-client-tls-" + suffix
-	err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
+	err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
 	if err != nil {
 		t.Fatal(err)
 	}

test/e2e/e2eutil/tls.go

@@ -28,8 +28,8 @@ import (
 )
 
 // PrepareTLS creates all the required tls certs for a given clusterName.
-func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error {
-	err := PreparePeerTLSSecret(clusterName, namespace, memberPeerTLSSecret)
+func PrepareTLS(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error {
+	err := PreparePeerTLSSecret(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret)
 	if err != nil {
 		return fmt.Errorf("failed to prepare peer TLS secret: %v", err)
 	}
@@ -45,7 +45,7 @@ func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecr
 	return nil
 }
 
-func PreparePeerTLSSecret(clusterName, ns, secretName string) error {
+func PreparePeerTLSSecret(clusterName, ns, kubeClusterDomain, secretName string) error {
 	dir, err := ioutil.TempDir("", "etcd-operator-tls-")
 	if err != nil {
 		return err
@@ -59,7 +59,7 @@ func PreparePeerTLSSecret(clusterName, ns, secretName string) error {
 		fmt.Sprintf("*.%s.%s.svc", clusterName, ns),
 		// Due to issue https://github.com/coreos/etcd/issues/8797,
 		// we need to provide FQDN in certs at the moment.
-		fmt.Sprintf("*.%s.%s.svc.cluster.local", clusterName, ns),
+		fmt.Sprintf("*.%s.%s.svc.%s", clusterName, ns, kubeClusterDomain),
 	}
 
 	err = prepareTLSCerts(certPath, keyPath, caPath, hosts)

test/e2e/framework/framework.go

@@ -48,17 +48,21 @@ const (
 )
 
 type Framework struct {
-	opImage    string
-	KubeClient kubernetes.Interface
-	CRClient   versioned.Interface
-	Namespace  string
+	opImage           string
+	KubeClient        kubernetes.Interface
+	KubeClusterDomain string
+	CRClient          versioned.Interface
+	Namespace         string
+	RetryAttempts     int
 }
 
 // Setup setups a test framework and points "Global" to it.
 func setup() error {
 	kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config")
+	kubeClusterDomain := flag.String("kube-cluster-domain", "cluster.local", "kube cluster domain")
 	opImage := flag.String("operator-image", "", "operator image, e.g. gcr.io/coreos-k8s-scale-testing/etcd-operator")
 	ns := flag.String("namespace", "default", "e2e test namespace")
+	retryAttemtps := flag.Int("retry-attempts", 6, "e2e cluster create retries (10 seconds per retry)")
 	flag.Parse()
 
 	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
@@ -71,10 +75,12 @@ func setup() error {
 	}
 
 	Global = &Framework{
-		KubeClient: cli,
-		CRClient:   client.MustNew(config),
-		Namespace:  *ns,
-		opImage:    *opImage,
+		KubeClient:        cli,
+		KubeClusterDomain: *kubeClusterDomain,
+		CRClient:          client.MustNew(config),
+		Namespace:         *ns,
+		opImage:           *opImage,
+		RetryAttempts:     *retryAttemtps,
 	}
 
 	// Skip the etcd-operator deployment setup if the operator image was not specified
@@ -125,7 +131,7 @@ func (f *Framework) SetupEtcdOperator() error {
 			Containers: []v1.Container{{
 				Name:            "etcd-operator",
 				Image:           f.opImage,
-				ImagePullPolicy: v1.PullAlways,
+				ImagePullPolicy: v1.PullIfNotPresent,
 				Command:         []string{"/usr/local/bin/etcd-operator"},
 				Env: []v1.EnvVar{
 					{
@@ -151,7 +157,7 @@ func (f *Framework) SetupEtcdOperator() error {
 			}, {
 				Name:            etcdBackupOperatorName,
 				Image:           f.opImage,
-				ImagePullPolicy: v1.PullAlways,
+				ImagePullPolicy: v1.PullIfNotPresent,
 				Command:         []string{"/usr/local/bin/etcd-backup-operator"},
 				Env: []v1.EnvVar{
 					{
@@ -166,7 +172,7 @@ func (f *Framework) SetupEtcdOperator() error {
 			}, {
 				Name:            etcdRestoreOperatorName,
 				Image:           f.opImage,
-				ImagePullPolicy: v1.PullAlways,
+				ImagePullPolicy: v1.PullIfNotPresent,
 				Command:         []string{"/usr/local/bin/etcd-restore-operator"},
 				Env: []v1.EnvVar{
 					{

test/e2e/recovery_test.go

@@ -39,7 +39,7 @@ func TestOneMemberRecovery(t *testing.T) {
 		}
 	}()
 
-	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd)
+	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
@@ -49,7 +49,7 @@ func TestOneMemberRecovery(t *testing.T) {
 	if err := e2eutil.KillMembers(f.KubeClient, f.Namespace, names[2]); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }

test/e2e/resize_test.go

@@ -39,7 +39,7 @@ func TestResizeCluster3to5(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
 	t.Log("reached to 3 members cluster")
@@ -51,7 +51,7 @@ func TestResizeCluster3to5(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 5 members etcd cluster: %v", err)
 	}
 }
@@ -72,7 +72,7 @@ func TestResizeCluster5to3(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 9, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 5 members etcd cluster: %v", err)
 	}
 	t.Log("reached to 5 members cluster")
@@ -84,7 +84,7 @@ func TestResizeCluster5to3(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }

test/e2e/tls_test.go

@@ -40,7 +40,7 @@ func TestTLS(t *testing.T) {
 	memberClientTLSSecret := "etcd-server-tls" + suffix
 	operatorClientTLSSecret := "etcd-client-tls" + suffix
 
-	err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
+	err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,7 +66,7 @@ func TestTLS(t *testing.T) {
 		}
 	}()
 
-	_, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, c)
+	_, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, c)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}