Run E2E tests

cbws · Nov 10, 2019 · 7f26c92 · 7f26c92
1 parent 30243de
commit 7f26c92
Show file tree

Hide file tree

Showing 12 changed files with 164 additions and 40 deletions.
diff --git a/.github/workflows/go-test-e2e.yml b/.github/workflows/go-test-e2e.yml
@@ -0,0 +1,115 @@
+name: Testing E2E
+on:
+  push:
+  schedule:
+    - cron: '0 1 * * *'
+jobs:
+  prepare:
+    name: Prepare
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: Docker login
+        run: docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+        env:
+          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+
+      - name: Prepare e2e image
+        run: |
+          go get -v -t -d ./...
+          go mod vendor
+          hack/build/e2e/docker_push
+        env:
+          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+
+      - name: Prepare operator image
+        run: |
+          hack/build/operator/build
+          hack/build/backup-operator/build
+          hack/build/restore-operator/build
+          IMAGE=${OPERATOR_IMAGE} hack/build/docker_push
+        env:
+          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+  test-e2e:
+    name: E2E
+    runs-on: ubuntu-latest
+    needs: prepare
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: KinD (Kubernetes in Docker) Action
+        uses: engineerd/setup-kind@v0.1.0
+
+      - name: Test
+        run: |
+          docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+          docker pull $TEST_IMAGE
+          export KUBECONFIG="$(kind get kubeconfig-path)"
+          kind load docker-image $TEST_IMAGE
+          kind load docker-image $OPERATOR_IMAGE
+          kubectl create secret docker-registry --docker-server=docker.pkg.github.com --docker-username=marlinc --docker-password=${GITHUB_PACKAGE_REGISTRY_TOKEN} github
+          kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "github"}]}'
+          kubectl config view
+          hack/ci/run_e2e
+        env:
+          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+          TEST_AWS_SECRET: na
+          TEST_S3_BUCKET: na
+          TEST_NAMESPACE: default
+          BUILD_IMAGE: false
+          BUILD_E2E: false
+          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+          PASSES: e2e
+  test-e2eslow:
+    name: E2E slow
+    runs-on: ubuntu-latest
+    needs: prepare
+    steps:
+      - name: Set up Go 1.13
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.13
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: KinD (Kubernetes in Docker) Action
+        uses: engineerd/setup-kind@v0.1.0
+
+      - name: Test
+        run: |
+          docker login docker.pkg.github.com -u marlinc -p "${GITHUB_PACKAGE_REGISTRY_TOKEN}"
+          docker pull $TEST_IMAGE
+          export KUBECONFIG="$(kind get kubeconfig-path)"
+          kind load docker-image $TEST_IMAGE
+          kind load docker-image $OPERATOR_IMAGE
+          kubectl create secret docker-registry --docker-server=docker.pkg.github.com --docker-username=marlinc --docker-password=${GITHUB_PACKAGE_REGISTRY_TOKEN} github
+          kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "github"}]}'
+          hack/ci/run_e2e
+        env:
+          GITHUB_PACKAGE_REGISTRY_TOKEN: ${{ secrets.GITHUB_PACKAGE_REGISTRY_TOKEN }}
+          OPERATOR_IMAGE: docker.pkg.github.com/cbws/etcd-operator/operator:${{github.sha}}
+          TEST_AWS_SECRET: na
+          TEST_S3_BUCKET: na
+          TEST_NAMESPACE: default
+          BUILD_IMAGE: false
+          BUILD_E2E: false
+          TEST_IMAGE: docker.pkg.github.com/cbws/etcd-operator/etcd-operator-e2e:${{github.sha}}
+          PASSES: e2eslow
diff --git a/hack/ci/get_dep b/hack/ci/get_dep
@@ -2,10 +2,12 @@
 
 set -o errexit
 
-DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator"
+#DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator"
+#
+#docker run --rm \
+#	-v "$PWD":"$DOCKER_REPO_ROOT" \
+#	-w "$DOCKER_REPO_ROOT" \
+#	gcr.io/coreos-k8s-scale-testing/etcd-operator-builder:0.4.1-2 \
+#	hack/update_vendor.sh
 
-docker run --rm \
-	-v "$PWD":"$DOCKER_REPO_ROOT" \
-	-w "$DOCKER_REPO_ROOT" \
-	gcr.io/coreos-k8s-scale-testing/etcd-operator-builder:0.4.1-2 \
-	hack/update_vendor.sh
+go get -v -t -d ./...
diff --git a/hack/ci/run_e2e b/hack/ci/run_e2e
@@ -36,7 +36,6 @@ fi
 BUILD_E2E=${BUILD_E2E:-true}
 if [[ ${BUILD_E2E} == "true" ]]; then
   TEST_IMAGE=${TEST_IMAGE:-"gcr.io/coreos-k8s-scale-testing/etcd-operator-e2e:${GIT_VERSION}"}
-  gcloud docker -a
   TEST_IMAGE=${TEST_IMAGE} hack/build/e2e/docker_push
 fi
 

diff --git a/test/e2e/basic_test.go b/test/e2e/basic_test.go
@@ -40,7 +40,7 @@ func TestCreateCluster(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
 }
@@ -63,7 +63,7 @@ func TestPauseControl(t *testing.T) {
 		}
 	}()
 
-	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd)
+	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
@@ -96,7 +96,7 @@ func TestPauseControl(t *testing.T) {
 		t.Fatalf("failed to resume control: %v", err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }
@@ -120,7 +120,7 @@ func TestEtcdUpgrade(t *testing.T) {
 		}
 	}()
 
-	err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, 6, testEtcd)
+	err = e2eutil.WaitSizeAndVersionReached(t, f.KubeClient, "3.1.10", 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}

diff --git a/test/e2e/e2eslow/backup_restore_test.go b/test/e2e/e2eslow/backup_restore_test.go
@@ -57,7 +57,7 @@ func TestBackupAndRestore(t *testing.T) {
 	memberPeerTLSSecret := "etcd-peer-tls-" + suffix
 	memberClientTLSSecret := "etcd-server-tls-" + suffix
 	operatorClientTLSSecret := "etcd-client-tls-" + suffix
-	err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
+	err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/test/e2e/e2eutil/tls.go b/test/e2e/e2eutil/tls.go
@@ -28,8 +28,8 @@ import (
 )
 
 // PrepareTLS creates all the required tls certs for a given clusterName.
-func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error {
-	err := PreparePeerTLSSecret(clusterName, namespace, memberPeerTLSSecret)
+func PrepareTLS(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret string) error {
+	err := PreparePeerTLSSecret(clusterName, namespace, kubeClusterDomain, memberPeerTLSSecret)
 	if err != nil {
 		return fmt.Errorf("failed to prepare peer TLS secret: %v", err)
 	}
@@ -45,7 +45,7 @@ func PrepareTLS(clusterName, namespace, memberPeerTLSSecret, memberClientTLSSecr
 	return nil
 }
 
-func PreparePeerTLSSecret(clusterName, ns, secretName string) error {
+func PreparePeerTLSSecret(clusterName, ns, kubeClusterDomain, secretName string) error {
 	dir, err := ioutil.TempDir("", "etcd-operator-tls-")
 	if err != nil {
 		return err
@@ -59,7 +59,7 @@ func PreparePeerTLSSecret(clusterName, ns, secretName string) error {
 		fmt.Sprintf("*.%s.%s.svc", clusterName, ns),
 		// Due to issue https://github.com/coreos/etcd/issues/8797,
 		// we need to provide FQDN in certs at the moment.
-		fmt.Sprintf("*.%s.%s.svc.cluster.local", clusterName, ns),
+		fmt.Sprintf("*.%s.%s.svc.%s", clusterName, ns, kubeClusterDomain),
 	}
 
 	err = prepareTLSCerts(certPath, keyPath, caPath, hosts)

diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
@@ -48,17 +48,21 @@ const (
 )
 
 type Framework struct {
-	opImage    string
-	KubeClient kubernetes.Interface
-	CRClient   versioned.Interface
-	Namespace  string
+	opImage           string
+	KubeClient        kubernetes.Interface
+	KubeClusterDomain string
+	CRClient          versioned.Interface
+	Namespace         string
+	RetryAttempts     int
 }
 
 // Setup setups a test framework and points "Global" to it.
 func setup() error {
 	kubeconfig := flag.String("kubeconfig", "", "kube config path, e.g. $HOME/.kube/config")
+	kubeClusterDomain := flag.String("kube-cluster-domain", "cluster.local", "kube cluster domain")
 	opImage := flag.String("operator-image", "", "operator image, e.g. gcr.io/coreos-k8s-scale-testing/etcd-operator")
 	ns := flag.String("namespace", "default", "e2e test namespace")
+	retryAttemtps := flag.Int("retry-attempts", 6, "e2e cluster create retries (10 seconds per retry)")
 	flag.Parse()
 
 	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
@@ -71,10 +75,12 @@ func setup() error {
 	}
 
 	Global = &Framework{
-		KubeClient: cli,
-		CRClient:   client.MustNew(config),
-		Namespace:  *ns,
-		opImage:    *opImage,
+		KubeClient:        cli,
+		KubeClusterDomain: *kubeClusterDomain,
+		CRClient:          client.MustNew(config),
+		Namespace:         *ns,
+		opImage:           *opImage,
+		RetryAttempts:     *retryAttemtps,
 	}
 
 	// Skip the etcd-operator deployment setup if the operator image was not specified

diff --git a/test/e2e/recovery_test.go b/test/e2e/recovery_test.go
@@ -39,7 +39,7 @@ func TestOneMemberRecovery(t *testing.T) {
 		}
 	}()
 
-	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd)
+	names, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
@@ -49,7 +49,7 @@ func TestOneMemberRecovery(t *testing.T) {
 	if err := e2eutil.KillMembers(f.KubeClient, f.Namespace, names[2]); err != nil {
 		t.Fatal(err)
 	}
-	if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilPodSizeReached(t, f.KubeClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }
diff --git a/test/e2e/resize_test.go b/test/e2e/resize_test.go
@@ -39,7 +39,7 @@ func TestResizeCluster3to5(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}
 	t.Log("reached to 3 members cluster")
@@ -51,7 +51,7 @@ func TestResizeCluster3to5(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 5 members etcd cluster: %v", err)
 	}
 }
@@ -72,7 +72,7 @@ func TestResizeCluster5to3(t *testing.T) {
 		}
 	}()
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, 9, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 5, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to create 5 members etcd cluster: %v", err)
 	}
 	t.Log("reached to 5 members cluster")
@@ -84,7 +84,7 @@ func TestResizeCluster5to3(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, testEtcd); err != nil {
+	if _, err := e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, testEtcd); err != nil {
 		t.Fatalf("failed to resize to 3 members etcd cluster: %v", err)
 	}
 }
diff --git a/test/e2e/tls_test.go b/test/e2e/tls_test.go
@@ -40,7 +40,7 @@ func TestTLS(t *testing.T) {
 	memberClientTLSSecret := "etcd-server-tls" + suffix
 	operatorClientTLSSecret := "etcd-client-tls" + suffix
 
-	err := e2eutil.PrepareTLS(clusterName, f.Namespace, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
+	err := e2eutil.PrepareTLS(clusterName, f.Namespace, f.KubeClusterDomain, memberPeerTLSSecret, memberClientTLSSecret, operatorClientTLSSecret)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -66,7 +66,7 @@ func TestTLS(t *testing.T) {
 		}
 	}()
 
-	_, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, 6, c)
+	_, err = e2eutil.WaitUntilSizeReached(t, f.CRClient, 3, f.RetryAttempts, c)
 	if err != nil {
 		t.Fatalf("failed to create 3 members etcd cluster: %v", err)
 	}

diff --git a/test/pod/run-test-pod b/test/pod/run-test-pod
@@ -33,7 +33,7 @@ RETRY_INTERVAL=5
 
 # Wait until pod is running or timeout
 echo "Waiting for test-pod to start runnning"
-TIMEOUT=90
+TIMEOUT=180
 ELAPSED=0
 POD_PHASE=""
 until [ "${POD_PHASE}" == "${PHASE_RUNNING}" ]
@@ -57,13 +57,11 @@ DOCKER_REPO_ROOT="/go/src/github.com/coreos/etcd-operator"
 mkdir -p _output/logs/
 cp $KUBECONFIG ./kubeconfig
 
-docker run --rm \
-    -v "$PWD":"$DOCKER_REPO_ROOT" \
-    -w "$DOCKER_REPO_ROOT" \
-    gcr.io/coreos-k8s-scale-testing/logcollector \
-    logcollector --kubeconfig=${DOCKER_REPO_ROOT}/kubeconfig --e2e-podname=${POD_NAME} \
+go run test/logcollector/main.go --kubeconfig=$KUBECONFIG --e2e-podname=${POD_NAME} \
     --namespace=${TEST_NAMESPACE} --logs-dir="_output/logs/"
 
+kubectl -n ${TEST_NAMESPACE} logs ${POD_NAME}
+
 # Check for pod success or failure
 POD_PHASE=$(kubectl -n ${TEST_NAMESPACE} get pod ${POD_NAME} -o jsonpath='{.status.phase}')
 if [ "${POD_PHASE}" == "${PHASE_SUCCEEDED}" ]; then

diff --git a/test/pod/test-pod-templ.yaml b/test/pod/test-pod-templ.yaml
@@ -4,14 +4,18 @@ metadata:
   name: <POD_NAME>
 spec:
   restartPolicy: Never
+  imagePullSecrets:
+    - name: github
   containers:
   - name: <POD_NAME>
     image: <TEST_IMAGE>
-    imagePullPolicy: Always
+    imagePullPolicy: IfNotPresent
     command: ["hack/test"]
     resources:
       requests:
-        cpu: 2
+        cpu: 250m
+      limits:
+        cpu: 1
     env:
       - name: TEST_NAMESPACE
         valueFrom: