From bb05ed390b538b2d4eb63bd038841ab8f6249bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Fri, 25 Jun 2021 17:08:44 +0800 Subject: [PATCH] Enable the E2E test on Github Action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Run the E2E test with kind(provision various versions of k8s cluster) and MinIO on Github Action 2. Bug fix: the variable "stdoutBuf" is assigned to both "installPluginCmd.Stdout" and "installPluginCmd.Stderr", this causes 'if !strings.Contains(stderrBuf.String(), "Duplicate value")' takes no effect as the "stderrBuf.String()" is always empty 3. Print the stdout and stderr for easy debugging Signed-off-by: Wenkai Yin(尹文开) --- .github/workflows/e2e-test-kind.yaml | 113 +++++++++++++++++++++++++++ changelogs/unreleased/3912-ywk253100 | 1 + test/e2e/kibishii_tests.go | 27 +++---- test/e2e/velero_utils.go | 2 +- 4 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/e2e-test-kind.yaml create mode 100644 changelogs/unreleased/3912-ywk253100 diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml new file mode 100644 index 0000000000..4d17c900eb --- /dev/null +++ b/.github/workflows/e2e-test-kind.yaml @@ -0,0 +1,113 @@ +name: "Run the E2E test on kind" +on: + push: + pull_request: + # Do not run when the change only includes these directories. + paths-ignore: + - "site/**" + - "design/**" +jobs: + # Build the Velero CLI and image once for all Kubernetes versions, and cache it so the fan-out workers can get it. + build: + runs-on: ubuntu-latest + steps: + # Look for a CLI that's made for this PR + - name: Fetch built CLI + id: cli-cache + uses: actions/cache@v2 + with: + path: ./_output/bin/linux/amd64/velero + # The cache key a combination of the current PR number and the commit SHA + key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Fetch built image + id: image-cache + uses: actions/cache@v2 + with: + path: ./velero.tar + # The cache key a combination of the current PR number and the commit SHA + key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Fetch cached go modules + uses: actions/cache@v2 + if: steps.cli-cache.outputs.cache-hit != 'true' + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Check out the code + uses: actions/checkout@v2 + if: steps.cli-cache.outputs.cache-hit != 'true' || steps.image-cache.outputs.cache-hit != 'true' + # If no binaries were built for this PR, build it now. + - name: Build Velero CLI + if: steps.cli-cache.outputs.cache-hit != 'true' + run: | + make local + # If no image were built for this PR, build it now. + - name: Build Velero Image + if: steps.image-cache.outputs.cache-hit != 'true' + run: | + IMAGE=velero VERSION=pr-test make container + docker save velero:pr-test -o ./velero.tar + # Run E2E test against all kubernetes versions on kind + run-e2e-test: + needs: build + runs-on: ubuntu-latest + strategy: + matrix: + k8s: + # doesn't cover 1.15 as 1.15 doesn't support "apiextensions.k8s.io/v1" that is needed for the case + #- 1.15.12 + - 1.16.15 + - 1.17.17 + - 1.18.15 + - 1.19.7 + - 1.20.2 + - 1.21.1 + fail-fast: false + steps: + - name: Check out the code + uses: actions/checkout@v2 + - name: Install MinIO + run: + docker run -d --rm -p 9000:9000 -e "MINIO_ACCESS_KEY=minio" -e "MINIO_SECRET_KEY=minio123" -e "MINIO_DEFAULT_BUCKETS=bucket,additional-bucket" bitnami/minio:2021.6.17-debian-10-r7 + - uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.11.1" + image: "kindest/node:v${{ matrix.k8s }}" + - name: Fetch built CLI + id: cli-cache + uses: actions/cache@v2 + with: + path: ./_output/bin/linux/amd64/velero + key: velero-cli-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Fetch built Image + id: image-cache + uses: actions/cache@v2 + with: + path: ./velero.tar + key: velero-image-${{ github.event.pull_request.number }}-${{ github.sha }} + - name: Load Velero Image + run: + kind load image-archive velero.tar + # always try to fetch the cached go modules as the e2e test needs it either + - name: Fetch cached go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Run E2E test + run: | + cat << EOF > /tmp/credential + [default] + aws_access_key_id=minio + aws_secret_access_key=minio123 + EOF + GOPATH=~/go CLOUD_PROVIDER=kind \ + OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ + ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ + VELERO_IMAGE=velero:pr-test \ + make -C test/e2e run \ No newline at end of file diff --git a/changelogs/unreleased/3912-ywk253100 b/changelogs/unreleased/3912-ywk253100 new file mode 100644 index 0000000000..81352675c7 --- /dev/null +++ b/changelogs/unreleased/3912-ywk253100 @@ -0,0 +1 @@ +Run the E2E test with kind(provision various versions of k8s cluster) and MinIO on Github Action \ No newline at end of file diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index cb3d4c5619..4f5899cc3e 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -18,7 +18,6 @@ package e2e import ( "fmt" - "os" "os/exec" "strconv" "time" @@ -40,29 +39,23 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform string // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/"+cloudPlatform) - - _, _, err := veleroexec.RunCommand(kibishiiInstallCmd) + _, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd) if err != nil { - return errors.Wrap(err, "failed to install kibishii") + return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr) } kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment", "-n", namespace, "-w", "--timeout=30m") - kibishiiSetWaitCmd.Stdout = os.Stdout - kibishiiSetWaitCmd.Stderr = os.Stderr - _, _, err = veleroexec.RunCommand(kibishiiSetWaitCmd) - + _, stderr, err = veleroexec.RunCommand(kibishiiSetWaitCmd) if err != nil { - return err + return errors.Wrapf(err, "failed to rollout, stderr=%s", stderr) } fmt.Printf("Waiting for kibishii jump-pad pod to be ready\n") jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad") - jumpPadWaitCmd.Stdout = os.Stdout - jumpPadWaitCmd.Stderr = os.Stderr - _, _, err = veleroexec.RunCommand(jumpPadWaitCmd) + _, stderr, err = veleroexec.RunCommand(jumpPadWaitCmd) if err != nil { - return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s", namespace, jumpPadPod) + return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s, stderr=%s", namespace, jumpPadPod, stderr) } return err @@ -75,9 +68,9 @@ func generateData(ctx context.Context, namespace string, levels int, filesPerLev strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes)) fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd) - _, _, err := veleroexec.RunCommand(kibishiiGenerateCmd) + _, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd) if err != nil { - return errors.Wrap(err, "failed to generate") + return errors.Wrapf(err, "failed to generate, stderr=%s", stderr) } return nil @@ -90,9 +83,9 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes)) fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd) - _, _, err := veleroexec.RunCommand(kibishiiVerifyCmd) + _, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) if err != nil { - return errors.Wrap(err, "failed to verify") + return errors.Wrapf(err, "failed to verify, stderr=%s", stderr) } return nil } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 67e0a2ca37..beb46645d6 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -426,7 +426,7 @@ func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) installPluginCmd.Stdout = stdoutBuf - installPluginCmd.Stderr = stdoutBuf + installPluginCmd.Stderr = stderrBuf err := installPluginCmd.Run()