Run e2e tests on AL2023 (with SELinux) and Ubuntu 20.04 as well #458
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | ||
| on: | ||
| push: | ||
| branches: ["main", "release-**"] | ||
| pull_request: | ||
| branches: ["main"] | ||
| paths: | ||
| - "tests/**" | ||
| - "pkg/**" | ||
| - "cmd/**" | ||
| - "charts/**" | ||
| - ".github/workflows/**" | ||
| - "Dockerfile" | ||
| # This workflow runs e2e tests and relies on existance of EKS cluster with a `s3-csi-driver-sa` service account | ||
| # already deployed to it, which provides the driver with access to s3. | ||
| # | ||
| # Since we have a single cluster for e2e tests, we ensure that no more than one instance of this workflow is | ||
| # running by `concurrency: e2e-cluster` option. | ||
| # | ||
| # Succesfull workflows triggered by push to main will upload tested image to the private repository "PROMOTED_IMAGE_NAME": | ||
| # - uploaded images will be tagged with main branch commit number | ||
| # - uploaded images will be later promoted to public repository by "release" workflow | ||
| concurrency: e2e-cluster | ||
| env: | ||
| AWS_REGION: "us-east-1" | ||
| COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }} | ||
| TMP_IMAGE_NAME: "s3-csi-driver-tmp" | ||
| PROMOTED_IMAGE_NAME: "s3-csi-driver" | ||
| BENCHMARK_RESULTS_BUCKET: "s3://mountpoint-s3-csi-driver-benchmark" | ||
| BENCHMARK_RESULTS_REGION: "us-east-1" | ||
| BENCHMARK_ARTIFACTS_FOLDER: ".github/artifacts" | ||
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | ||
| jobs: | ||
| build: | ||
| # this is to prevent the job to run at forked projects | ||
| if: github.repository == 'awslabs/mountpoint-s3-csi-driver' | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version-file: "go.mod" | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v2 | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v2 | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@master | ||
| with: | ||
| role-to-assume: ${{ secrets.TEST_IAM_ROLE }} | ||
| aws-region: ${{ env.AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Build, tag, and push docker image to Amazon ECR Private Repository | ||
| env: | ||
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | ||
| IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }} | ||
| run: | | ||
| export PLATFORM=linux/amd64,linux/arm64 | ||
| export TAG=${{ env.COMMIT_ID }} | ||
| make -j `nproc` all-push | ||
| test: | ||
| needs: build | ||
| strategy: | ||
| matrix: | ||
| cluster-type: ["eksctl", "kops"] | ||
| arch: ["x86", "arm"] | ||
| family: | ||
| ["AmazonLinux2", "AmazonLinux2023", "Bottlerocket", "Ubuntu2004"] | ||
| exclude: | ||
| - cluster-type: "kops" | ||
| family: "Bottlerocket" | ||
| - cluster-type: "eksctl" | ||
| arch: "arm" | ||
| family: "Bottlerocket" | ||
| - cluster-type: "eksctl" | ||
| family: "Ubuntu2004" | ||
| # We're mainly using AL2023 to test our CSI driver with enforced SELinux, | ||
| # AL2023 supports switching to enforcing mode via cloud-init, but, | ||
| # passing user-data (the way we configure cloud-init) is not supported on eksctl for AL2023. | ||
| # So, we're only running SELinux tests with kops. | ||
| - cluster-type: "eksctl" | ||
| family: "AmazonLinux2023" | ||
| include: | ||
| # Since we only enable enforcing mode for SELinux in AL2023, it's easier to list it in "include" | ||
| # field rather than trying to exclude all other variants. | ||
| - cluster-type: "kops" | ||
| arch: "arm" | ||
| family: "AmazonLinux2023" | ||
| seLinuxMode: "enforcing" | ||
| # TODO: This is temporary, we wanted to run our tests on SELinux enforced hosts even before we support it. | ||
| # Our tests will be passing on SELinux enforced hosts as well once we support it and we will remove this configuration. | ||
| continue-on-error: ${{ matrix.seLinuxMode == "enforcing" }} | ||
|
Check failure on line 101 in .github/workflows/e2e-tests.yaml
|
||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Go | ||
| uses: actions/setup-go@v4 | ||
| with: | ||
| go-version-file: "go.mod" | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: "3.10" | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@master | ||
| with: | ||
| role-to-assume: ${{ secrets.TEST_IAM_ROLE }} | ||
| aws-region: ${{ env.AWS_REGION }} | ||
| - name: Install tools | ||
| run: | | ||
| export ACTION=install_tools | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Create cluster | ||
| if: matrix.cluster-type == 'kops' | ||
| run: | | ||
| export ACTION=create_cluster | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Update kubeconfig | ||
| run: | | ||
| export ACTION=update_kubeconfig | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Install the driver | ||
| run: | | ||
| export ACTION=install_driver | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export IMAGE_NAME=${{ env.TMP_IMAGE_NAME }} | ||
| export TAG=${{ env.COMMIT_ID }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Pre e2e cleanup | ||
| if: matrix.cluster-type == 'eksctl' | ||
| run: | | ||
| export ACTION=e2e_cleanup | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Run E2E Tests | ||
| run: | | ||
| export ACTION=run_tests | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export TAG=${{ env.COMMIT_ID }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Run Performance Tests | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| run: | | ||
| export ACTION=run_perf | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export TAG=${{ env.COMMIT_ID }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Download previous benchmark results | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| run: | | ||
| mkdir -p ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} | ||
| aws s3 cp --region ${{ env.BENCHMARK_RESULTS_REGION }} ${{ env.BENCHMARK_RESULTS_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true | ||
| - name: Update benchmark result file | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| uses: benchmark-action/github-action-benchmark@v1 | ||
| with: | ||
| tool: "customBiggerIsBetter" | ||
| output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json | ||
| alert-threshold: "200%" | ||
| fail-on-alert: true | ||
| external-data-json-path: ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json | ||
| max-items-in-chart: 20 | ||
| - name: Store benchmark result | ||
| if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' | ||
| run: | | ||
| tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json | ||
| aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} s3://mountpoint-s3-csi-driver-benchmark --recursive | ||
| - name: Post e2e cleanup | ||
| if: always() | ||
| run: | | ||
| export ACTION=e2e_cleanup | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Uninstall the driver | ||
| if: always() | ||
| run: | | ||
| export ACTION=uninstall_driver | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| - name: Delete cluster | ||
| if: always() && matrix.cluster-type == 'kops' | ||
| run: | | ||
| export ACTION=delete_cluster | ||
| export AWS_REGION=${{ env.AWS_REGION }} | ||
| export CLUSTER_TYPE=${{ matrix.cluster-type }} | ||
| export ARCH=${{ matrix.arch }} | ||
| export AMI_FAMILY=${{ matrix.family }} | ||
| tests/e2e-kubernetes/scripts/run.sh | ||
| promote: | ||
| if: startsWith(github.ref_name, 'release') | ||
| needs: test | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - name: Set up crane | ||
| uses: imjasonh/setup-crane@v0.1 | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/configure-aws-credentials@master | ||
| with: | ||
| role-to-assume: ${{ secrets.TEST_IAM_ROLE }} | ||
| aws-region: ${{ env.AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/amazon-ecr-login@v2 | ||
| - name: Promote image for release branch | ||
| env: | ||
| REGISTRY: ${{ steps.login-ecr.outputs.registry }} | ||
| run: | | ||
| export TMP_IMAGE_NAME=${REGISTRY}/${{ env.TMP_IMAGE_NAME }}:${{ env.COMMIT_ID }} | ||
| export NEW_IMAGE_NAME=${REGISTRY}/${{ env.PROMOTED_IMAGE_NAME }}:${{ env.COMMIT_ID }} | ||
| crane copy ${TMP_IMAGE_NAME} ${NEW_IMAGE_NAME} | ||
