Integration tests (#15)

Add e2e tests
awslabs · Oct 26, 2023 · 783da90 · 783da90
1 parent b0112e3
commit 783da90
Show file tree

Hide file tree

Showing 11 changed files with 1,464 additions and 2 deletions.
diff --git a/.github/workflows/container-image.yaml b/.github/workflows/container-image.yaml
@@ -48,7 +48,6 @@ jobs:
           else
             GIT_TAG=$BRANCH_OR_TAG
           fi
-          echo $REGISTRY
           export PLATFORM=linux/amd64
           export TAG=$GIT_TAG
           make build_image

diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
@@ -0,0 +1,67 @@
+name: E2E Tests
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+# This workflow runs e2e tests and relies on existance of EKS cluster with a `s3-csi-driver-sa` service account
+# already deployed to it, which provides the driver with access to s3.
+#
+# Since we have a single cluster for e2e tests, we ensure that no more than one instance of this workflow is
+# running by `concurrency: e2e-cluster` option.
+concurrency: e2e-cluster
+env:
+  AWS_REGION : "us-east-1"
+  KUBECONFIG: "/tmp/kubeconfig"
+  COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }}
+jobs:
+  build:
+    # this is to prevent the job to run at forked projects
+    if: github.repository == 'awslabs/aws-s3-csi-driver'
+    runs-on: ubuntu-latest
+    environment: PR Integration Tests
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@master
+        with:
+          role-to-assume: arn:aws:iam::239424963615:role/S3CSIDriverE2ETestsRole
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Build, tag, and push docker image to Amazon ECR Public Repository
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          BRANCH_OR_TAG=$(echo $GITHUB_REF | cut -d'/' -f3)
+          export PLATFORM=linux/amd64
+          export TAG=${{ env.COMMIT_ID }}
+          make build_image
+          make push_image
+      - name: Install the driver
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        run: |
+          export EKS_REGION=${{ env.AWS_REGION }}
+          export EKS_CLUSTER_NAME=s3-csi-cluster
+          export KUBECONFIG=${{ env.KUBECONFIG }}
+          export TAG=${{ env.COMMIT_ID }}
+          tests/e2e-kubernetes/install.sh
+      - name: Run E2E Tests
+        run: make e2e E2E_KUBECONFIG=${{ env.KUBECONFIG }} E2E_COMMIT_ID=${{ env.COMMIT_ID }}
diff --git a/Makefile b/Makefile
@@ -11,6 +11,7 @@
 #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #See the License for the specific language governing permissions and
 #limitations under the License.
+SHELL = /bin/bash
 
 VERSION=0.1.0
 
@@ -26,12 +27,17 @@ GOPATH=$(shell go env GOPATH)
 GOOS=$(shell go env GOOS)
 GOBIN=$(shell pwd)/bin
 
-REGISTRY?=151381207180.dkr.ecr.eu-west-1.amazonaws.com
+REGISTRY?=""
 IMAGE?=$(REGISTRY)/s3-csi-driver
 TAG?=$(GIT_COMMIT)
 
 PLATFORM?=linux/amd64,linux/arm64
 
+# region is expected to be the same where cluster is created
+E2E_REGION?=us-east-1
+E2E_COMMIT_ID?=local
+E2E_KUBECONFIG?=""
+
 .EXPORT_ALL_VARIABLES:
 
 .PHONY: build_image
@@ -57,6 +63,14 @@ test:
 	# skipping controller test cases because we don't implement controller for static provisioning, this is a known limitation of sanity testing package: https://github.com/kubernetes-csi/csi-test/issues/214
 	go test -v ./tests/sanity/... -ginkgo.skip="ControllerGetCapabilities" -ginkgo.skip="ValidateVolumeCapabilities"
 
+.PHONY: e2e
+e2e:
+	pushd tests/e2e-kubernetes; \
+	KUBECONFIG=${E2E_KUBECONFIG} go test -ginkgo.vv --bucket-region=${E2E_REGION} --commit-id=${E2E_COMMIT_ID}; \
+	EXIT_CODE=$$?; \
+	popd; \
+	exit $$EXIT_CODE
+
 .PHONY: fmt
 fmt:
 	go fmt ./...

diff --git a/go.work b/go.work
@@ -0,0 +1,6 @@
+go 1.21
+
+use (
+    ./tests/e2e-kubernetes
+    .
+)
diff --git a/go.work.sum b/go.work.sum
diff --git a/tests/e2e-kubernetes/README.md b/tests/e2e-kubernetes/README.md
@@ -0,0 +1,13 @@
+
+## Usage
+From repository root:
+```
+make e2e E2E_KUBECONFIG=~/.kube/config E2E_REGION=eu-west-1
+```
+
+## Prerequisites
+- existing k8s cluster (e.g. EKS)
+- `kubectl` in $PATH
+- `kubeconfig` setting up access to k8s cluster
+- driver deployed in the cluster
+- aws credentials with access to s3 (create/delete buckets, read/write/list objects)
diff --git a/tests/e2e-kubernetes/e2e_test.go b/tests/e2e-kubernetes/e2e_test.go
@@ -0,0 +1,57 @@
+package e2e
+
+import (
+	"flag"
+	"testing"
+
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	f "k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
+	"k8s.io/kubernetes/test/e2e/storage/utils"
+)
+
+func init() {
+	testing.Init()
+	f.RegisterClusterFlags(flag.CommandLine) // configures --kubeconfig flag
+	f.RegisterCommonFlags(flag.CommandLine)  // configures --kubectl flag
+	f.AfterReadingAllFlags(&f.TestContext)
+
+	flag.StringVar(&CommitId, "commit-id", "local", "commit id will be used to name buckets")
+	flag.StringVar(&BucketRegion, "bucket-region", "us-east-1", "region where temporary buckets will be created")
+	flag.Parse()
+}
+
+func TestE2E(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "S3 CSI E2E Suite")
+}
+
+var CSITestSuites = []func() framework.TestSuite{
+	// testsuites.InitCapacityTestSuite,
+	testsuites.InitVolumesTestSuite, // success: writes 53 bytes to index.html file, reads and verifies content from another pod
+	// testsuites.InitVolumeIOTestSuite,   // tries to open a file for writing multiple times, which is unsupported by MP
+	// testsuites.InitVolumeModeTestSuite, // fail: tries to mount in block mode, success: check unused volume is not mounted
+	// testsuites.InitSubPathTestSuite,
+	// testsuites.InitProvisioningTestSuite,
+	// testsuites.InitMultiVolumeTestSuite,
+	// testsuites.InitVolumeExpandTestSuite,
+	// testsuites.InitDisruptiveTestSuite,
+	// testsuites.InitVolumeLimitsTestSuite,
+	// testsuites.InitTopologyTestSuite,
+	// testsuites.InitVolumeStressTestSuite,
+	// testsuites.InitFsGroupChangePolicyTestSuite,
+	// testsuites.InitSnapshottableTestSuite,
+	// testsuites.InitSnapshottableStressTestSuite,
+	// testsuites.InitVolumePerformanceTestSuite,
+	// testsuites.InitReadWriteOncePodTestSuite,
+}
+
+// This executes testSuites for csi volumes.
+var _ = utils.SIGDescribe("CSI Volumes", func() {
+	curDriver := initS3Driver()
+	ginkgo.Context(framework.GetDriverNameWithFeatureTags(curDriver), func() {
+		framework.DefineTestSuites(curDriver, CSITestSuites)
+	})
+})
diff --git a/tests/e2e-kubernetes/go.mod b/tests/e2e-kubernetes/go.mod
@@ -0,0 +1,158 @@
+module github.com/awslabs/aws-s3-csi-driver/tests/e2e-kubernetes
+
+go 1.21
+
+require (
+	github.com/onsi/ginkgo/v2 v2.9.4
+	github.com/onsi/gomega v1.27.6
+	k8s.io/apimachinery v0.28.3
+	k8s.io/kubernetes v1.28.3
+)
+
+require (
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/aws/aws-sdk-go v1.46.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/cel-go v0.16.1 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.6.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/selinux v1.10.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.9 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect
+	go.opentelemetry.io/otel v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
+	go.opentelemetry.io/otel/metric v0.31.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.10.0 // indirect
+	go.opentelemetry.io/otel/trace v1.10.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.19.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.8.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
+	google.golang.org/grpc v1.54.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/api v0.28.3 // indirect
+	k8s.io/apiextensions-apiserver v0.0.0 // indirect
+	k8s.io/apiserver v0.28.3 // indirect
+	k8s.io/client-go v0.28.3 // indirect
+	k8s.io/cloud-provider v0.0.0 // indirect
+	k8s.io/component-base v0.28.3 // indirect
+	k8s.io/component-helpers v0.28.3 // indirect
+	k8s.io/controller-manager v0.28.3 // indirect
+	k8s.io/csi-translation-lib v0.0.0 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kms v0.28.3 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/kubectl v0.0.0 // indirect
+	k8s.io/kubelet v0.0.0 // indirect
+	k8s.io/mount-utils v0.0.0 // indirect
+	k8s.io/pod-security-admission v0.0.0 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
+	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+// a hack to consume kubernetes as a library: https://github.com/kubernetes/kubernetes/issues/79384
+replace (
+	k8s.io/api => k8s.io/api v0.28.3
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.28.3
+	k8s.io/apiserver => k8s.io/apiserver v0.28.3
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.3
+	k8s.io/client-go => k8s.io/client-go v0.28.3
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.3
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.3
+	k8s.io/code-generator => k8s.io/code-generator v0.28.3
+	k8s.io/component-base => k8s.io/component-base v0.28.3
+	k8s.io/component-helpers => k8s.io/component-helpers v0.28.3
+	k8s.io/controller-manager => k8s.io/controller-manager v0.28.3
+	k8s.io/cri-api => k8s.io/cri-api v0.28.3
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.3
+	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.28.3
+	k8s.io/kms => k8s.io/kms v0.28.3
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.3
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.3
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.3
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.3
+	k8s.io/kubectl => k8s.io/kubectl v0.28.3
+	k8s.io/kubelet => k8s.io/kubelet v0.28.3
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.3
+	k8s.io/metrics => k8s.io/metrics v0.28.3
+	k8s.io/mount-utils => k8s.io/mount-utils v0.28.3
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.3
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.3
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.28.3
+	k8s.io/sample-controller => k8s.io/sample-controller v0.28.3
+)