From 8a257be70492d88c548999a6790baf3371e407a4 Mon Sep 17 00:00:00 2001 From: Oleg Kunitsyn <114359669+hiddenmarten@users.noreply.github.com> Date: Sat, 13 Apr 2024 19:42:40 +0200 Subject: [PATCH] [104] Simple e2e test case (#168) --- .github/workflows/make-test-e2e.yaml | 21 ++- .github/workflows/make-test.yaml | 13 +- ...lease-drafter.yml => release-drafter.yaml} | 2 +- Makefile | 5 +- examples/manifests/etcdcluster-simple.yaml | 1 - go.mod | 13 +- go.sum | 21 +++ .../v0.1/contribution-guidelines/_index.md | 14 +- test/e2e/e2e_test.go | 118 +++++++-------- test/utils/utils.go | 143 +++++++++--------- 10 files changed, 203 insertions(+), 148 deletions(-) rename .github/workflows/{release-drafter.yml => release-drafter.yaml} (94%) diff --git a/.github/workflows/make-test-e2e.yaml b/.github/workflows/make-test-e2e.yaml index 46615a03..cd91bf3f 100644 --- a/.github/workflows/make-test-e2e.yaml +++ b/.github/workflows/make-test-e2e.yaml @@ -8,16 +8,29 @@ on: - synchronize jobs: - build: + test-e2e: + name: test-e2e on k8s ${{ matrix.k8s.attribute }} version # Pull request has label 'ok-to-test' or the author is a member of the organization if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(fromJSON('["COLLABORATOR", "MEMBER", "OWNER"]'), github.event.pull_request.author_association) + strategy: + matrix: + k8s: + - version: 1.27.1 + attribute: penultimate + - version: 1.28.0 + attribute: previous + - version: 1.29.0 + attribute: latest runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4.1.1 - uses: actions/setup-go@v5.0.0 with: go-version: 1.22.2 - - uses: helm/kind-action@v1.9.0 + - uses: docker/setup-buildx-action@v3.3.0 + - uses: tale/kubectl-action@v1.4.0 with: - cluster_name: kind - - run: make test-e2e KIND_CLUSTER_NAME=kind + kubectl-version: v1.29.3 + # Empty kubeconfig file + base64-kube-config: "YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCnByZWZlcmVuY2VzOiB7fQo=" + - run: ENVTEST_K8S_VERSION=${{ matrix.k8s.version }} make test-e2e diff --git a/.github/workflows/make-test.yaml b/.github/workflows/make-test.yaml index 62e3c5f0..c7e33521 100644 --- a/.github/workflows/make-test.yaml +++ b/.github/workflows/make-test.yaml @@ -8,16 +8,23 @@ on: - synchronize jobs: - build: + test: + name: test on k8s ${{ matrix.k8s.attribute }} version # Pull request has label 'ok-to-test' or the author is a member of the organization if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(fromJSON('["COLLABORATOR", "MEMBER", "OWNER"]'), github.event.pull_request.author_association) strategy: matrix: - k8s_version: [1.27.1,1.28.0,1.29.0] + k8s: + - version: 1.27.1 + attribute: penultimate + - version: 1.28.0 + attribute: previous + - version: 1.29.0 + attribute: latest runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4.1.1 - uses: actions/setup-go@v5.0.0 with: go-version: 1.22.2 - - run: ENVTEST_K8S_VERSION=${{ matrix.k8s_version }} make test + - run: ENVTEST_K8S_VERSION=${{ matrix.k8s.version }} make test diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yaml similarity index 94% rename from .github/workflows/release-drafter.yml rename to .github/workflows/release-drafter.yaml index 6f0155dd..e536a801 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yaml @@ -8,7 +8,7 @@ on: - main jobs: - update_release_draft: + release-drafter: runs-on: ubuntu-22.04 steps: - uses: release-drafter/release-drafter@v6.0.0 diff --git a/Makefile b/Makefile index 88802e92..48918941 100644 --- a/Makefile +++ b/Makefile @@ -165,9 +165,10 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified $(KUSTOMIZE) build config/crd | $(KUBECTL) delete -n $(NAMESPACE) --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy -deploy: manifests kustomize kind-load ## Deploy controller to the K8s cluster specified in ~/.kube/config. +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/aenix-io/etcd-operator=${IMG} $(KUSTOMIZE) build config/default | $(KUBECTL) -n $(NAMESPACE) apply -f - + $(KUBECTL) wait deployment.apps/etcd-operator-controller-manager --for condition=Available --namespace $(NAMESPACE) --timeout 5m .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. @@ -198,8 +199,10 @@ kind-delete: kind ## Create kubernetes cluster using Kind. kind-prepare: kind-create # Install prometheus operator $(KUBECTL) apply --server-side -f "https://github.com/prometheus-operator/prometheus-operator/releases/download/$(PROMETHEUS_OPERATOR_VERSION)/bundle.yaml" + $(KUBECTL) wait deployment.apps/prometheus-operator --for condition=Available --namespace default --timeout 5m # Install cert-manager operator $(KUBECTL) apply --server-side -f "https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml" + $(KUBECTL) wait deployment.apps/cert-manager-webhook --for condition=Available --namespace cert-manager --timeout 5m ##@ Dependencies diff --git a/examples/manifests/etcdcluster-simple.yaml b/examples/manifests/etcdcluster-simple.yaml index 7368c297..f1bff9e4 100644 --- a/examples/manifests/etcdcluster-simple.yaml +++ b/examples/manifests/etcdcluster-simple.yaml @@ -3,6 +3,5 @@ apiVersion: etcd.aenix.io/v1alpha1 kind: EtcdCluster metadata: name: test - namespace: default spec: replicas: 3 diff --git a/go.mod b/go.mod index 91d78ce6..e87eb740 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aenix-io/etcd-operator -go 1.21 +go 1.22.2 require ( github.com/onsi/ginkgo/v2 v2.17.1 @@ -15,6 +15,8 @@ require ( require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect @@ -32,7 +34,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -47,6 +49,9 @@ require ( github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.etcd.io/etcd/api/v3 v3.5.13 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect + go.etcd.io/etcd/client/v3 v3.5.13 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect @@ -59,6 +64,10 @@ require ( golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 1092e282..4a32bba0 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,10 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -30,6 +34,7 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -53,6 +58,8 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJY github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -116,6 +123,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= +go.etcd.io/etcd/api/v3 v3.5.13/go.mod h1:gBqlqkcMMZMVTMm4NDZloEVJzxQOQIls8splbqBDa0c= +go.etcd.io/etcd/client/pkg/v3 v3.5.13 h1:RVZSAnWWWiI5IrYAXjQorajncORbS0zI48LQlE2kQWg= +go.etcd.io/etcd/client/pkg/v3 v3.5.13/go.mod h1:XxHT4u1qU12E2+po+UVPrEeL94Um6zL58ppuJWXSAB8= +go.etcd.io/etcd/client/v3 v3.5.13 h1:o0fHTNJLeO0MyVbc7I3fsCf6nrOqn5d+diSarKnB2js= +go.etcd.io/etcd/client/v3 v3.5.13/go.mod h1:cqiAeY8b5DEEcpxvgWKsbLIWNM/8Wy2xJSDMtioMcoI= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -176,6 +189,14 @@ gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/site/content/en/docs/v0.1/contribution-guidelines/_index.md b/site/content/en/docs/v0.1/contribution-guidelines/_index.md index abf64cce..619175ff 100644 --- a/site/content/en/docs/v0.1/contribution-guidelines/_index.md +++ b/site/content/en/docs/v0.1/contribution-guidelines/_index.md @@ -12,8 +12,10 @@ more. ### Easy way Using this way, you don't be able to debug the controller locally. After every change you will have to redeploy changes. + #### Pre-requisites -- [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) +- Any docker-like container tool, "docker" by default. For more information search for: `CONTAINER_TOOL` in Makefile. +- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) **Steps** 1. Create and prepare kind cluster: @@ -21,19 +23,19 @@ Using this way, you don't be able to debug the controller locally. After every c make kind-prepare ``` -2. Install CRDs into kind cluster +2. Build image and load it into kind cluster: ```shell - make install + make kind-load ``` -3. Build image and load it into kind cluster, deploy etcd-operator, RBAC, webhook certs +3. Deploy CRDs, etcd-operator, RBAC, webhook certs into kind cluster: ```shell make deploy ``` -4. To deploy your code changes, redeploy etcd-operator: +4. To deploy your code changes, load a new image and redeploy etcd-operator: ```shell - make redeploy + make kind-load && make redeploy ``` 5. To clean up after all, delete kind cluster: diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index c0d5ebca..4866bae1 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,89 +17,87 @@ limitations under the License. package e2e import ( - "fmt" "os/exec" - "time" + "strconv" + "sync" + "github.com/aenix-io/etcd-operator/test/utils" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "github.com/aenix-io/etcd-operator/test/utils" ) -const namespace = "etcd-operator-system" +var _ = Describe("etcd-operator", Ordered, func() { -var _ = Describe("controller", Ordered, func() { BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) - - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) + var err error + By("prepare kind environment") + cmd := exec.Command("make", "kind-prepare") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("upload latest etcd-operator docker image to kind cluster") + cmd = exec.Command("make", "kind-load") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("deploy etcd-operator") + cmd = exec.Command("make", "deploy") + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) }) AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() - - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() - - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) + By("Delete kind environment") + cmd := exec.Command("make", "kind-delete") _, _ = utils.Run(cmd) }) - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string + Context("Simple", func() { + It("should deploy etcd cluster", func() { var err error + const namespace = "test-simple-etcd-cluster" + var wg sync.WaitGroup + wg.Add(1) - By("deploying the controller-manager") - cmd := exec.Command("make", "deploy") + By("create namespace") + cmd := exec.Command("kubectl", "create", "namespace", namespace) _, err = utils.Run(cmd) ExpectWithOffset(1, err).NotTo(HaveOccurred()) - By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - - cmd = exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) + By("apply simple etcd cluster manifest") + dir, _ := utils.GetProjectDir() + cmd = exec.Command("kubectl", "apply", + "--filename", dir+"/examples/manifests/etcdcluster-simple.yaml", + "--namespace", namespace, + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) - podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } - controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + By("wait for statefulset is ready") + cmd = exec.Command("kubectl", "wait", + "statefulset/test", + "--for", "jsonpath={.status.availableReplicas}=3", + "--namespace", namespace, + "--timeout", "5m", + ) + _, err = utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) - // Validate pod status - cmd = exec.Command("kubectl", "get", - "pods", controllerPodName, "-o", "jsonpath={.status.phase}", - "-n", namespace, + By("port-forward service to localhost") + port, _ := utils.GetFreePort() + go func() { + defer GinkgoRecover() + defer wg.Done() + cmd = exec.Command("kubectl", "port-forward", + "service/test-client", strconv.Itoa(port)+":2379", + "--namespace", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil - } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + _, err = utils.Run(cmd) + }() + By("check etcd cluster is healthy") + endpoints := []string{"localhost:" + strconv.Itoa(port)} + for i := 0; i < 3; i++ { + Expect(utils.IsEtcdClusterHealthy(endpoints)).To(BeTrue()) + } }) }) }) diff --git a/test/utils/utils.go b/test/utils/utils.go index 638436ab..cba3878f 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -17,35 +17,21 @@ limitations under the License. package utils import ( + "context" "fmt" + + clientv3 "go.etcd.io/etcd/client/v3" + + "log" + "net" "os" "os/exec" "strings" + "time" . "github.com/onsi/ginkgo/v2" //nolint:golint,revive ) -const ( - prometheusOperatorVersion = "v0.73.0" - prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + - "releases/download/%s/bundle.yaml" - - certmanagerVersion = "v1.14.4" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" -) - -func warnError(err error) { - fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err -} - // Run executes the provided command within this context func Run(cmd *exec.Cmd) ([]byte, error) { dir, _ := GetProjectDir() @@ -66,55 +52,6 @@ func Run(cmd *exec.Cmd) ([]byte, error) { return output, nil } -// UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// UninstallCertManager uninstalls the cert manager -func UninstallCertManager() { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } -} - -// InstallCertManager installs the cert manager bundle. -func InstallCertManager() error { - url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion) - cmd := exec.Command("kubectl", "apply", "-f", url) - if _, err := Run(cmd); err != nil { - return err - } - // Wait for cert-manager-webhook to be ready, which can take time if cert-manager - // was re-installed after uninstalling on a cluster. - cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook", - "--for", "condition=Available", - "--namespace", "cert-manager", - "--timeout", "5m", - ) - - _, err := Run(cmd) - return err -} - -// LoadImageToKindCluster loads a local docker image to the kind cluster -func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command("kind", kindOptions...) - _, err := Run(cmd) - return err -} - // GetNonEmptyLines converts given command output string into individual objects // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string { @@ -138,3 +75,69 @@ func GetProjectDir() (string, error) { wd = strings.Replace(wd, "/test/e2e", "", -1) return wd, nil } + +// GetFreePort asks the kernel for a free open port that is ready to use. +func GetFreePort() (port int, err error) { + var a *net.TCPAddr + if a, err = net.ResolveTCPAddr("tcp", "localhost:0"); err == nil { + var l *net.TCPListener + if l, err = net.ListenTCP("tcp", a); err == nil { + defer func(l *net.TCPListener) { + err := l.Close() + if err != nil { + log.Fatal(err) + } + }(l) + return l.Addr().(*net.TCPAddr).Port, nil + } + } + return +} + +// GetEtcdClient creates client for interacting with etcd. +func GetEtcdClient(endpoints []string) *clientv3.Client { + cli, err := clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + }) + if err != nil { + log.Fatal(err) + } + return cli +} + +// IsEtcdClusterHealthy checks etcd cluster health. +func IsEtcdClusterHealthy(endpoints []string) bool { + // Should be changed when etcd is healthy + health := false + + // Configure client + client := GetEtcdClient(endpoints) + defer func(client *clientv3.Client) { + err := client.Close() + if err != nil { + log.Fatal(err) + } + }(client) + + // Prepare the maintenance client + maint := clientv3.NewMaintenance(client) + + // Context for the call + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // Perform the status call to check health + for i := range endpoints { + resp, err := maint.Status(ctx, endpoints[i]) + if err != nil { + log.Fatalf("Failed to get endpoint health: %v", err) + } else { + if resp.Errors == nil { + fmt.Printf("Endpoint is healthy: %s\n", resp.Version) + health = true + } + } + } + return health +}