From 35350a7e8ffb0b7164b3052efe7ddb59bde30ed1 Mon Sep 17 00:00:00 2001 From: Jonathan Tong Date: Tue, 30 Jan 2024 18:06:04 -0500 Subject: [PATCH] Add CAPI test framework and initial e2e test --- .gitignore | 8 + Makefile | 50 +++- go.mod | 7 + go.sum | 22 ++ hack/common-vars.sh | 25 ++ hack/gen-flavors.sh | 43 ++++ scripts/ci-e2e.sh | 96 ++------ test/e2e/capi_test.go | 52 +++++ test/e2e/common.go | 114 ++++++++++ test/e2e/config/helm.yaml | 144 ++++++++++++ .../addons-helm/v1beta1/bases/calico.yaml | 31 +++ .../v1beta1/bases/cluster-with-kcp.yaml | 91 ++++++++ .../data/addons-helm/v1beta1/bases/md.yaml | 51 +++++ .../addons-helm/v1beta1/cluster-template.yaml | 165 ++++++++++++++ .../cluster-template/kustomization.yaml | 4 + .../shared/v1beta1-provider/metadata.yaml | 8 + test/e2e/data/shared/v1beta1/metadata.yaml | 24 ++ test/e2e/e2e_suite_test.go | 215 ++++++++++++++++++ test/e2e/e2e_suite_vars.go | 79 +++++++ test/e2e/helpers.go | 158 +++++++++++++ test/e2e/log.go | 57 +++++ 21 files changed, 1360 insertions(+), 84 deletions(-) create mode 100644 hack/common-vars.sh create mode 100755 hack/gen-flavors.sh create mode 100644 test/e2e/capi_test.go create mode 100644 test/e2e/common.go create mode 100644 test/e2e/config/helm.yaml create mode 100644 test/e2e/data/addons-helm/v1beta1/bases/calico.yaml create mode 100644 test/e2e/data/addons-helm/v1beta1/bases/cluster-with-kcp.yaml create mode 100644 test/e2e/data/addons-helm/v1beta1/bases/md.yaml create mode 100644 test/e2e/data/addons-helm/v1beta1/cluster-template.yaml create mode 100644 test/e2e/data/addons-helm/v1beta1/cluster-template/kustomization.yaml create mode 100644 test/e2e/data/shared/v1beta1-provider/metadata.yaml create mode 100644 test/e2e/data/shared/v1beta1/metadata.yaml create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/e2e_suite_vars.go create mode 100644 test/e2e/helpers.go create mode 100644 test/e2e/log.go diff --git a/.gitignore b/.gitignore index 058370b6..1934fa2e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,19 @@ testbin/* # Output of the go coverage tool, specifically when used with LiteIDE *.out +go.work* # Kubernetes Generated files - skip generated files, except for vendored files !vendor/**/zz_generated.* +# e2e output +test/e2e/junit.e2e_suite.*.xml +test/e2e/caaph-e2e-*.yaml +test/e2e/config/helm-envsubst.yaml +test/e2e/logs/* +_artifacts + # editor and IDE paraphernalia *.code-workspace .idea diff --git a/Makefile b/Makefile index e91a27a8..6d422249 100644 --- a/Makefile +++ b/Makefile @@ -81,7 +81,8 @@ GINKGO_NODES ?= 1 GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m -# E2E_CONF_FILE ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/config/docker.yaml +E2E_CONF_FILE ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/config/helm.yaml +E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/helm-envsubst.yaml SKIP_RESOURCE_CLEANUP ?= false USE_EXISTING_CLUSTER ?= false GINKGO_NOCOLOR ?= false @@ -263,6 +264,16 @@ generate-modules: ## Run go mod tidy to ensure modules are up to date go mod tidy cd $(TOOLS_DIR); go mod tidy +DOCKER_TEMPLATES := test/e2e/data/addons-helm + +.PHONY: generate-e2e-templates +generate-e2e-templates: $(KUSTOMIZE) ## Generate templates for e2e tests + $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1beta1/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1beta1/cluster-template.yaml + +.PHONY: generate-flavors +generate-flavors: $(KUSTOMIZE) ## Generate template flavors. + ./hack/gen-flavors.sh + ## -------------------------------------- ## Lint / Verify ## -------------------------------------- @@ -383,15 +394,34 @@ test-cover: ## Run unit and integration tests and generate a coverage report go tool cover -func=out/coverage.out -o out/coverage.txt go tool cover -html=out/coverage.out -o out/coverage.html -# .PHONY: test-e2e -# test-e2e: $(GINKGO) generate-e2e-templates generate-test-extension-deployment ## Run the end-to-end tests -# $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ -# -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ -# $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ -# --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ -# -e2e.artifacts-folder="$(ARTIFACTS)" \ -# -e2e.config="$(E2E_CONF_FILE)" \ -# -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + +.PHONY: test-e2e +test-e2e: ## Run "docker-build" and "docker-push" rules then run e2e tests. + PULL_POLICY=IfNotPresent MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ + $(MAKE) docker-build docker-push \ + test-e2e-run + +.PHONY: test-e2e-run-skip-manifest +test-e2e-run-skip-manifest: $(GINKGO) $(ENVSUBST) generate-e2e-templates ## Run the end-to-end tests + $(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \ + $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ + -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ + $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ + --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(ROOT_DIR)/$(TEST_DIR)/e2e -- \ + -e2e.artifacts-folder="$(ARTIFACTS)" \ + -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + +.PHONY: test-e2e-run +test-e2e-run: + $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/default/manager_image_patch.yaml" + $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/default/manager_pull_policy.yaml" PULL_POLICY=IfNotPresent + MANAGER_IMAGE=$(CONTROLLER_IMG)-$(ARCH):$(TAG) \ + $(MAKE) test-e2e-run-skip-manifest + +.PHONY: get-e2e-kubeconfig +get-e2e-kubeconfig: + @kind get kubeconfig --name caaph-e2e ## -------------------------------------- ## Deployment diff --git a/go.mod b/go.mod index 54614f08..1500f8b0 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( k8s.io/klog/v2 v2.100.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/cluster-api v1.6.0 + sigs.k8s.io/cluster-api/test v1.6.0 sigs.k8s.io/controller-runtime v0.16.3 ) @@ -32,10 +33,12 @@ require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/adrg/xdg v0.4.0 // indirect + github.com/alessio/shellescape v1.4.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -86,6 +89,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.0 // indirect @@ -127,6 +131,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/prometheus/client_golang v1.17.0 // indirect @@ -169,6 +174,7 @@ require ( go.uber.org/zap v1.25.0 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect golang.org/x/sync v0.5.0 // indirect @@ -198,6 +204,7 @@ require ( oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kind v0.20.0 // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index d5f24b8a..5c9a8c5f 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -80,6 +81,8 @@ github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -137,6 +140,7 @@ github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -185,6 +189,7 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBF github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= @@ -323,6 +328,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -369,8 +376,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= @@ -422,6 +431,7 @@ github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -467,6 +477,7 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= @@ -475,6 +486,9 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -541,6 +555,7 @@ github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -806,6 +821,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1002,6 +1018,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1067,15 +1084,20 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6U sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0= sigs.k8s.io/cluster-api v1.6.0 h1:2bhVSnUbtWI8taCjd9lGiHExsRUpKf7Z1fXqi/IwYx4= sigs.k8s.io/cluster-api v1.6.0/go.mod h1:LB7u/WxiWj4/bbpHNOa1oQ8nq0MQ5iYlD0pGfRSBGLI= +sigs.k8s.io/cluster-api/test v1.6.0 h1:hvqUpSYxXCvs4FiEfsDpFZAfZ7i4kkP/59mVdFHlzSI= +sigs.k8s.io/cluster-api/test v1.6.0/go.mod h1:DJtbkrnrH77cd3PnXeKCQDMtCGVCrHZHPOjMvEsLB2U= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kind v0.20.0 h1:f0sc3v9mQbGnjBUaqSFST1dwIuiikKVGgoTwpoP33a8= +sigs.k8s.io/kind v0.20.0/go.mod h1:aBlbxg08cauDgZ612shr017/rZwqd7AS563FvpWKPVs= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3/go.mod h1:9n16EZKMhXBNSiUC5kSdFQJkdH3zbxS/JoO619G1VAY= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U= sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/common-vars.sh b/hack/common-vars.sh new file mode 100644 index 00000000..d2fe090b --- /dev/null +++ b/hack/common-vars.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +# shellcheck disable=SC2034 +KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl" +# shellcheck disable=SC2034 +KUSTOMIZE="${REPO_ROOT}/hack/tools/bin/kustomize" +# shellcheck disable=SC2034 +ENVSUBST="${REPO_ROOT}/hack/tools/bin/envsubst" +# shellcheck disable=SC2034 +YQ="${REPO_ROOT}/hack/tools/bin/yq" diff --git a/hack/gen-flavors.sh b/hack/gen-flavors.sh new file mode 100755 index 00000000..5a6e0f3c --- /dev/null +++ b/hack/gen-flavors.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +# shellcheck source=hack/common-vars.sh +source "${REPO_ROOT}/hack/common-vars.sh" + +make --directory="${REPO_ROOT}" "${KUSTOMIZE##*/}" + +flavors_dir="${REPO_ROOT}/templates/flavors/" +ci_dir="${REPO_ROOT}/templates/test/ci/" +dev_dir="${REPO_ROOT}/templates/test/dev/" + +for name in $(find "${flavors_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v base); do + ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone --reorder none "${flavors_dir}${name}" > "${REPO_ROOT}/templates/cluster-template-${name}.yaml" +done +# move the default template to the default file expected by clusterctl +mv "${REPO_ROOT}/templates/cluster-template-default.yaml" "${REPO_ROOT}/templates/cluster-template.yaml" + +for name in $(find "${ci_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches); do + ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone --reorder none "${ci_dir}${name}" > "${ci_dir}cluster-template-${name}.yaml" +done + +for name in $(find "${dev_dir}"* -maxdepth 0 -type d -print0 | xargs -0 -I {} basename {} | grep -v patches); do + ${KUSTOMIZE} build --load-restrictor LoadRestrictionsNone --reorder none "${dev_dir}${name}" > "${dev_dir}cluster-template-${name}.yaml" +done diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index 82d2fced..13769d0a 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,87 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: This needs to be set up for this repo once e2e tests have been implemented. +############################################################################### set -o errexit +set -o nounset set -o pipefail -REPO_ROOT=$(git rev-parse --show-toplevel) -cd "${REPO_ROOT}" || exit 1 +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. -# shellcheck source=./scripts/ci-e2e-lib.sh -source "${REPO_ROOT}/scripts/ci-e2e-lib.sh" - -# shellcheck source=./hack/ensure-go.sh +# shellcheck source=hack/ensure-go.sh source "${REPO_ROOT}/hack/ensure-go.sh" -# shellcheck source=./hack/ensure-kubectl.sh -source "${REPO_ROOT}/hack/ensure-kubectl.sh" -# shellcheck source=./hack/ensure-kind.sh -source "${REPO_ROOT}/hack/ensure-kind.sh" - -# Make sure the tools binaries are on the path. -export PATH="${REPO_ROOT}/hack/tools/bin:${PATH}" - -# Builds CAPI (and CAPD) images. -capi:buildDockerImages - -# Prepare kindest/node images for all the required Kubernetes version; this implies -# 1. Kubernetes version labels (e.g. latest) to the corresponding version numbers. -# 2. Pre-pulling the corresponding kindest/node image if available; if not, building the image locally. -# Following variables are currently checked (if defined): -# - KUBERNETES_VERSION -# - KUBERNETES_VERSION_UPGRADE_TO -# - KUBERNETES_VERSION_UPGRADE_FROM -k8s::prepareKindestImages - -# pre-pull all the images that will be used in the e2e, thus making the actual test run -# less sensible to the network speed. This includes: -# - cert-manager images -kind:prepullAdditionalImages - -# Configure e2e tests -export GINKGO_NODES=3 -export GINKGO_NOCOLOR=true -export GINKGO_ARGS="--fail-fast" # Other ginkgo args that need to be appended to the command. -export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/docker.yaml" -export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}" -export SKIP_RESOURCE_CLEANUP=false -export USE_EXISTING_CLUSTER=false - -# Setup local output directory -ARTIFACTS_LOCAL="${ARTIFACTS}/localhost" -mkdir -p "${ARTIFACTS_LOCAL}" -echo "This folder contains logs from the local host where the tests ran." > "${ARTIFACTS_LOCAL}/README.md" - -# Configure the containerd socket, otherwise 'ctr' would not work -export CONTAINERD_ADDRESS=/var/run/docker/containerd/containerd.sock -# ensure we retrieve additional info for debugging when we leave the script -cleanup() { - # shellcheck disable=SC2046 - kill $(pgrep -f 'docker events') || true - # shellcheck disable=SC2046 - kill $(pgrep -f 'ctr -n moby events') || true +export LOCAL_ONLY=${LOCAL_ONLY:-"true"} +export USE_LOCAL_KIND_REGISTRY=${USE_LOCAL_KIND_REGISTRY:-${LOCAL_ONLY}} +export BUILD_MANAGER_IMAGE=${BUILD_MANAGER_IMAGE:-"true"} - cp /var/log/docker.log "${ARTIFACTS_LOCAL}/docker.log" || true - docker ps -a > "${ARTIFACTS_LOCAL}/docker-ps.txt" || true - docker images > "${ARTIFACTS_LOCAL}/docker-images.txt" || true - docker info > "${ARTIFACTS_LOCAL}/docker-info.txt" || true - docker system df > "${ARTIFACTS_LOCAL}/docker-system-df.txt" || true - docker version > "${ARTIFACTS_LOCAL}/docker-version.txt" || true +export REGISTRY=${REGISTRY:-"localhost:5000/ci-e2e"} - ctr namespaces list > "${ARTIFACTS_LOCAL}/containerd-namespaces.txt" || true - ctr -n moby tasks list > "${ARTIFACTS_LOCAL}/containerd-tasks.txt" || true - ctr -n moby containers list > "${ARTIFACTS_LOCAL}/containerd-containers.txt" || true - ctr -n moby images list > "${ARTIFACTS_LOCAL}/containerd-images.txt" || true - ctr -n moby version > "${ARTIFACTS_LOCAL}/containerd-version.txt" || true -} -trap "cleanup" EXIT SIGINT +if [[ "${BUILD_MANAGER_IMAGE}" == "true" ]]; then + defaultTag=$(date -u '+%Y%m%d%H%M%S') + export TAG="${defaultTag:-dev}" +fi -docker events > "${ARTIFACTS_LOCAL}/docker-events.txt" 2>&1 & -ctr -n moby events > "${ARTIFACTS_LOCAL}/containerd-events.txt" 2>&1 & +export GINKGO_NODES=10 -# Run e2e tests -mkdir -p "$ARTIFACTS" -echo "+ run tests!" -make test-e2e +# Image is configured as `${CONTROLLER_IMG}-${ARCH}:${TAG}` where `CONTROLLER_IMG` is defaulted to `${REGISTRY}/${IMAGE_NAME}`. +if [[ "${BUILD_MANAGER_IMAGE}" == "false" ]]; then + # Load an existing image, skip docker-build and docker-push. + make test-e2e-run +else + # Build an image and push to the registry. TAG is set to `$(date -u '+%Y%m%d%H%M%S')`. + make test-e2e +fi diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go new file mode 100644 index 00000000..17b89e0d --- /dev/null +++ b/test/e2e/capi_test.go @@ -0,0 +1,52 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" +) + +var _ = Describe("Running the Cluster API E2E tests", func() { + + AfterEach(func() { + CheckTestBeforeCleanup() + }) + + Context("Running the quick-start spec [PR-Blocking]", func() { + capi_e2e.QuickStartSpec(context.TODO(), func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To(""), + ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{ + WaitForControlPlaneInitialized: EnsureControlPlaneInitialized, + }, + } + }) + }) +}) diff --git a/test/e2e/common.go b/test/e2e/common.go new file mode 100644 index 00000000..1905adaf --- /dev/null +++ b/test/e2e/common.go @@ -0,0 +1,114 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + kubeadmv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// EnsureControlPlaneInitialized waits for the cluster KubeadmControlPlane object to be initialized +// and then installs cloud-provider-azure components via Helm. +// Fulfills the clusterctl.Waiter type so that it can be used as ApplyClusterTemplateAndWaitInput data +// in the flow of a clusterctl.ApplyClusterTemplateAndWait E2E test scenario. +func EnsureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, result *clusterctl.ApplyCustomClusterTemplateAndWaitResult) { + getter := input.ClusterProxy.GetClient() + cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{ + Getter: getter, + Name: input.ClusterName, + Namespace: input.Namespace, + }) + kubeadmControlPlane := &kubeadmv1.KubeadmControlPlane{} + key := client.ObjectKey{ + Namespace: cluster.Spec.ControlPlaneRef.Namespace, + Name: cluster.Spec.ControlPlaneRef.Name, + } + + By("Ensuring KubeadmControlPlane is initialized") + Eventually(func(g Gomega) { + g.Expect(getter.Get(ctx, key, kubeadmControlPlane)).To(Succeed(), "Failed to get KubeadmControlPlane object %s/%s", cluster.Spec.ControlPlaneRef.Namespace, cluster.Spec.ControlPlaneRef.Name) + g.Expect(kubeadmControlPlane.Status.Initialized).To(BeTrue(), "KubeadmControlPlane is not yet initialized") + }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "KubeadmControlPlane object %s/%s was not initialized in time", cluster.Spec.ControlPlaneRef.Namespace, cluster.Spec.ControlPlaneRef.Name) + + By("Ensuring API Server is reachable before querying Helm charts") + Eventually(func(g Gomega) { + ns := &corev1.Namespace{} + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) + g.Expect(clusterProxy.GetClient().Get(ctx, client.ObjectKey{Name: kubesystem}, ns)).To(Succeed(), "Failed to get kube-system namespace") + }, input.WaitForControlPlaneIntervals...).Should(Succeed(), "API Server was not reachable in time") + + By("Ensure calico is ready after control plane is initialized") + EnsureCalicoIsReady(ctx, input) + + result.ControlPlane = framework.DiscoveryAndWaitForControlPlaneInitialized(ctx, framework.DiscoveryAndWaitForControlPlaneInitializedInput{ + Lister: input.ClusterProxy.GetClient(), + Cluster: result.Cluster, + }, input.WaitForControlPlaneIntervals...) +} + +const ( + calicoHelmChartRepoURL string = "https://docs.tigera.io/calico/charts" + calicoOperatorNamespace string = "tigera-operator" + CalicoSystemNamespace string = "calico-system" + CalicoAPIServerNamespace string = "calico-apiserver" + calicoHelmReleaseName string = "projectcalico" + calicoHelmChartName string = "tigera-operator" +) + +// EnsureCalicoIsReady verifies that the calico deployments exist and and are available on the workload cluster. +func EnsureCalicoIsReady(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput) { + specName := "ensure-calico" + + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) + + By("Waiting for Ready tigera-operator deployment pods") + for _, d := range []string{"tigera-operator"} { + waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, calicoOperatorNamespace, specName) + WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) + } + + By("Waiting for Ready calico-system deployment pods") + for _, d := range []string{"calico-kube-controllers", "calico-typha"} { + waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, CalicoSystemNamespace, specName) + WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) + } + By("Waiting for Ready calico-apiserver deployment pods") + for _, d := range []string{"calico-apiserver"} { + waitInput := GetWaitForDeploymentsAvailableInput(ctx, clusterProxy, d, CalicoAPIServerNamespace, specName) + WaitForDeploymentsAvailable(ctx, waitInput, e2eConfig.GetIntervals(specName, "wait-deployment")...) + } +} + +// CheckTestBeforeCleanup checks to see if the current running Ginkgo test failed, and prints +// a status message regarding cleanup. +func CheckTestBeforeCleanup() { + if CurrentSpecReport().State.Is(types.SpecStateFailureStates) { + Logf("FAILED!") + } + Logf("Cleaning up after \"%s\" spec", CurrentSpecReport().FullText()) +} diff --git a/test/e2e/config/helm.yaml b/test/e2e/config/helm.yaml new file mode 100644 index 00000000..2105fbf2 --- /dev/null +++ b/test/e2e/config/helm.yaml @@ -0,0 +1,144 @@ +managementClusterName: caaph-e2e + +images: + - name: ${MANAGER_IMAGE} + loadBehavior: mustLoad + - name: registry.k8s.io/cluster-api/cluster-api-controller:v1.6.0 + loadBehavior: tryLoad + - name: registry.k8s.io/cluster-api/kubeadm-bootstrap-controller:v1.6.0 + loadBehavior: tryLoad + - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.0 + loadBehavior: tryLoad + # Note: This pulls the CAPD image from the staging repo instead of the official registry. + - name: gcr.io/k8s-staging-cluster-api/capd-manager:v1.6.0 + loadBehavior: tryLoad + +providers: +- name: cluster-api + type: CoreProvider + versions: + - name: v1.6.0 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/core-components.yaml + type: url + contract: v1beta1 + files: + - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + + +- name: kubeadm + type: BootstrapProvider + versions: + - name: v1.6.0 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/bootstrap-components.yaml + type: url + contract: v1beta1 + files: + - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + +- name: kubeadm + type: ControlPlaneProvider + versions: + - name: v1.6.0 + value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/control-plane-components.yaml + type: url + contract: v1beta1 + files: + - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + +- name: docker + type: InfrastructureProvider + versions: + - name: "v1.6.0" # latest published release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only. + value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.0/infrastructure-components-development.yaml" + type: "url" + contract: v1beta1 + replacements: + - old: --metrics-addr=127.0.0.1:8080 + new: --metrics-addr=:8080 + files: + - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1/metadata.yaml" + - sourcePath: "${PWD}/test/e2e/data/addons-helm/v1beta1/cluster-template.yaml" + +- name: helm + type: AddonProvider + versions: + - name: v0.2.99 # "vNext"; use manifests from local source files + value: "${PWD}/config/default" + contract: v1beta1 + files: + - sourcePath: "${PWD}/test/e2e/data/shared/v1beta1-provider/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - old: "--v=0" + new: "--v=2" + +variables: + # Default variables for the e2e test; those values could be overridden via env variables, thus + # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. + # The following Kubernetes versions should be the latest versions with already published kindest/node images. + # This avoids building node images in the default case which improves the test duration significantly. + KUBERNETES_VERSION_MANAGEMENT: "v1.29.0" + KUBERNETES_VERSION: "v1.29.0" + KUBERNETES_VERSION_UPGRADE_FROM: "v1.28.0" + KUBERNETES_VERSION_UPGRADE_TO: "v1.29.0" + ETCD_VERSION_UPGRADE_TO: "3.5.10-0" + COREDNS_VERSION_UPGRADE_TO: "v1.11.1" + DOCKER_SERVICE_DOMAIN: "cluster.local" + IP_FAMILY: "dual" + DOCKER_SERVICE_CIDRS: "10.128.0.0/12" + DOCKER_POD_CIDRS: "192.168.0.0/16" + DOCKER_SERVICE_IPV6_CIDRS: "fd00:100:64::/108" + DOCKER_POD_IPV6_CIDRS: "fd00:100:96::/48" + CNI: "./data/cni/kindnet/kindnet.yaml" + KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" + AUTOSCALER_WORKLOAD: "./data/autoscaler/autoscaler-to-workload-workload.yaml" + NODE_DRAIN_TIMEOUT: "60s" + # Enabling the feature flags by setting the env variables. + EXP_CLUSTER_RESOURCE_SET: "true" + EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION: "true" + EXP_MACHINE_POOL: "true" + CLUSTER_TOPOLOGY: "true" + EXP_RUNTIME_SDK: "true" + EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true" + CAPI_DIAGNOSTICS_ADDRESS: ":8080" + CAPI_INSECURE_DIAGNOSTICS: "true" + +intervals: + default/wait-controllers: ["3m", "10s"] + default/wait-cluster: ["5m", "10s"] + default/wait-control-plane: ["10m", "10s"] + default/wait-worker-nodes: ["5m", "10s"] + default/wait-machine-pool-nodes: ["5m", "10s"] + default/wait-delete-cluster: ["3m", "10s"] + default/wait-machine-upgrade: ["20m", "10s"] + default/wait-machine-pool-upgrade: ["5m", "10s"] + default/wait-nodes-ready: ["10m", "10s"] + default/wait-machine-remediation: ["5m", "10s"] + default/wait-autoscaler: ["5m", "10s"] + default/wait-deployment: ["15m", "10s"] + default/wait-nsg-update: [ "20m", "10s" ] + default/wait-daemonset: [ "15m", "10s" ] + default/wait-deployment-available: [ "15m", "10s" ] + default/wait-job: [ "5m", "10s" ] + default/wait-service: [ "15m", "10s" ] + default/wait-private-cluster: ["30m", "10s"] + node-drain/wait-deployment-available: ["3m", "10s"] + node-drain/wait-control-plane: ["15m", "10s"] + node-drain/wait-machine-deleted: ["2m", "10s"] + kcp-remediation/wait-machines: ["5m", "10s"] + kcp-remediation/check-machines-stable: ["30s", "5s"] + kcp-remediation/wait-machine-provisioned: ["5m", "10s"] + # Giving a bit more time during scale tests, we analyze independently if everything works quickly enough. + scale/wait-cluster: ["10m", "10s"] + scale/wait-control-plane: ["20m", "10s"] + scale/wait-worker-nodes: ["20m", "10s"] diff --git a/test/e2e/data/addons-helm/v1beta1/bases/calico.yaml b/test/e2e/data/addons-helm/v1beta1/bases/calico.yaml new file mode 100644 index 00000000..d9948af0 --- /dev/null +++ b/test/e2e/data/addons-helm/v1beta1/bases/calico.yaml @@ -0,0 +1,31 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico +spec: + clusterSelector: + matchLabels: + cni: calico + repoURL: https://docs.tigera.io/calico/charts + chartName: tigera-operator + # version: ${CALICO_VERSION} + releaseName: projectcalico + namespace: tigera-operator + valuesTemplate: | + installation: + cni: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + registry: mcr.microsoft.com/oss + # Image and registry configuration for the tigera/operator pod. + tigeraOperator: + image: tigera/operator + registry: mcr.microsoft.com/oss + calicoctl: + image: mcr.microsoft.com/oss/calico/ctl diff --git a/test/e2e/data/addons-helm/v1beta1/bases/cluster-with-kcp.yaml b/test/e2e/data/addons-helm/v1beta1/bases/cluster-with-kcp.yaml new file mode 100644 index 00000000..38498bd4 --- /dev/null +++ b/test/e2e/data/addons-helm/v1beta1/bases/cluster-with-kcp.yaml @@ -0,0 +1,91 @@ +--- +# DockerCluster object referenced by the Cluster object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: '${CLUSTER_NAME}' +spec: + failureDomains: + fd1: + controlPlane: true + fd2: + controlPlane: true + fd3: + controlPlane: true + fd4: + controlPlane: false + fd5: + controlPlane: false + fd6: + controlPlane: false + fd7: + controlPlane: false + fd8: + controlPlane: false +--- +# Cluster object with +# - Reference to the KubeadmControlPlane object +# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet. +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: '${CLUSTER_NAME}' + labels: + cni: calico +spec: + clusterNetwork: + services: + cidrBlocks: ['${DOCKER_SERVICE_CIDRS}'] + pods: + cidrBlocks: ['${DOCKER_POD_CIDRS}'] + serviceDomain: '${DOCKER_SERVICE_DOMAIN}' + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: '${CLUSTER_NAME}' + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +# DockerMachineTemplate object referenced by the KubeadmControlPlane object +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" + # The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration. + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} +--- +# KubeadmControlPlane referenced by the Cluster object with +# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test. +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" + labels: + kcp-adoption.step2: "" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: {enable-hostpath-provisioner: 'true'} + apiServer: + # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] + initConfiguration: + nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use. + joinConfiguration: + nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use. + version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/addons-helm/v1beta1/bases/md.yaml b/test/e2e/data/addons-helm/v1beta1/bases/md.yaml new file mode 100644 index 00000000..5d42a2cf --- /dev/null +++ b/test/e2e/data/addons-helm/v1beta1/bases/md.yaml @@ -0,0 +1,51 @@ +--- +# DockerMachineTemplate referenced by the MachineDeployment and with +# - extraMounts for the docker sock, thus allowing self-hosting test +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" + # The DOCKER_PRELOAD_IMAGES variable gets set in self-hosted E2E tests to the list of images of the E2E configuration. + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} +--- +# KubeadmConfigTemplate referenced by the MachineDeployment +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: {} # node registration parameters are automatically injected by CAPD according to the kindest/node image in use. +--- +# MachineDeployment object +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + failureDomain: fd4 diff --git a/test/e2e/data/addons-helm/v1beta1/cluster-template.yaml b/test/e2e/data/addons-helm/v1beta1/cluster-template.yaml new file mode 100644 index 00000000..76a47722 --- /dev/null +++ b/test/e2e/data/addons-helm/v1beta1/cluster-template.yaml @@ -0,0 +1,165 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: calico +spec: + chartName: tigera-operator + clusterSelector: + matchLabels: + cni: calico + namespace: tigera-operator + releaseName: projectcalico + repoURL: https://docs.tigera.io/calico/charts + valuesTemplate: |- + installation: + cni: + type: Calico + calicoNetwork: + bgp: Disabled + mtu: 1350 + ipPools: + ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} + - cidr: {{ $cidr }} + encapsulation: VXLAN{{end}} + registry: mcr.microsoft.com/oss + # Image and registry configuration for the tigera/operator pod. + tigeraOperator: + image: tigera/operator + registry: mcr.microsoft.com/oss + calicoctl: + image: mcr.microsoft.com/oss/calico/ctl +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + joinConfiguration: + nodeRegistration: {} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni: calico + name: ${CLUSTER_NAME} +spec: + clusterNetwork: + pods: + cidrBlocks: + - ${DOCKER_POD_CIDRS} + serviceDomain: ${DOCKER_SERVICE_DOMAIN} + services: + cidrBlocks: + - ${DOCKER_SERVICE_CIDRS} + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: ${CLUSTER_NAME} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + failureDomain: fd4 + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + labels: + kcp-adoption.step2: "" + name: ${CLUSTER_NAME}-control-plane +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + certSANs: + - localhost + - 127.0.0.1 + - 0.0.0.0 + - host.docker.internal + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + initConfiguration: + nodeRegistration: {} + joinConfiguration: + nodeRegistration: {} + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: ${CLUSTER_NAME} +spec: + failureDomains: + fd1: + controlPlane: true + fd2: + controlPlane: true + fd3: + controlPlane: true + fd4: + controlPlane: false + fd5: + controlPlane: false + fd6: + controlPlane: false + fd7: + controlPlane: false + fd8: + controlPlane: false +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + preLoadImages: ${DOCKER_PRELOAD_IMAGES:-[]} diff --git a/test/e2e/data/addons-helm/v1beta1/cluster-template/kustomization.yaml b/test/e2e/data/addons-helm/v1beta1/cluster-template/kustomization.yaml new file mode 100644 index 00000000..d6e8ff2c --- /dev/null +++ b/test/e2e/data/addons-helm/v1beta1/cluster-template/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../bases/calico.yaml +- ../bases/md.yaml +- ../bases/cluster-with-kcp.yaml diff --git a/test/e2e/data/shared/v1beta1-provider/metadata.yaml b/test/e2e/data/shared/v1beta1-provider/metadata.yaml new file mode 100644 index 00000000..b753fc9b --- /dev/null +++ b/test/e2e/data/shared/v1beta1-provider/metadata.yaml @@ -0,0 +1,8 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 + - major: 0 + minor: 2 + contract: v1beta1 diff --git a/test/e2e/data/shared/v1beta1/metadata.yaml b/test/e2e/data/shared/v1beta1/metadata.yaml new file mode 100644 index 00000000..23faa295 --- /dev/null +++ b/test/e2e/data/shared/v1beta1/metadata.yaml @@ -0,0 +1,24 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 1 + minor: 6 + contract: v1beta1 + - major: 1 + minor: 5 + contract: v1beta1 + - major: 1 + minor: 4 + contract: v1beta1 + - major: 1 + minor: 3 + contract: v1beta1 + - major: 1 + minor: 2 + contract: v1beta1 + - major: 1 + minor: 1 + contract: v1beta1 + - major: 1 + minor: 0 + contract: v1beta1 diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 00000000..6bcd280a --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,215 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/gob" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + ctrl "sigs.k8s.io/controller-runtime" +) + +func init() { + flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored") + flag.BoolVar(&useCIArtifacts, "kubetest.use-ci-artifacts", false, "use the latest build from the main branch of the Kubernetes repository. Set KUBERNETES_VERSION environment variable to latest-1.xx to use the build from 1.xx release branch.") + flag.BoolVar(&usePRArtifacts, "kubetest.use-pr-artifacts", false, "use the build from a PR of the Kubernetes repository") + flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") + flag.BoolVar(&skipLogCollection, "e2e.skip-log-collection", false, "if true, the log collection after tests will be skipped") + flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") + flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file") + flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-path", "", "path to the kubetest repo-list path") +} + +func TestE2E(t *testing.T) { + ctrl.SetLogger(klog.Background()) + RegisterFailHandler(Fail) + RunSpecs(t, "caaph-e2e") +} + +// Using a SynchronizedBeforeSuite for controlling how to create resources shared across ParallelNodes (~ginkgo threads). +// The local clusterctl repository & the bootstrap cluster are created once and shared across all the tests. +var _ = SynchronizedBeforeSuite(func() []byte { + // Before all ParallelNodes. + + Expect(configPath).To(BeAnExistingFile(), "Invalid test suite argument. e2e.config should be an existing file.") + Expect(os.MkdirAll(artifactFolder, 0o755)).To(Succeed(), "Invalid test suite argument. Can't create e2e.artifacts-folder %q", artifactFolder) + + By("Initializing a runtime.Scheme with all the GVK relevant for this test") + scheme := initScheme() + + Byf("Loading the e2e test configuration from %q", configPath) + e2eConfig = loadE2EConfig(configPath) + + Byf("Creating a clusterctl local repository into %q", artifactFolder) + clusterctlConfigPath = createClusterctlLocalRepository(e2eConfig, filepath.Join(artifactFolder, "repository")) + + By("Setting up the bootstrap cluster") + bootstrapClusterProvider, bootstrapClusterProxy = setupBootstrapCluster(e2eConfig, scheme, useExistingCluster) + + By("Initializing the bootstrap cluster") + initBootstrapCluster(bootstrapClusterProxy, e2eConfig, clusterctlConfigPath, artifactFolder) + + // encode the e2e config into the byte array. + var configBuf bytes.Buffer + enc := gob.NewEncoder(&configBuf) + Expect(enc.Encode(e2eConfig)).To(Succeed()) + configStr := base64.StdEncoding.EncodeToString(configBuf.Bytes()) + + return []byte( + strings.Join([]string{ + artifactFolder, + clusterctlConfigPath, + configStr, + bootstrapClusterProxy.GetKubeconfigPath(), + }, ","), + ) +}, func(data []byte) { + // Before each ParallelNode. + + parts := strings.Split(string(data), ",") + Expect(parts).To(HaveLen(4)) + + artifactFolder = parts[0] + clusterctlConfigPath = parts[1] + + // Decode the e2e config + configBytes, err := base64.StdEncoding.DecodeString(parts[2]) + Expect(err).NotTo(HaveOccurred()) + buf := bytes.NewBuffer(configBytes) + dec := gob.NewDecoder(buf) + Expect(dec.Decode(&e2eConfig)).To(Succeed()) + + // we unset Kubernetes version variables to make sure we use the ones resolved from the first Ginkgo ParallelNode in the e2e config. + os.Unsetenv(capi_e2e.KubernetesVersion) + os.Unsetenv(capi_e2e.KubernetesVersionUpgradeFrom) + os.Unsetenv(capi_e2e.KubernetesVersionUpgradeTo) + + kubeconfigPath := parts[3] + bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(), framework.WithMachineLogCollector(framework.DockerLogCollector{})) +}) + +// Using a SynchronizedAfterSuite for controlling how to delete resources shared across ParallelNodes (~ginkgo threads). +// The bootstrap cluster is shared across all the tests, so it should be deleted only after all ParallelNodes completes. +// The local clusterctl repository is preserved like everything else created into the artifact folder. +var _ = SynchronizedAfterSuite(func() { + // After each ParallelNode. +}, func() { + // After all ParallelNodes. + + By("Tearing down the management cluster") + if !skipCleanup { + tearDown(bootstrapClusterProvider, bootstrapClusterProxy) + } +}) + +func loadE2EConfig(configPath string) *clusterctl.E2EConfig { + config := clusterctl.LoadE2EConfig(context.TODO(), clusterctl.LoadE2EConfigInput{ConfigPath: configPath}) + Expect(config).NotTo(BeNil(), "Failed to load E2E config from %s", configPath) + + return config +} + +func createClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFolder string) string { + createRepositoryInput := clusterctl.CreateRepositoryInput{ + E2EConfig: config, + RepositoryFolder: repositoryFolder, + } + + // // Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file as in place of the CNI_RESOURCES envSubst variable. + // Expect(config.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath) + // cniPath := config.GetVariable(capi_e2e.CNIPath) + // Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", capi_e2e.CNIPath) + // createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, capi_e2e.CNIResources) + + clusterctlConfig := clusterctl.CreateRepository(context.TODO(), createRepositoryInput) + Expect(clusterctlConfig).To(BeAnExistingFile(), "The clusterctl config file does not exists in the local repository %s", repositoryFolder) + return clusterctlConfig +} + +func initScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + framework.TryAddDefaultSchemes(scheme) + Expect(addonsv1alpha1.AddToScheme(scheme)).To(Succeed()) + return scheme +} + +func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme, useExistingCluster bool) (bootstrap.ClusterProvider, framework.ClusterProxy) { + var clusterProvider bootstrap.ClusterProvider + kubeconfigPath := "" + if !useExistingCluster { + clusterProvider = bootstrap.CreateKindBootstrapClusterAndLoadImages(context.TODO(), bootstrap.CreateKindBootstrapClusterAndLoadImagesInput{ + Name: config.ManagementClusterName, + RequiresDockerSock: config.HasDockerProvider(), + Images: config.Images, + }) + Expect(clusterProvider).NotTo(BeNil(), "Failed to create a bootstrap cluster") + + kubeconfigPath = clusterProvider.GetKubeconfigPath() + Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster") + } else { + // Loading image for already created cluster + imagesInput := bootstrap.LoadImagesToKindClusterInput{ + Name: "caaph-e2e", + Images: config.Images, + } + err := bootstrap.LoadImagesToKindCluster(context.TODO(), imagesInput) + Expect(err).To(BeNil(), "Failed to load images to the bootstrap cluster: %s", err) + } + + clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme) + Expect(clusterProxy).NotTo(BeNil(), "Failed to get a bootstrap cluster proxy") + return clusterProvider, clusterProxy +} + +func initBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config *clusterctl.E2EConfig, clusterctlConfig, artifactFolder string) { + clusterctl.InitManagementClusterAndWatchControllerLogs(context.TODO(), clusterctl.InitManagementClusterAndWatchControllerLogsInput{ + ClusterProxy: bootstrapClusterProxy, + ClusterctlConfigPath: clusterctlConfig, + InfrastructureProviders: config.InfrastructureProviders(), + AddonProviders: config.AddonProviders(), + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) +} + +func tearDown(bootstrapClusterProvider bootstrap.ClusterProvider, bootstrapClusterProxy framework.ClusterProxy) { + if bootstrapClusterProxy != nil { + bootstrapClusterProxy.Dispose(context.TODO()) + } + if bootstrapClusterProvider != nil { + bootstrapClusterProvider.Dispose(context.TODO()) + } +} diff --git a/test/e2e/e2e_suite_vars.go b/test/e2e/e2e_suite_vars.go new file mode 100644 index 00000000..ce5561fa --- /dev/null +++ b/test/e2e/e2e_suite_vars.go @@ -0,0 +1,79 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/bootstrap" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" +) + +const ( + kubesystem = "kube-system" + activitylog = "azure-activity-logs" + nodesDir = "nodes" +) + +// Test suite flags +var ( + // configPath is the path to the e2e config file. + configPath string + + // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). + useExistingCluster bool + + // artifactFolder is the folder to store e2e test artifacts. + artifactFolder string + + // skipCleanup prevents cleanup of test resources e.g. for debug purposes. + skipCleanup bool + + // skipLogCollection prevents the log collection process from running. + skipLogCollection bool +) + +// Test suite global vars +var ( + // e2eConfig to be used for this test, read from configPath. + e2eConfig *clusterctl.E2EConfig + + // clusterctlConfigPath to be used for this test, created by generating a clusterctl local repository + // with the providers specified in the configPath. + clusterctlConfigPath string + + // bootstrapClusterProvider manages provisioning of the the bootstrap cluster to be used for the e2e tests. + // Please note that provisioning will be skipped if e2e.use-existing-cluster is provided. + bootstrapClusterProvider bootstrap.ClusterProvider + + // bootstrapClusterProxy allows to interact with the bootstrap cluster to be used for the e2e tests. + bootstrapClusterProxy framework.ClusterProxy + + // kubetestConfigFilePath is the path to the kubetest configuration file + kubetestConfigFilePath string + + // kubetestRepoListPath + kubetestRepoListPath string + + // useCIArtifacts specifies whether or not to use the latest build from the main branch of the Kubernetes repository + useCIArtifacts bool + + // usePRArtifacts specifies whether or not to use the build from a PR of the Kubernetes repository + usePRArtifacts bool +) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go new file mode 100644 index 00000000..11dc5188 --- /dev/null +++ b/test/e2e/helpers.go @@ -0,0 +1,158 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "text/tabwriter" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + sshPort = "22" + deleteOperationTimeout = 20 * time.Minute + retryableOperationTimeout = 30 * time.Second + retryableDeleteOperationTimeout = 3 * time.Minute + retryableOperationSleepBetweenRetries = 3 * time.Second + helmInstallTimeout = 3 * time.Minute + sshConnectionTimeout = 30 * time.Second +) + +func Byf(format string, a ...interface{}) { + By(fmt.Sprintf(format, a...)) +} + +// deploymentsClientAdapter adapts a Deployment to work with WaitForDeploymentsAvailable. +type deploymentsClientAdapter struct { + client typedappsv1.DeploymentInterface +} + +// Get fetches the deployment named by the key and updates the provided object. +func (c deploymentsClientAdapter) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + deployment, err := c.client.Get(ctx, key.Name, metav1.GetOptions{}) + if deployObj, ok := obj.(*appsv1.Deployment); ok { + deployment.DeepCopyInto(deployObj) + } + return err +} + +// WaitForDeploymentsAvailableInput is the input for WaitForDeploymentsAvailable. +type WaitForDeploymentsAvailableInput struct { + Getter framework.Getter + Deployment *appsv1.Deployment + Clientset *kubernetes.Clientset +} + +// WaitForDeploymentsAvailable waits until the Deployment has status.Available = True, that signals that +// all the desired replicas are in place. +// This can be used to check if Cluster API controllers installed in the management cluster are working. +func WaitForDeploymentsAvailable(ctx context.Context, input WaitForDeploymentsAvailableInput, intervals ...interface{}) { + start := time.Now() + namespace, name := input.Deployment.GetNamespace(), input.Deployment.GetName() + Byf("waiting for deployment %s/%s to be available", namespace, name) + Log("starting to wait for deployment to become available") + Eventually(func() bool { + key := client.ObjectKey{Namespace: namespace, Name: name} + if err := input.Getter.Get(ctx, key, input.Deployment); err == nil { + for _, c := range input.Deployment.Status.Conditions { + if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue { + return true + } + } + } + return false + }, intervals...).Should(BeTrue(), func() string { return DescribeFailedDeployment(ctx, input) }) + Logf("Deployment %s/%s is now available, took %v", namespace, name, time.Since(start)) +} + +// GetWaitForDeploymentsAvailableInput is a convenience func to compose a WaitForDeploymentsAvailableInput +func GetWaitForDeploymentsAvailableInput(ctx context.Context, clusterProxy framework.ClusterProxy, name, namespace string, specName string) WaitForDeploymentsAvailableInput { + Expect(clusterProxy).NotTo(BeNil()) + cl := clusterProxy.GetClient() + var d = &appsv1.Deployment{} + Eventually(func() error { + return cl.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, d) + }, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed()) + clientset := clusterProxy.GetClientSet() + return WaitForDeploymentsAvailableInput{ + Deployment: d, + Clientset: clientset, + Getter: cl, + } +} + +// DescribeFailedDeployment returns detailed output to help debug a deployment failure in e2e. +func DescribeFailedDeployment(ctx context.Context, input WaitForDeploymentsAvailableInput) string { + namespace, name := input.Deployment.GetNamespace(), input.Deployment.GetName() + b := strings.Builder{} + b.WriteString(fmt.Sprintf("Deployment %s/%s failed", + namespace, name)) + b.WriteString(fmt.Sprintf("\nDeployment:\n%s\n", prettyPrint(input.Deployment))) + b.WriteString(describeEvents(ctx, input.Clientset, namespace, name)) + return b.String() +} + +// describeEvents returns a string summarizing recent events involving the named object(s). +func describeEvents(ctx context.Context, clientset *kubernetes.Clientset, namespace, name string) string { + b := strings.Builder{} + if clientset == nil { + b.WriteString("clientset is nil, so skipping output of relevant events") + } else { + opts := metav1.ListOptions{ + FieldSelector: fmt.Sprintf("involvedObject.name=%s", name), + Limit: 20, + } + evts, err := clientset.CoreV1().Events(namespace).List(ctx, opts) + if err != nil { + b.WriteString(err.Error()) + } else { + w := tabwriter.NewWriter(&b, 0, 4, 2, ' ', tabwriter.FilterHTML) + fmt.Fprintln(w, "LAST SEEN\tTYPE\tREASON\tOBJECT\tMESSAGE") + for _, e := range evts.Items { + fmt.Fprintf(w, "%s\t%s\t%s\t%s/%s\t%s\n", e.LastTimestamp, e.Type, e.Reason, + strings.ToLower(e.InvolvedObject.Kind), e.InvolvedObject.Name, e.Message) + } + w.Flush() + } + } + return b.String() +} + +// prettyPrint returns a formatted JSON version of the object given. +func prettyPrint(v interface{}) string { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err.Error() + } + return string(b) +} diff --git a/test/e2e/log.go b/test/e2e/log.go new file mode 100644 index 00000000..db3c658b --- /dev/null +++ b/test/e2e/log.go @@ -0,0 +1,57 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" +) + +// This code was inspired from kubernetes/kubernetes, specifically https://github.com/oomichi/kubernetes/blob/master/test/e2e/framework/log.go + +func nowStamp() string { + return time.Now().Format(time.StampMilli) +} + +func logf(level string, format string, args ...interface{}) { + fmt.Fprintf(ginkgo.GinkgoWriter, nowStamp()+": "+level+": "+format+"\n", args...) +} + +// Logf prints info logs with a timestamp and formatting. +func Logf(format string, args ...interface{}) { + logf("INFO", format, args...) +} + +// LogWarningf prints warning logs with a timestamp and formatting. +func LogWarningf(format string, args ...interface{}) { + logf("WARNING", format, args...) +} + +// Log prints info logs with a timestamp. +func Log(message string) { + logf("INFO", message) +} + +// LogWarning prints warning logs with a timestamp. +func LogWarning(message string) { + logf("WARNING", message) +}