ci(e2e): simplify e2e tests by build a config file

Currently a lot of parameters are scattered around the codebase as global variables.
This aims to reduce some of the global parameters.

Signed-off-by: Charly Molter <charly.molter@konghq.com>

.circleci/config.yml

@@ -337,20 +337,17 @@ jobs:
     - run:
         name: "Run E2E tests"
         command: |
-          export KUMA_UNIVERSAL_IMAGE="${ECR_REGISTRY}/kuma-universal:${TAG}";
           export E2E_PKG_LIST="./test/e2e/helm/...";
           export API_VERSION="<< parameters.api_version >>"
-
-          make test/e2e/test \
-            KUMA_GLOBAL_IMAGE_REGISTRY="${ECR_REGISTRY}" \
-            KUMA_GLOBAL_IMAGE_TAG="${TAG}" \
-            KUMA_CNI_CONF_NAME="10-aws.conflist" \
-            KUMA_USE_LOAD_BALANCER=true \
-            KUMA_USE_HOSTNAME_INSTEAD_OF_IP=true \
-            KUMA_UNIVERSAL_IMAGE="${KUMA_UNIVERSAL_IMAGE}" \
-            KUMA_DEFAULT_RETRIES=60 \
-            KUMA_DEFAULT_TIMEOUT="6s"
-
+          export KUMA_GLOBAL_IMAGE_REGISTRY="${ECR_REGISTRY}"
+          export KUMA_GLOBAL_IMAGE_TAG="${TAG}"
+          export KUMA_K8S_TYPE="aws"
+          export KUMA_USE_LOAD_BALANCER=true
+          export KUMA_USE_HOSTNAME_INSTEAD_OF_IP=true
+          export KUMA_DEFAULT_RETRIES=60
+          export KUMA_DEFAULT_TIMEOUT="6s"
+
+          make test/e2e/test
     - run:
         name: "Cleanup"
         when: always
@@ -571,19 +568,14 @@ jobs:
     - run:
         name: "Run E2E tests"
         command: |
-          export KUMA_UNIVERSAL_IMAGE="${AZURE_ACR_REGISTRY_FULLNAME}/kuma-universal:${TAG}";
-          export E2E_PKG_LIST="./test/e2e/helm/...";
+          export E2E_PKG_LIST="./test/e2e/helm/..."
           export API_VERSION="<< parameters.api_version >>"
+          export KUMA_GLOBAL_IMAGE_REGISTRY="${AZURE_ACR_REGISTRY_FULLNAME}"
+          export KUMA_GLOBAL_IMAGE_TAG="${TAG}"
+          export KUMA_K8S_TYPE="azure"
+          export KUMA_USE_LOAD_BALANCER=true
 
-          make test/e2e/test \
-            KUMA_GLOBAL_IMAGE_REGISTRY="${AZURE_ACR_REGISTRY_FULLNAME}" \
-            KUMA_GLOBAL_IMAGE_TAG="${TAG}" \
-            KUMA_CNI_CONF_NAME="10-azure.conflist" \
-            KUMA_UNIVERSAL_IMAGE="${KUMA_UNIVERSAL_IMAGE}" \
-            KUMA_DEFAULT_RETRIES=30 \
-            KUMA_DEFAULT_TIMEOUT="3s" \
-            KUMA_USE_LOAD_BALANCER=true
-
+          make test/e2e/test
     - run:
         name: "Cleanup"
         when: always

mk/e2e.new.mk

@@ -3,73 +3,24 @@ K8SCLUSTERS_START_TARGETS = $(addprefix test/e2e/k8s/start/cluster/, $(K8SCLUSTE
 K8SCLUSTERS_STOP_TARGETS  = $(addprefix test/e2e/k8s/stop/cluster/, $(K8SCLUSTERS))
 K8SCLUSTERS_LOAD_IMAGES_TARGETS  = $(addprefix test/e2e/k8s/load/images/, $(K8SCLUSTERS))
 K8SCLUSTERS_WAIT_TARGETS  = $(addprefix test/e2e/k8s/wait/, $(K8SCLUSTERS))
-API_VERSION ?= v3
 # export `IPV6=true` to enable IPv6 testing
 
-HELM_CHART_PATH ?=
-KUMA_GLOBAL_IMAGE_TAG ?=
-KUMA_GLOBAL_IMAGE_REGISTRY ?=
-KUMA_CP_IMAGE_REPOSITORY ?=
-KUMA_DP_IMAGE_REPOSITORY ?=
-KUMA_DP_INIT_IMAGE_REPOSITORY ?=
-KUMA_USE_LOAD_BALANCER ?=
-KUMA_USE_HOSTNAME_INSTEAD_OF_IP ?=
-KUMA_DEFAULT_RETRIES ?=
-KUMA_DEFAULT_TIMEOUT ?=
+# Targets to run prior to running the tests
+E2E_DEPS_TARGETS ?=
+# Environment veriables the tests should run with
+E2E_ENV_VARS ?=
 
 ifdef CI
 # In circleCI all this was built from previous targets let's reuse them!
-E2E_DEPS_TARGETS=docker/load
+E2E_DEPS_TARGETS+= docker/load
 else
-E2E_DEPS_TARGETS=build/kumactl images
+E2E_DEPS_TARGETS+= build/kumactl images
 endif
 
 ifndef KUMA_UNIVERSAL_IMAGE
 	KUMA_UNIVERSAL_IMAGE=$(KUMA_UNIVERSAL_DOCKER_IMAGE)
 endif
-
-ENV_VARS ?= API_VERSION="$(API_VERSION)"
-ENV_VARS += KUMA_UNIVERSAL_IMAGE=$(KUMA_UNIVERSAL_IMAGE)
-
-ifdef HELM_CHART_PATH
-	ENV_VARS += HELM_CHART_PATH=$(HELM_CHART_PATH)
-endif
-
-ifdef KUMA_GLOBAL_IMAGE_TAG
-	ENV_VARS += KUMA_GLOBAL_IMAGE_TAG=$(KUMA_GLOBAL_IMAGE_TAG)
-endif
-
-ifdef KUMA_GLOBAL_IMAGE_REGISTRY
-	ENV_VARS += KUMA_GLOBAL_IMAGE_REGISTRY=$(KUMA_GLOBAL_IMAGE_REGISTRY)
-endif
-
-ifdef KUMA_CP_IMAGE_REPOSITORY
-	ENV_VARS += KUMA_CP_IMAGE_REPOSITORY=$(KUMA_CP_IMAGE_REPOSITORY)
-endif
-
-ifdef KUMA_DP_IMAGE_REPOSITORY
-	ENV_VARS += KUMA_DP_IMAGE_REPOSITORY=$(KUMA_DP_IMAGE_REPOSITORY)
-endif
-
-ifdef KUMA_DP_INIT_IMAGE_REPOSITORY
-	ENV_VARS += KUMA_DP_INIT_IMAGE_REPOSITORY=$(KUMA_DP_INIT_IMAGE_REPOSITORY)
-endif
-
-ifdef KUMA_USE_LOAD_BALANCER
-	ENV_VARS += KUMA_USE_LOAD_BALANCER=$(KUMA_USE_LOAD_BALANCER)
-endif
-
-ifdef KUMA_USE_HOSTNAME_INSTEAD_OF_IP
-	ENV_VARS += KUMA_USE_HOSTNAME_INSTEAD_OF_IP=$(KUMA_USE_HOSTNAME_INSTEAD_OF_IP)
-endif
-
-ifdef KUMA_DEFAULT_RETRIES
-	ENV_VARS += KUMA_DEFAULT_RETRIES=$(KUMA_DEFAULT_RETRIES)
-endif
-
-ifdef KUMA_DEFAULT_TIMEOUT
-	ENV_VARS += KUMA_DEFAULT_TIMEOUT=$(KUMA_DEFAULT_TIMEOUT)
-endif
+E2E_ENV_VARS += KUMA_UNIVERSAL_IMAGE=$(KUMA_UNIVERSAL_IMAGE)
 
 # We don't use `go list` here because Ginkgo requires disk path names,
 # not Go packages names.
@@ -81,6 +32,7 @@ GO_TEST_E2E:=ginkgo $(GOFLAGS) $(LD_FLAGS) $(GINKGO_E2E_FLAGS)
 
 ifdef K3D
 K8S_CLUSTER_TOOL=k3d
+E2E_ENV_VARS += KUMA_K8S_TYPE=k3d
 else
 K8S_CLUSTER_TOOL=kind
 endif
@@ -112,6 +64,10 @@ endef
 
 $(foreach cluster, $(K8SCLUSTERS), $(eval $(call gen-k8sclusters,$(cluster))))
 
+ifdef K8SCLUSTERS
+E2E_ENV_VARS += K8SCLUSTERS="$(K8SCLUSTERS)"
+endif
+E2E_ENV_VARS += KUMACTLBIN=${BUILD_ARTIFACTS_DIR}/kumactl/kumactl
 .PHONY: test/e2e/list
 test/e2e/list:
 	@echo $(ALL_TESTS)
@@ -126,10 +82,7 @@ test/e2e/k8s/stop: $(K8SCLUSTERS_STOP_TARGETS)
 .PHONY: test/e2e/test
 test/e2e/test:
 	for t in $(E2E_PKG_LIST); do \
-		K8SCLUSTERS="$(K8SCLUSTERS)" \
-		KUMACTLBIN=${BUILD_ARTIFACTS_DIR}/kumactl/kumactl \
-		$(ENV_VARS) \
-		$(GO_TEST_E2E) -v -timeout=45m $$t || exit; \
+		$(E2E_ENV_VARS) $(GO_TEST_E2E) -v -timeout=45m $$t || exit; \
 	done
 
 # test/e2e/debug is used for quicker feedback of E2E tests (ex. debugging flaky tests)
@@ -139,9 +92,7 @@ test/e2e/test:
 # We run ginkgo instead of "go test" to fail fast (builtin "go test" fail fast does not seem to work with individual ginkgo tests)
 .PHONY: test/e2e/debug
 test/e2e/debug: build/kumactl images test/e2e/k8s/start
-	K8SCLUSTERS="$(K8SCLUSTERS)" \
-	KUMACTLBIN=${BUILD_ARTIFACTS_DIR}/kumactl/kumactl \
-	$(ENV_VARS) \
+	$(E2E_ENV_VARS) \
 	GINKGO_EDITOR_INTEGRATION=true \
 		$(GO_TEST_E2E) --failFast $(E2E_PKG_LIST)
 	$(MAKE) test/e2e/k8s/stop
@@ -156,9 +107,7 @@ test/e2e/debug-fast:
 	$(MAKE) build/kumactl
 	$(MAKE) $(K8SCLUSTERS_LOAD_IMAGES_TARGETS) # K3D is able to load images before the cluster is ready. It retries if cluster is not able to handle images yet.
 	$(MAKE) $(K8SCLUSTERS_WAIT_TARGETS) # there is no easy way of waiting for processes in the background so just wait for K8S clusters
-	K8SCLUSTERS="$(K8SCLUSTERS)" \
-	KUMACTLBIN=${BUILD_ARTIFACTS_DIR}/kumactl/kumactl \
-	$(ENV_VARS) \
+	$(E2E_ENV_VARS) \
 	GINKGO_EDITOR_INTEGRATION=true \
 		$(GO_TEST_E2E) --failFast $(E2E_PKG_LIST)
 	$(MAKE) test/e2e/k8s/stop
@@ -167,9 +116,7 @@ test/e2e/debug-fast:
 # and doesn't start Kind clusters
 .PHONY: test/e2e/debug-universal
 test/e2e/debug-universal: build/kumactl images/test
-	K8SCLUSTERS="" \
-	KUMACTLBIN=${BUILD_ARTIFACTS_DIR}/kumactl/kumactl \
-	$(ENV_VARS) \
+	$(E2E_ENV_VARS) \
 	GINKGO_EDITOR_INTEGRATION=true \
 		$(GO_TEST_E2E) --failFast $(E2E_PKG_LIST)
 

pkg/config/display.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"encoding/json"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
 	"reflect"
 )
 
@@ -27,3 +29,16 @@ func copyConfig(cfg Config) (Config, error) {
 	}
 	return newCfg, nil
 }
+
+func DumpToFile(filename string, cfg Config) (error) {
+	if err := cfg.Validate(); err != nil {
+		return err
+	}
+
+	b, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filename, b, 0666)
+}

test/e2e/auth/auth_universal.go

@@ -10,14 +10,12 @@ import (
 
 func AuthUniversal() {
 	var cluster Cluster
-	var deployOptsFuncs []KumaDeploymentOption
 
 	BeforeEach(func() {
 		cluster = NewUniversalCluster(NewTestingT(), Kuma3, Silent)
-		deployOptsFuncs = KumaUniversalDeployOpts
 
 		err := NewClusterSetup().
-			Install(Kuma(core.Standalone, deployOptsFuncs...)).
+			Install(Kuma(core.Standalone)).
 			Setup(cluster)
 		Expect(err).ToNot(HaveOccurred())
 		err = cluster.VerifyKuma()
@@ -28,7 +26,7 @@ func AuthUniversal() {
 		if ShouldSkipCleanup() {
 			return
 		}
-		Expect(cluster.DeleteKuma(deployOptsFuncs...)).To(Succeed())
+		Expect(cluster.DeleteKuma()).To(Succeed())
 		Expect(cluster.DismissCluster()).To(Succeed())
 	})
 

test/e2e/auth/dp/dp_auth_universal.go

@@ -10,22 +10,20 @@ import (
 
 func DpAuthUniversal() {
 	var cluster Cluster
-	var deployOptsFuncs []KumaDeploymentOption
 
 	E2EBeforeSuite(func() {
 		cluster = NewUniversalCluster(NewTestingT(), Kuma3, Silent)
-		deployOptsFuncs = KumaUniversalDeployOpts
 
 		err := NewClusterSetup().
-			Install(Kuma(core.Standalone, deployOptsFuncs...)).
+			Install(Kuma(core.Standalone)).
 			Setup(cluster)
 		Expect(err).ToNot(HaveOccurred())
 		err = cluster.VerifyKuma()
 		Expect(err).ToNot(HaveOccurred())
 	})
 
 	E2EAfterSuite(func() {
-		Expect(cluster.DeleteKuma(deployOptsFuncs...)).To(Succeed())
+		Expect(cluster.DeleteKuma()).To(Succeed())
 		Expect(cluster.DismissCluster()).To(Succeed())
 	})
 

test/e2e/compatibility/cp_compatibility_kubernetes_multizone.go

@@ -15,16 +15,11 @@ import (
 	. "github.com/kumahq/kuma/test/framework"
 )
 
-var OldChart = "0.7.0"
-var UpstreamImageRegistry = "kumahq"
-
 func CpCompatibilityMultizoneKubernetes() {
 	var globalCluster Cluster
 	var globalReleaseName string
-	var globalDeployOptsFuncs = KumaK8sDeployOpts
 
 	var zoneCluster Cluster
-	var zoneDeployOptsFuncs = KumaZoneK8sDeployOpts
 	var zoneReleaseName string
 
 	// Ensure that the upstream Kuma help repository is configured
@@ -38,7 +33,7 @@ func CpCompatibilityMultizoneKubernetes() {
 		// `--force-update` flag prevents heml emitting an error
 		// in this case.
 		_, err := helm.RunHelmCommandAndGetOutputE(t, &opts,
-			"repo", "add", "--force-update", "kuma", "https://kumahq.github.io/charts")
+			"repo", "add", "--force-update", "kuma", Config.HelmRepoUrl)
 		Expect(err).To(Succeed())
 
 		_, err = helm.RunHelmCommandAndGetOutputE(t, &opts, "repo", "update")
@@ -56,17 +51,16 @@ func CpCompatibilityMultizoneKubernetes() {
 			strings.ToLower(random.UniqueId()),
 		)
 
-		globalDeployOptsFuncs = append(globalDeployOptsFuncs,
-			WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
-			WithInstallationMode(HelmInstallationMode),
-			WithHelmChartPath(HelmRepo),
-			WithHelmReleaseName(globalReleaseName),
-			WithHelmChartVersion(OldChart),
-			WithoutHelmOpt("global.image.tag"),
-			WithHelmOpt("global.image.registry", UpstreamImageRegistry))
-
 		err := NewClusterSetup().
-			Install(Kuma(core.Global, globalDeployOptsFuncs...)).
+			Install(Kuma(core.Global,
+				WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
+				WithInstallationMode(HelmInstallationMode),
+				WithHelmChartPath(Config.HelmChartPath),
+				WithHelmReleaseName(globalReleaseName),
+				WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion),
+				WithoutHelmOpt("global.image.tag"),
+				WithHelmOpt("global.image.registry", Config.KumaImageRegistry),
+			)).
 			Setup(globalCluster)
 		Expect(err).ToNot(HaveOccurred())
 
@@ -82,20 +76,18 @@ func CpCompatibilityMultizoneKubernetes() {
 			strings.ToLower(random.UniqueId()),
 		)
 
-		zoneDeployOptsFuncs = append(zoneDeployOptsFuncs,
-			WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
-			WithInstallationMode(HelmInstallationMode),
-			WithHelmChartPath(HelmRepo),
-			WithHelmReleaseName(zoneReleaseName),
-			WithHelmChartVersion(OldChart),
-			WithoutHelmOpt("global.image.tag"),
-			WithHelmOpt("global.image.registry", UpstreamImageRegistry),
-			WithGlobalAddress(globalCluster.GetKuma().GetKDSServerAddress()),
-			WithHelmOpt("ingress.enabled", "true"),
-		)
-
 		err = NewClusterSetup().
-			Install(Kuma(core.Zone, zoneDeployOptsFuncs...)).
+			Install(Kuma(core.Zone,
+				WithEnv("KUMA_API_SERVER_AUTH_ALLOW_FROM_LOCALHOST", "true"),
+				WithInstallationMode(HelmInstallationMode),
+				WithHelmChartPath(Config.HelmChartName),
+				WithHelmReleaseName(zoneReleaseName),
+				WithHelmChartVersion(Config.SuiteConfig.Compatibility.HelmVersion),
+				WithoutHelmOpt("global.image.tag"),
+				WithHelmOpt("global.image.registry", Config.KumaImageRegistry),
+				WithGlobalAddress(globalCluster.GetKuma().GetKDSServerAddress()),
+				WithHelmOpt("ingress.enabled", "true"),
+			)).
 			Install(NamespaceWithSidecarInjectionOnAnnotation(TestNamespace)).
 			Setup(zoneCluster)
 		Expect(err).ToNot(HaveOccurred())
@@ -108,17 +100,16 @@ func CpCompatibilityMultizoneKubernetes() {
 			return
 		}
 
-		Expect(zoneCluster.DeleteKuma(zoneDeployOptsFuncs...)).To(Succeed())
+		Expect(zoneCluster.DeleteKuma()).To(Succeed())
 		Expect(zoneCluster.DismissCluster()).To(Succeed())
 
-		Expect(globalCluster.DeleteKuma(globalDeployOptsFuncs...)).To(Succeed())
+		Expect(globalCluster.DeleteKuma()).To(Succeed())
 		Expect(globalCluster.DismissCluster()).To(Succeed())
 	})
 
 	It("should sync resources between new global and old zone", func() {
 		// when global is upgraded
-		upgradeOptsFuncs := append(KumaK8sDeployOpts, WithHelmReleaseName(globalReleaseName))
-		err := globalCluster.(*K8sCluster).UpgradeKuma(core.Global, upgradeOptsFuncs...)
+		err := globalCluster.(*K8sCluster).UpgradeKuma(core.Global, WithHelmReleaseName(globalReleaseName))
 		Expect(err).ToNot(HaveOccurred())
 
 		// and new resource is created on Global