Merge pull request #787 from devigned/e2e-exp

🌱 Enable experimental features in e2e testing

.tiltignore

@@ -0,0 +1 @@
+templates

Makefile

@@ -119,10 +119,6 @@ SKIP_CREATE_MGMT_CLUSTER ?= false
 # Build time versioning details.
 LDFLAGS := $(shell hack/version.sh)
 
-# Allow overriding the feature gates
-EXP_MACHINE_POOL ?= false
-FEATURE_GATES_JSON_PATCH := [{"op": "add", "path": "/spec/template/spec/containers/1/args/-", "value": "--feature-gates=MachinePool=$(EXP_MACHINE_POOL)"}]
-
 CLUSTER_TEMPLATE ?= cluster-template.yaml
 MANAGED_CLUSTER_TEMPLATE ?= cluster-template-aks.yaml
 
@@ -387,18 +383,12 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST)
 	kubectl wait --for=condition=Available --timeout=5m apiservice v1beta1.webhook.cert-manager.io
 
 	# Deploy CAPI
-	kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.7/cluster-api-components.yaml
+	curl -sSL https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.7/cluster-api-components.yaml | $(ENVSUBST) | kubectl apply -f -
 
 	# Deploy CAPZ
 	kind load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=capz
 	$(KUSTOMIZE) build config | $(ENVSUBST) | kubectl apply -f -
 
-	# Patch controllers with feature gates flag
-	kubectl patch deployment capi-controller-manager -n capi-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)'
-	kubectl patch deployment capi-kubeadm-bootstrap-controller-manager -n capi-kubeadm-bootstrap-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)'
-	kubectl patch deployment capz-controller-manager -n capz-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)'
-	kubectl patch deployment capi-controller-manager -n capi-webhook-system --type=json -p='$(FEATURE_GATES_JSON_PATCH)'
-
 	# Wait for CAPI deployments
 	kubectl wait --for=condition=Available --timeout=5m -n capi-system deployment -l cluster.x-k8s.io/provider=cluster-api
 	kubectl wait --for=condition=Available --timeout=5m -n capi-kubeadm-bootstrap-system deployment -l cluster.x-k8s.io/provider=bootstrap-kubeadm
@@ -458,7 +448,7 @@ kind-create: ## create capz kind cluster if needed
 	./scripts/kind-with-registry.sh
 
 .PHONY: tilt-up
-tilt-up: kind-create ## start tilt and build kind cluster if needed
+tilt-up: $(ENVSUBST) $(KUSTOMIZE) kind-create ## start tilt and build kind cluster if needed
 	tilt up
 
 .PHONY: delete-cluster

Tiltfile

@@ -1,7 +1,11 @@
 # -*- mode: Python -*-
 
-# set defaults
+kustomize_cmd = "./hack/tools/bin/kustomize"
+envsubst_cmd = "./hack/tools/bin/envsubst"
+
+update_settings(k8s_upsert_timeout_secs=60)  # on first tilt up, often can take longer than 30 seconds
 
+# set defaults
 settings = {
     "allowed_contexts": [
         "kind-capz"
@@ -53,7 +57,9 @@ def deploy_cert_manager():
 # deploy CAPI
 def deploy_capi():
     version = settings.get("capi_version")
-    local("kubectl apply -f https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version))
+    capi_uri = "https://github.com/kubernetes-sigs/cluster-api/releases/download/{}/cluster-api-components.yaml".format(version)
+    cmd = "curl -sSL {} | {} | kubectl apply -f -".format(capi_uri, envsubst_cmd)
+    local(cmd, quiet=True)
     if settings.get("extra_args"):
         extra_args = settings.get("extra_args")
         if extra_args.get("core"):
@@ -142,6 +148,7 @@ COPY --from=tilt-helper /restart.sh .
 COPY manager .
 """
 
+
 # Build CAPZ and add feature gates
 def capz():
     # Apply the kustomized yaml for this provider
@@ -164,7 +171,7 @@ def capz():
     local_resource(
         "manager",
         cmd = 'mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager',
-        deps = ["./api", "./main.go", "./pkg", "./controllers", "./cloud", "./exp"]
+        deps = ["api", "cloud", "config", "controllers", "exp", "feature", "pkg", "go.mod", "go.sum", "main.go"]
     )
 
     dockerfile_contents = "\n".join([
@@ -187,11 +194,13 @@ def capz():
         entrypoint = entrypoint,
         only = "manager",
         live_update = [
-            sync("./.tiltbuild/manager", "/manager"),
+            sync(".tiltbuild/manager", "/manager"),
             run("sh /restart.sh"),
         ],
+        ignore = ["templates"]
     )
 
+    yaml = envsubst(yaml)
     k8s_yaml(blob(yaml))
 
 
@@ -270,11 +279,12 @@ def deploy_worker_templates(flavor, substitutions):
         value = substitutions[substitution]
         yaml = yaml.replace("${" + substitution + "}", value)
 
+    yaml = envsubst(yaml)
     yaml = yaml.replace('"', '\\"')     # add escape character to double quotes in yaml
 
     local_resource(
         "worker-" + flavor,
-        cmd = "make generate-flavors; echo \"" + yaml + "\" > ./.tiltbuild/worker-" + flavor + ".yaml; kubectl apply -f ./.tiltbuild/worker-" + flavor + ".yaml",
+        cmd = "make generate-flavors; echo \"" + yaml + "\" > ./.tiltbuild/worker-" + flavor + ".yaml; cat ./.tiltbuild/worker-" + flavor + ".yaml | " + envsubst_cmd + " | kubectl apply -f -",
         auto_init = False,
         trigger_mode = TRIGGER_MODE_MANUAL
     )
@@ -323,6 +333,11 @@ def base64_decode(to_decode):
     decode_blob = local("echo '{}' | base64 --decode -".format(to_decode), quiet=True)
     return str(decode_blob)
 
+
+def envsubst(yaml):
+    yaml = yaml.replace('"', '\\"')
+    return str(local("echo \"{}\" | {}".format(yaml, envsubst_cmd), quiet=True))
+
 ##############################
 # Actual work happens here
 ##############################

config/manager/manager.yaml

@@ -18,6 +18,7 @@ spec:
       containers:
         - args:
             - --enable-leader-election
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false}"
           image: controller:latest
           imagePullPolicy: Always
           name: manager

config/manager/manager_auth_proxy_patch.yaml

@@ -23,3 +23,4 @@ spec:
           args:
             - "--metrics-addr=127.0.0.1:8080"
             - "--enable-leader-election"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false}"

config/webhook/manager_webhook_patch.yaml

@@ -11,6 +11,7 @@ spec:
           args:
             - "--metrics-addr=127.0.0.1:8080"
             - "--webhook-port=9443"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=false},AKS=${EXP_AKS:=false}"
           ports:
             - containerPort: 9443
               name: webhook-server

test/e2e/config/azure-dev.yaml

@@ -55,6 +55,9 @@ variables:
   KUBERNETES_VERSION_UPGRADE_FROM: "${KUBERNETES_VERSION_UPGRADE_FROM:-v1.17.4}"
   CNI: "${PWD}/templates/addons/calico.yaml"
   REDACT_LOG_SCRIPT: "${PWD}/hack/log/redact.sh"
+  EXP_CLUSTER_RESOURCE_SET: "true"
+  EXP_AKS: "true"
+  EXP_MACHINE_POOL: "true"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]