Add E2E Conformance tests using k8s ci artifacts

go.mod

@@ -16,9 +16,9 @@ require (
 	k8s.io/client-go v0.23.0
 	k8s.io/klog/v2 v2.30.0
 	k8s.io/utils v0.0.0-20211208161948-7d6a63dca704
-	sigs.k8s.io/cluster-api v1.1.0
-	sigs.k8s.io/cluster-api/test v1.1.0
-	sigs.k8s.io/controller-runtime v0.11.0
+	sigs.k8s.io/cluster-api v1.1.2
+	sigs.k8s.io/cluster-api/test v1.1.2
+	sigs.k8s.io/controller-runtime v0.11.1
 )
 
 require (
@@ -115,6 +115,4 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.1.0
-
-replace sigs.k8s.io/cluster-api/test => sigs.k8s.io/cluster-api/test v1.1.0-beta.2.0.20220211164825-cca725c1ca50
+replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.1.2

go.sum

@@ -1568,12 +1568,12 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.25/go.mod h1:Mlj9PNLmG9bZ6BHFwFKDo5afkpWyUISkb9Me0GnK66I=
-sigs.k8s.io/cluster-api v1.1.0 h1:zJJzIUezEveYK3Ljmk/jku+DYd96RP5bZ181l2W9gVg=
-sigs.k8s.io/cluster-api v1.1.0/go.mod h1:aSxmmNwDjmg9rDaL0RhRELpGX9uSTTwLF02AMtk/XGY=
-sigs.k8s.io/cluster-api/test v1.1.0-beta.2.0.20220211164825-cca725c1ca50 h1:kLrXadWcHlDAYdoKzLfD48Onfwl6hlJ2k9hgmGuHT6s=
-sigs.k8s.io/cluster-api/test v1.1.0-beta.2.0.20220211164825-cca725c1ca50/go.mod h1:ct7zQrWXsYIAhL/lMLFPfCAqGLDBTEzx+AtwIXpI45o=
-sigs.k8s.io/controller-runtime v0.11.0 h1:DqO+c8mywcZLFJWILq4iktoECTyn30Bkj0CwgqMpZWQ=
-sigs.k8s.io/controller-runtime v0.11.0/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
+sigs.k8s.io/cluster-api v1.1.2 h1:v00hk4crISOo2sKBhvOq0PC375BPk79Cpflt3Jtn7k8=
+sigs.k8s.io/cluster-api v1.1.2/go.mod h1:aq0b2tkMHZDTnuLEU7KOZOiQ5Pg82s3vh/KH/X6c/mM=
+sigs.k8s.io/cluster-api/test v1.1.2 h1:7kGGYqQc1Vn0p/geYXBDOypXJOwLQOcRz9WrFrTHmBY=
+sigs.k8s.io/cluster-api/test v1.1.2/go.mod h1:dk1BBIkLLcvOPuwgKWJ4zfJryGbfCFAZJtWRYo9QrZw=
+sigs.k8s.io/controller-runtime v0.11.1 h1:7YIHT2QnHJArj/dk9aUkYhfqfK5cIxPOX5gPECfdZLU=
+sigs.k8s.io/controller-runtime v0.11.1/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
 sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA=

scripts/ci-conformance.sh

@@ -32,6 +32,9 @@ source "${REPO_ROOT}/hack/ensure-doctl.sh"
 
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/digitalocean-ci.yaml"
+KUBERNETES_VERSION=${KUBERNETES_VERSION:-}
+KUBERNETES_MAJOR_VERSION=${KUBERNETES_MAJOR_VERSION:-}
+KUBERNETES_MAJOR_VERSION=${KUBERNETES_MAJOR_VERSION:-}
 
 SSH_KEY_NAME=capdo-conf-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 12 ; echo '')
 SSH_KEY_PATH=/tmp/${SSH_KEY_NAME}
@@ -58,5 +61,19 @@ trap 'remove_ssh_key ${SSH_KEY_FINGERPRINT}' EXIT
 export DO_SSH_KEY_FINGERPRINT=${SSH_KEY_FINGERPRINT}
 
 export GINKGO_FOCUS="Conformance Tests"
+
+if [[ "${E2E_ARGS:-}" == "-kubetest.use-ci-artifacts" ]]; then
+    CI_VERSION=${CI_VERSION:-$(curl -sSL https://dl.k8s.io/ci/latest.txt)}
+    KUBERNETES_VERSION=${CI_VERSION}
+    KUBERNETES_MAJOR_VERSION=$(echo "${KUBERNETES_VERSION}" | cut -d '.' -f1 - | sed 's/v//')
+    KUBERNETES_MINOR_VERSION=$(echo "${KUBERNETES_VERSION}" | cut -d '.' -f2 -)
+
+    export CI_VERSION
+    export KUBERNETES_VERSION
+    export KUBERNETES_MAJOR_VERSION
+    export KUBERNETES_MAJOR_VERSION
+fi
+echo "Will use K8s version $KUBERNETES_VERSION"
+
 make test-conformance
 test_status="${?}"

test/e2e/config/digitalocean-ci.yaml

@@ -7,8 +7,8 @@ providers:
   - name: cluster-api
     type: CoreProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/core-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"
@@ -20,8 +20,8 @@ providers:
   - name: kubeadm
     type: BootstrapProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/bootstrap-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"
@@ -33,8 +33,8 @@ providers:
   - name: kubeadm
     type: ControlPlaneProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/control-plane-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"
@@ -58,10 +58,13 @@ providers:
         targetName: "metadata.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-digitalocean/cluster-template.yaml"
         targetName: "cluster-template.yaml"
+      - sourcePath: "${PWD}/test/e2e/data/infrastructure-digitalocean/cluster-template-prow-ci-version.yaml"
+        targetName: "cluster-template-conformance-ci-artifacts.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-digitalocean/cluster-template-md-remediation.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-digitalocean/cluster-template-kcp-remediation.yaml"
       - sourcePath: "${PWD}/test/e2e/data/infrastructure-digitalocean/cluster-template-upgrades.yaml"
 
+
 variables:
   REDACT_LOG_SCRIPT: "${PWD}/hack/log/redact.sh"
   KUBERNETES_VERSION: "v1.22.6"
@@ -77,8 +80,7 @@ variables:
   DO_NODE_MACHINE_TYPE: "s-2vcpu-2gb"
   # Cluster API Kubernetes v1.22.6 on Ubuntu 20.04
   DO_NODE_MACHINE_IMAGE: "101591787"
-  CONFORMANCE_CI_ARTIFACTS_KUBERNETES_VERSION: "v1.23.3"
-  CONFORMANCE_WORKER_MACHINE_COUNT: "5"
+  CONFORMANCE_WORKER_MACHINE_COUNT: "2"
   CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT: "1"
   KUBETEST_CONFIGURATION: "${PWD}/test/e2e/data/kubetest/conformance.yaml"
   ETCD_VERSION_UPGRADE_TO: "3.5.1-0"

test/e2e/config/digitalocean-dev.yaml

@@ -7,8 +7,8 @@ providers:
   - name: cluster-api
     type: CoreProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/core-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/core-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"
@@ -20,8 +20,8 @@ providers:
   - name: kubeadm
     type: BootstrapProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/bootstrap-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/bootstrap-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"
@@ -33,8 +33,8 @@ providers:
   - name: kubeadm
     type: ControlPlaneProvider
     versions:
-    - name: v1.1.1
-      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.1/control-plane-components.yaml
+    - name: v1.1.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.1.2/control-plane-components.yaml
       type: "url"
       files:
       - sourcePath: "${PWD}/test/e2e/data/metadata/cluster-api/metadata.yaml"

test/e2e/conformance_test.go

@@ -60,7 +60,11 @@ var _ = Describe("Conformance Tests", func() {
 		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.KubernetesVersion))
 		Expect(e2eConfig.Variables).To(HaveKey(capi_e2e.CNIPath))
 
-		clusterName = fmt.Sprintf("capdo-conf-%s", util.RandomString(6))
+		clusterName = os.Getenv("CLUSTER_NAME")
+		if clusterName == "" {
+			clusterName = fmt.Sprintf("capdo-conf-%s", util.RandomString(6))
+		}
+		fmt.Fprintf(GinkgoWriter, "INFO: Cluster name is %s\n", clusterName)
 
 		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
 		namespace, cancelWatches = setupSpecNamespace(ctx, specName, bootstrapClusterProxy, artifactFolder)
@@ -78,6 +82,11 @@ var _ = Describe("Conformance Tests", func() {
 	Measure(specName, func(b Benchmarker) {
 		var err error
 
+		flavor := clusterctl.DefaultFlavor
+		if useCIArtifacts {
+			flavor = "conformance-ci-artifacts"
+		}
+
 		workerMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_WORKER_MACHINE_COUNT"), 10, 64)
 		Expect(err).NotTo(HaveOccurred())
 		controlPlaneMachineCount, err := strconv.ParseInt(e2eConfig.GetVariable("CONFORMANCE_CONTROL_PLANE_MACHINE_COUNT"), 10, 64)
@@ -91,7 +100,7 @@ var _ = Describe("Conformance Tests", func() {
 					ClusterctlConfigPath:     clusterctlConfigPath,
 					KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
 					InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
-					Flavor:                   clusterctl.DefaultFlavor,
+					Flavor:                   flavor,
 					Namespace:                namespace.Name,
 					ClusterName:              clusterName,
 					KubernetesVersion:        e2eConfig.GetVariable(capi_e2e.KubernetesVersion),

test/e2e/data/ccm/digitalocean-cloud-controller-manager.yaml

@@ -8,59 +8,49 @@ stringData:
   access-token: ${DIGITALOCEAN_ACCESS_TOKEN}
 ---
 apiVersion: apps/v1
-kind: DaemonSet
+kind: Deployment
 metadata:
   name: digitalocean-cloud-controller-manager
   namespace: kube-system
 spec:
+  replicas: 1
+  revisionHistoryLimit: 2
   selector:
     matchLabels:
-      app.kubernetes.io/name: digitalocean-cloud-controller-manager
+      app: digitalocean-cloud-controller-manager
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: digitalocean-cloud-controller-manager
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
+        app: digitalocean-cloud-controller-manager
     spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      serviceAccountName: cloud-controller-manager
+      priorityClassName: system-cluster-critical
       dnsPolicy: Default
       hostNetwork: true
+      serviceAccountName: cloud-controller-manager
       tolerations:
+        # this taint is set by all kubelets running `--cloud-provider=external`
+        # so we should tolerate it to schedule the digitalocean ccm
         - key: "node.cloudprovider.kubernetes.io/uninitialized"
           value: "true"
           effect: "NoSchedule"
         - key: "CriticalAddonsOnly"
           operator: "Exists"
+        # cloud controller manages should be able to run on masters
         - key: "node-role.kubernetes.io/master"
           effect: NoSchedule
-        - effect: NoExecute
-          key: node.kubernetes.io/not-ready
-          operator: Exists
-          tolerationSeconds: 300
-        - effect: NoExecute
-          key: node.kubernetes.io/unreachable
-          operator: Exists
-          tolerationSeconds: 300
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
       containers:
       - image: digitalocean/digitalocean-cloud-controller-manager:v0.1.36
         name: digitalocean-cloud-controller-manager
         command:
           - "/bin/digitalocean-cloud-controller-manager"
-          - "--leader-elect=true"
+          - "--leader-elect=false"
         resources:
           requests:
             cpu: 100m
             memory: 50Mi
         env:
-          - name: KUBERNETES_SERVICE_HOST
-            valueFrom:
-              fieldRef:
-                fieldPath: status.hostIP
-          - name: KUBERNETES_SERVICE_PORT
-            value: "6443"
           - name: DO_ACCESS_TOKEN
             valueFrom:
               secretKeyRef:

test/e2e/data/cni/calico/calico.yaml

@@ -4347,6 +4347,8 @@ spec:
           operator: Exists
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        - key: node-role.kubernetes.io/control-plane
+          effect: NoSchedule
       serviceAccountName: calico-kube-controllers
       priorityClassName: system-cluster-critical
       containers: