🌱 Add Kubernetes conformance E2E test

test/e2e/config/docker.yaml

@@ -85,6 +85,7 @@ variables:
   CNI: "./data/cni/kindnet/kindnet.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
   EXP_MACHINE_POOL: "true"
+  KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]

test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml

@@ -61,7 +61,8 @@ spec:
       controllerManager:
         extraArgs: {enable-hostpath-provisioner: 'true'}
       apiServer:
-        certSANs: [localhost, 127.0.0.1, 0.0.0.0]
+        # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
+        certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
     initConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock

test/e2e/data/kubetest/conformance-fast.yaml

@@ -0,0 +1,8 @@
+ginkgo.focus: \[Conformance\]
+ginkgo.skip: \[sig-scheduling\].*\[Serial\]
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 3
+ginkgo.trace: true
+ginkgo.v: true

test/e2e/data/kubetest/conformance.yaml

@@ -0,0 +1,7 @@
+ginkgo.focus: \[Conformance\]
+disable-log-dump: true
+ginkgo.progress: true
+ginkgo.slowSpecThreshold: 120.0
+ginkgo.flakeAttempts: 3
+ginkgo.trace: true
+ginkgo.v: true

test/e2e/e2e_suite_test.go

@@ -21,15 +21,12 @@ package e2e
 import (
 	"context"
 	"flag"
-	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
-	"github.com/onsi/ginkgo/config"
-	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -84,8 +81,8 @@ func TestE2E(t *testing.T) {
 	}
 
 	RegisterFailHandler(Fail)
-	junitPath := filepath.Join(artifactFolder, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode))
-	junitReporter := reporters.NewJUnitReporter(junitPath)
+
+	junitReporter := framework.CreateJUnitReporterForProw(artifactFolder)
 	RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{junitReporter})
 }
 

test/e2e/k8s_conformance.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/test/framework/kubetest"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// K8SConformanceSpecInput is the input for K8SConformanceSpec.
+type K8SConformanceSpecInput struct {
+	E2EConfig             *clusterctl.E2EConfig
+	ClusterctlConfigPath  string
+	BootstrapClusterProxy framework.ClusterProxy
+	ArtifactFolder        string
+	SkipCleanup           bool
+}
+
+// K8SConformanceSpec implements a spec that creates a cluster and runs Kubernetes conformance suite.
+func K8SConformanceSpec(ctx context.Context, inputGetter func() K8SConformanceSpecInput) {
+	const (
+		kubetestConfigurationVariable = "KUBETEST_CONFIGURATION"
+	)
+	var (
+		specName               = "k8s-conformance"
+		input                  K8SConformanceSpecInput
+		namespace              *corev1.Namespace
+		cancelWatches          context.CancelFunc
+		clusterResources       *clusterctl.ApplyClusterTemplateAndWaitResult
+		kubetestConfigFilePath string
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
+		Expect(input.E2EConfig.Variables).To(HaveKey(kubetestConfigurationVariable), "% spec requires a %s variable to be defined in the config file", specName, kubetestConfigurationVariable)
+		kubetestConfigFilePath = input.E2EConfig.GetVariable(kubetestConfigurationVariable)
+		Expect(kubetestConfigFilePath).To(BeAnExistingFile(), "%s should be a valid kubetest config file")
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+	})
+
+	It("Should create a workload cluster and run kubetest", func() {
+
+		By("Creating a workload cluster")
+
+		// NOTE: The number of CP nodes does not have relevance for conformance; instead, the number of workers allows
+		// better parallelism of tests and thus a lower execution time.
+		var workerMachineCount int64 = 5
+
+		clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+			ClusterProxy: input.BootstrapClusterProxy,
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+				Flavor:                   clusterctl.DefaultFlavor,
+				Namespace:                namespace.Name,
+				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+				ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				WorkerMachineCount:       pointer.Int64Ptr(workerMachineCount),
+			},
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		})
+
+		workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name)
+
+		// Start running conformance test suites.
+		err := kubetest.Run(
+			ctx,
+			kubetest.RunInput{
+				ClusterProxy:       workloadProxy,
+				NumberOfNodes:      int(workerMachineCount),
+				ArtifactsDirectory: input.ArtifactFolder,
+				ConfigFilePath:     kubetestConfigFilePath,
+				GinkgoNodes:        int(workerMachineCount),
+			},
+		)
+		Expect(err).ToNot(HaveOccurred(), "Failed to run Kubernetes conformance")
+
+		By("PASSED!")
+	})
+
+	AfterEach(func() {
+		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+	})
+}

test/e2e/k8s_conformance_test.go

@@ -0,0 +1,39 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = Describe("When testing K8S conformance", func() {
+
+	K8SConformanceSpec(context.TODO(), func() K8SConformanceSpecInput {
+		return K8SConformanceSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+		}
+	})
+
+})