From eef7bb9227377c31df1d20a60c4d8d778346fa68 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Tue, 1 Dec 2020 15:03:36 +0100 Subject: [PATCH] Add Kubernetes conformance E2E test --- test/e2e/config/docker.yaml | 1 + .../v1alpha3/bases/cluster-with-kcp.yaml | 3 +- test/e2e/data/kubetest/conformance-fast.yaml | 8 ++ test/e2e/data/kubetest/conformance.yaml | 7 + test/e2e/e2e_suite_test.go | 7 +- test/e2e/k8s_conformance.go | 125 ++++++++++++++++++ test/e2e/k8s_conformance_test.go | 39 ++++++ 7 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 test/e2e/data/kubetest/conformance-fast.yaml create mode 100644 test/e2e/data/kubetest/conformance.yaml create mode 100644 test/e2e/k8s_conformance.go create mode 100644 test/e2e/k8s_conformance_test.go diff --git a/test/e2e/config/docker.yaml b/test/e2e/config/docker.yaml index 89fadcdb4682..560eed4ca7d3 100644 --- a/test/e2e/config/docker.yaml +++ b/test/e2e/config/docker.yaml @@ -85,6 +85,7 @@ variables: CNI: "./data/cni/kindnet/kindnet.yaml" EXP_CLUSTER_RESOURCE_SET: "true" EXP_MACHINE_POOL: "true" + KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" intervals: default/wait-controllers: ["3m", "10s"] diff --git a/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml index 5fb45f7a77ea..24f099f48f1e 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml @@ -61,7 +61,8 @@ spec: controllerManager: extraArgs: {enable-hostpath-provisioner: 'true'} apiServer: - certSANs: [localhost, 127.0.0.1, 0.0.0.0] + # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock diff --git a/test/e2e/data/kubetest/conformance-fast.yaml b/test/e2e/data/kubetest/conformance-fast.yaml new file mode 100644 index 000000000000..6f7936378b14 --- /dev/null +++ b/test/e2e/data/kubetest/conformance-fast.yaml @@ -0,0 +1,8 @@ +ginkgo.focus: \[Conformance\] +ginkgo.skip: \[sig-scheduling\].*\[Serial\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/data/kubetest/conformance.yaml b/test/e2e/data/kubetest/conformance.yaml new file mode 100644 index 000000000000..d748f432888b --- /dev/null +++ b/test/e2e/data/kubetest/conformance.yaml @@ -0,0 +1,7 @@ +ginkgo.focus: \[Conformance\] +disable-log-dump: true +ginkgo.progress: true +ginkgo.slowSpecThreshold: 120.0 +ginkgo.flakeAttempts: 3 +ginkgo.trace: true +ginkgo.v: true diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0b0159eb9334..13c1648ba9c2 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -21,15 +21,12 @@ package e2e import ( "context" "flag" - "fmt" "os" "path/filepath" "strings" "testing" . "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" @@ -84,8 +81,8 @@ func TestE2E(t *testing.T) { } RegisterFailHandler(Fail) - junitPath := filepath.Join(artifactFolder, fmt.Sprintf("junit.e2e_suite.%d.xml", config.GinkgoConfig.ParallelNode)) - junitReporter := reporters.NewJUnitReporter(junitPath) + + junitReporter := framework.CreateJUnitReporterForProw(artifactFolder) RunSpecsWithDefaultAndCustomReporters(t, "capi-e2e", []Reporter{junitReporter}) } diff --git a/test/e2e/k8s_conformance.go b/test/e2e/k8s_conformance.go new file mode 100644 index 000000000000..0c29da077873 --- /dev/null +++ b/test/e2e/k8s_conformance.go @@ -0,0 +1,125 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/test/framework/kubetest" + "sigs.k8s.io/cluster-api/util" +) + +// K8SConformanceSpecInput is the input for K8SConformanceSpec. +type K8SConformanceSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool +} + +// K8SConformanceSpec implements a spec that creates a cluster and runs Kubernetes conformance suite. +func K8SConformanceSpec(ctx context.Context, inputGetter func() K8SConformanceSpecInput) { + const ( + kubetestConfigurationVariable = "KUBETEST_CONFIGURATION" + ) + var ( + specName = "k8s-conformance" + input K8SConformanceSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + kubetestConfigFilePath string + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + Expect(input.E2EConfig.Variables).To(HaveKey(kubetestConfigurationVariable), "% spec requires a %s variable to be defined in the config file", specName, kubetestConfigurationVariable) + kubetestConfigFilePath = input.E2EConfig.GetVariable(kubetestConfigurationVariable) + Expect(kubetestConfigFilePath).To(BeAnExistingFile(), "%s should be a valid kubetest config file") + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + }) + + It("Should create a workload cluster and run kubetest", func() { + + By("Creating a workload cluster") + + // NOTE: The number of CP nodes does not have relevance for conformance; instead, the number of workers allows + // better parallelism of tests and thus a lower execution time. + var workerMachineCount int64 = 5 + + clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(workerMachineCount), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name) + + // Start running conformance test suites. + err := kubetest.Run( + ctx, + kubetest.RunInput{ + ClusterProxy: workloadProxy, + NumberOfNodes: int(workerMachineCount), + ArtifactsDirectory: input.ArtifactFolder, + ConfigFilePath: kubetestConfigFilePath, + GinkgoNodes: int(workerMachineCount), + }, + ) + Expect(err).ToNot(HaveOccurred(), "Failed to run Kubernetes conformance") + + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/k8s_conformance_test.go b/test/e2e/k8s_conformance_test.go new file mode 100644 index 000000000000..9db3e099dfa2 --- /dev/null +++ b/test/e2e/k8s_conformance_test.go @@ -0,0 +1,39 @@ +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing K8S conformance", func() { + + K8SConformanceSpec(context.TODO(), func() K8SConformanceSpecInput { + return K8SConformanceSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) + +})