Skip to content

Commit

Permalink
e2e framework: Add conformance testing to test framework
Browse files Browse the repository at this point in the history
  • Loading branch information
randomvariable committed Sep 3, 2020
1 parent 3462a5c commit 73e9d34
Show file tree
Hide file tree
Showing 61 changed files with 2,655 additions and 245 deletions.
109 changes: 75 additions & 34 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -44,25 +44,38 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
BIN_DIR := bin
E2E_FRAMEWORK_DIR := test/framework
CAPD_DIR := test/infrastructure/docker
RELEASE_NOTES_BIN := bin/release-notes
RELEASE_NOTES := $(TOOLS_DIR)/$(RELEASE_NOTES_BIN)
LINK_CHECKER_BIN := bin/liche
LINK_CHECKER := $(TOOLS_DIR)/$(LINK_CHECKER_BIN)
GO_APIDIFF_BIN := bin/go-apidiff
GO_APIDIFF := $(TOOLS_DIR)/$(GO_APIDIFF_BIN)
ENVSUBST_BIN := bin/envsubst
ENVSUBST := $(TOOLS_DIR)/$(ENVSUBST_BIN)

PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
export PATH

# Binaries.
# Need to use abspath so we can invoke these from subdirectories
KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/kustomize)
CONTROLLER_GEN := $(abspath $(TOOLS_BIN_DIR)/controller-gen)
GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint)
CONVERSION_GEN := $(abspath $(TOOLS_BIN_DIR)/conversion-gen)
ENVSUBST := $(abspath $(TOOLS_BIN_DIR)/envsubst)

KUSTOMIZE_BIN := $(TOOLS_BIN_DIR)/kustomize
CONTROLLER_GEN_BIN := $(TOOLS_BIN_DIR)/controller-gen
GOLANGCI_LINT_BIN := $(TOOLS_BIN_DIR)/golangci-lint
CONVERSION_GEN_BIN := $(TOOLS_BIN_DIR)/conversion-gen
DEFAULTER_GEN_BIN := $(TOOLS_BIN_DIR)/defaulter-gen
GINKGO_BIN := $(TOOLS_BIN_DIR)/ginkgo
ENVSUBST_BIN := $(TOOLS_BIN_DIR)/envsubst
GOBINDATA_BIN := $(TOOLS_BIN_DIR)/go-bindata
RELEASE_NOTES_BIN := $(TOOLS_BIN_DIR)/release-notes
GO_APIDIFF_BIN := $(TOOLS_BIN_DIR)/go-apidiff
LINK_CHECKER_BIN := $(TOOLS_BIN_DIR)/link-checker

KUSTOMIZE := $(abspath $(KUSTOMIZE_BIN))
CONTROLLER_GEN := $(abspath $(CONTROLLER_GEN))
GOLANGCI_LINT := $(abspath $(GOLANGCI_LINT))
CONVERSION_GEN := $(abspath $(CONVERSION_GEN))
DEFAULTER_GEN := $(abspath $(DEFAULTER_GEN))
GINKGO := $(abspath $(GINKGO_BIN))
ENVSUBST := $(abspath $(ENVSUBST_BIN))
GOBINDATA := $(abspath $(GOBINDATA))
RELEASE_NOTES := $(abspath $(RELEASE_NOTES_BIN))
GO_APIDIFF := $(abspath $(GO_APIDIFF_BIN))
LINK_CHECKER := $(abspath $(LINK_CHECKER_BIN))

# Bindata.
GOBINDATA := $(abspath $(TOOLS_BIN_DIR)/go-bindata)
GOBINDATA_CLUSTERCTL_DIR := cmd/clusterctl/config
CLOUDINIT_PKG_DIR := bootstrap/kubeadm/internal/cloudinit
CLOUDINIT_GENERATED := $(CLOUDINIT_PKG_DIR)/zz_generated.bindata.go
Expand Down Expand Up @@ -112,7 +125,7 @@ help: ## Display this help
## --------------------------------------

.PHONY: test
test: ## Run tests.
test: ## Run tests.
source ./scripts/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v ./... $(TEST_ARGS)

.PHONY: test-cover
Expand All @@ -127,9 +140,13 @@ docker-build-e2e: ## Rebuild all Cluster API provider images to be used in the e
$(MAKE) -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-cluster-api

.PHONY: test-e2e
test-e2e: ## Run the e2e tests
test-e2e: $(GINKGO_BIN) ## Run the e2e tests
$(MAKE) -C test/e2e run

.PHONY: test-conformance
test-conformance: $(GINKGO_BIN) $(KUSTOMIZE_BIN) ## Run the e2e tests
$(MAKE) -C test/e2e conformance

## --------------------------------------
## Binaries
## --------------------------------------
Expand All @@ -156,32 +173,51 @@ managers: ## Build all managers
clusterctl: ## Build clusterctl binary
go build -ldflags "$(LDFLAGS)" -o bin/clusterctl sigs.k8s.io/cluster-api/cmd/clusterctl

$(KUSTOMIZE): $(TOOLS_DIR)/go.mod # Build kustomize from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/kustomize sigs.k8s.io/kustomize/kustomize/v3
## --------------------------------------
## Tooling Binaries
## --------------------------------------

$(TOOLS_BIN_DIR):
mkdir -p $@

$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen
$(TOOLS_SHARE_DIR):
mkdir -p $@

$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint
$(GOBINDATA_BIN): $(TOOLS_DIR)/go.mod # Build go-bindata from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/go-bindata/go-bindata/go-bindata

$(CONVERSION_GEN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/conversion-gen k8s.io/code-generator/cmd/conversion-gen
$(LINK_CHECKER_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/raviqqe/liche

$(GOBINDATA): $(TOOLS_DIR)/go.mod # Build go-bindata from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(BIN_DIR)/go-bindata github.com/go-bindata/go-bindata/go-bindata
$(GO_APIDIFF_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/joelanford/go-apidiff

$(RELEASE_NOTES): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(RELEASE_NOTES_BIN) ./release
$(CONTROLLER_GEN_BIN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) sigs.k8s.io/controller-tools/cmd/controller-gen

$(LINK_CHECKER): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(LINK_CHECKER_BIN) github.com/raviqqe/liche
$(ENVSUBST_BIN): $(TOOLS_DIR)/go.mod # Build envsubst from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/a8m/envsubst/cmd/envsubst

$(GO_APIDIFF): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(GO_APIDIFF_BIN) github.com/joelanford/go-apidiff
$(GOLANGCI_LINT_BIN): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/golangci/golangci-lint/cmd/golangci-lint

$(ENVSUBST): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(ENVSUBST_BIN) github.com/drone/envsubst/cmd/envsubst
$(MOCKGEN_BIN): $(TOOLS_DIR)/go.mod # Build mockgen from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) github.com/golang/mock/mockgen

$(CONVERSION_GEN_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) k8s.io/code-generator/cmd/conversion-gen

$(DEFAULTER_GEN_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) k8s.io/code-generator/cmd/defaulter-gen

$(KUSTOMIZE_BIN): $(TOOLS_DIR)/go.mod # Build kustomize from tools folder.
cd $(TOOLS_DIR); go build -tags=tools -o $(subst hack/tools/,,$@) sigs.k8s.io/kustomize/kustomize/v3

$(RELEASE_NOTES_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags tools -o $(subst hack/tools/,,$@) sigs.k8s.io/cluster-api/hack/tools/release

$(GINKGO_BIN): $(TOOLS_DIR)/go.mod
cd $(TOOLS_DIR) && go build -tags=tools -o $(subst hack/tools/,,$@) github.com/onsi/ginkgo/ginkgo

envsubst: $(ENVSUBST)

Expand Down Expand Up @@ -523,6 +559,11 @@ docker-build-example-provider: ## Build the docker image for example provider
clean: ## Remove all generated files
$(MAKE) clean-bin
$(MAKE) clean-book
$(MAKE) clean-artifacts

.PHONY: clean-artifacts
clean-bin: ## Remove all test artifacts
rm -rf _artifacts

.PHONY: clean-bin
clean-bin: ## Remove all generated binaries
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ require (
k8s.io/cluster-bootstrap v0.17.8
k8s.io/component-base v0.17.9
k8s.io/klog v1.0.0
k8s.io/klog/v2 v2.0.0
k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19
sigs.k8s.io/controller-runtime v0.5.10
sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802
Expand Down
1 change: 1 addition & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -661,6 +661,7 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
sigs.k8s.io/cluster-api/test/framework v0.0.0-20200304170348-97097699f713 h1:5xCiWE7khnVT5JhwKd2XnJP71wxlnX1aRaWQsNJd1xw=
sigs.k8s.io/controller-runtime v0.5.10 h1:IJ2zO+BeNvTJEo2W1Kho1+X756QroQjYCKIzEYYqsI8=
sigs.k8s.io/controller-runtime v0.5.10/go.mod h1:OTqxLuz7gVcrq+BHGUgedRu6b2VIKCEc7Pu4Jbwui0A=
sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 h1:L6/8hETA7jvdx3xBcbDifrIN2xaYHE7tA58n+Kdp2Zw=
Expand Down
8 changes: 8 additions & 0 deletions test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ TEST_E2E_DIR := $(REPO_ROOT)/test/e2e
GINKGO_FOCUS ?=
GINKGO_NODES ?= 1
E2E_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev.yaml
CONFORMANCE_CONF_FILE ?= ${REPO_ROOT}/test/e2e/config/docker-dev-conformance.yaml
ARTIFACTS ?= ${REPO_ROOT}/_artifacts
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
Expand All @@ -63,3 +64,10 @@ run: ginkgo ## Run the end-to-end tests
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)

.PHONY: conformance
conformance:
cd $(TEST_E2E_DIR); $(GINKGO) -v -trace -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) ./conformance -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(CONFORMANCE_CONF_FILE)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
52 changes: 3 additions & 49 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,11 @@ limitations under the License.
package e2e

import (
"context"
"fmt"
"path/filepath"

. "github.com/onsi/ginkgo"

"github.com/blang/semver"
"github.com/onsi/gomega/types"
corev1 "k8s.io/api/core/v1"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/util"
"sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
)

// Test suite constants for e2e config variables
Expand All @@ -42,48 +35,9 @@ const (
CoreDNSVersionUpgradeTo = "COREDNS_VERSION_UPGRADE_TO"
)

// Byf is deprecated. Use "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" as dot import instead.
func Byf(format string, a ...interface{}) {
By(fmt.Sprintf(format, a...))
}

func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) {
Byf("Creating a namespace for hosting the %q test spec", specName)
namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
Creator: clusterProxy.GetClient(),
ClientSet: clusterProxy.GetClientSet(),
Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()),
})

return namespace, cancelWatches
}

func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) {
Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name)
// Dump all Cluster API related resources to artifacts before deleting them.
framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{
Lister: clusterProxy.GetClient(),
Namespace: namespace.Name,
LogPath: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"),
})

if !skipCleanup {
Byf("Deleting cluster %s/%s", cluster.Namespace, cluster.Name)
// While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance
// that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait
// instead of DeleteClusterAndWait
framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{
Client: clusterProxy.GetClient(),
Namespace: namespace.Name,
}, intervalsGetter(specName, "wait-delete-cluster")...)

Byf("Deleting namespace used for hosting the %q test spec", specName)
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
Deleter: clusterProxy.GetClient(),
Name: namespace.Name,
})
}
cancelWatches()
ginkgoextensions.Byf(format, a...)
}

// HaveValidVersion succeeds if version is a valid semver version
Expand Down
1 change: 1 addition & 0 deletions test/e2e/config/docker-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ variables:
COREDNS_VERSION_UPGRADE_TO: "1.6.7"
KUBERNETES_VERSION_UPGRADE_TO: "v1.18.2"
KUBERNETES_VERSION_UPGRADE_FROM: "v1.17.2"
KUBERNETES_CI_ARTIFACTS_START_VERSION: "v1.19.0"
DOCKER_SERVICE_DOMAIN: "cluster.local"
DOCKER_SERVICE_CIDRS: "10.128.0.0/12"
# IMPORTANT! This values should match the one used by the CNI provider
Expand Down
124 changes: 124 additions & 0 deletions test/e2e/config/docker-dev-conformance.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
---
# E2E test scenario using local dev images and manifests built from the source tree for following providers:
# - cluster-api
# - bootstrap kubeadm
# - control-plane kubeadm
# - docker

# For creating local dev images built from the source tree;
# - `make docker-build REGISTRY=gcr.io/k8s-staging-cluster-api` to build the cluster-api, bootstrap kubeadm, control-plane kubeadm provider images.
# - `make -C test/infrastructure/docker docker-build REGISTRY=gcr.io/k8s-staging-cluster-api` to build the docker provider images.

images:
# Use local dev images built source tree;
- name: gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/kubeadm-bootstrap-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/kubeadm-control-plane-controller-amd64:dev
loadBehavior: mustLoad
- name: gcr.io/k8s-staging-cluster-api/capd-manager-amd64:dev
loadBehavior: mustLoad
- name: quay.io/jetstack/cert-manager-cainjector:v0.16.1
loadBehavior: tryLoad
- name: quay.io/jetstack/cert-manager-webhook:v0.16.1
loadBehavior: tryLoad
- name: quay.io/jetstack/cert-manager-controller:v0.16.1
loadBehavior: tryLoad
# If using Calico uncomment following lines to speed up test by pre-loading required images on nodes
# - name: calico/kube-controllers:v3.13.1
# loadBehavior: tryLoad
# - name: calico/cni:v3.13.1
# loadBehavior: tryLoad
# - name: calico/pod2daemon-flexvol:v3.13.1
# loadBehavior: tryLoad
# - name: calico/node:v3.13.1
# loadBehavior: tryLoad

providers:

- name: cluster-api
type: CoreProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../config
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: BootstrapProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../bootstrap/kubeadm/config
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: kubeadm
type: ControlPlaneProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../controlplane/kubeadm/config
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080

- name: docker
type: InfrastructureProvider
versions:
- name: v0.3.0
# Use manifest from source files
value: ../../../test/infrastructure/docker/config
replacements:
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "--enable-leader-election"
new: "--enable-leader-election=false"
- old: --metrics-addr=127.0.0.1:8080
new: --metrics-addr=:8080
files:
# Add a metadata for docker provider
- sourcePath: "../data/infrastructure-docker/metadata.yaml"
# Add cluster templates
- sourcePath: "../data/infrastructure-docker/cluster-template.yaml"
- sourcePath: "../data/infrastructure-docker/cluster-template-kcp-adoption.yaml"

variables:
KUBERNETES_VERSION: "v1.19.0"
ETCD_VERSION_UPGRADE_TO: "3.4.3-0"
COREDNS_VERSION_UPGRADE_TO: "1.6.7"
KUBERNETES_VERSION_UPGRADE_TO: "v1.18.2"
KUBERNETES_VERSION_UPGRADE_FROM: "v1.17.2"
KUBERNETES_CI_ARTIFACTS_START_VERSION: "v1.19.0"
DOCKER_SERVICE_DOMAIN: "cluster.local"
DOCKER_SERVICE_CIDRS: "10.128.0.0/12"
# IMPORTANT! This values should match the one used by the CNI provider
DOCKER_POD_CIDRS: "192.168.0.0/16"
#CNI: "./data/cni/calico/calico.yaml"
CNI: "../data/cni/kindnet/kindnet.yaml"
EXP_CLUSTER_RESOURCE_SET: "true"

intervals:
default/wait-controllers: ["3m", "10s"]
default/wait-cluster: ["3m", "10s"]
default/wait-control-plane: ["10m", "10s"]
default/wait-worker-nodes: ["5m", "10s"]
default/wait-delete-cluster: ["3m", "10s"]
default/wait-machine-upgrade: ["20m", "10s"]
default/wait-machine-remediation: ["5m", "10s"]
Loading

0 comments on commit 73e9d34

Please sign in to comment.