WIP: kubetest

Signed-off-by: Naadir Jeewa <jeewan@vmware.com>

.gitignore

@@ -63,3 +63,5 @@ clusterctl-settings.json
 
 # test results
 _artifacts
+
+*.sentinel

common.mk

@@ -0,0 +1,65 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include $(ROOT_DIR_RELATIVE)/versions.mk
+
+# Ensure Make is run with bash shell as some syntax below is bash-specific
+SHELL:=bash
+.ONESHELL:
+.SHELLFLAGS := -eu -o pipefail -c
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+
+TOOLS_DIR := $(ROOT_DIR_RELATIVE)/hack/tools
+TOOLS_DIR_DEPS := $(TOOLS_DIR)/go.sum $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/Makefile
+TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
+
+PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
+export PATH
+
+UID := $(shell id -u)
+GID := $(shell id -g)
+
+rwildcard=$(foreach d,$(wildcard $(1:=/*)),$(call rwildcard,$d,$2) $(filter $(subst *,%,$2),$d))
+
+# Hosts running SELinux need :z added to volume mounts
+SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0)
+
+ifeq ($(SELINUX_ENABLED),1)
+  DOCKER_VOL_OPTS?=:z
+endif
+
+# This option is for running docker manifest command
+export DOCKER_CLI_EXPERIMENTAL := enabled
+
+# Use GOPROXY environment variable if set
+GOPROXY := $(shell go env GOPROXY)
+ifeq ($(GOPROXY),)
+GOPROXY := https://proxy.golang.org
+endif
+export GOPROXY
+
+
+$(TOOLS_BIN_DIR)/%: $(TOOLS_DIR_DEPS)
+	make -C $(TOOLS_DIR) $(@:hack/tools/%=%)
+
+## --------------------------------------
+## Help
+## --------------------------------------
+
+.DEFAULT_GOAL:=help
+
+help:  ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

hack/tools/Makefile

@@ -0,0 +1,115 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ROOT_DIR_RELATIVE := ../..
+include $(ROOT_DIR_RELATIVE)/common.mk
+
+UNAME := $(shell uname -s)
+
+# Directories.
+BIN_DIR := bin
+SHARE_DIR := share
+
+OS := $(shell go env GOOS)
+RUST_TARGET := unknown-$(OS)-gnu
+MDBOOK_EXTRACT_COMMAND := tar xfvz $(SHARE_DIR)/mdbook.tar.gz -C bin
+MDBOOK_ARCHIVE_EXT := .tar.gz
+
+ifeq ($(OS), windows)
+	RUST_TARGET := pc-windows-msvc
+	MDBOOK_ARCHIVE_EXT := .zip
+	MDBOOK_EXTRACT_COMMAND := unzip -d /tmp
+endif
+
+ifeq ($(OS), darwin)
+	RUST_TARGET := apple-darwin
+endif
+
+## --------------------------------------
+## Tooling Binaries
+## --------------------------------------
+
+.PHONY: modules
+modules: ## Runs go mod to ensure modules are up to date.
+	go mod tidy
+
+$(BIN_DIR):
+	mkdir -p $@
+
+$(SHARE_DIR):
+	mkdir -p $@
+
+CONTROLLER_GEN := $(BIN_DIR)/controller-gen
+$(CONTROLLER_GEN): $(BIN_DIR) go.mod go.sum # Build controller-gen from tools folder.
+	go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen
+
+CONVERSION_GEN := $(BIN_DIR)/conversion-gen
+$(CONVERSION_GEN): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ k8s.io/code-generator/cmd/conversion-gen
+
+DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen
+$(DEFAULTER_GEN): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ k8s.io/code-generator/cmd/defaulter-gen
+
+ENVSUBST := $(BIN_DIR)/envsubst
+$(ENVSUBST): $(BIN_DIR) go.mod go.sum # Build envsubst from tools folder.
+	go build -tags=tools -o $@ github.com/a8m/envsubst/cmd/envsubst
+
+GINKGO := $(BIN_DIR)/ginkgo
+$(GINKGO): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ github.com/onsi/ginkgo/ginkgo
+
+GOLANGCI_LINT := $(BIN_DIR)/golangci-lint
+$(GOLANGCI_LINT): $(BIN_DIR) go.mod go.sum # Build golangci-lint from tools folder.
+	go build -tags=tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint
+
+KIND := $(BIN_DIR)/kind
+$(KIND): $(BIN_DIR) go.mod go.sum
+	go build -tags tools -o $@ sigs.k8s.io/kind
+
+KUSTOMIZE := $(BIN_DIR)/kustomize
+$(KUSTOMIZE): $(BIN_DIR) go.mod go.sum # Build kustomize from tools folder.
+	go build -tags=tools -o $@ sigs.k8s.io/kustomize/kustomize/v3
+
+MDBOOK_EMBED := $(BIN_DIR)/mdbook-embed
+$(MDBOOK_EMBED): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $(BIN_DIR)/mdbook-embed ./mdbook/embed
+
+MDBOOK_RELEASELINK := $(BIN_DIR)/mdbook-releaselink
+$(MDBOOK_RELEASELINK): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $(BIN_DIR)/mdbook-releaselink ./mdbook/releaselink
+
+MDBOOK_TABULATE := $(BIN_DIR)/mdbook-tabulate
+$(MDBOOK_TABULATE): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $(BIN_DIR)/mdbook-tabulate ./mdbook/tabulate
+
+MOCKGEN := $(BIN_DIR)/mockgen
+$(MOCKGEN): $(BIN_DIR) go.mod go.sum # Build mockgen from tools folder.
+	go build -tags=tools -o $@ github.com/golang/mock/mockgen
+
+RELEASE_NOTES := $(BIN_DIR)/release-notes
+$(RELEASE_NOTES): $(BIN_DIR) go.mod go.sum
+	go build -tags tools -o $@ ./release
+
+LINK_CHECKER := $(BIN_DIR)/link-checker
+$(LINK_CHECKER): $(BIN_DIR) go.mod go.sum
+	go build -tags tools -o $@ github.com/raviqqe/liche
+
+GOBINDATA := $(BIN_DIR)/go-bindata
+$(GOBINDATA): $(BIN_DIR) go.mod go.sum
+	go build -tags tools -o $@ github.com/go-bindata/go-bindata/go-bindata
+
+GOAPIDIFF := $(BIN_DIR)/go-apidiff
+$(GOAPIDIFF): $(BIN_DIR) go.mod go.sum
+	go build -tags tools -o $@ github.com/joelanford/go-apidiff

hack/tools/tools.mk

@@ -0,0 +1,12 @@
+# Binaries.
+KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
+CONTROLLER_GEN:= $(TOOLS_BIN_DIR)/controller-gen
+GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
+CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen
+DEFAULTER_GEN := $(TOOLS_BIN_DIR)/defaulter-gen
+GINKGO := $(TOOLS_BIN_DIR)/ginkgo
+ENVSUBST := $(TOOLS_BIN_DIR)/envsubst
+GOBINDATA := $(TOOLS_BIN_DIR)/go-bindata
+RELEASE_NOTES := $(TOOLS_BIN_DIR)/release-notes
+GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff
+LINK_CHECKER := $(TOOLS_BIN_DIR)/link-checker

scripts/ci-e2e.sh

@@ -61,4 +61,4 @@ export USE_EXISTING_CLUSTER=false
 
 # Run e2e tests
 mkdir -p "$ARTIFACTS"
-make -C test/e2e/ run
+make test-e2e

test/e2e/common.go

@@ -17,18 +17,11 @@ limitations under the License.
 package e2e
 
 import (
-	"context"
 	"fmt"
-	"path/filepath"
-
-	. "github.com/onsi/ginkgo"
 
 	"github.com/blang/semver"
 	"github.com/onsi/gomega/types"
-	corev1 "k8s.io/api/core/v1"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
-	"sigs.k8s.io/cluster-api/test/framework"
-	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
 )
 
 // Test suite constants for e2e config variables
@@ -42,54 +35,9 @@ const (
 	CoreDNSVersionUpgradeTo      = "COREDNS_VERSION_UPGRADE_TO"
 )
 
+// Byf is deprecated. Use "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" as dot import instead.
 func Byf(format string, a ...interface{}) {
-	By(fmt.Sprintf(format, a...))
-}
-
-func setupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) {
-	Byf("Creating a namespace for hosting the %q test spec", specName)
-	namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
-		Creator:   clusterProxy.GetClient(),
-		ClientSet: clusterProxy.GetClientSet(),
-		Name:      fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
-		LogFolder: filepath.Join(artifactFolder, "clusters", clusterProxy.GetName()),
-	})
-
-	return namespace, cancelWatches
-}
-
-func dumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string, namespace *corev1.Namespace, cancelWatches context.CancelFunc, cluster *clusterv1.Cluster, intervalsGetter func(spec, key string) []interface{}, skipCleanup bool) {
-	Byf("Dumping logs from the %q workload cluster", cluster.Name)
-
-	// Dump all the logs from the workload cluster before deleting them.
-	clusterProxy.CollectWorkloadClusterLogs(ctx, cluster.Namespace, cluster.Name, filepath.Join(artifactFolder, "clusters", cluster.Name, "machines"))
-
-	Byf("Dumping all the Cluster API resources in the %q namespace", namespace.Name)
-
-	// Dump all Cluster API related resources to artifacts before deleting them.
-	framework.DumpAllResources(ctx, framework.DumpAllResourcesInput{
-		Lister:    clusterProxy.GetClient(),
-		Namespace: namespace.Name,
-		LogPath:   filepath.Join(artifactFolder, "clusters", clusterProxy.GetName(), "resources"),
-	})
-
-	if !skipCleanup {
-		Byf("Deleting cluster %s/%s", cluster.Namespace, cluster.Name)
-		// While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance
-		// that cluster variable is not set even if the cluster exists, so we are calling DeleteAllClustersAndWait
-		// instead of DeleteClusterAndWait
-		framework.DeleteAllClustersAndWait(ctx, framework.DeleteAllClustersAndWaitInput{
-			Client:    clusterProxy.GetClient(),
-			Namespace: namespace.Name,
-		}, intervalsGetter(specName, "wait-delete-cluster")...)
-
-		Byf("Deleting namespace used for hosting the %q test spec", specName)
-		framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
-			Deleter: clusterProxy.GetClient(),
-			Name:    namespace.Name,
-		})
-	}
-	cancelWatches()
+	ginkgoextensions.Byf(format, a...)
 }
 
 // HaveValidVersion succeeds if version is a valid semver version

test/e2e/config/docker-ci.yaml

@@ -35,6 +35,8 @@ providers:
     replacements:
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
+    - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master"
+      new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci"
 
 - name: kubeadm
   type: BootstrapProvider
@@ -45,6 +47,8 @@ providers:
     replacements:
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
+    - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master"
+      new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci"
 
 - name: kubeadm
   type: ControlPlaneProvider
@@ -55,6 +59,8 @@ providers:
     replacements:
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
+    - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master"
+      new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci"
 
 - name: docker
   type: InfrastructureProvider
@@ -65,6 +71,8 @@ providers:
     replacements:
     - old: --metrics-addr=127.0.0.1:8080
       new: --metrics-addr=:8080
+    - old: "gcr.io/k8s-staging-cluster-api/cluster-api-controller:master"
+      new: "gcr.io/k8s-staging-cluster-api/cluster-api-controller-amd64:ci"
   files:
   # Add a metadata for docker provider
   - sourcePath: "../data/infrastructure-docker/metadata.yaml"