Skip to content

Commit

Permalink
WIP: kubetest
Browse files Browse the repository at this point in the history
Signed-off-by: Naadir Jeewa <jeewan@vmware.com>
  • Loading branch information
randomvariable committed Sep 16, 2020
1 parent cd8ac05 commit a7aba7b
Show file tree
Hide file tree
Showing 71 changed files with 3,019 additions and 465 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Expand Up @@ -63,3 +63,5 @@ clusterctl-settings.json

# test results
_artifacts

*.sentinel
236 changes: 120 additions & 116 deletions Makefile

Large diffs are not rendered by default.

65 changes: 65 additions & 0 deletions common.mk
@@ -0,0 +1,65 @@
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

include $(ROOT_DIR_RELATIVE)/versions.mk

# Ensure Make is run with bash shell as some syntax below is bash-specific
SHELL:=bash
.ONESHELL:
.SHELLFLAGS := -eu -o pipefail -c
.DELETE_ON_ERROR:
MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules

TOOLS_DIR := $(ROOT_DIR_RELATIVE)/hack/tools
TOOLS_DIR_DEPS := $(TOOLS_DIR)/go.sum $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/Makefile
TOOLS_BIN_DIR := $(TOOLS_DIR)/bin

PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
export PATH

UID := $(shell id -u)
GID := $(shell id -g)

rwildcard=$(foreach d,$(wildcard $(1:=/*)),$(call rwildcard,$d,$2) $(filter $(subst *,%,$2),$d))

# Hosts running SELinux need :z added to volume mounts
SELINUX_ENABLED := $(shell cat /sys/fs/selinux/enforce 2> /dev/null || echo 0)

ifeq ($(SELINUX_ENABLED),1)
DOCKER_VOL_OPTS?=:z
endif

# This option is for running docker manifest command
export DOCKER_CLI_EXPERIMENTAL := enabled

# Use GOPROXY environment variable if set
GOPROXY := $(shell go env GOPROXY)
ifeq ($(GOPROXY),)
GOPROXY := https://proxy.golang.org
endif
export GOPROXY


$(TOOLS_BIN_DIR)/%: $(TOOLS_DIR_DEPS)
make -C $(TOOLS_DIR) $(@:hack/tools/%=%)

## --------------------------------------
## Help
## --------------------------------------

.DEFAULT_GOAL:=help

help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
58 changes: 35 additions & 23 deletions docs/book/src/developer/e2e.md
Expand Up @@ -9,33 +9,33 @@ Following guidelines should be followed when developing E2E tests:
- Define test spec reflecting real user workflow, e.g. [Cluster API quick start].
- Unless you are testing provider specific features, ensure your test can run with
different infrastructure providers (see [Writing Portable Tests](#writing-portable-e2e-tests)).

The [Cluster API test framework] provides you a set of helpers method for getting your test in place
quickly; the [test E2E package] provide examples of how this can be achieved and reusable
quickly; the [test E2E package] provide examples of how this can be achieved and reusable
test specs for the most common Cluster API use cases.

## Prerequisites

Each E2E test requires a set of artifacts to be available:

- Binaries & docker images for Kubernetes, CNI, CRI & CSI
- Binaries & docker images for Kubernetes, CNI, CRI & CSI
- Manifests & docker images for the Cluster API core components
- Manifests & docker images for the Cluster API infrastructure provider; in most cases
also machine images are required (AMI, OVA etc.)
- Credentials for the target infrastructure provider
- Credentials for the target infrastructure provider
- Other support tools (e.g. kustomize, gsutil etc.)

The Cluster API test framework provides support for building and retrieving the manifest
files for Cluster API core components and for the Cluster API infrastructure provider
(see [Setup](#setup))

For the remaining tasks you can find examples of
how this can be implemented e.g. in [CAPA E2E tests] and [CAPG E2E tests].
how this can be implemented e.g. in [CAPA E2E tests] and [CAPG E2E tests].

## Setup

In order to run E2E tests it is required to create a Kubernetes cluster with a
complete set of Cluster API providers installed. Setting up those elements is
In order to run E2E tests it is required to create a Kubernetes cluster with a
complete set of Cluster API providers installed. Setting up those elements is
usually implemented in a `BeforeSuite` function, and it consists of two steps:

- Defining an E2E config file
Expand All @@ -48,7 +48,7 @@ setting up a management cluster.

Using the config file it is possible to:

- Define the list of providers to be installed in the management cluster. Most notably,
- Define the list of providers to be installed in the management cluster. Most notably,
for each provider it is possible to define:
- One or more versions of the providers manifest (built from the sources, or pulled from a
remote location).
Expand All @@ -68,14 +68,14 @@ An [example E2E config file] can be found here.
<h1>Deprecated E2E config file format</h1>

The [Cluster API test framework] includes also a [deprecated E2E config file] implementation,
that was used before the introduction of clusterctl. This might be removed in future releases
that was used before the introduction of clusterctl. This might be removed in future releases
of the test framework.

</aside>

### Creating the management cluster and installing providers

In order to run Cluster API E2E tests, you need a Kubernetes cluster; the [NewKindClusterProvider] gives you a
In order to run Cluster API E2E tests, you need a Kubernetes cluster; the [NewKindClusterProvider] gives you a
type that can be used to create a local kind cluster and pre-load images into it, but also existing clusters can
be used if available.

Expand All @@ -92,7 +92,7 @@ This method:
<h1>Deprecated InitManagementCluster method</h1>

The [Cluster API test framework] includes also a [deprecated InitManagementCluster method] implementation,
that was used before the introduction of clusterctl. This might be removed in future releases
that was used before the introduction of clusterctl. This might be removed in future releases
of the test framework.

</aside>
Expand All @@ -109,22 +109,22 @@ A typical test spec is a sequence of:
### Creating Namespaces

The [CreateNamespaceAndWatchEvents method] provides a convenient way to create a namespace and setup
watches for capturing namespaces events
watches for capturing namespaces events

### Creating objects

There are two possible approaches for creating objects in the management cluster:

- Create object by object: create the `Cluster` object, then `AwsCluster`, `Machines`, `AwsMachines` etc.
- Apply a `cluster-templates.yaml` file thus creating all the objects this file contains.
- Apply a `cluster-templates.yaml` file thus creating all the objects this file contains.

The first approaches leverage on the [controller-runtime Client] and gives you full control, but it comes with
some drawbacks as well, because this method does not reflect directly real user workflows, and most importantly,
the resulting tests are not as reusable with other infrastructure providers. (See [writing portable tests](#writing-portable-e2e-tests)).

We recommend using the [ClusterTemplate method] and the [Apply method] for creating objects in the cluster.
This methods mimics the recommended user workflows, and it is based on `cluster-templates.yaml` files that can be
provided via the [E2E config file], and thus easily swappable when changing the target infrastructure provider.
provided via the [E2E config file], and thus easily swappable when changing the target infrastructure provider.

<aside class="note">

Expand All @@ -143,12 +143,12 @@ infrastructure to be provisioned, e.g. [WaitForClusterToProvision], [WaitForKube

### Exec operations

You can use [Cluster API test framework] methods to modify Cluster API objects, as a last option, use
You can use [Cluster API test framework] methods to modify Cluster API objects, as a last option, use
the [controller-runtime Client].

The [Cluster API test framework] includes also methods for executing clusterctl operations, like e.g.
The [Cluster API test framework] includes also methods for executing clusterctl operations, like e.g.
the [ClusterTemplate method], the [ClusterctlMove method] etc.; in order to improve observability,
each clusterctl operation creates a detailed log.
each clusterctl operation creates a detailed log.

After using clusterctl operations, you can rely on the `Get` and on the `Wait` methods
defined in the [Cluster API test framework] to check if the operation completed successfully.
Expand All @@ -165,7 +165,7 @@ Those task are usually implemented in the `AfterSuite`, and again the [Cluster A
you useful methods for those tasks.

Please note that despite the fact that test specs are expected to delete objects in the management cluster and
wait for the corresponding infrastructure to be terminated, it can happen that the test spec
wait for the corresponding infrastructure to be terminated, it can happen that the test spec
fails before starting object deletion or that objects deletion itself fails.

As a consequence, when scheduling/running a test suite, it is required to ensure all the generated
Expand All @@ -178,8 +178,8 @@ changing the test configuration file.

Following recommendations should be followed to write portable E2E tests:

- Create different [E2E config file], one for each target infrastructure provider,
providing different sets of env variables and timeout intervals.
- Create different [E2E config file], one for each target infrastructure provider,
providing different sets of env variables and timeout intervals.
- Use the [InitManagementCluster method] for setting up the management cluster.
- Use the [ClusterTemplate method] and the [Apply method]
for creating objects in the cluster using `cluster-templates.yaml` files instead
Expand All @@ -201,8 +201,18 @@ baseline for Cluster API conformance.
However, creating such suite is something that can provide a huge value for the
long term success of the project.

The [test E2E package] provide examples of how this can be achieved implemeting a set of and reusable
test specs for the most common Cluster API use cases.
The [test E2E package] provide examples of how this can be achieved implemeting a set of and reusable
test specs for the most common Cluster API use cases.

## Kubernetes conformance tests

The [kubetest package] provides an API to run the Kubernetes conformance suite for a given cluster.
You may also want to test the latest CI release of Kubernetes from the main branch of the k/k repository.
Helper functions are available in the [kubernetesversions package] to fetch these releases as well as
inject shellscripts into kubeadm templates to download the CI artifacts.

Examples of use can be found in the [test E2E package].


<!-- links -->
[Cluster API quick start]: https://cluster-api.sigs.k8s.io/user/quick-start.html
Expand All @@ -225,4 +235,6 @@ test specs for the most common Cluster API use cases.
[InfrastructureProvider method]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework/clusterctl?tab=doc#E2EConfig.InfrastructureProviders
[GetIntervals method]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework/clusterctl?tab=doc#E2EConfig.GetIntervals
[test E2E package]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/e2e?tab=doc
[CreateNamespaceAndWatchEvents method]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework?tab=doc#CreateNamespaceAndWatchEvents
[kubetest package]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework/kubetest?tab=do
[kubernetesversions package]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework/kubernetesversions?tab=doc
[CreateNamespaceAndWatchEvents method]: https://pkg.go.dev/sigs.k8s.io/cluster-api/test/framework?tab=doc#CreateNamespaceAndWatchEvents
115 changes: 115 additions & 0 deletions hack/tools/Makefile
@@ -0,0 +1,115 @@
# Copyright 2020 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

ROOT_DIR_RELATIVE := ../..
include $(ROOT_DIR_RELATIVE)/common.mk

UNAME := $(shell uname -s)

# Directories.
BIN_DIR := bin
SHARE_DIR := share

OS := $(shell go env GOOS)
RUST_TARGET := unknown-$(OS)-gnu
MDBOOK_EXTRACT_COMMAND := tar xfvz $(SHARE_DIR)/mdbook.tar.gz -C bin
MDBOOK_ARCHIVE_EXT := .tar.gz

ifeq ($(OS), windows)
RUST_TARGET := pc-windows-msvc
MDBOOK_ARCHIVE_EXT := .zip
MDBOOK_EXTRACT_COMMAND := unzip -d /tmp
endif

ifeq ($(OS), darwin)
RUST_TARGET := apple-darwin
endif

## --------------------------------------
## Tooling Binaries
## --------------------------------------

.PHONY: modules
modules: ## Runs go mod to ensure modules are up to date.
go mod tidy

$(BIN_DIR):
mkdir -p $@

$(SHARE_DIR):
mkdir -p $@

CONTROLLER_GEN := $(BIN_DIR)/controller-gen
$(CONTROLLER_GEN): $(BIN_DIR) go.mod go.sum # Build controller-gen from tools folder.
go build -tags=tools -o $@ sigs.k8s.io/controller-tools/cmd/controller-gen

CONVERSION_GEN := $(BIN_DIR)/conversion-gen
$(CONVERSION_GEN): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $@ k8s.io/code-generator/cmd/conversion-gen

DEFAULTER_GEN := $(BIN_DIR)/defaulter-gen
$(DEFAULTER_GEN): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $@ k8s.io/code-generator/cmd/defaulter-gen

ENVSUBST := $(BIN_DIR)/envsubst
$(ENVSUBST): $(BIN_DIR) go.mod go.sum # Build envsubst from tools folder.
go build -tags=tools -o $@ github.com/a8m/envsubst/cmd/envsubst

GINKGO := $(BIN_DIR)/ginkgo
$(GINKGO): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $@ github.com/onsi/ginkgo/ginkgo

GOLANGCI_LINT := $(BIN_DIR)/golangci-lint
$(GOLANGCI_LINT): $(BIN_DIR) go.mod go.sum # Build golangci-lint from tools folder.
go build -tags=tools -o $@ github.com/golangci/golangci-lint/cmd/golangci-lint

KIND := $(BIN_DIR)/kind
$(KIND): $(BIN_DIR) go.mod go.sum
go build -tags tools -o $@ sigs.k8s.io/kind

KUSTOMIZE := $(BIN_DIR)/kustomize
$(KUSTOMIZE): $(BIN_DIR) go.mod go.sum # Build kustomize from tools folder.
go build -tags=tools -o $@ sigs.k8s.io/kustomize/kustomize/v3

MDBOOK_EMBED := $(BIN_DIR)/mdbook-embed
$(MDBOOK_EMBED): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $(BIN_DIR)/mdbook-embed ./mdbook/embed

MDBOOK_RELEASELINK := $(BIN_DIR)/mdbook-releaselink
$(MDBOOK_RELEASELINK): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $(BIN_DIR)/mdbook-releaselink ./mdbook/releaselink

MDBOOK_TABULATE := $(BIN_DIR)/mdbook-tabulate
$(MDBOOK_TABULATE): $(BIN_DIR) go.mod go.sum
go build -tags=tools -o $(BIN_DIR)/mdbook-tabulate ./mdbook/tabulate

MOCKGEN := $(BIN_DIR)/mockgen
$(MOCKGEN): $(BIN_DIR) go.mod go.sum # Build mockgen from tools folder.
go build -tags=tools -o $@ github.com/golang/mock/mockgen

RELEASE_NOTES := $(BIN_DIR)/release-notes
$(RELEASE_NOTES): $(BIN_DIR) go.mod go.sum
go build -tags tools -o $@ ./release

LINK_CHECKER := $(BIN_DIR)/link-checker
$(LINK_CHECKER): $(BIN_DIR) go.mod go.sum
go build -tags tools -o $@ github.com/raviqqe/liche

GOBINDATA := $(BIN_DIR)/go-bindata
$(GOBINDATA): $(BIN_DIR) go.mod go.sum
go build -tags tools -o $@ github.com/go-bindata/go-bindata/go-bindata

GOAPIDIFF := $(BIN_DIR)/go-apidiff
$(GOAPIDIFF): $(BIN_DIR) go.mod go.sum
go build -tags tools -o $@ github.com/joelanford/go-apidiff
12 changes: 12 additions & 0 deletions hack/tools/tools.mk
@@ -0,0 +1,12 @@
# Binaries.
KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
CONTROLLER_GEN:= $(TOOLS_BIN_DIR)/controller-gen
GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen
DEFAULTER_GEN := $(TOOLS_BIN_DIR)/defaulter-gen
GINKGO := $(TOOLS_BIN_DIR)/ginkgo
ENVSUBST := $(TOOLS_BIN_DIR)/envsubst
GOBINDATA := $(TOOLS_BIN_DIR)/go-bindata
RELEASE_NOTES := $(TOOLS_BIN_DIR)/release-notes
GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff
LINK_CHECKER := $(TOOLS_BIN_DIR)/link-checker

0 comments on commit a7aba7b

Please sign in to comment.