From ba92180e9b9576ff1d186bc1042e6139be1a5722 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Mon, 7 Jan 2019 12:57:47 +0800 Subject: [PATCH] - support gke provider - docs on how to run e2e tests --- README.md | 5 +- hack/e2e.sh | 163 ++++++++++++++++++++++++++++++++++++------------ hack/run-e2e.sh | 23 ++++--- 3 files changed, 141 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index fa527080a..471e65120 100644 --- a/README.md +++ b/README.md @@ -273,9 +273,8 @@ to "Block". ## E2E Tests ### Running -``` console -go run hack/e2e.go -- -v --test --test_args="--ginkgo.focus=PersistentVolumes-local" -``` + +Run `./hack/e2e.sh -h` to view help. ### View CI Results [GCE](https://k8s-testgrid.appspot.com/sig-storage#gce&include-filter-by-regex=PersistentVolumes-local) diff --git a/hack/e2e.sh b/hack/e2e.sh index 038d37598..246a2ca5c 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -18,42 +18,7 @@ # # It uses kubetest to setup/test/teardown kubernetes cluster. # -# Examples: -# -# 1) To run against existing local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh -# -# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh -# -# Optionally, you can add extra test args, e.g. -# -# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh --test-cmd-args=-ginkgo.focus='.*discovery.*' -# -# 2) To run against new local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh -# -# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes sudo -E env "PATH=$PATH" ./hack/e2e.sh -# -# Note that current kubetest needs root permission to cleanup. -# -# 3) To run against cluster with GCE provider locally, specify following environments: -# -# export GOOGLE_APPLICATION_CREDENTIALS= -# export GCP_ZONE= -# export GCP_PROJECT= -# -# and create ssh keypair at ~/.ssh/google_compute_engine or specifc ssh keypair -# with following environments: -# -# export JENKINS_GCE_SSH_PRIVATE_KEY_FILE= -# export JENKINS_GCE_SSH_PUBLIC_KEY_FILE= -# -# 4) To run against cluster with GCE provider in test-infra/prow job, add -# `preset-service-account: "true"` and `preset-k8s-ssh: "true"` labels in your -# prow job. -# -# The first label will set `GOOGLE_APPLICATION_CREDENTIALS` environment for -# you, and `kubetest` will acquire GCP project and zone from boskos -# automatically. The latter will prepare SSH key pair. -# +# Run `./hack/e2e.sh -h` to see help. set -o errexit set -o nounset @@ -64,11 +29,104 @@ cd $ROOT source "$ROOT/hack/lib.sh" +function usage() { + cat <<'EOF' +This script is entrypoint to run e2e tests. + +Usage: hack/e2e.sh + + -h show this message and exit + +Environments: + + PROVIDER local/gce/gke (detect automatically if not specified) + GCP_ZONE (for gce/gke) GCP zone + GCP_PROJECT (for gce/gke) GCP project + EXTRACT_STRATEGY kubetest extract strategy, see k8s.io/test-infra/kubetest/README.md for explanation + KUBERNETES_SRC if specified, kubetest will skip extracting kubernetes src from GCS, use it instead + DEPLOYMENT none/bash/gke + GKE_ENVIRONMENT (gke only) test/staging/prod + GOOGLE_APPLICATION_CREDENTIALS (for gce/gke) google applcation credentials which is used to access google cloud platform + JENKINS_GCE_SSH_PRIVATE_KEY_FILE (for gce/gke) GCP ssh key private file + JENKINS_GCE_SSH_PUBLIC_KEY_FILE (for gce/gke) GCP ssh key public file + +Examples: + +1) To run against existing local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh + + KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh + + Optionally, you can add extra test args, e.g. + + KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh --test-cmd-args=-ginkgo.focus='.*discovery.*' + +2) To run against new local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh + + KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes sudo -E env "PATH=$PATH" ./hack/e2e.sh + + Note that current kubetest needs root permission to cleanup. + +3) To run against cluster with GCE provider locally + + You need install Google Cloud SDK first, then prepare google application + credentials and configure ssh key pairs. + + You can create ssh keypair with ssh-keygen at ~/.ssh/google_compute_engine + or specifc existing ssh keypair with following environments: + + export JENKINS_GCE_SSH_PRIVATE_KEY_FILE= + export JENKINS_GCE_SSH_PUBLIC_KEY_FILE= + + Then run with following environments: + + export GOOGLE_APPLICATION_CREDENTIALS= + export GCP_ZONE= + export GCP_PROJECT= + ./hack/e2e.sh + +4) To run against cluster with GKE provider locally + + Prepare same as with GCE provider. + + Then run with following environments: + + export GOOGLE_APPLICATION_CREDENTIALS= + export GCP_ZONE= + export GCP_PROJECT= + export PROVIDER=gke + export GKE_ENVIRONMENT=prod + export EXTRACT_STRATEGY=gke-default + ./hack/e2e.sh + +5) To run against cluster with GCE/GKE provider in test-infra/prow job + + Almost same as running locally, you can add `preset-service-account: + "true"` and `preset-k8s-ssh: "true"` labels in your prow job to use + test-infra google application credentiails and GCP ssh key pair. + + The first label will set `GOOGLE_APPLICATION_CREDENTIALS` environment for + you, and `kubetest` will acquire GCP project and zone from boskos + automatically. The latter will prepare SSH key pair. + +EOF +} + +while getopts "h?" opt; do + case "$opt" in + h|\?) + usage + exit 0 + ;; + esac +done + PROVIDER=${PROVIDER:-} GCP_ZONE=${GCP_ZONE:-} GCP_PROJECT=${GCP_PROJECT:-} EXTRACT_STRATEGY=${EXTRACT_STRATEGY:-ci/latest} DEPLOYMENT=${DEPLOYMENT:-} +CLUSTER=${CLUSTER:-e2e} +GKE_ENVIRONMENT=${GKE_ENVIRONMENT:-prod} if [ -z "${KUBECTL:-}" ]; then KUBECTL=$(which kubectl 2>/dev/null || true) fi @@ -114,7 +172,8 @@ if [ -n "$KUBERNETES_SRC" ]; then cd $KUBERNETES_SRC fi -if [ "$PROVIDER" == "gce" ]; then +if [ "$PROVIDER" == "gce" -o "$PROVIDER" == "gke" ]; then + # GCP configurations for gce/gke if [ -n "$GCP_PROJECT" ]; then kubetest_args+=( --gcp-project "$GCP_PROJECT" @@ -127,6 +186,12 @@ if [ "$PROVIDER" == "gce" ]; then ) fi + if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then + kubetest_args+=( + --gcp-service-account "$GOOGLE_APPLICATION_CREDENTIALS" + ) + fi + # kubetest needs ssh keypair to ssh into nodes if [ ! -d ~/.ssh ]; then mkdir ~/.ssh @@ -142,15 +207,33 @@ if [ "$PROVIDER" == "gce" ]; then chmod 0600 ~/.ssh/google_compute_engine.pub fi - if [ -z "$DEPLOYMENT" ]; then - DEPLOYMENT=bash + if [ "$PROVIDER" == "gce" ]; then + if [ -z "$DEPLOYMENT" ]; then + DEPLOYMENT=bash + fi + else + # gke requires DEPLOYMENT=gke + if [ -z "$DEPLOYMENT" ]; then + DEPLOYMENT=gke + fi + if [ "$DEPLOYMENT" != "gke" ]; then + echo "error: only DEPLOYMENT=gke is supported, found '$DEPLOYMENT'" >&2 + exit + fi + # gke required + kubetest_args+=( + --cluster "$CLUSTER" + --gke-environment "$GKE_ENVIRONMENT" + --gcp-node-image "cos" + --gcp-network "e2e" + ) fi elif [ "$PROVIDER" == "local" ]; then if [ -z "$DEPLOYMENT" ]; then DEPLOYMENT=local fi else - echo "error: unsupported provider '$KUBERNETES_PROVIDER'" >&2 + echo "error: unsupported provider '$PROVIDER'" >&2 exit 1 fi diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index e92ef2e31..f46de6a85 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -31,31 +31,40 @@ source "${KUBE_ROOT}/cluster/kube-util.sh" KUBERNETES_SRC=${KUBE_ROOT} KUBECTL=${KUBE_ROOT}/cluster/kubectl.sh KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-} # e.g. local, gce +KUBERNETES_CONFORMANCE_TEST=${KUBERNETES_CONFORMANCE_TEST:-} +KUBERNETES_CONFORMANCE_PROVIDER=${KUBERNETES_CONFORMANCE_PROVIDER:-} KUBE_GCE_ZONE=${KUBE_GCE_ZONE:-} # Available when provider is gce PROJECT=${PROJECT:-} # Available when provider is gce KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG} echo "KUBERNETES_SRC: $KUBERNETES_SRC" >&2 echo "KUBERNETES_PROVIDER: $KUBERNETES_PROVIDER" >&2 +echo "KUBERNETES_CONFORMANCE_PROVIDER: $KUBERNETES_CONFORMANCE_PROVIDER" >&2 +echo "KUBERNETES_CONFORMANCE_TEST: $KUBERNETES_CONFORMANCE_TEST" >&2 echo "KUBE_GCE_ZONE: $KUBE_GCE_ZONE" >&2 echo "PROJECT: $PROJECT" >&2 echo "KUBECTL: $KUBECTL" >&2 echo "KUBECONFIG: $KUBECONFIG" >&2 -if [ -z "$KUBERNETES_PROVIDER" ]; then - echo "error: KUBERNETES_PROVIDER not set" >&2 +if [ -z "$KUBERNETES_PROVIDER" -a -z "$KUBERNETES_CONFORMANCE_PROVIDER" ]; then + echo "error: KUBERNETES_PROVIDER/KUBERNETES_CONFORMANCE_PROVIDER not set" >&2 exit 1 fi -echo "Testing cluster with provider: ${KUBERNETES_PROVIDER}" >&2 - -prepare-e2e -detect-master >/dev/null +if [ -n "$KUBERNETES_CONFORMANCE_TEST" ]; then + echo "Conformance test: not doing test setup." +else + echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"." + prepare-e2e + detect-master >/dev/null +fi # build image make -if [ "$KUBERNETES_PROVIDER" == "gce" ]; then +# Why we use KUBERNETES_CONFORMANCE_PROVIDER here, see +# https://github.com/kubernetes/test-infra/blob/5475440d76f9039f7e1a5fa86c2f85ea8414b093/kubetest/gke.go#L210-L229. +if [ "$KUBERNETES_PROVIDER" == "gce" -o "$KUBERNETES_CONFORMANCE_PROVIDER" == "gke" ]; then if [ -z "$PROJECT" ]; then echo "error: PROJECT is required" >&2 exit 1