Merge pull request #32 from cofyc/fix27

Support gke provider and docs how to run e2e tests

README.md

@@ -273,9 +273,8 @@ to "Block".
 ## E2E Tests
 
 ### Running
-``` console
-go run hack/e2e.go -- -v --test --test_args="--ginkgo.focus=PersistentVolumes-local"
-```
+
+Run `./hack/e2e.sh -h` to view help.
 
 ### View CI Results
 [GCE](https://k8s-testgrid.appspot.com/sig-storage#gce&include-filter-by-regex=PersistentVolumes-local)

hack/e2e.sh

@@ -18,42 +18,7 @@
 #
 # It uses kubetest to setup/test/teardown kubernetes cluster.
 #
-# Examples:
-#
-# 1) To run against existing local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh
-#
-# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh
-#
-# Optionally, you can add extra test args, e.g.
-#
-# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh --test-cmd-args=-ginkgo.focus='.*discovery.*'
-#
-# 2) To run against new local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh
-#
-# KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes sudo -E env "PATH=$PATH" ./hack/e2e.sh
-#
-# Note that current kubetest needs root permission to cleanup.
-#
-# 3) To run against cluster with GCE provider locally, specify following environments:
-#
-# export GOOGLE_APPLICATION_CREDENTIALS=<path-to-your-google-application-credentials>
-# export GCP_ZONE=<gcp-zone>
-# export GCP_PROJECT=<gcp-project>
-#
-# and create ssh keypair at ~/.ssh/google_compute_engine or specifc ssh keypair
-# with following environments:
-#
-# export JENKINS_GCE_SSH_PRIVATE_KEY_FILE=<path-to-your-ssh-private-key>
-# export JENKINS_GCE_SSH_PUBLIC_KEY_FILE=<path-to-your-ssh-public-key>
-#
-# 4) To run against cluster with GCE provider in test-infra/prow job, add
-# `preset-service-account: "true"` and `preset-k8s-ssh: "true"` labels in your
-# prow job.
-#
-# The first label will set `GOOGLE_APPLICATION_CREDENTIALS` environment for
-# you, and `kubetest` will acquire GCP project and zone from boskos
-# automatically. The latter will prepare SSH key pair.
-#
+# Run `./hack/e2e.sh -h` to see help.
 
 set -o errexit
 set -o nounset
@@ -64,11 +29,104 @@ cd $ROOT
 
 source "$ROOT/hack/lib.sh"
 
+function usage() {
+    cat <<'EOF'
+This script is entrypoint to run e2e tests.
+
+Usage: hack/e2e.sh
+
+    -h      show this message and exit
+
+Environments:
+
+    PROVIDER                            local/gce/gke (detect automatically if not specified)
+    GCP_ZONE                            (for gce/gke) GCP zone
+    GCP_PROJECT                         (for gce/gke) GCP project
+    EXTRACT_STRATEGY                    kubetest extract strategy, see k8s.io/test-infra/kubetest/README.md for explanation
+    KUBERNETES_SRC                      if specified, kubetest will skip extracting kubernetes src from GCS, use it instead
+    DEPLOYMENT                          none/bash/gke
+    GKE_ENVIRONMENT                     (gke only) test/staging/prod
+    GOOGLE_APPLICATION_CREDENTIALS      (for gce/gke) google applcation credentials which is used to access google cloud platform
+    JENKINS_GCE_SSH_PRIVATE_KEY_FILE    (for gce/gke) GCP ssh key private file
+    JENKINS_GCE_SSH_PUBLIC_KEY_FILE     (for gce/gke) GCP ssh key public file
+
+Examples:
+
+1) To run against existing local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh
+
+    KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh
+
+  Optionally, you can add extra test args, e.g.
+
+    KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes DEPLOYMENT=none ./hack/e2e.sh --test-cmd-args=-ginkgo.focus='.*discovery.*'
+
+2) To run against new local cluster started by k8s.io/kubernetes/hack/local-up-cluster.sh
+
+    KUBERNETES_SRC=$GOPATH/src/k8s.io/kubernetes sudo -E env "PATH=$PATH" ./hack/e2e.sh
+
+  Note that current kubetest needs root permission to cleanup.
+
+3) To run against cluster with GCE provider locally
+
+  You need install Google Cloud SDK first, then prepare google application
+  credentials and configure ssh key pairs.
+
+  You can create ssh keypair with ssh-keygen at  ~/.ssh/google_compute_engine
+  or specifc existing ssh keypair with following environments:
+
+    export JENKINS_GCE_SSH_PRIVATE_KEY_FILE=<path-to-your-ssh-private-key>
+    export JENKINS_GCE_SSH_PUBLIC_KEY_FILE=<path-to-your-ssh-public-key>
+
+  Then run with following environments:
+
+    export GOOGLE_APPLICATION_CREDENTIALS=<path-to-your-google-application-credentials>
+    export GCP_ZONE=<your-gcp-zone>
+    export GCP_PROJECT=<your-gcp-project>
+    ./hack/e2e.sh
+
+4) To run against cluster with GKE provider locally
+
+  Prepare same as with GCE provider.
+
+  Then run with following environments:
+
+    export GOOGLE_APPLICATION_CREDENTIALS=<path-to-your-google-application-credentials>
+    export GCP_ZONE=<your-gcp-zone>
+    export GCP_PROJECT=<your-gcp-project>
+    export PROVIDER=gke
+    export GKE_ENVIRONMENT=prod
+    export EXTRACT_STRATEGY=gke-default
+    ./hack/e2e.sh
+
+5) To run against cluster with GCE/GKE provider in test-infra/prow job
+
+  Almost same as running locally, you can add `preset-service-account:
+  "true"` and `preset-k8s-ssh: "true"` labels in your prow job to use
+  test-infra google application credentiails and GCP ssh key pair.
+
+  The first label will set `GOOGLE_APPLICATION_CREDENTIALS` environment for
+  you, and `kubetest` will acquire GCP project and zone from boskos
+  automatically. The latter will prepare SSH key pair.
+
+EOF
+}
+
+while getopts "h?" opt; do
+    case "$opt" in
+    h|\?)
+        usage
+        exit 0
+        ;;
+    esac
+done
+
 PROVIDER=${PROVIDER:-}
 GCP_ZONE=${GCP_ZONE:-}
 GCP_PROJECT=${GCP_PROJECT:-}
 EXTRACT_STRATEGY=${EXTRACT_STRATEGY:-ci/latest}
 DEPLOYMENT=${DEPLOYMENT:-}
+CLUSTER=${CLUSTER:-e2e}
+GKE_ENVIRONMENT=${GKE_ENVIRONMENT:-prod}
 if [ -z "${KUBECTL:-}" ]; then
     KUBECTL=$(which kubectl 2>/dev/null || true)
 fi
@@ -114,7 +172,8 @@ if [ -n "$KUBERNETES_SRC" ]; then
     cd $KUBERNETES_SRC
 fi
 
-if [ "$PROVIDER" == "gce" ]; then
+if [ "$PROVIDER" == "gce" -o "$PROVIDER" == "gke" ]; then
+    # GCP configurations for gce/gke
     if [ -n "$GCP_PROJECT" ]; then
         kubetest_args+=(
             --gcp-project "$GCP_PROJECT"
@@ -127,6 +186,12 @@ if [ "$PROVIDER" == "gce" ]; then
         )
     fi
 
+    if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
+        kubetest_args+=(
+            --gcp-service-account "$GOOGLE_APPLICATION_CREDENTIALS"
+        )
+    fi
+
     # kubetest needs ssh keypair to ssh into nodes
     if [ ! -d ~/.ssh ]; then
         mkdir ~/.ssh
@@ -142,15 +207,33 @@ if [ "$PROVIDER" == "gce" ]; then
         chmod 0600 ~/.ssh/google_compute_engine.pub
     fi
 
-    if [ -z "$DEPLOYMENT" ]; then
-        DEPLOYMENT=bash
+    if [ "$PROVIDER" == "gce" ]; then
+        if [ -z "$DEPLOYMENT" ]; then
+            DEPLOYMENT=bash
+        fi
+    else
+        # gke requires DEPLOYMENT=gke
+        if [ -z "$DEPLOYMENT" ]; then
+            DEPLOYMENT=gke
+        fi
+        if [ "$DEPLOYMENT" != "gke" ]; then
+            echo "error: only DEPLOYMENT=gke is supported, found '$DEPLOYMENT'" >&2
+            exit
+        fi
+        # gke required
+        kubetest_args+=(
+            --cluster "$CLUSTER"
+            --gke-environment "$GKE_ENVIRONMENT"
+            --gcp-node-image "cos"
+            --gcp-network "e2e"
+        )
     fi
 elif [ "$PROVIDER" == "local" ]; then
     if [ -z "$DEPLOYMENT" ]; then
         DEPLOYMENT=local
     fi
 else
-    echo "error: unsupported provider '$KUBERNETES_PROVIDER'" >&2
+    echo "error: unsupported provider '$PROVIDER'" >&2
     exit 1
 fi
 

hack/run-e2e.sh

@@ -31,31 +31,40 @@ source "${KUBE_ROOT}/cluster/kube-util.sh"
 KUBERNETES_SRC=${KUBE_ROOT}
 KUBECTL=${KUBE_ROOT}/cluster/kubectl.sh
 KUBERNETES_PROVIDER=${KUBERNETES_PROVIDER:-} # e.g. local, gce
+KUBERNETES_CONFORMANCE_TEST=${KUBERNETES_CONFORMANCE_TEST:-}
+KUBERNETES_CONFORMANCE_PROVIDER=${KUBERNETES_CONFORMANCE_PROVIDER:-}
 KUBE_GCE_ZONE=${KUBE_GCE_ZONE:-} # Available when provider is gce
 PROJECT=${PROJECT:-} # Available when provider is gce
 KUBECONFIG=${KUBECONFIG:-$DEFAULT_KUBECONFIG}
 
 echo "KUBERNETES_SRC: $KUBERNETES_SRC" >&2
 echo "KUBERNETES_PROVIDER: $KUBERNETES_PROVIDER" >&2
+echo "KUBERNETES_CONFORMANCE_PROVIDER: $KUBERNETES_CONFORMANCE_PROVIDER" >&2
+echo "KUBERNETES_CONFORMANCE_TEST: $KUBERNETES_CONFORMANCE_TEST" >&2
 echo "KUBE_GCE_ZONE: $KUBE_GCE_ZONE" >&2
 echo "PROJECT: $PROJECT" >&2
 echo "KUBECTL: $KUBECTL" >&2
 echo "KUBECONFIG: $KUBECONFIG" >&2
 
-if [ -z "$KUBERNETES_PROVIDER" ]; then
-    echo "error: KUBERNETES_PROVIDER not set" >&2
+if [ -z "$KUBERNETES_PROVIDER" -a -z "$KUBERNETES_CONFORMANCE_PROVIDER" ]; then
+    echo "error: KUBERNETES_PROVIDER/KUBERNETES_CONFORMANCE_PROVIDER not set" >&2
     exit 1
 fi
 
-echo "Testing cluster with provider: ${KUBERNETES_PROVIDER}" >&2
-
-prepare-e2e
-detect-master >/dev/null
+if [ -n "$KUBERNETES_CONFORMANCE_TEST" ]; then
+    echo "Conformance test: not doing test setup."
+else
+    echo "Setting up for KUBERNETES_PROVIDER=\"${KUBERNETES_PROVIDER}\"."
+    prepare-e2e
+    detect-master >/dev/null
+fi
 
 # build image
 make
 
-if [ "$KUBERNETES_PROVIDER" == "gce" ]; then
+# Why we use KUBERNETES_CONFORMANCE_PROVIDER here, see
+# https://github.com/kubernetes/test-infra/blob/5475440d76f9039f7e1a5fa86c2f85ea8414b093/kubetest/gke.go#L210-L229.
+if [ "$KUBERNETES_PROVIDER" == "gce" -o "$KUBERNETES_CONFORMANCE_PROVIDER" == "gke" ]; then
     if [ -z "$PROJECT" ]; then
         echo "error: PROJECT is required" >&2
         exit 1