From 0834721eb915ae82f905510449ce96c6b8a8312a Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 24 Nov 2015 16:16:00 -0800 Subject: [PATCH 1/8] configure_upgrade_step in e2e.sh --- hack/jenkins/e2e.sh | 157 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 1e35f2e466a8..86d140975462 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -67,6 +67,163 @@ function get_latest_trusty_image() { rm -rf .gsutil &> /dev/null } +# Properly configure globals for an upgrade step in a GKE or GCE upgrade suite +# +# These suites: +# step1: launch a cluster at $old_version, +# step2: upgrades the master to $new_version, +# step3: runs $old_version e2es, +# step4: upgrades the rest of the cluster, +# step5: runs $old_version e2es again, then +# step6: runs $new_version e2es and tears down the cluster. +# +# Assumes globals: +# $JOB_NAME +# $KUBERNETES_PROVIDER +# $GKE_DEFAULT_SKIP_TESTS +# $GCE_DEFAULT_SKIP_TESTS +# $GCE_FLAKY_TESTS +# $GCE_SLOW_TESTS +# $GKE_FLAKY_TESTS +# +# Args: +# $1 old_version: the version to deploy a cluster at, and old e2e tests to run +# against the upgraded cluster (should be something like +# 'release/latest', to work with JENKINS_PUBLISHED_VERSION logic) +# $2 new_version: the version to upgrade the cluster to, and new e2e tests to run +# against the upgraded cluster (should be something like +# 'ci/latest', to work with JENKINS_PUBLISHED_VERSION logic) +# $3 cluster_name: determines E2E_CLUSTER_NAME and E2E_NETWORK +# $4 project: determines PROJECT + +function configure_upgrade_step() { + local -r old_version="$1" + local -r new_version="$2" + local -r cluster_name="$3" + local -r project="$4" + + [[ "${JOB_NAME}" =~ .*-(step[1-6])-.* ]] || { + echo "JOB_NAME ${JOB_NAME} is not a valid upgrade job name, could not parse" + exit 1 + } + local -r step="${BASH_REMATCH[1]}" + + local -r gce_test_args="--ginkgo.skip=$(join_regex_allow_empty \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ + )" + local -r gke_test_args="--ginkgo.skip=$(join_regex_allow_empty \ + ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ + ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ + ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ + ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ + )" + + if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then + DOGFOOD_GCLOUD="true" + GKE_API_ENDPOINT="https://test-container.sandbox.googleapis.com/" + fi + + E2E_CLUSTER_NAME="$cluster_name" + E2E_NETWORK="$cluster_name" + PROJECT="$project" + + case $step in + step1) + # Deploy at old version + JENKINS_PUBLISHED_VERSION="${old_version}" + + E2E_UP="true" + E2E_TEST="false" + E2E_DOWN="false" + + if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then + E2E_SET_CLUSTER_API_VERSION=y + fi + ;; + + step2) + # Use upgrade logic of version we're upgrading to. + JENKINS_PUBLISHED_VERSION="${new_version}" + JENKINS_FORCE_GET_TARS=y + + E2E_OPT="--check_version_skew=false" + E2E_UP="false" + E2E_TEST="true" + E2E_DOWN="false" + GINKGO_TEST_ARGS="--ginkgo.focus=Cluster\sUpgrade.*upgrade-master --upgrade-target=${new_version}" + ;; + + step3) + # Run old e2es + JENKINS_PUBLISHED_VERSION="${old_version}" + JENKINS_FORCE_GET_TARS=y + + E2E_OPT="--check_version_skew=false" + E2E_UP="false" + E2E_TEST="true" + E2E_DOWN="false" + + if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then + GINKGO_TEST_ARGS="${gke_test_args}" + else + GINKGO_TEST_ARGS="${gce_test_args}" + fi + ;; + + step4) + # Use upgrade logic of version we're upgrading to. + JENKINS_PUBLISHED_VERSION="${new_version}" + JENKINS_FORCE_GET_TARS=y + + E2E_OPT="--check_version_skew=false" + E2E_UP="false" + E2E_TEST="true" + E2E_DOWN="false" + GINKGO_TEST_ARGS="--ginkgo.focus=Cluster\sUpgrade.*upgrade-cluster --upgrade-target=${new_version}" + ;; + + step5) + # Run old e2es + JENKINS_PUBLISHED_VERSION="${old_version}" + JENKINS_FORCE_GET_TARS=y + + E2E_OPT="--check_version_skew=false" + E2E_UP="false" + E2E_TEST="true" + E2E_DOWN="false" + + if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then + GINKGO_TEST_ARGS="${gke_test_args}" + else + GINKGO_TEST_ARGS="${gce_test_args}" + fi + ;; + + step6) + # Run new e2es + JENKINS_PUBLISHED_VERSION="${new_version}" + JENKINS_FORCE_GET_TARS=y + + # TODO(15011): these really shouldn't be (very) version skewed, but + # because we have to get ci/latest again, it could get slightly out of + # whack. + E2E_OPT="--check_version_skew=false" + E2E_UP="false" + E2E_TEST="true" + E2E_DOWN="true" + + if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then + GINKGO_TEST_ARGS="${gke_test_args}" + else + GINKGO_TEST_ARGS="${gce_test_args}" + fi + ;; + esac +} + echo "--------------------------------------------------------------------------------" echo "Initial Environment:" printenv | sort From 164c7e592f84019c082e2ceb43436e53e5e5f481 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 1 Dec 2015 11:08:48 -0800 Subject: [PATCH 2/8] Remove GKE_FLAKY_TESTS; not in this branch --- hack/jenkins/e2e.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 86d140975462..e08749100eef 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -84,7 +84,6 @@ function get_latest_trusty_image() { # $GCE_DEFAULT_SKIP_TESTS # $GCE_FLAKY_TESTS # $GCE_SLOW_TESTS -# $GKE_FLAKY_TESTS # # Args: # $1 old_version: the version to deploy a cluster at, and old e2e tests to run @@ -118,7 +117,6 @@ function configure_upgrade_step() { ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \ ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \ ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \ - ${GKE_FLAKY_TESTS[@]:+${GKE_FLAKY_TESTS[@]}} \ )" if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then From 7a2021dd73b6ce22363c8f8def3d94da46400195 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 1 Dec 2015 11:01:53 -0800 Subject: [PATCH 3/8] Add jobs for kubernetes-upgrade-gke-1.0-release --- hack/jenkins/e2e.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index e08749100eef..53be682601d7 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -762,6 +762,22 @@ case ${JOB_NAME} in # Needed to be able to create PD from the e2e test : ${AWS_SHARED_CREDENTIALS_FILE=/var/lib/jenkins/.aws/credentials} ;; + + # kubernetes-upgrade-gke-1.0-release + # + # Configurations for step1, step3, and step5 live in the release-1.0 branch. + + kubernetes-upgrade-1.0-release-gke-step2-upgrade-master) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-1.0-release-gke-step4-upgrade-cluster) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-1.0-release-gke-step6-e2e-new) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + ;; esac # AWS variables From ebbe6fd3a5be9e9c04aea94b949b22b4e93613df Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 1 Dec 2015 14:57:15 -0800 Subject: [PATCH 4/8] Make job names consistent --- hack/jenkins/e2e.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 53be682601d7..e65e7248e993 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -767,16 +767,16 @@ case ${JOB_NAME} in # # Configurations for step1, step3, and step5 live in the release-1.0 branch. - kubernetes-upgrade-1.0-release-gke-step2-upgrade-master) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-release-step2-upgrade-master) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-1.0-release-gke-step4-upgrade-cluster) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-release-step4-upgrade-cluster) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-1.0-release-gke-step6-e2e-new) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'gke-upgrade-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-release-step6-e2e-new) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' ;; esac From 39c156dd7aefe0139d2b69072ad84f5ac2f845d6 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 1 Dec 2015 15:05:27 -0800 Subject: [PATCH 5/8] Add 1.1-master jobs to release-1.1 --- hack/jenkins/e2e.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index e65e7248e993..8dac58e66faf 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -763,6 +763,22 @@ case ${JOB_NAME} in : ${AWS_SHARED_CREDENTIALS_FILE=/var/lib/jenkins/.aws/credentials} ;; + # kubernetes-upgrade-gke-1.1-master + # + # Configurations for step2, step4, and step6 live in master. + + kubernetes-upgrade-gke-1.1-master-step1-deploy) + configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-1.1-master-step3-e2e-old) + configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-1.1-master-step5-e2e-old) + configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade' + ;; + # kubernetes-upgrade-gke-1.0-release # # Configurations for step1, step3, and step5 live in the release-1.0 branch. From a3990de38de15f9205ed288e8cb602478e1ce640 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Tue, 1 Dec 2015 15:12:04 -0800 Subject: [PATCH 6/8] Add stable-release jobs to release-1.1 --- hack/jenkins/e2e.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 8dac58e66faf..e1b9a2fe3cd6 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -794,6 +794,32 @@ case ${JOB_NAME} in kubernetes-upgrade-gke-1.0-release-step6-e2e-new) configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' ;; + + # kubernetes-upgrade-gke-stable-release + + kubernetes-upgrade-gke-stable-release-step1-deploy) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-stable-release-step2-upgrade-master) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-stable-release-step3-e2e-old) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-stable-release-step4-upgrade-cluster) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-stable-release-step5-e2e-old) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; + + kubernetes-upgrade-gke-stable-release-step6-e2e-new) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + ;; esac # AWS variables From e5b90240ba3ce2fea56bf6c068588c6222219b25 Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Wed, 2 Dec 2015 15:47:22 -0800 Subject: [PATCH 7/8] Change release to current-release --- hack/jenkins/e2e.sh | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index e1b9a2fe3cd6..37175244d575 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -779,46 +779,46 @@ case ${JOB_NAME} in configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade' ;; - # kubernetes-upgrade-gke-1.0-release + # kubernetes-upgrade-gke-1.0-current-release # # Configurations for step1, step3, and step5 live in the release-1.0 branch. - kubernetes-upgrade-gke-1.0-release-step2-upgrade-master) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-current-release-step2-upgrade-master) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-1.0-release-step4-upgrade-cluster) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-current-release-step4-upgrade-cluster) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-1.0-release-step6-e2e-new) - configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-1.0-current-release-step6-e2e-new) + configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - # kubernetes-upgrade-gke-stable-release + # kubernetes-upgrade-gke-stable-current-release - kubernetes-upgrade-gke-stable-release-step1-deploy) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step1-deploy) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-stable-release-step2-upgrade-master) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step2-upgrade-master) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-stable-release-step3-e2e-old) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step3-e2e-old) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-stable-release-step4-upgrade-cluster) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step4-upgrade-cluster) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-stable-release-step5-e2e-old) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step5-e2e-old) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; - kubernetes-upgrade-gke-stable-release-step6-e2e-new) - configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-release' 'kubernetes-jenkins-gke-upgrade' + kubernetes-upgrade-gke-stable-current-release-step6-e2e-new) + configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade' ;; esac From 0dd74bf2c32c30d205737cc6c41d34ada4beba0a Mon Sep 17 00:00:00 2001 From: Isaac Hollander McCreery Date: Wed, 2 Dec 2015 15:49:43 -0800 Subject: [PATCH 8/8] Add comments --- hack/jenkins/e2e.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hack/jenkins/e2e.sh b/hack/jenkins/e2e.sh index 37175244d575..0264f21f4a50 100755 --- a/hack/jenkins/e2e.sh +++ b/hack/jenkins/e2e.sh @@ -765,6 +765,8 @@ case ${JOB_NAME} in # kubernetes-upgrade-gke-1.1-master # + # Test upgrades from the latest release-1.1 build to the latest master build. + # # Configurations for step2, step4, and step6 live in master. kubernetes-upgrade-gke-1.1-master-step1-deploy) @@ -781,6 +783,9 @@ case ${JOB_NAME} in # kubernetes-upgrade-gke-1.0-current-release # + # Test upgrades from the latest release-1.0 build to the latest current + # release build. + # # Configurations for step1, step3, and step5 live in the release-1.0 branch. kubernetes-upgrade-gke-1.0-current-release-step2-upgrade-master) @@ -796,6 +801,8 @@ case ${JOB_NAME} in ;; # kubernetes-upgrade-gke-stable-current-release + # + # Test upgrades from the stable build to the latest current release build. kubernetes-upgrade-gke-stable-current-release-step1-deploy) configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'