Configure upgrade jobs from 1.0 to 1.1 and stable to current-release on release-1.1

hack/jenkins/e2e.sh

@@ -67,6 +67,161 @@ function get_latest_trusty_image() {
     rm -rf .gsutil &> /dev/null
 }
 
+# Properly configure globals for an upgrade step in a GKE or GCE upgrade suite
+#
+# These suites:
+#   step1: launch a cluster at $old_version,
+#   step2: upgrades the master to $new_version,
+#   step3: runs $old_version e2es,
+#   step4: upgrades the rest of the cluster,
+#   step5: runs $old_version e2es again, then
+#   step6: runs $new_version e2es and tears down the cluster.
+#
+# Assumes globals:
+#   $JOB_NAME
+#   $KUBERNETES_PROVIDER
+#   $GKE_DEFAULT_SKIP_TESTS
+#   $GCE_DEFAULT_SKIP_TESTS
+#   $GCE_FLAKY_TESTS
+#   $GCE_SLOW_TESTS
+#
+# Args:
+#   $1 old_version:  the version to deploy a cluster at, and old e2e tests to run
+#                    against the upgraded cluster (should be something like
+#                    'release/latest', to work with JENKINS_PUBLISHED_VERSION logic)
+#   $2 new_version:  the version to upgrade the cluster to, and new e2e tests to run
+#                    against the upgraded cluster (should be something like
+#                    'ci/latest', to work with JENKINS_PUBLISHED_VERSION logic)
+#   $3 cluster_name: determines E2E_CLUSTER_NAME and E2E_NETWORK
+#   $4 project:      determines PROJECT
+
+function configure_upgrade_step() {
+  local -r old_version="$1"
+  local -r new_version="$2"
+  local -r cluster_name="$3"
+  local -r project="$4"
+
+  [[ "${JOB_NAME}" =~ .*-(step[1-6])-.* ]] || {
+    echo "JOB_NAME ${JOB_NAME} is not a valid upgrade job name, could not parse"
+    exit 1
+  }
+  local -r step="${BASH_REMATCH[1]}"
+
+  local -r gce_test_args="--ginkgo.skip=$(join_regex_allow_empty \
+        ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+        ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+        ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \
+        )"
+  local -r gke_test_args="--ginkgo.skip=$(join_regex_allow_empty \
+        ${GKE_DEFAULT_SKIP_TESTS[@]:+${GKE_DEFAULT_SKIP_TESTS[@]}} \
+        ${GCE_DEFAULT_SKIP_TESTS[@]:+${GCE_DEFAULT_SKIP_TESTS[@]}} \
+        ${GCE_FLAKY_TESTS[@]:+${GCE_FLAKY_TESTS[@]}} \
+        ${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}} \
+        )"
+
+  if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
+    DOGFOOD_GCLOUD="true"
+    GKE_API_ENDPOINT="https://test-container.sandbox.googleapis.com/"
+  fi
+
+  E2E_CLUSTER_NAME="$cluster_name"
+  E2E_NETWORK="$cluster_name"
+  PROJECT="$project"
+
+  case $step in
+    step1)
+      # Deploy at old version
+      JENKINS_PUBLISHED_VERSION="${old_version}"
+
+      E2E_UP="true"
+      E2E_TEST="false"
+      E2E_DOWN="false"
+
+      if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
+        E2E_SET_CLUSTER_API_VERSION=y
+      fi
+      ;;
+
+    step2)
+      # Use upgrade logic of version we're upgrading to.
+      JENKINS_PUBLISHED_VERSION="${new_version}"
+      JENKINS_FORCE_GET_TARS=y
+
+      E2E_OPT="--check_version_skew=false"
+      E2E_UP="false"
+      E2E_TEST="true"
+      E2E_DOWN="false"
+      GINKGO_TEST_ARGS="--ginkgo.focus=Cluster\sUpgrade.*upgrade-master --upgrade-target=${new_version}"
+      ;;
+
+    step3)
+      # Run old e2es
+      JENKINS_PUBLISHED_VERSION="${old_version}"
+      JENKINS_FORCE_GET_TARS=y
+
+      E2E_OPT="--check_version_skew=false"
+      E2E_UP="false"
+      E2E_TEST="true"
+      E2E_DOWN="false"
+
+      if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
+        GINKGO_TEST_ARGS="${gke_test_args}"
+      else
+        GINKGO_TEST_ARGS="${gce_test_args}"
+      fi
+      ;;
+
+    step4)
+      # Use upgrade logic of version we're upgrading to.
+      JENKINS_PUBLISHED_VERSION="${new_version}"
+      JENKINS_FORCE_GET_TARS=y
+
+      E2E_OPT="--check_version_skew=false"
+      E2E_UP="false"
+      E2E_TEST="true"
+      E2E_DOWN="false"
+      GINKGO_TEST_ARGS="--ginkgo.focus=Cluster\sUpgrade.*upgrade-cluster --upgrade-target=${new_version}"
+      ;;
+
+    step5)
+      # Run old e2es
+      JENKINS_PUBLISHED_VERSION="${old_version}"
+      JENKINS_FORCE_GET_TARS=y
+
+      E2E_OPT="--check_version_skew=false"
+      E2E_UP="false"
+      E2E_TEST="true"
+      E2E_DOWN="false"
+
+      if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
+        GINKGO_TEST_ARGS="${gke_test_args}"
+      else
+        GINKGO_TEST_ARGS="${gce_test_args}"
+      fi
+      ;;
+
+    step6)
+      # Run new e2es
+      JENKINS_PUBLISHED_VERSION="${new_version}"
+      JENKINS_FORCE_GET_TARS=y
+
+      # TODO(15011): these really shouldn't be (very) version skewed, but
+      # because we have to get ci/latest again, it could get slightly out of
+      # whack.
+      E2E_OPT="--check_version_skew=false"
+      E2E_UP="false"
+      E2E_TEST="true"
+      E2E_DOWN="true"
+
+      if [[ "${KUBERNETES_PROVIDER}" == "gke" ]]; then
+        GINKGO_TEST_ARGS="${gke_test_args}"
+      else
+        GINKGO_TEST_ARGS="${gce_test_args}"
+      fi
+      ;;
+  esac
+}
+
 echo "--------------------------------------------------------------------------------"
 echo "Initial Environment:"
 printenv | sort
@@ -607,6 +762,71 @@ case ${JOB_NAME} in
     # Needed to be able to create PD from the e2e test
     : ${AWS_SHARED_CREDENTIALS_FILE=/var/lib/jenkins/.aws/credentials}
     ;;
+
+  # kubernetes-upgrade-gke-1.1-master
+  #
+  # Test upgrades from the latest release-1.1 build to the latest master build.
+  #
+  # Configurations for step2, step4, and step6 live in master.
+
+  kubernetes-upgrade-gke-1.1-master-step1-deploy)
+    configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-1.1-master-step3-e2e-old)
+    configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-1.1-master-step5-e2e-old)
+    configure_upgrade_step 'ci/latest-1.1' 'configured-in-master' 'upgrade-gke-1-1-master' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  # kubernetes-upgrade-gke-1.0-current-release
+  #
+  # Test upgrades from the latest release-1.0 build to the latest current
+  # release build.
+  #
+  # Configurations for step1, step3, and step5 live in the release-1.0 branch.
+
+  kubernetes-upgrade-gke-1.0-current-release-step2-upgrade-master)
+    configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-1.0-current-release-step4-upgrade-cluster)
+    configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-1.0-current-release-step6-e2e-new)
+    configure_upgrade_step 'configured-in-release-1.0' 'ci/latest-1.1' 'upgrade-gke-1-0-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  # kubernetes-upgrade-gke-stable-current-release
+  #
+  # Test upgrades from the stable build to the latest current release build.
+
+  kubernetes-upgrade-gke-stable-current-release-step1-deploy)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-stable-current-release-step2-upgrade-master)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-stable-current-release-step3-e2e-old)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-stable-current-release-step4-upgrade-cluster)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-stable-current-release-step5-e2e-old)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
+
+  kubernetes-upgrade-gke-stable-current-release-step6-e2e-new)
+    configure_upgrade_step 'release/stable-1.1' 'ci/latest-1.1' 'upgrade-gke-stable-current-release' 'kubernetes-jenkins-gke-upgrade'
+    ;;
 esac
 
 # AWS variables