Merge pull request #11432 from justinsb/fix_upgrade_ab_2

e2e upgrade-ab: a few more fixes and notes on how to run locally

tests/e2e/scenarios/upgrade-ab/README.md

@@ -0,0 +1,26 @@
+### Running locally
+
+If you want run locally:
+
+First set your KOPS_STATE_STORE:
+
+```
+export KOPS_STATE_STORE=...
+```
+
+Then set the configuration for the test run:
+
+```
+export KOPS_VERSION_A="v1.18.3"
+export KOPS_VERSION_B="v1.19.2"
+export K8S_VERSION_A="v1.18.18"
+export K8S_VERSION_B="v1.18.18"
+export ADMIN_ACCESS="0.0.0.0/0" # Or use your IPv4 with /32
+
+export CLOUD_PROVIDER=aws
+export CLUSTER_NAME=upgrade-ab.k8s.local
+
+export PATH=${GOPATH}/bin:$PATH
+
+tests/e2e/scenarios/upgrade-ab/run-test.sh
+```

tests/e2e/scenarios/upgrade-ab/run-test.sh

@@ -18,6 +18,9 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# Print all commands
+set -o xtrace
+
 echo "CLOUD_PROVIDER=${CLOUD_PROVIDER}"
 
 if [ -z "$KOPS_VERSION_A" ] || [ -z "$K8S_VERSION_A" ] || [ -z "$KOPS_VERSION_B" ] || [ -z "$K8S_VERSION_B" ]; then
@@ -31,16 +34,9 @@ REPO_ROOT=$(git rev-parse --show-toplevel);
 WORKDIR=$(mktemp -d)
 
 KOPS_A=${WORKDIR}/kops-${KOPS_VERSION_A}
-wget -O "${KOPS_A}" "https://github.com/kubernetes/kops/releases/download/v$KOPS_VERSION_A/kops-$(go env GOOS)-$(go env GOARCH)"
+wget -O "${KOPS_A}" "https://github.com/kubernetes/kops/releases/download/$KOPS_VERSION_A/kops-$(go env GOOS)-$(go env GOARCH)"
 chmod +x "${KOPS_A}"
 
-KOPS_B=${WORKDIR}/kops-${KOPS_VERSION_B}
-if [[ "${KOPS_VERSION_B}" == "source" ]]; then
-  cp "${REPO_ROOT}/bazel-bin/cmd/kops/linux-amd64/kops" "${KOPS_B}"
-else
-  wget -O "${KOPS_B}" "https://github.com/kubernetes/kops/releases/download/v$KOPS_VERSION_B/kops-$(go env GOOS)-$(go env GOARCH)"
-  chmod +x "${KOPS_B}"
-fi
 
 KUBETEST2="kubetest2 kops -v=2 --cloud-provider=${CLOUD_PROVIDER} --cluster-name=${CLUSTER_NAME:-}"
 KUBETEST2="${KUBETEST2} --admin-access=${ADMIN_ACCESS:-}"
@@ -52,7 +48,13 @@ go install sigs.k8s.io/kubetest2
 go install ./kubetest2-kops
 go install ./kubetest2-tester-kops
 
-${KUBETEST2} --build --kops-root="${REPO_ROOT}" --stage-location="${STAGE_LOCATION:-}" --kops-binary-path="${KOPS_B}"
+KOPS_B=${WORKDIR}/kops-${KOPS_VERSION_B}
+if [[ "${KOPS_VERSION_B}" == "source" ]]; then
+  ${KUBETEST2} --build --kops-root="${REPO_ROOT}" --stage-location="${STAGE_LOCATION:-}" --kops-binary-path="${KOPS_B}"
+else
+  wget -O "${KOPS_B}" "https://github.com/kubernetes/kops/releases/download/$KOPS_VERSION_B/kops-$(go env GOOS)-$(go env GOARCH)"
+  chmod +x "${KOPS_B}"
+fi
 
 # Always tear-down the cluster when we're done
 function finish {
@@ -66,6 +68,14 @@ ${KUBETEST2} \
 		--kubernetes-version="${K8S_VERSION_A}" \
 		--create-args="--networking calico"
 
+# Export kubeconfig-a
+KUBECONFIG_A="${WORKDIR}/kubeconfig-a"
+# TODO: Add --admin if 1.19 or higher...
+# Note: --kubeconfig flag not in 1.18
+KUBECONFIG="${KUBECONFIG_A}" "${KOPS_A}" export kubecfg --name "${CLUSTER_NAME}"
+
+# Verify kubeconfig-a
+KUBECONFIG="${KUBECONFIG_A}" kubectl get nodes -owide
 
 "${KOPS_B}" set cluster "${CLUSTER_NAME}" "cluster.spec.kubernetesVersion=${K8S_VERSION_B}"
 
@@ -74,11 +84,18 @@ ${KUBETEST2} \
 # Verify no additional changes
 "${KOPS_B}" update cluster
 
+sleep 300
+# Verify kubeconfig-a still works
+KUBECONFIG="${KUBECONFIG_A}" kubectl get nodes -owide
+
 "${KOPS_B}" rolling-update cluster
 "${KOPS_B}" rolling-update cluster --yes --validation-timeout 30m
 
 "${KOPS_B}" validate cluster
 
+# Verify kubeconfig-a still works
+KUBECONFIG="${KUBECONFIG_A}" kubectl get nodes -owide
+
 ${KUBETEST2} \
 		--cloud-provider="${CLOUD_PROVIDER}" \
 		--kops-binary-path="${KOPS_B}" \