From 3d7e2497d6627ac4986f0c44a1dcc1f80825671c Mon Sep 17 00:00:00 2001 From: Ilya Dmitrichenko Date: Mon, 27 Apr 2020 15:40:27 +0100 Subject: [PATCH] test: Improve `gke/{select,release}-cluster.sh` scripts - use explicit paths - make better use of `gcloud` features, reduce external dependencies - replace `get-cluster-version.sh` with a version file - use region instead of zone to enable regional spread and avoid availibity issue due to a zonal outage; namely we can create clusters in any region/zone and the Jenkins job is not tied into any particular one - list node pools instead of assuming default pool Signed-off-by: Ilya Dmitrichenko --- jenkinsfiles/ginkgo-gke.Jenkinsfile | 3 +- test/gke/get-cluster-version.sh | 11 -------- test/gke/release-cluster.sh | 27 ++++++++++++++---- test/gke/select-cluster.sh | 43 ++++++++++++++++++++--------- 4 files changed, 52 insertions(+), 32 deletions(-) delete mode 100755 test/gke/get-cluster-version.sh diff --git a/jenkinsfiles/ginkgo-gke.Jenkinsfile b/jenkinsfiles/ginkgo-gke.Jenkinsfile index 3fd143bcb8f2..ec23d6a2d223 100644 --- a/jenkinsfiles/ginkgo-gke.Jenkinsfile +++ b/jenkinsfiles/ginkgo-gke.Jenkinsfile @@ -14,7 +14,6 @@ pipeline { TESTDIR="${WORKSPACE}/${PROJ_PATH}/test" GOPATH="${WORKSPACE}" GKE_KEY=credentials('gke-key') - GKE_ZONE="us-west1-a" TAG="${GIT_COMMIT}" HOME="${WORKSPACE}" } @@ -138,7 +137,7 @@ pipeline { )}""" K8S_VERSION= """${sh( returnStdout: true, - script: 'cd ${TESTDIR}; gke/get-cluster-version.sh' + script: 'cat ${TESTDIR}/gke/cluster-version' )}""" FOCUS= """${sh( returnStdout: true, diff --git a/test/gke/get-cluster-version.sh b/test/gke/get-cluster-version.sh deleted file mode 100755 index d21246b2a61e..000000000000 --- a/test/gke/get-cluster-version.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# get-cluster-version.sh prints the version of the selected cluster. It expects -# the file 'cluster-name' to be populated. That file is created in -# select-cluster.sh - -set -e - -CLUSTER_NAME=$(cat ./gke/cluster-name) -K8S_VERSION=$(gcloud container clusters list --zone $GKE_ZONE --filter "name:${CLUSTER_NAME}" | awk '{print $3}' | grep -v MASTER_VERSION | sed -E 's/([0-9]+\.[0-9]+)\..*/\1/') -echo -n ${K8S_VERSION} diff --git a/test/gke/release-cluster.sh b/test/gke/release-cluster.sh index cc76e931a55c..8e70b769acdd 100755 --- a/test/gke/release-cluster.sh +++ b/test/gke/release-cluster.sh @@ -1,13 +1,22 @@ #!/bin/bash -export KUBECONFIG=gke-kubeconfig -cluster=$(cat cluster-name) +test_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +project="cilium-ci" +# this is only needs to be set as some of gcloud commands requires it, +# but as this script uses resource URIs clusters in all locations are +# going to be discovered and used +region="us-west2" + +export KUBECONFIG="${script_dir}/gke-kubeconfig" +cluster_uri="$(cat "${script_dir}/cluster-uri")" # Create a function to unlock the cluster. We then execute this on script exit. # This should occur even if the script is interrupted, by a jenkins timeout, # for example. unlock() { - echo "releasing cluster lock from $cluster" + echo "releasing cluster lock from ${cluster_uri}" kubectl annotate deployment lock lock- } trap unlock EXIT @@ -24,7 +33,13 @@ echo "deleting terminating namespaces" set -e -echo "scaling $cluster ng to 0" -yes | gcloud container clusters resize $cluster --node-pool default-pool --num-nodes 0 --zone $GKE_ZONE +echo "scaling ${cluster_uri} to 0" +node_pools=($(gcloud container node-pools list --project "${project}" --region "${region}" --cluster "${cluster_uri}" --uri)) +if [ "${#node_pools[@]}" -ne 1 ] ; then + echo "expected 1 node pool, found ${#node_pools[@]}" + exit 1 +fi + +gcloud container clusters resize --project "${project}" --region "${region}" --node-pool "${node_pools[1]}" --num-nodes 2 --quiet "${cluster_uri}" -rm -f cluster-name +rm -f "${script_dir}/cluster-uri" "${script_dir}/cluster-name" "${script_dir}/cluster-version" diff --git a/test/gke/select-cluster.sh b/test/gke/select-cluster.sh index d9e0abaa2f64..4562f863c28e 100755 --- a/test/gke/select-cluster.sh +++ b/test/gke/select-cluster.sh @@ -1,22 +1,31 @@ #!/bin/bash +test_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +project="cilium-ci" +# this is only needs to be set as some of gcloud commands requires it, +# but as this script uses resource URIs clusters in all locations are +# going to be discovered and used +region="us-west2" + set -e locked=1 -export KUBECONFIG=gke-kubeconfig +export KUBECONFIG="${script_dir}/gke-kubeconfig" while [ $locked -ne 0 ]; do - rm gke-kubeconfig || true + rm -f "${KUBECONFIG}" echo "selecting random cluster" - cluster=$(gcloud container clusters list --zone $GKE_ZONE | grep cilium-ci | sort -R | head -n 1 | awk '{print $1}') + cluster_uri="$(gcloud container clusters list --project "${project}" --filter="name ~ ^cilium-ci-" --uri | sort -R | head -n 1)" - echo "getting kubeconfig for $cluster" - gcloud container clusters get-credentials --zone $GKE_ZONE $cluster + echo "getting kubeconfig for ${cluster_uri} (will store in ${KUBECONFIG})" + gcloud container clusters get-credentials --project "${project}" --region "${region}" "${cluster_uri}" echo "aquiring cluster lock" set +e - kubectl create -f lock.yaml + kubectl create -f "${script_dir}/lock.yaml" kubectl annotate deployment lock lock=1 locked=$? @@ -24,18 +33,26 @@ while [ $locked -ne 0 ]; do set -e done -echo "lock acquired on cluster $cluster" -# cluster-name is used in get-cluster-version.sh, which runs after this in CI. -echo $cluster > cluster-name +echo "lock acquired on cluster ${cluster_uri}" +echo "${cluster_uri}" > "${script_dir}/cluster-uri" +gcloud container clusters describe --project "${project}" --region "${region}" --format='value(name)' "${cluster_uri}" > "${script_dir}/cluster-name" +gcloud container clusters describe --project "${project}" --region "${region}" --format='value(currentMasterVersion)' "${cluster_uri}" \ + | sed -E 's/([0-9]+\.[0-9]+)\..*/\1/' > "${script_dir}/cluster-version" echo "creating cilium ns" kubectl create ns cilium || true echo "deleting terminating namespaces" -./delete-terminating-namespaces.sh +${script_dir}/delete-terminating-namespaces.sh echo "scaling $cluster to 2" -yes | gcloud container clusters resize $cluster --node-pool default-pool --num-nodes 2 --zone $GKE_ZONE +node_pools=($(gcloud container node-pools list --project "${project}" --region "${region}" --cluster "${cluster_uri}" --uri)) +if [ "${#node_pools[@]}" -ne 1 ] ; then + echo "expected 1 node pool, found ${#node_pools[@]}" + exit 1 +fi + +gcloud container clusters resize --project "${project}" --region "${region}" --node-pool "${node_pools[1]}" --num-nodes 2 --quiet "${cluster_uri}" echo "labeling nodes" index=1 @@ -46,5 +63,5 @@ do done echo "adding node registry as trusted" -helm template registry-adder ../k8sT/manifests/registry-adder-gke --set IP="$(../print-node-ip.sh)" > registry-adder.yaml -kubectl apply -f registry-adder.yaml +helm template registry-adder "${test_dir}/k8sT/manifests/registry-adder-gke" --set IP="$(${test_dir}/print-node-ip.sh)" > "${script_dir}/registry-adder.yaml" +kubectl apply -f "${script_dir}/registry-adder.yaml"