test: Improve gke/{select,release}-cluster.sh scripts

- use explicit paths
- make better use of `gcloud` features, reduce external
  dependencies
- replace `get-cluster-version.sh` with a version file
- use region instead of zone to enable regional spread
  and avoid availibity issue due to a zonal outage;
  namely we can create clusters in any region/zone and
  the Jenkins job is not tied into any particular one
- list node pools instead of assuming default pool

Signed-off-by: Ilya Dmitrichenko <errordeveloper@gmail.com>

jenkinsfiles/ginkgo-gke.Jenkinsfile

@@ -14,7 +14,6 @@ pipeline {
         TESTDIR="${WORKSPACE}/${PROJ_PATH}/test"
         GOPATH="${WORKSPACE}"
         GKE_KEY=credentials('gke-key')
-        GKE_ZONE="us-west1-a"
         TAG="${GIT_COMMIT}"
         HOME="${WORKSPACE}"
     }
@@ -138,7 +137,7 @@ pipeline {
                         )}"""
                 K8S_VERSION= """${sh(
                         returnStdout: true,
-                        script: 'cd ${TESTDIR}; gke/get-cluster-version.sh'
+                        script: 'cat ${TESTDIR}/gke/cluster-version'
                         )}"""
                 FOCUS= """${sh(
                         returnStdout: true,

test/gke/release-cluster.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
 
-export KUBECONFIG=gke-kubeconfig
-cluster=$(cat cluster-name)
+test_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+project="cilium-ci"
+# this is only needs to be set as some of gcloud commands requires it,
+# but as this script uses resource URIs clusters in all locations are
+# going to be discovered and used
+region="us-west2"
+
+export KUBECONFIG="${script_dir}/gke-kubeconfig"
+cluster_uri="$(cat "${script_dir}/cluster-uri")"
 
 # Create a function to unlock the cluster. We then execute this on script exit.
 # This should occur even if the script is interrupted, by a jenkins timeout,
 # for example.
 unlock() {    
-    echo "releasing cluster lock from $cluster"
+    echo "releasing cluster lock from ${cluster_uri}"
     kubectl annotate deployment lock lock-
 }
 trap unlock EXIT
@@ -24,7 +33,13 @@ echo "deleting terminating namespaces"
 
 set -e
 
-echo "scaling $cluster ng to 0"
-yes | gcloud container clusters resize $cluster --node-pool default-pool --num-nodes 0 --zone $GKE_ZONE
+echo "scaling ${cluster_uri} to 0"
+node_pools=($(gcloud container node-pools list --project "${project}" --region "${region}" --cluster "${cluster_uri}" --uri))
+if [ "${#node_pools[@]}" -ne 1 ] ; then
+  echo "expected 1 node pool, found ${#node_pools[@]}"
+  exit 1
+fi
+
+gcloud container clusters resize --project "${project}" --region "${region}" --node-pool "${node_pools[1]}" --num-nodes 2 --quiet "${cluster_uri}"
 
-rm -f cluster-name
+rm -f "${script_dir}/cluster-uri" "${script_dir}/cluster-name" "${script_dir}/cluster-version"

test/gke/select-cluster.sh

@@ -1,41 +1,58 @@
 #!/bin/bash
 
+test_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+project="cilium-ci"
+# this is only needs to be set as some of gcloud commands requires it,
+# but as this script uses resource URIs clusters in all locations are
+# going to be discovered and used
+region="us-west2"
+
 set -e
 
 locked=1
 
-export KUBECONFIG=gke-kubeconfig
+export KUBECONFIG="${script_dir}/gke-kubeconfig"
 
 while [ $locked -ne 0 ]; do
-    rm gke-kubeconfig || true
+    rm -f "${KUBECONFIG}"
     echo "selecting random cluster"
-    cluster=$(gcloud container clusters list --zone $GKE_ZONE | grep cilium-ci | sort -R | head -n 1 | awk '{print $1}')
+    cluster_uri="$(gcloud container clusters list --project "${project}" --filter="name ~ ^cilium-ci-" --uri | sort -R | head -n 1)"
 
-    echo "getting kubeconfig for $cluster"
-    gcloud container clusters get-credentials --zone $GKE_ZONE $cluster
+    echo "getting kubeconfig for ${cluster_uri} (will store in ${KUBECONFIG})"
+    gcloud container clusters get-credentials --project "${project}" --region "${region}" "${cluster_uri}"
 
     echo "aquiring cluster lock"
     set +e
-    kubectl create -f lock.yaml
+    kubectl create -f "${script_dir}/lock.yaml"
 
     kubectl annotate deployment lock lock=1
     locked=$?
     echo $locked
     set -e
 done
 
-echo "lock acquired on cluster $cluster"
-# cluster-name is used in get-cluster-version.sh, which runs after this in CI.
-echo $cluster > cluster-name
+echo "lock acquired on cluster ${cluster_uri}"
+echo "${cluster_uri}" > "${script_dir}/cluster-uri"
+gcloud container clusters describe --project "${project}" --region "${region}" --format='value(name)' "${cluster_uri}" > "${script_dir}/cluster-name"
+gcloud container clusters describe --project "${project}" --region "${region}" --format='value(currentMasterVersion)' "${cluster_uri}" \
+  | sed -E 's/([0-9]+\.[0-9]+)\..*/\1/' > "${script_dir}/cluster-version"
 
 echo "creating cilium ns"
 kubectl create ns cilium || true
 
 echo "deleting terminating namespaces"
-./delete-terminating-namespaces.sh
+${script_dir}/delete-terminating-namespaces.sh
 
 echo "scaling $cluster to 2"
-yes | gcloud container clusters resize $cluster --node-pool default-pool --num-nodes 2 --zone $GKE_ZONE
+node_pools=($(gcloud container node-pools list --project "${project}" --region "${region}" --cluster "${cluster_uri}" --uri))
+if [ "${#node_pools[@]}" -ne 1 ] ; then
+  echo "expected 1 node pool, found ${#node_pools[@]}"
+  exit 1
+fi
+
+gcloud container clusters resize --project "${project}" --region "${region}" --node-pool "${node_pools[1]}" --num-nodes 2 --quiet "${cluster_uri}"
 
 echo "labeling nodes"
 index=1
@@ -46,5 +63,5 @@ do
 done
 
 echo "adding node registry as trusted"
-helm template registry-adder ../k8sT/manifests/registry-adder-gke --set IP="$(../print-node-ip.sh)" > registry-adder.yaml
-kubectl apply -f registry-adder.yaml
+helm template registry-adder "${test_dir}/k8sT/manifests/registry-adder-gke" --set IP="$(${test_dir}/print-node-ip.sh)" > "${script_dir}/registry-adder.yaml"
+kubectl apply -f "${script_dir}/registry-adder.yaml"