Add script to use gcloud to print GCP resources used, and call in Jenkins runs

cluster/gce/list-resources.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# Copyright 2015 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Calls gcloud to print out a variety of Google Cloud Platform resources used by
+# Kubernetes. Can be run before/after test runs and compared to track leaking
+# resources.
+
+# PROJECT must be set in the environment.
+# If ZONE, KUBE_GCE_INSTANCE_PREFIX, CLUSTER_NAME, KUBE_GCE_NETWORK, or
+# KUBE_GKE_NETWORK is set, they will be used to filter the results.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ZONE=${ZONE:-}
+REGION=${ZONE%-*}
+INSTANCE_PREFIX=${KUBE_GCE_INSTANCE_PREFIX:-${CLUSTER_NAME:-}}
+NETWORK=${KUBE_GCE_NETWORK:-${KUBE_GKE_NETWORK:-}}
+
+# Usage: gcloud-compute-list <resource> <additional parameters to gcloud...>
+# GREP_REGEX is applied to the output of gcloud if set
+GREP_REGEX=""
+function gcloud-compute-list() {
+  local -r resource=$1
+  echo -e "\n\n[ ${resource} ]"
+  gcloud compute ${resource} list --project=${PROJECT} ${@:2} | grep "${GREP_REGEX}"
+}
+
+echo "Project: ${PROJECT}"
+echo "Region: ${REGION}"
+echo "Zone: ${ZONE}"
+echo "Instance prefix: ${INSTANCE_PREFIX:-}"
+echo "Network: ${NETWORK:-}"
+
+# List resources related to instances, filtering by the instance prefix if
+# provided.
+gcloud-compute-list instance-templates --regexp="${INSTANCE_PREFIX}.*"
+gcloud-compute-list instance-groups ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*"
+gcloud-compute-list instances ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*"
+
+# List disk resources, filterying by instance prefix if provided.
+gcloud-compute-list disks ${ZONE:+"--zone=${ZONE}"} --regexp="${INSTANCE_PREFIX}.*"
+
+# List network resources. We include names starting with "a", corresponding to
+# those that Kubernetes creates.
+gcloud-compute-list addresses ${REGION:+"--region=${REGION}"} --regexp="a.*|${INSTANCE_PREFIX}.*"
+# Match either the header or a line with the specified e2e network.
+# This assumes that the network name is the second field in the output.
+GREP_REGEX="^NAME\|^[^\s]\+\s\+\(default\|${NETWORK}\)\s"
+gcloud-compute-list routes --regexp="default.*|${INSTANCE_PREFIX}.*"
+gcloud-compute-list firewall-rules --regexp="default.*|k8s-fw.*|${INSTANCE_PREFIX}.*"
+GREP_REGEX=""
+gcloud-compute-list forwarding-rules ${REGION:+"--region=${REGION}"}
+gcloud-compute-list target-pools ${REGION:+"--region=${REGION}"}

hack/jenkins/e2e.sh

@@ -807,6 +807,9 @@ cd kubernetes
 ARTIFACTS=${WORKSPACE}/_artifacts
 mkdir -p ${ARTIFACTS}
 export E2E_REPORT_DIR=${ARTIFACTS}
+declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt"
+declare -r gcp_resources_cluster_up="${ARTIFACTS}/gcp-resources-cluster-up.txt"
+declare -r gcp_resources_after="${ARTIFACTS}/gcp-resources-after.txt"
 
 ### Pre Set Up ###
 # Install gcloud from a custom path if provided. Used to test GKE with gcloud
@@ -826,8 +829,14 @@ fi
 ### Set up ###
 if [[ "${E2E_UP,,}" == "true" ]]; then
     go run ./hack/e2e.go ${E2E_OPT} -v --down
+    if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then
+      ./cluster/gce/list-resources.sh > "${gcp_resources_before}"
+    fi
     go run ./hack/e2e.go ${E2E_OPT} -v --up
     go run ./hack/e2e.go -v --ctl="version --match-server-version=false"
+    if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then
+      ./cluster/gce/list-resources.sh > "${gcp_resources_cluster_up}"
+    fi
 fi
 
 ### Run tests ###
@@ -862,14 +871,6 @@ if [[ "${USE_KUBEMARK:-}" == "true" ]]; then
   unset NUM_MINIONS_BKP
 fi
 
-# TODO(zml): We have a bunch of legacy Jenkins configs that are
-# expecting junit*.xml to be in ${WORKSPACE} root and it's Friday
-# afternoon, so just put the junit report where it's expected.
-# If link already exists, non-zero return code should not cause build to fail.
-for junit in ${ARTIFACTS}/junit*.xml; do
-  ln -s -f ${junit} ${WORKSPACE} || true
-done
-
 ### Clean up ###
 if [[ "${E2E_DOWN,,}" == "true" ]]; then
     # Sleep before deleting the cluster to give the controller manager time to
@@ -880,4 +881,11 @@ if [[ "${E2E_DOWN,,}" == "true" ]]; then
     # for the wait between attempts.
     sleep 30
     go run ./hack/e2e.go ${E2E_OPT} -v --down
+    if [[ ${KUBERNETES_PROVIDER} == "gce" || ${KUBERNETES_PROVIDER} == "gke" ]]; then
+      ./cluster/gce/list-resources.sh > "${gcp_resources_after}"
+    fi
+fi
+
+if [[ -f "${gcp_resources_before}" && -f "${gcp_resources_after}" ]]; then
+  diff -sw -U0 -F'^\[.*\]$' "${gcp_resources_before}" "${gcp_resources_after}" || true
 fi