Grab images as part of update e2e test

cluster/gce/util.sh

@@ -131,26 +131,23 @@ function kube-up {
   # Detect the project into $PROJECT if it isn't set
   detect-project
 
+  # This will take us up to the git repo root
+  local base_dir=$(dirname "${BASH_SOURCE}")/../..
+
   # Build up start up script for master
   KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX)
-  trap "rm -rf ${KUBE_TEMP}" EXIT
+  trap 'rm -rf "${KUBE_TEMP}"' EXIT
 
   get-password
   echo "Using password: $user:$passwd"
-  python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd
-  HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd)
-
-  (
-    echo "#! /bin/bash"
-    echo "MASTER_NAME=${MASTER_NAME}"
-    echo "MASTER_RELEASE_TAR=${RELEASE_NORMALIZED}/master-release.tgz"
-    echo "MASTER_HTPASSWD='${HTPASSWD}'"
-    grep -v "^#" $(dirname $0)/templates/download-release.sh
-    grep -v "^#" $(dirname $0)/templates/salt-master.sh
-  ) > ${KUBE_TEMP}/master-start.sh
+  python "${base_dir}/third_party/htpasswd/htpasswd.py" -b \
+    -c "${KUBE_TEMP}/htpasswd" $user $passwd
+  HTPASSWD=$(cat "${KUBE_TEMP}/htpasswd")
 
   if ! gcutil getnetwork "${NETWORK}"; then
     echo "Creating new network for: ${NETWORK}"
+    # The network needs to be created synchronously or we have a race. The
+    # firewalls can be added concurrent with instance creation.
     gcutil addnetwork "${NETWORK}" --range "10.240.0.0/16"
     gcutil addfirewall "${NETWORK}-default-internal" \
       --norespect_terminal_width \
@@ -174,6 +171,15 @@ function kube-up {
     --target_tags ${MASTER_TAG} \
     --allowed tcp:443 &
 
+  (
+    echo "#! /bin/bash"
+    echo "MASTER_NAME='${MASTER_NAME}'"
+    echo "MASTER_RELEASE_TAR=${RELEASE_NORMALIZED}/master-release.tgz"
+    echo "MASTER_HTPASSWD='${HTPASSWD}'"
+    grep -v "^#" "${base_dir}/cluster/templates/download-release.sh"
+    grep -v "^#" "${base_dir}/cluster/templates/salt-master.sh"
+  ) > "${KUBE_TEMP}/master-start.sh"
+
   gcutil addinstance ${MASTER_NAME}\
     --norespect_terminal_width \
     --project ${PROJECT} \
@@ -184,14 +190,14 @@ function kube-up {
     --network ${NETWORK} \
     --service_account_scopes="storage-ro,compute-rw" \
     --automatic_restart \
-    --metadata_from_file startup-script:${KUBE_TEMP}/master-start.sh &
+    --metadata_from_file "startup-script:${KUBE_TEMP}/master-start.sh" &
 
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     (
       echo "#! /bin/bash"
-      echo "MASTER_NAME=${MASTER_NAME}"
+      echo "MASTER_NAME='${MASTER_NAME}'"
       echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}"
-      grep -v "^#" $(dirname $0)/templates/salt-minion.sh
+      grep -v "^#" "${base_dir}/cluster/templates/salt-minion.sh"
     ) > ${KUBE_TEMP}/minion-start-${i}.sh
 
     gcutil addfirewall ${MINION_NAMES[$i]}-all \
@@ -212,7 +218,7 @@ function kube-up {
       --service_account_scopes=${MINION_SCOPES} \
       --automatic_restart \
       --can_ip_forward \
-      --metadata_from_file startup-script:${KUBE_TEMP}/minion-start-${i}.sh &
+      --metadata_from_file "startup-script:${KUBE_TEMP}/minion-start-${i}.sh" &
 
     gcutil addroute ${MINION_NAMES[$i]} ${MINION_IP_RANGES[$i]} \
     --norespect_terminal_width \

hack/e2e-suite/update.sh

@@ -19,7 +19,6 @@
 set -o errexit
 set -o nounset
 set -o pipefail
-set -x
 
 source "${KUBE_REPO_ROOT}/cluster/kube-env.sh"
 source "${KUBE_REPO_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh"
@@ -28,42 +27,53 @@ source "${KUBE_REPO_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh"
 CONTROLLER_NAME=update-demo
 
 function validate() {
-  NUM_REPLICAS=$1
-  CONTAINER_IMAGE_VERSION=$2
-  POD_ID_LIST=$($KUBECFG '-template={{range.Items}}{{.ID}} {{end}}' -l simpleService=${CONTROLLER_NAME} list pods)
-  POD_ARR=($POD_ID_LIST)
-  while [ ${#POD_ARR[@]} -ne $NUM_REPLICAS ]; do
-    echo "Waiting for the right number of containers"
-    sleep 5
-    POD_ID_LIST=$($KUBECFG '-template={{range.Items}}{{.ID}} {{end}}' -l simpleService=${CONTROLLER_NAME} list pods)
-    POD_ARR=($POD_ID_LIST)
-  done
+  local num_replicas=$1
+  local container_image_version=$2
 
   # Container turn up on a clean cluster can take a while for the docker image pull.
-  ALL_RUNNING=0
-  while [ $ALL_RUNNING -ne 1 ]; do
-    echo "Waiting for all containers in pod to come up."
-    sleep 5
-    ALL_RUNNING=1
-    for id in $POD_ID_LIST; do
-      TEMPLATE_STRING="{{and ((index .CurrentState.Info \"${CONTROLLER_NAME}\").State.Running) .CurrentState.Info.net.State.Running}}"
-      CURRENT_STATUS=$($KUBECFG -template "${TEMPLATE_STRING}" get pods/$id)
-      if [ "$CURRENT_STATUS" != "{}" ]; then
-        ALL_RUNNING=0
-      else
-        CURRENT_IMAGE=$($KUBECFG -template "{{(index .CurrentState.Info \"${CONTROLLER_NAME}\").DetailInfo.Config.Image}}" get pods/$id)
-        if [ "$CURRENT_IMAGE" != "${DOCKER_HUB_USER}/update-demo:${CONTAINER_IMAGE_VERSION}" ]; then
-          ALL_RUNNING=0
-        fi
+  local num_running=0
+  while [[ $num_running -ne $num_replicas ]]; do
+    echo "Waiting for all containers in pod to come up. Currently: ${num_running}/${num_replicas}"
+    sleep 2
+
+    local pod_id_list
+    pod_id_list=($($KUBECFG -template='{{range.Items}}{{.ID}} {{end}}' -l simpleService="${CONTROLLER_NAME}" list pods))
+
+    echo "  ${#pod_id_list[@]} out of ${num_replicas} created"
+
+    local id
+    num_running=0
+    for id in "${pod_id_list[@]}"; do
+      local template_string current_status current_image host_ip
+      template_string="{{and ((index .CurrentState.Info \"${CONTROLLER_NAME}\").State.Running) .CurrentState.Info.net.State.Running}}"
+      current_status=$($KUBECFG -template="${template_string}" get "pods/$id")
+      if [[ "$current_status" != "{}" ]]; then
+        echo "  $id is created but not running"
+        continue
+      fi
+
+      template_string="{{(index .CurrentState.Info \"${CONTROLLER_NAME}\").DetailInfo.Config.Image}}"
+      current_image=$($KUBECFG -template="${template_string}" get "pods/$id")
+      if [[ "$current_image" != "${DOCKER_HUB_USER}/update-demo:${container_image_version}" ]]; then
+        echo "  ${id} is created but running wrong image"
+        continue
       fi
+
+
+      host_ip=$($KUBECFG -template='{{.CurrentState.HostIP}}' get pods/$id)
+      curl -s --max-time 5 --fail http://${host_ip}:8080/data.json \
+          | grep -q ${container_image_version} || {
+        echo "  ${id} is running the right image but curl to contents failed or returned wrong info"
+        continue
+
+      }
+
+      echo "  ${id} is verified up and running"
+
+      ((num_running++)) || true
     done
   done
-
-  ids=($POD_ID_LIST)
-  if [ ${#ids[@]} -ne $NUM_REPLICAS ]; then
-    echo "Unexpected number of pods: ${#ids[@]}.  Expected $NUM_REPLICAS"
-    exit 1
-  fi
+  return 0
 }
 
 export DOCKER_HUB_USER=jbeda

hack/e2e-test.sh

@@ -20,6 +20,9 @@
 # Use testing config
 export KUBE_CONFIG_FILE="config-test.sh"
 export KUBE_REPO_ROOT="$(dirname $0)/.."
+
+# TODO(jbeda): This will break on usage if there is a space in
+# ${KUBE_REPO_ROOT}.  Covert to an array?  Or an exported function?
 export KUBECFG="${KUBE_REPO_ROOT}/cluster/kubecfg.sh -expect_version_match"
 
 source $(dirname $0)/../cluster/kube-env.sh