From 4ae7d7ccd7869e006c0516dea8dfe548e7396b64 Mon Sep 17 00:00:00 2001
From: Ritesh H Shukla <sritesh@vmware.com>
Date: Fri, 21 Oct 2016 00:42:48 +0000
Subject: [PATCH] Wait for cbr0 configuration to complete before setting up
 routes. - fixes #34248, #31310 - Remove optimizations for salt status check
 till flakyness of install is addressed - fix indentation

---
 cluster/vsphere/config-default.sh |  2 +-
 cluster/vsphere/util.sh           | 51 ++++++++++++++++++++++++-------
 2 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh
index e646dfa5000b..c9fe02985a25 100755
--- a/cluster/vsphere/config-default.sh
+++ b/cluster/vsphere/config-default.sh
@@ -27,7 +27,7 @@ MASTER_MEMORY_MB=1024
 MASTER_CPU=1
 
 NODE_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_NODES}}))
-NODE_IP_RANGES="10.244.0.0/16"
+NODE_IP_RANGES="10.244.0.0/16" # Min Prefix supported is 16
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
 NODE_MEMORY_MB=2048
 NODE_CPU=1
diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh
index e32379670c25..042ac94cc527 100755
--- a/cluster/vsphere/util.sh
+++ b/cluster/vsphere/util.sh
@@ -320,23 +320,52 @@ function setup-pod-routes {
   # identify the subnet assigned to the node by the kubernertes controller manager.
   KUBE_NODE_BRIDGE_NETWORK=()
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-     printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}\n"
-     network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1')
-     KUBE_NODE_BRIDGE_NETWORK+=("${network}")
-  done
+    printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}\n"
+
+    network=""
+    top2_octets_final=$(echo $NODE_IP_RANGES | awk -F "." '{ print $1 "." $2 }') # Assume that a 24 bit mask per node
+
+    attempt=0
+    max_attempt=60
+    while true ; do
+      attempt=$(($attempt+1))
+
+      network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1')
+      top2_octets_read=$(echo $network | awk -F "." '{ print $1 "." $2 }')
+
+      if [[ "$top2_octets_read" == "$top2_octets_final" ]]; then
+        break
+      fi
+
+      if (( $attempt == $max_attempt )); then
+        echo
+        echo "(Failed) Waiting for cbr0 bridge to come up @ ${NODE_NAMES[$i]}"
+        echo
+        exit 1
+      fi
 
+      printf "."
+      sleep 5
+    done
+
+    printf "\n"
+    KUBE_NODE_BRIDGE_NETWORK+=("${network}")
+  done
 
   # Make the pods visible to each other and to the master.
   # The master needs have routes to the pods for the UI to work.
   local j
   for (( i=0; i<${#NODE_NAMES[@]}; i++)); do
-     printf "setting up routes for ${NODE_NAMES[$i]}"
-     kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
-     for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
-        if [[ $i != $j ]]; then
-           kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}"
-        fi
-      done
+    printf "setting up routes for ${NODE_NAMES[$i]}\n"
+    printf " adding route to ${MASTER_NAME} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}\n"
+    kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}"
+    for (( j=0; j<${#NODE_NAMES[@]}; j++)); do
+      if [[ $i != $j ]]; then
+        printf " adding route to ${NODE_NAMES[$j]} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}\n" 
+        kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}"
+      fi
+    done
+    printf "\n"
   done
 }