Wait for first rabbitmq node

It is to avoid race-condition
during autoclustering.

Change-Id: I1dac81833dcee836abb8e7f6cbf3d366224d7552
Closes-Bug: #1764925
(cherry picked from commit c701bfc)

containers/external/rabbitmq/contrail-entrypoint.sh

@@ -3,6 +3,7 @@
 # In all in one deployment there is the race between vhost0 initialization
 # and own IP detection, so there is 10 retries
 for i in {1..10} ; do
+  server_names_list=()
   cluster_nodes='{['
   local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
   IFS=',' read -ra server_list <<< "${RABBITMQ_NODES}"
@@ -23,6 +24,7 @@ for i in {1..10} ; do
       echo "WARNING: hostname for $server is not resolved properly, cluster setup will not be functional."
     fi
     cluster_nodes+="'contrail@${server_hostname}',"
+    server_names_list=($server_names_list $server_hostname)
     if [[ "$local_ips" =~ ",$server," ]] ; then
       my_ip=$server
       my_node=$server_hostname
@@ -46,14 +48,38 @@ if [[ "$RABBITMQ_NODE_PORT" != '' ]] ; then
   export RABBITMQ_NODE_PORT=${RABBITMQ_NODE_PORT}
 fi
 if (( ${#server_list[@]} > 1 )); then
-  export RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS="-rabbit cluster_nodes $cluster_nodes"
+  RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=${RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS:-}
+  RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS+="-rabbit cluster_nodes $cluster_nodes"
+  export RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS
 fi
 
 echo "INFO: RABBITMQ_NODENAME=$RABBITMQ_NODENAME, RABBITMQ_NODE_PORT=$RABBITMQ_NODE_PORT"
 echo "INFO: RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=$RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS"
-echo "INFO: /docker-entrypoint.sh $@"
 
 # to be able to run rabbitmqctl without params
 echo "RABBITMQ_NODENAME=contrail@$my_node" > /etc/rabbitmq/rabbitmq-env.conf
 
+# save cookie to file (otherwise rabbitmqctl set it to wrong value)
+if [[ -n "$RABBITMQ_ERLANG_COOKIE" ]] ; then
+  cookie_file="/var/lib/rabbitmq/.erlang.cookie"
+  echo $RABBITMQ_ERLANG_COOKIE > $cookie_file
+  chmod 600 $cookie_file
+  chown rabbitmq:rabbitmq $cookie_file
+fi
+
+# It looks that there is a race in rabbitmq auto-cluster discovery that
+# leads to a split-brain on cluster start and cluster is setup
+# incorrectly - some of nodes decide to form own cluster instead to join to others.
+if [[ "${server_names_list[0]}" != "$my_node" ]] ; then
+  echo "INFO: delay node $my_node start until first node starts: ${i}/10..."
+  for i in {1..20} ; do
+    sleep 3
+    echo "INFO: check if the node contrail@${server_names_list[0]} started: ${i}/20..."
+    if rabbitmqctl -q -n contrail@${server_names_list[0]} cluster_status ; then
+      break
+    fi
+  done
+fi
+
+echo "INFO: $(date): /docker-entrypoint.sh $@"
 exec /docker-entrypoint.sh "$@"