SSL related fixes.

- wait till cert files appear, because
node-init may not finished before a container starts
- provision vrouter with host ip taken from vhost0
- non empty default value for SERVER_CA_CERTFILE
- retries for detecting own IP in 3rd party containers,
in case of all-in-one there is a race with vhost0 initialization
(at this moment there is no own ip detected)

Change-Id: Id08030a70b859b660ef9843a37256b2326176963
Partial-Bug: #1760051

containers/base/common.sh

@@ -27,9 +27,7 @@ SSL_ENABLE=${SSL_ENABLE:-False}
 SSL_INSECURE=${SSL_INSECURE:-True}
 SERVER_CERTFILE=${SERVER_CERTFILE:-'/etc/contrail/ssl/certs/server.pem'}
 SERVER_KEYFILE=${SERVER_KEYFILE:-'/etc/contrail/ssl/private/server-privkey.pem'}
-# Used self-signed by default
-#SERVER_CA_CERTFILE=${SERVER_CA_CERTFILE:-'/etc/contrail/ssl/certs/ca-cert.pem'}
-SERVER_CA_CERTFILE=${SERVER_CA_CERTFILE:-''}
+SERVER_CA_CERTFILE=${SERVER_CA_CERTFILE:-'/etc/contrail/ssl/certs/ca-cert.pem'}
 
 CONTROLLER_NODES=${CONTROLLER_NODES:-${DEFAULT_LOCAL_IP}}
 

containers/base/contrail-functions.sh

@@ -1,15 +1,57 @@
 #!/bin/bash
 
+
+function is_ssl_enabled() {
+  is_enabled "$SSL_ENABLE" \
+   || is_enabled "$XMPP_SSL_ENABLE" \
+   || is_enabled "$INTROSPECT_SSL_ENABLE" \
+   || is_enabled "$SANDESH_SSL_ENABLE"
+}
+
+function wait_files() {
+  local file1=$1
+  local file2=$2
+  local count=0
+  while (true) ; do
+    if [[ -f "$file1" && -f "$file2" ]] ; then
+      return
+    fi
+    (( count+=1 ))
+    if (( count == 60 ))  ; then
+      break
+    fi
+    sleep 1
+  done
+  return 1
+}
+
+function wait_certs_if_ssl_enabled() {
+  if ! is_ssl_enabled ; then
+    return
+  fi
+
+  is_enabled $SSL_ENABLE && wait_files "$SERVER_KEYFILE" "$SERVER_CERTFILE"
+  if [[ "$SERVER_KEYFILE" != "$XMPP_SERVER_CERTFILE" ]] ; then
+    is_enabled $XMPP_SSL_ENABLE && wait_files "$XMPP_SERVER_CERTFILE" "$XMPP_SERVER_KEYFILE"
+  fi
+  if [[ "$SERVER_KEYFILE" != "$INTROSPECT_CERTFILE" ]] ; then
+    is_enabled $INTROSPECT_SSL_ENABLE && wait_files "$INTROSPECT_CERTFILE" "$INTROSPECT_KEYFILE"
+  fi
+  if [[ "$SERVER_KEYFILE" != "$SANDESH_CERTFILE" ]] ; then
+    is_enabled $SANDESH_SSL_ENABLE && wait_files "$SANDESH_CERTFILE" "$SANDESH_KEYFILE"
+  fi
+}
+
 function pre_start_init() {
-    :
+  wait_certs_if_ssl_enabled
 }
 
 function is_tsn() {
-    [[ $TSN_EVPN_MODE =~ ^[Tt][Rr][Uu][Ee]$ ]]
+  [[ $TSN_EVPN_MODE =~ ^[Tt][Rr][Uu][Ee]$ ]]
 }
 
 function is_dpdk() {
-    test "$AGENT_MODE" == 'dpdk'
+   test "$AGENT_MODE" == 'dpdk'
 }
 
 function set_third_party_auth_config(){

containers/external/cassandra/contrail-entrypoint.sh

@@ -1,32 +1,40 @@
 #!/bin/bash -e
 
-IFS=',' read -ra srv_list <<< "$CASSANDRA_SEEDS"
-local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
-for srv in "${srv_list[@]}"; do
-  if [[ "$local_ips" =~ ",$srv," ]] ; then
-    echo "INFO: found '$srv' in local IPs '$local_ips'"
-    my_ip=$srv
+# In all in one deployment there is the race between vhost0 initialization
+# and own IP detection, so there is 10 retries
+for i in {1..10} ; do
+  my_ip=''
+  IFS=',' read -ra srv_list <<< "$CASSANDRA_SEEDS"
+  local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
+  for srv in "${srv_list[@]}"; do
+    if [[ "$local_ips" =~ ",$srv," ]] ; then
+      echo "INFO: found '$srv' in local IPs '$local_ips'"
+      my_ip=$srv
+      break
+    fi
+  done
+  if [ -n "$my_ip" ]; then
     break
   fi
+  sleep 1
 done
 
-
 if [ -z "$my_ip" ]; then
   echo "ERROR: Cannot find self ips ('$local_ips') in Cassandra nodes ('$CASSANDRA_SEEDS')"
-  exit
+  exit -1
 fi
 
 # use first two servers as seeds
 export CASSANDRA_SEEDS=$(echo $CASSANDRA_SEEDS | cut -d ',' -f 1,2)
 export CASSANDRA_LISTEN_ADDRESS=$my_ip
 
-export JVM_EXTRA_OPTS="-Dcassandra.rpc_port=${CASSANDRA_PORT:-9160} \
+export JVM_EXTRA_OPTS="${JVM_EXTRA_OPTS} -Dcassandra.rpc_port=${CASSANDRA_PORT:-9160} \
   -Dcassandra.native_transport_port=${CASSANDRA_CQL_PORT:-9042} \
   -Dcassandra.ssl_storage_port=${CASSANDRA_SSL_STORAGE_PORT:-7011} \
   -Dcassandra.storage_port=${CASSANDRA_STORAGE_PORT:-7010} \
   -Dcassandra.jmx.local.port=${CASSANDRA_JMX_LOCAL_PORT:-7200}"
 
-echo "INFO: CASSANDRA_SEEDS=$CASSANDRA_SEEDS CASSANDRA_LISTEN_ADDRESS=$CASSANDRA_LISTEN_ADDRESS"
+echo "INFO: CASSANDRA_SEEDS=$CASSANDRA_SEEDS CASSANDRA_LISTEN_ADDRESS=$CASSANDRA_LISTEN_ADDRESS JVM_EXTRA_OPTS=$JVM_EXTRA_OPTS"
 echo "INFO: exec /docker-entrypoint.sh $@"
 
 exec /docker-entrypoint.sh "$@"

containers/external/kafka/docker-entrypoint.sh

@@ -20,14 +20,23 @@ ZOOKEEPER_ANALYTICS_PORT=${ZOOKEEPER_ANALYTICS_PORT:-2182}
 : ${KAFKA_LISTEN_ADDRESS='auto'}
 my_index=1
 if [ "$KAFKA_LISTEN_ADDRESS" = 'auto' ]; then
-  IFS=',' read -ra server_list <<< "$KAFKA_NODES"
-  for server in "${server_list[@]}"; do
-    if [[ "$local_ips" =~ ",$server," ]] ; then
-      echo "INFO: found '$server' in local IPs '$local_ips'"
-      my_ip=$server
+  # In all in one deployment there is the race between vhost0 initialization
+  # and own IP detection, so there is 10 retries
+  for i in {1..10} ; do
+    my_ip=''
+    IFS=',' read -ra server_list <<< "$KAFKA_NODES"
+    for server in "${server_list[@]}"; do
+      if [[ "$local_ips" =~ ",$server," ]] ; then
+        echo "INFO: found '$server' in local IPs '$local_ips'"
+        my_ip=$server
+        break
+      fi
+      (( my_index+=1 ))
+    done
+    if [ -n "$my_ip" ]; then
       break
     fi
-    (( my_index+=1 ))
+    sleep 1
   done
 
   if [ -z "$my_ip" ]; then

containers/external/rabbitmq/contrail-entrypoint.sh

@@ -1,36 +1,44 @@
 #!/bin/bash -e
 
-cluster_nodes='{['
-local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
-IFS=',' read -ra server_list <<< "${RABBITMQ_NODES}"
-my_ip=''
-my_node=''
-rabbit_node_list=''
-for server in ${server_list[@]}; do
-  server_hostname=''
-  if getent hosts $server ; then
-    server_hostname=$(getent hosts $server | awk '{print $2}' | awk -F '.' '{print $1}')
-  else
-    if host -4 $server ; then
-      server_hostname=$(host -4 $server | cut -d" " -f5 | awk '{print $1}')
-      server_hostname=${server_hostname::-1}
+# In all in one deployment there is the race between vhost0 initialization
+# and own IP detection, so there is 10 retries
+for i in {1..10} ; do
+  cluster_nodes='{['
+  local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
+  IFS=',' read -ra server_list <<< "${RABBITMQ_NODES}"
+  my_ip=''
+  my_node=''
+  rabbit_node_list=''
+  for server in ${server_list[@]}; do
+    server_hostname=''
+    if getent hosts $server ; then
+      server_hostname=$(getent hosts $server | awk '{print $2}' | awk -F '.' '{print $1}')
+    else
+      if host -4 $server ; then
+        server_hostname=$(host -4 $server | cut -d" " -f5 | awk '{print $1}')
+        server_hostname=${server_hostname::-1}
+      fi
     fi
+    if [[ "$server_hostname" == '' ]] ; then
+      echo "WARNING: hostname for $server is not resolved properly, cluster setup will not be functional."
+    fi
+    cluster_nodes+="'contrail@${server_hostname}',"
+    if [[ "$local_ips" =~ ",$server," ]] ; then
+      my_ip=$server
+      my_node=$server_hostname
+      echo "INFO: my_ip=$server my_node=$server_hostname"
+    fi
+  done
+  if [ -n "$my_ip" ] ; then
+    break
   fi
-  if [[ "$server_hostname" == '' ]] ; then
-    echo "WARNING: hostname for $server is not resolved properly, cluster setup will not be functional."
-  fi
-  cluster_nodes+="'contrail@${server_hostname}',"
-  if [[ "$local_ips" =~ ",$server," ]] ; then
-    my_ip=$server
-    my_node=$server_hostname
-    echo "INFO: my_ip=$server my_node=$server_hostname"
-  fi
+  sleep 1
 done
 
 cluster_nodes=${cluster_nodes::-1}'],disc}'
 if [ -z "$my_ip" ] ; then
   echo "ERROR: Cannot find self ips ('$local_ips') in RabbitMQ nodes ('$RABBITMQ_NODES')"
-  exit
+  exit -1
 fi
 
 export RABBITMQ_NODENAME=contrail@$my_node

containers/external/zookeeper/contrail-entrypoint.sh

@@ -3,25 +3,33 @@
 ZOOKEEPER_PORT=${ZOOKEEPER_PORT:-2181}
 ZOOKEEPER_PORTS=${ZOOKEEPER_PORTS:-'2888:3888'}
 
-ord=1
-my_ord=0
-IFS=',' read -ra srv_list <<< "$ZOOKEEPER_NODES"
-local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
-zoo_servers=''
-for srv in "${srv_list[@]}"; do
-  if [[ -z "$ZOO_SERVERS" ]] ; then
-    zoo_servers+="server.${ord}=${srv}:${ZOOKEEPER_PORTS} "
+# In all in one deployment there is the race between vhost0 initialization
+# and own IP detection, so there is 10 retries
+for i in {1..10} ; do
+  ord=1
+  my_ord=0
+  IFS=',' read -ra srv_list <<< "$ZOOKEEPER_NODES"
+  local_ips=",$(cat "/proc/net/fib_trie" | awk '/32 host/ { print f } {f=$2}' | tr '\n' ','),"
+  zoo_servers=''
+  for srv in "${srv_list[@]}"; do
+    if [[ -z "$ZOO_SERVERS" ]] ; then
+      zoo_servers+="server.${ord}=${srv}:${ZOOKEEPER_PORTS} "
+    fi
+    if [[ "$local_ips" =~ ",$srv," ]] ; then
+      echo "INFO: found '$srv' in local IPs '$local_ips'"
+      my_ord=$ord
+    fi
+    ord=$((ord+1))
+  done
+  if (( $my_ord > 0 && $my_ord <= "${#srv_list[@]}" )); then
+    break
   fi
-  if [[ "$local_ips" =~ ",$srv," ]] ; then
-    echo "INFO: found '$srv' in local IPs '$local_ips'"
-    my_ord=$ord
-  fi
-  ord=$((ord+1))
+  sleep 1
 done
 
 if (( $my_ord < 1 || $my_ord > "${#srv_list[@]}" )); then
   echo "ERROR: Cannot find self ips ('$local_ips') in Zookeeper nodes ('$ZOOKEEPER_NODES')"
-  exit
+  exit -1
 fi
 
 # If ZOO_SERVERS is provided then just use it, because it is an interface of

containers/node-init/certs-init.sh

@@ -2,14 +2,20 @@
 
 source /common.sh
 
-if [ ! is_enabled "$SSL_ENABLE" ] \
-   && [ ! is_enabled "$XMPP_SSL_ENABLE" ] \
-   && [ ! is_enabled "$INTROSPECT_SSL_ENABLE" ] \
-   && [ ! is_enabled "$SANDESH_SSL_ENABLE" ] ; then
+if ! is_ssl_enabled ; then
   echo "INFO: No SSL Parameters Enabled, nothing to do"
   exit 0
 fi
 
+FORCE_GENERATE_CERT=${FORCE_GENERATE_CERT:-'false'}
+if [[ -f "$SERVER_CERTFILE" && -f "$SERVER_KEYFILE" ]] ; then
+  if ! is_enabled $FORCE_GENERATE_CERT ; then
+    echo "INFO: cert and key files are already exist"
+    exit 0
+  fi
+  echo "WARNING: cert and key files are already exist, but force generation is set"
+fi
+
 function fail() {
   local msg="$@"
   echo "ERROR: $msg"
@@ -25,19 +31,9 @@ for ip in $(get_local_ips) ; do
   (( alt_name_num+=1 ))
 done
 
-[[ -z "$SERVER_CA_CERTFILE" && -z "$SERVER_CA_KEYFILE" ]] \
-  || [[ -n "$SERVER_CA_CERTFILE" && -n "$SERVER_CA_KEYFILE" ]] \
-  || fail "SERVER_CA_CERTFILE and SERVER_CA_KEYFILE must be either both empty or both set"
-
 working_dir='/tmp/contrail_ssl_gen'
-ca_file="${SERVER_CA_CERTFILE}"
-if [[ -z "${ca_file}" || ! -f "${ca_file}" ]] ; then
-  ca_file="$working_dir/certs/ca.crt.pem"
-fi
-ca_key_file=${SERVER_CA_KEYFILE}
-if [[ -z "${ca_key_file}" || ! -f "${ca_key_file}" ]] ; then
-  ca_key_file="$working_dir/certs/ca.key.pem"
-fi
+ca_file=${SERVER_CA_CERTFILE:-"$working_dir/certs/ca.crt.pem"}
+ca_key_file=${SERVER_CA_KEYFILE:-"$working_dir/certs/ca.key.pem"}
 
 rm -rf $working_dir
 mkdir -p $working_dir/certs
@@ -122,19 +118,23 @@ mkdir -p $(dirname $SERVER_CERTFILE)
 mkdir -p $(dirname $SERVER_KEYFILE)
 
 #generate local self-signed CA if requested
-if [[ ! -f "${ca_file}" && ! -f "${ca_key_file}" ]] ; then
+if [[ ! -f "${ca_key_file}" ]] ; then
   openssl genrsa -out $ca_key_file 4096 || fail "Failed to generate CA key file"
   chmod 600 $ca_key_file || fail "Failed to to chmod 600 on $ca_key_file"
+fi
+if [[ ! -f "${ca_file}" ]] ; then
   openssl req -config $openssl_config_file -new -x509 -days 365 -extensions v3_ca -key $ca_key_file -out $ca_file || fail "Failed to generate CA cert"
   chmod 644 $ca_file || fail "Failed to chmod 644 on $ca_file"
-else
-  [[ -f "${ca_file}" && -f "${ca_key_file}" ]] || fail "'${ca_file}' or '${ca_key_file}' doesnt exist"
 fi
+[[ -f "${ca_file}" && -f "${ca_key_file}" ]] || fail "'${ca_file}' or '${ca_key_file}' doesnt exist"
 
 # generate server certificate
 csr_file="${working_dir}/server.pem.csr"
-openssl genrsa -out $SERVER_KEYFILE 2048 || fail "Failed to generate server key file $SERVER_KEYFILE"
-chmod 600 $SERVER_KEYFILE || fail "Failed to chmod 600 on $SERVER_KEYFILE"
-openssl req -config $openssl_config_file -key $SERVER_KEYFILE -new  -out $csr_file || fail "Failed to create CSR"
-yes | openssl ca -config $openssl_config_file -extensions v3_req -days 365 -in $csr_file -out $SERVER_CERTFILE || fail "Failed to sign certificate"
-chmod 644 $SERVER_CERTFILE || fail "Failed to chmod 644 on $SERVER_CERTFILE"
+openssl genrsa -out ${SERVER_KEYFILE}.tmp 2048 || fail "Failed to generate server key file ${SERVER_KEYFILE}.tmp"
+chmod 600 ${SERVER_KEYFILE}.tmp || fail "Failed to chmod 600 on ${SERVER_KEYFILE}.tmp"
+openssl req -config $openssl_config_file -key ${SERVER_KEYFILE}.tmp -new  -out $csr_file || fail "Failed to create CSR"
+yes | openssl ca -config $openssl_config_file -extensions v3_req -days 365 -in $csr_file -out ${SERVER_CERTFILE}.tmp || fail "Failed to sign certificate"
+chmod 644 ${SERVER_CERTFILE}.tmp || fail "Failed to chmod 644 on ${SERVER_CERTFILE}.tmp"
+
+mv ${SERVER_KEYFILE}.tmp ${SERVER_KEYFILE}
+mv ${SERVER_CERTFILE}.tmp ${SERVER_CERTFILE}

containers/nodemgr/provision.sh

@@ -104,7 +104,17 @@ control)
   ;;
 
 vrouter)
-  host_ip=$(get_default_ip)
+  host_ip=''
+  for _ in {1..30} ; do
+    host_ip=$(get_listen_ip_for_nic vhost0)
+    if [[ -n "$host_ip" ]] ; then
+      break
+    fi
+    sleep 1
+  done
+  if [[ -z "$host_ip" ]] ; then
+    host_ip=$(get_default_ip)
+  fi
   params=''
   if is_dpdk ; then
     params="$params --dpdk_enabled"