For introduce net-watchdog for dpdk case.

This container periodically checks that vhost0 is up and if it is check that it has IP, if there is no IP then it assign it. This is a qorkaround for now because vhost0 can dissappear any time if dpdk container restarts. For non-dpdk case the vhost0 init actions are done in agent container. Change-Id: I5051b7c1cf123aee0f2b875be1e6c59b5c736486 Partial-Bug: #1743610
Juniper · Feb 15, 2018 · 3c29a32 · 3c29a32
1 parent 6c986ad
commit 3c29a32
Show file tree

Hide file tree

Showing 16 changed files with 445 additions and 227 deletions.
diff --git a/README.md b/README.md
@@ -129,3 +129,5 @@ Please refer to the README-HELM.md
 43. ~~vrouter agent entrypoint: add ability to configure metadata_proxy_secret=contrail in [METADATA]~~
 44. Sort out with KUBERNETES_public_fip_pool
 45. ~~Ubuntu containers~~
+46. Introduce base container for agent and move agent specific functions to it
+47. Remove net-watchdog contrail as the problem with vhost0 rest by agent be solved (dpdk case)
diff --git a/common.env.sample b/common.env.sample
@@ -156,7 +156,7 @@
 
 # ---
 # Agent mode. For DPDK case should be set to 'dpdk'.
-#AGENT_MODE=${AGENT_MODE:-'vrouter'}
+#AGENT_MODE=${AGENT_MODE:-'nic'}
 
 # ---
 # Cpu coremask, it is used for DPDK mode for cores pinning

diff --git a/containers/agent/net-watchdog/Dockerfile b/containers/agent/net-watchdog/Dockerfile
@@ -0,0 +1,18 @@
+ARG CONTRAIL_REGISTRY=localhost:5000
+ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
+FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
+
+RUN yum install -y \
+        ethtool pciutils initscripts NetworkManager \
+        contrail-vrouter-utils \
+    yum clean all && \
+    rm -rf /var/cache/yum && \
+    ldconfig
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+LABEL net.juniper.nodemgr.filter.name=contrail-vrouter-network-init
+LABEL net.juniper.contrail=agent
+LABEL net.juniper.node=agent
diff --git a/containers/agent/net-watchdog/entrypoint.sh b/containers/agent/net-watchdog/entrypoint.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+source /common.sh
+source /agent-functions.sh
+
+TRACK_VHOST0=${TRACK_VHOST0:-'true'}
+TRACK_VHOST0_PAUSE=${TRACK_VHOST0_PAUSE:-5}
+
+while (true) ; do
+
+    echo "INFO: ip address show:"
+    ip address show
+
+    init_vhost0
+
+    if [[ "$TRACK_VHOST0" != 'true' ]] ; then
+        echo "INFO: TRACK_VHOST0 is $TRACK_VHOST0. Stop stracking."
+        break
+    fi
+    echo "INFO: wait for $TRACK_VHOST0_PAUSE seconds..."
+    sleep $TRACK_VHOST0_PAUSE
+done
+
+exec $@
diff --git a/containers/agent/vrouter-dpdk/entrypoint.sh b/containers/agent/vrouter-dpdk/entrypoint.sh
@@ -3,13 +3,25 @@
 source /common.sh
 
 echo "INFO: agent started in $AGENT_MODE mode"
-IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac)
-pci_address=$(get_pci_address_for_nic $phys_int)
+
+function assert_file() {
+    local file=$1
+    if [[ ! -f "$file" ]] ; then
+        echo "ERROR: there is no file $file"
+        exit -1
+    fi
+}
+
+phys_int=`get_default_physical_iface`
+binding_data_dir='/var/run/vrouter'
+assert_file "$binding_data_dir/${phys_int}_mac"
+phys_int_mac=`cat "$binding_data_dir/${phys_int}_mac"`
+assert_file "$binding_data_dir/${phys_int}_pci"
+pci_address=`cat "$binding_data_dir/${phys_int}_pci"`
 echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address"
 
-VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic $phys_int`}
-vrouter_cidr=$(get_cidr_for_nic $phys_int)
-echo "INFO: $phys_int cidr $vrouter_cidr, gateway $VROUTER_GATEWAY"
+# ensure device is bind to dpdk driver
+wait_device_for_driver $DPDK_UIO_DRIVER $pci_address
 
 # TODO: consider to avoid taskset here and leave to manage by Docker
 cmd="$@"
@@ -33,23 +45,7 @@ mkdir -p -m 777 /var/crashes
 rm -f '/run/.rte_config'
 
 # set maximum socket buffer size to (max hold flows entries * 9160 bytes)
-sysctl -w net.core.wmem_max=9160000
-
-# bind iface to dpdk uio driver before start dpdk agent
-bind_dev_to_driver $DPDK_UIO_DRIVER $phys_int
-
-function background_init() {
-    wait_dpdk_agent_start
-    if [[ -n $cmd ]] ; then
-        local pname=`echo $cmd | cut -d ' ' -f 1`
-        enable_hugepages_to_coredump "$pname"
-    fi
-    create_vhost0_dpdk $phys_int $phys_int_mac $vrouter_cidr $VROUTER_GATEWAY
-}
-
-# TODO: tart backgound job to enable hugepase to coredump
-# and initialize vhost0
-background_init &
+set_ctl net.core.wmem_max 9160000
 
 echo "INFO: exec '$real_cmd'"
 exec $real_cmd
diff --git a/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos b/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos
@@ -2,7 +2,10 @@ ARG CONTRAIL_REGISTRY=localhost:5000
 ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
 FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
 
-RUN yum clean all -y && \
+RUN yum install -y \
+        ethtool pciutils sysvinit-tools \
+        contrail-vrouter-utils contrail-vrouter-dpdk-init && \
+    yum clean all -y && \
     rm -rf /var/cache/yum
 
 COPY entrypoint.sh /

diff --git a/containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh b/containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh
@@ -1,60 +1,15 @@
-#!/bin/bash -x
+#!/bin/bash
 
 source /common.sh
 
 HUGE_PAGES_DIR=${HUGE_PAGES_DIR:-'/dev/hugepages'}
-if [[ ! -d "$HUGE_PAGES_DIR" ]] ; then
-    echo "WARNING: There is no $HUGE_PAGES_DIR mounted from host. Try to create and mount hugetlbfs."
-    if ! mkdir -p $HUGE_PAGES_DIR ; then
-        echo "ERROR: failed to create $HUGE_PAGES_DIR"
-        exit -1
-    fi
-    if ! mount -t hugetlbfs hugetlbfs $HUGE_PAGES_DIR ; then
-        echo "ERROR: failed to mount hugetlbfs to $HUGE_PAGES_DIR"
-        exit -1
-    fi
-fi
-
-if [[ ! -d "$HUGE_PAGES_DIR" ]]  ; then
-    echo "ERROR: There is no $HUGE_PAGES_DIR. Probably HugeTables are anuvailable on the host."
-    exit -1
-fi
-
-function set_ctl() {
-    local var=$1
-    local value=$2
-    if grep -q "^$var" /etc/sysctl.conf ; then
-        sed -i "s/^$var.*=.*/$var=$value/g"  /etc/sysctl.conf
-    else
-        echo "$var=$value" >> /etc/sysctl.conf
-    fi
-}
+ensure_hugepages $HUGE_PAGES_DIR
 
 set_ctl vm.nr_hugepages ${HUGE_PAGES}
 set_ctl vm.max_map_count 128960
 set_ctl net.ipv4.tcp_keepalive_time 5
 set_ctl net.ipv4.tcp_keepalive_probes 5
 set_ctl net.ipv4.tcp_keepalive_intvl 1
-sysctl --system
-
-function load_kernel_module() {
-    local module=$1
-    shift 1
-    local opts=$@
-    echo "INFO: load $module kernel module"
-    if ! modprobe -v "$module" $opts ; then
-        echo "ERROR: failed to load $module driver"
-        exit -1
-    fi
-}
-
-function unload_kernel_module() {
-    local module=$1
-    echo "INFO: unload $module kernel module"
-    if ! rmmod $module ; then
-        echo "WARNING: Failed to unload $module driver"
-    fi
-}
 
 load_kernel_module uio
 load_kernel_module "$DPDK_UIO_DRIVER"
@@ -64,4 +19,31 @@ if ! is_ubuntu_xenial && ! is_centos; then
     load_kernel_module rte_kni kthread_mode=multiple
 fi
 
-exec "$@"
+echo "INFO: agent $AGENT_MODE mode"
+IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac)
+pci_address=$(get_pci_address_for_nic $phys_int)
+default_gw_metric=`get_default_gateway_for_nic_metric $phys_int`
+echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address"
+
+# save data for next usage in network init container
+# TODO: check that data valid for the case if container is re-run again by some reason
+addrs=$(get_ips_for_nic $phys_int)
+gateway=${VROUTER_GATEWAY:-"$default_gw_metric"}
+binding_data_dir='/var/run/vrouter'
+mkdir -p $binding_data_dir
+echo "INFO: addrs=[$addrs], gateway=$gateway"
+echo "$phys_int" > $binding_data_dir/${phys_int}_nic
+echo "$phys_int_mac" > $binding_data_dir/${phys_int}_mac
+echo "$pci_address" > $binding_data_dir/${phys_int}_pci
+echo "$addrs" > $binding_data_dir/${phys_int}_ip_addresses
+echo "$gateway" > $binding_data_dir/${phys_int}_gateway
+
+if [[ "$phys_int" == "vhost0" ]] ; then
+    echo "ERROR: it is not expected the vhost0 is up and running"
+    exit -1
+fi
+
+# bind iface to dpdk uio driver before start dpdk agent
+bind_devs_to_driver $DPDK_UIO_DRIVER $phys_int
+
+exec $@
diff --git a/containers/agent/vrouter-init-kernel/Dockerfile.centos b/containers/agent/vrouter-init-kernel/Dockerfile.centos
@@ -2,7 +2,7 @@ ARG CONTRAIL_REGISTRY=localhost:5000
 ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
 FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
 
-RUN yum install -y contrail-vrouter contrail-vrouter-utils initscripts dhclient && \
+RUN yum install -y contrail-vrouter contrail-vrouter-utils && \
     yum clean all && \
     rm -rf /var/cache/yum
 

diff --git a/containers/agent/vrouter-init-kernel/entrypoint.sh b/containers/agent/vrouter-init-kernel/entrypoint.sh
@@ -55,88 +55,4 @@ else
   fi
 fi
 
-# VRouter specific code starts here
-function pkt_setup () {
-    for f in /sys/class/net/$1/queues/rx-*
-    do
-        q="$(echo $f | cut -d '-' -f2)"
-        r=$(($q%32))
-        s=$(($q/32))
-        ((mask=1<<$r))
-        str=(`printf "%x" $mask`)
-        if [ $s -gt 0 ]; then
-            for ((i=0; i < $s; i++))
-            do
-                str+=,00000000
-            done
-        fi
-        echo $str > $f/rps_cpus
-    done
-    ip link set dev $1 up
-}
-
-function insert_vrouter() {
-    if [ -f /sys/class/net/pkt1/queues/rx-0/rps_cpus ]; then
-        pkt_setup pkt1
-    fi
-    if [ -f /sys/class/net/pkt2/queues/rx-0/rps_cpus ]; then
-        pkt_setup pkt2
-    fi
-    if [ -f /sys/class/net/pkt3/queues/rx-0/rps_cpus ]; then
-        pkt_setup pkt3
-    fi
-    vif --create vhost0 --mac $phys_int_mac
-    vif --add $phys_int --mac $phys_int_mac --vrf 0 --vhost-phys --type physical
-    vif --add vhost0 --mac $phys_int_mac --vrf 0 --type vhost --xconnect $phys_int
-    return 0
-}
-
-echo "INFO: ip address show:"
-ip address show
-
-IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac)
-echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac"
-
-# Probe vhost0
-vrouter_cidr="$(get_cidr_for_nic vhost0)"
-
-if [[ -e /etc/sysconfig/network-scripts/ifcfg-${phys_int} && ! -e /etc/sysconfig/network-scripts/ifcfg-vhost0 ]]; then
-    echo "INFO: creating vhost0"
-    insert_vrouter
-    ifdown ${phys_int}
-    cp -f /etc/sysconfig/network-scripts/ifcfg-${phys_int} /etc/sysconfig/network-scripts/ifcfg-vhost0
-    sed -i "s/${phys_int}/vhost0/g" /etc/sysconfig/network-scripts/ifcfg-vhost0
-    sed -ri "/(DEVICE|ONBOOT|NM_CONTROLLED)/! s/.*/#commented_by_contrail& /" /etc/sysconfig/network-scripts/ifcfg-${phys_int}
-    ifup ${phys_int}
-    ifup vhost0
-    while IFS= read -r line
-    do
-      ip route del $line
-    done < <(ip route sh |grep ${phys_int})
-elif [[ "$vrouter_cidr" == '' ]] ; then
-    echo "INFO: creating vhost0"
-    addrs=$(ip addr show dev $phys_int | grep "inet" | grep -oP "[0-9a-f\:\.]*/[0-9]* brd [0-9\.]*|[0-9a-f\:\.]*/[0-9]*")
-    default_gw=`ip route show dev $phys_int | grep default | head -n 1 | awk '{print $3}'`
-    default_gw_metric=`ip route show dev $phys_int | grep default | head -1 | grep -o "metric [0-9]*"`
-    VROUTER_GATEWAY=${VROUTER_GATEWAY:-"$default_gw $default_gw_metric"}
-    insert_vrouter
-
-    # TODO: switch off dhcp on phys_int
-    echo "INFO: Changing physical interface to vhost in ip table"
-    ip link set vhost0 up
-    echo "$addrs" | while IFS= read -r line ; do
-        echo "Processing $line"
-        addr_to_del=`echo $line | cut -d ' ' -f 1`
-        addr_to_add=`echo $line | sed 's/brd/broadcast/'`
-        ip address delete $addr_to_del dev $phys_int
-        ip address add $addr_to_add dev vhost0
-        if [[ -n "$VROUTER_GATEWAY" ]]; then
-            echo "INFO: set default gateway"
-            ip route add default via $VROUTER_GATEWAY
-        fi
-    done
-else
-    echo "INFO: vhost0 is already up"
-fi
-
 exec $@
diff --git a/containers/agent/vrouter/Dockerfile.centos b/containers/agent/vrouter/Dockerfile.centos
@@ -2,7 +2,8 @@ ARG CONTRAIL_REGISTRY=localhost:5000
 ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
 FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
 
-RUN yum install -y ethtool contrail-vrouter-agent contrail-vrouter-utils \
+RUN yum install -y ethtool \
+      contrail-vrouter-agent contrail-vrouter-utils \
       sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \
     yum clean all && \
     rm -rf /var/cache/yum && \