Initial DPDK support. Single iface.

Change-Id: I67dc67436bd91feabd735f37dc874bf67e6a4ae8
Partial-Bug: #1743610

common.env.sample

@@ -154,6 +154,24 @@
 #   - 16.04     if LINUX_DISTR=ubuntu
 #LINUX_DISTR_VER=
 
+# ---
+# Agent mode. For DPDK case should be set to 'dpdk'.
+#AGENT_MODE=${AGENT_MODE:-'vrouter'}
+
+# ---
+# Cpu coremask, it is used for DPDK mode for cores pinning
+#CPU_CORE_MASK='0x01'
+
+# ---
+# Number of huge pages to be set if provided
+# (optional)
+#HUGE_PAGES=1024
+
+# ---
+# DPDK UIO driver
+#DPDK_UIO_DRIVER='uio_pci_generic'
+
+
 # ---
 # TLS
 #SERVER_CERT=${XMPP_SERVER_CERT:-'/etc/contrail/ssl/certs/server.pem'}

containers/agent/vrouter-dpdk/10-core-pattern.conf

@@ -0,0 +1 @@
+kernel.core_pattern = /var/crashes/core.%e.%p.%h.%t

containers/agent/vrouter-dpdk/Dockerfile.centos

@@ -0,0 +1,20 @@
+ARG CONTRAIL_REGISTRY=localhost:5000
+ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
+FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
+
+RUN yum install -y  ethtool pciutils sysvinit-tools \
+                    contrail-vrouter-dpdk contrail-vrouter-dpdk-init contrail-vrouter-utils \
+                    sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \
+    yum clean all && \
+    rm -rf /var/cache/yum && \
+    ldconfig
+
+COPY 10-core-pattern.conf /etc/sysctl.d/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/usr/bin/contrail-vrouter-dpdk", "--no-daemon", "--socket-mem 1024"]
+
+LABEL net.juniper.nodemgr.filter.name=contrail-vrouter-agent
+LABEL net.juniper.contrail=agent
+LABEL net.juniper.node=agent

containers/agent/vrouter-dpdk/Dockerfile.ubuntu

@@ -0,0 +1,22 @@
+ARG CONTRAIL_REGISTRY=localhost:5000
+ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton
+FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get install -y  ethtool pciutils sysvinit-utils \
+                        contrail-vrouter-dpdk contrail-vrouter-dpdk-init contrail-vrouter-utils \
+                        sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \
+    apt-get autoremove && \
+    apt-get clean && \
+    rm -rf /var/cache/apt && \
+    ldconfig
+
+COPY 10-core-pattern.conf /etc/sysctl.d/
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/usr/bin/contrail-vrouter-dpdk", "--no-daemon", "--socket-mem 1024"]
+
+LABEL net.juniper.nodemgr.filter.name=contrail-vrouter-agent
+LABEL net.juniper.contrail=agent
+LABEL net.juniper.node=agent

containers/agent/vrouter-dpdk/entrypoint.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+source /common.sh
+
+echo "INFO: agent started in $AGENT_MODE mode"
+IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac)
+pci_address=$(get_pci_address_for_nic $phys_int)
+echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address"
+
+VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic $phys_int`}
+vrouter_cidr=$(get_cidr_for_nic $phys_int)
+echo "INFO: $phys_int cidr $vrouter_cidr, gateway $VROUTER_GATEWAY"
+
+# TODO: consider to avoid taskset here and leave to manage by Docker
+cmd="$@"
+real_cmd=$cmd
+if [[ -n "$CPU_CORE_MASK" ]] ; then
+    taskset_param="$CPU_CORE_MASK"
+    if [[ "${CPU_CORE_MASK}" =~ '[,-]' ]]; then
+        taskset_param="-c $CPU_CORE_MASK"
+    fi
+    if is_ubuntu ; then
+        real_cmd="/usr/bin/taskset $taskset_param $cmd"
+    else
+        real_cmd="/bin/taskset $taskset_param $cmd"
+    fi
+fi
+
+mkdir -p -m 777 /var/crashes
+
+# remove rte configuration file (for case if vRouter has crashed)
+# TODO: most probably not needed.. since crash means container re-created
+rm -f '/run/.rte_config'
+
+# set maximum socket buffer size to (max hold flows entries * 9160 bytes)
+sysctl -w net.core.wmem_max=9160000
+
+# bind iface to dpdk uio driver before start dpdk agent
+bind_dev_to_driver $DPDK_UIO_DRIVER $phys_int
+
+function background_init() {
+    wait_dpdk_agent_start
+    if [[ -n $cmd ]] ; then
+        local pname=`echo $cmd | cut -d ' ' -f 1`
+        enable_hugepages_to_coredump "$pname"
+    fi
+    create_vhost0_dpdk $phys_int $phys_int_mac $vrouter_cidr $VROUTER_GATEWAY
+}
+
+# TODO: tart backgound job to enable hugepase to coredump
+# and initialize vhost0
+background_init &
+
+echo "INFO: exec '$real_cmd'"
+exec $real_cmd

containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos

@@ -0,0 +1,13 @@
+ARG CONTRAIL_REGISTRY=localhost:5000
+ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
+FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
+
+RUN yum clean all -y && \
+    rm -rf /var/cache/yum
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+LABEL net.juniper.contrail=vrouter-init-kernel
+LABEL net.juniper.node=agent

containers/agent/vrouter-init-kernel-dpdk/Dockerfile.ubuntu

@@ -0,0 +1,16 @@
+ARG CONTRAIL_REGISTRY=localhost:5000
+ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton
+FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
+
+RUN export DEBIAN_FRONTEND=noninteractive && \ 
+    apt-get install -y kmod && \
+    apt-get autoremove && \
+    apt-get clean && \
+    rm -rf /var/cache/apt
+
+COPY entrypoint.sh /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+LABEL net.juniper.contrail=vrouter-init-kernel
+LABEL net.juniper.node=agent

containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh

@@ -0,0 +1,67 @@
+#!/bin/bash -x
+
+source /common.sh
+
+HUGE_PAGES_DIR=${HUGE_PAGES_DIR:-'/dev/hugepages'}
+if [[ ! -d "$HUGE_PAGES_DIR" ]] ; then
+    echo "WARNING: There is no $HUGE_PAGES_DIR mounted from host. Try to create and mount hugetlbfs."
+    if ! mkdir -p $HUGE_PAGES_DIR ; then
+        echo "ERROR: failed to create $HUGE_PAGES_DIR"
+        exit -1
+    fi
+    if ! mount -t hugetlbfs hugetlbfs $HUGE_PAGES_DIR ; then
+        echo "ERROR: failed to mount hugetlbfs to $HUGE_PAGES_DIR"
+        exit -1
+    fi
+fi
+
+if [[ ! -d "$HUGE_PAGES_DIR" ]]  ; then
+    echo "ERROR: There is no $HUGE_PAGES_DIR. Probably HugeTables are anuvailable on the host."
+    exit -1
+fi
+
+function set_ctl() {
+    local var=$1
+    local value=$2
+    if grep -q "^$var" /etc/sysctl.conf ; then
+        sed -i "s/^$var.*=.*/$var=$value/g"  /etc/sysctl.conf
+    else
+        echo "$var=$value" >> /etc/sysctl.conf
+    fi
+}
+
+set_ctl vm.nr_hugepages ${HUGE_PAGES}
+set_ctl vm.max_map_count 128960
+set_ctl net.ipv4.tcp_keepalive_time 5
+set_ctl net.ipv4.tcp_keepalive_probes 5
+set_ctl net.ipv4.tcp_keepalive_intvl 1
+sysctl --system
+
+function load_kernel_module() {
+    local module=$1
+    shift 1
+    local opts=$@
+    echo "INFO: load $module kernel module"
+    if ! modprobe -v "$module" $opts ; then
+        echo "ERROR: failed to load $module driver"
+        exit -1
+    fi
+}
+
+function unload_kernel_module() {
+    local module=$1
+    echo "INFO: unload $module kernel module"
+    if ! rmmod $module ; then
+        echo "WARNING: Failed to unload $module driver"
+    fi
+}
+
+load_kernel_module uio
+load_kernel_module "$DPDK_UIO_DRIVER"
+if ! is_ubuntu_xenial && ! is_centos; then
+    # multiple kthreads for port monitoring
+    # TODO: for centos if failes
+    load_kernel_module rte_kni kthread_mode=multiple
+fi
+
+exec "$@"

containers/agent/vrouter/Dockerfile.centos

@@ -2,7 +2,7 @@ ARG CONTRAIL_REGISTRY=localhost:5000
 ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton
 FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
 
-RUN yum install -y contrail-vrouter-agent contrail-vrouter-utils \
+RUN yum install -y ethtool contrail-vrouter-agent contrail-vrouter-utils \
       sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \
     yum clean all && \
     rm -rf /var/cache/yum && \

containers/agent/vrouter/Dockerfile.ubuntu

@@ -3,7 +3,7 @@ ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton
 FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG}
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get install -y contrail-vrouter-agent contrail-vrouter-utils && \
+    apt-get install -y ethtool contrail-vrouter-agent contrail-vrouter-utils && \
     apt-get autoremove && \
     apt-get clean && \
     rm -rf /var/cache/apt && \

containers/agent/vrouter/entrypoint.sh

@@ -5,26 +5,41 @@ source /common.sh
 HYPERVISOR_TYPE=${HYPERVISOR_TYPE:-'kvm'}
 VROUTER_HOSTNAME=${VROUTER_HOSTNAME:-${DEFAULT_HOSTNAME}}
 
+echo "INFO: agent started in $AGENT_MODE mode"
 IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac)
-echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac"
-
-# It is expected that vhost0 is up and running here
-VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic vhost0`}
-vrouter_cidr=$(get_cidr_for_nic vhost0)
-echo "INFO: vhost0 cidr $vrouter_cidr, gateway $VROUTER_GATEWAY"
-
-# It is expected that vhost0 is up and running here
+pci_address=$(get_pci_address_for_nic $phys_int)
+echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address"
+
+# For dpdk case is is expected vhost0 is down here
+# but for regular case it shold be up
+if is_dpdk ; then
+    nic_to_read_net_parameters=$phys_int
+else
+    nic_to_read_net_parameters='vhost0'
+fi
+VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic $nic_to_read_net_parameters`}
+vrouter_cidr=$(get_cidr_for_nic $nic_to_read_net_parameters)
+echo "INFO: $nic_to_read_net_parameters cidr $vrouter_cidr, gateway $VROUTER_GATEWAY"
 if [[ -z "$vrouter_cidr" ]] ; then
-    echo "ERROR: vhost0 interface is down or has no assigned IP"
+    echo "ERROR: $nic_to_read_net_parameters interface is down or has no assigned IP"
     exit -1
 fi
 vrouter_ip=${vrouter_cidr%/*}
-
 if [[ -z "$VROUTER_GATEWAY" ]] ; then
-    echo "ERROR: VROUTER_GATEWAY is empty or there is no default route for vhost0"
+    echo "ERROR: VROUTER_GATEWAY is empty or there is no default route for $nic_to_read_net_parameters"
     exit -1
 fi
 
+agent_mode_options="physical_interface_mac = $phys_int_mac"
+if is_dpdk ; then
+    read -r -d '' agent_mode_options << EOM
+platform=${AGENT_MODE}
+physical_interface_mac = $phys_int_mac
+physical_interface_address = $pci_address
+physical_uio_driver=${DPDK_UIO_DRIVER}
+EOM
+fi
+
 mkdir -p -m 777 /var/crashes
 
 echo "INFO: Preparing /etc/contrail/contrail-vrouter-agent.conf"
@@ -44,7 +59,7 @@ xmpp_server_cert=${XMPP_SERVER_CERT}
 xmpp_server_key=${XMPP_SERVER_KEY}
 xmpp_ca_cert=${XMPP_SERVER_CA_CERT}
 
-physical_interface_mac = $phys_int_mac
+$agent_mode_options
 
 $sandesh_client_config
 
@@ -76,11 +91,11 @@ cat /etc/contrail/contrail-vrouter-agent.conf
 
 set_vnc_api_lib_ini
 
-# Prepare default_pmac
-echo $phys_int_mac > /etc/contrail/default_pmac
-
-wait_for_contrail_api
+function provision_node_background() {
+    wait_for_contrail_api
+    provision_node provision_vrouter.py $vrouter_ip $VROUTER_HOSTNAME
+}
 
-provision_node provision_vrouter.py $vrouter_ip $VROUTER_HOSTNAME
+provision_node_background &
 
 exec "$@"

containers/base/common.sh

@@ -159,3 +159,7 @@ sandesh_certfile=${SANDESH_CERTFILE}
 sandesh_ca_cert=${SANDESH_CA_CERT}
 EOM
 
+AGENT_MODE=${AGENT_MODE:-'vrouter'}
+DPDK_UIO_DRIVER=${DPDK_UIO_DRIVER:-'uio_pci_generic'}
+CPU_CORE_MASK=${CPU_CORE_MASK:-'0x01'}
+HUGE_PAGES=${HUGE_PAGES:-1024}