From e210e7f0c23b8675956db8db6f37a455db5e67cc Mon Sep 17 00:00:00 2001 From: Andrey Pavlov Date: Mon, 12 Feb 2018 14:13:03 +0300 Subject: [PATCH] Initial DPDK support. Single iface. Change-Id: I67dc67436bd91feabd735f37dc874bf67e6a4ae8 Partial-Bug: #1743610 --- common.env.sample | 18 + .../agent/vrouter-dpdk/10-core-pattern.conf | 1 + .../agent/vrouter-dpdk/Dockerfile.centos | 20 + .../agent/vrouter-dpdk/Dockerfile.ubuntu | 22 + containers/agent/vrouter-dpdk/entrypoint.sh | 55 + .../Dockerfile.centos | 13 + .../Dockerfile.ubuntu | 16 + .../vrouter-init-kernel-dpdk/entrypoint.sh | 67 ++ containers/agent/vrouter/Dockerfile.centos | 2 +- containers/agent/vrouter/Dockerfile.ubuntu | 2 +- containers/agent/vrouter/entrypoint.sh | 49 +- containers/base/common.sh | 4 + containers/general-base/functions.sh | 164 +++ docker-compose/docker-compose-dpdk.yaml | 348 +++++++ .../manifests/contrail-template-dpdk.yaml | 966 ++++++++++++++++++ parse-env.sh | 4 + 16 files changed, 1732 insertions(+), 19 deletions(-) create mode 100644 containers/agent/vrouter-dpdk/10-core-pattern.conf create mode 100644 containers/agent/vrouter-dpdk/Dockerfile.centos create mode 100644 containers/agent/vrouter-dpdk/Dockerfile.ubuntu create mode 100755 containers/agent/vrouter-dpdk/entrypoint.sh create mode 100644 containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos create mode 100644 containers/agent/vrouter-init-kernel-dpdk/Dockerfile.ubuntu create mode 100755 containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh create mode 100644 docker-compose/docker-compose-dpdk.yaml create mode 100644 kubernetes/manifests/contrail-template-dpdk.yaml diff --git a/common.env.sample b/common.env.sample index a2503d8b3..6f87319c5 100644 --- a/common.env.sample +++ b/common.env.sample @@ -154,6 +154,24 @@ # - 16.04 if LINUX_DISTR=ubuntu #LINUX_DISTR_VER= +# --- +# Agent mode. For DPDK case should be set to 'dpdk'. +#AGENT_MODE=${AGENT_MODE:-'vrouter'} + +# --- +# Cpu coremask, it is used for DPDK mode for cores pinning +#CPU_CORE_MASK='0x01' + +# --- +# Number of huge pages to be set if provided +# (optional) +#HUGE_PAGES=1024 + +# --- +# DPDK UIO driver +#DPDK_UIO_DRIVER='uio_pci_generic' + + # --- # TLS #SERVER_CERT=${XMPP_SERVER_CERT:-'/etc/contrail/ssl/certs/server.pem'} diff --git a/containers/agent/vrouter-dpdk/10-core-pattern.conf b/containers/agent/vrouter-dpdk/10-core-pattern.conf new file mode 100644 index 000000000..8e74dc9f1 --- /dev/null +++ b/containers/agent/vrouter-dpdk/10-core-pattern.conf @@ -0,0 +1 @@ +kernel.core_pattern = /var/crashes/core.%e.%p.%h.%t diff --git a/containers/agent/vrouter-dpdk/Dockerfile.centos b/containers/agent/vrouter-dpdk/Dockerfile.centos new file mode 100644 index 000000000..e05f6fd93 --- /dev/null +++ b/containers/agent/vrouter-dpdk/Dockerfile.centos @@ -0,0 +1,20 @@ +ARG CONTRAIL_REGISTRY=localhost:5000 +ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton +FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} + +RUN yum install -y ethtool pciutils sysvinit-tools \ + contrail-vrouter-dpdk contrail-vrouter-dpdk-init contrail-vrouter-utils \ + sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \ + yum clean all && \ + rm -rf /var/cache/yum && \ + ldconfig + +COPY 10-core-pattern.conf /etc/sysctl.d/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/contrail-vrouter-dpdk", "--no-daemon", "--socket-mem 1024"] + +LABEL net.juniper.nodemgr.filter.name=contrail-vrouter-agent +LABEL net.juniper.contrail=agent +LABEL net.juniper.node=agent diff --git a/containers/agent/vrouter-dpdk/Dockerfile.ubuntu b/containers/agent/vrouter-dpdk/Dockerfile.ubuntu new file mode 100644 index 000000000..cc1c5c640 --- /dev/null +++ b/containers/agent/vrouter-dpdk/Dockerfile.ubuntu @@ -0,0 +1,22 @@ +ARG CONTRAIL_REGISTRY=localhost:5000 +ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton +FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y ethtool pciutils sysvinit-utils \ + contrail-vrouter-dpdk contrail-vrouter-dpdk-init contrail-vrouter-utils \ + sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \ + apt-get autoremove && \ + apt-get clean && \ + rm -rf /var/cache/apt && \ + ldconfig + +COPY 10-core-pattern.conf /etc/sysctl.d/ +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/contrail-vrouter-dpdk", "--no-daemon", "--socket-mem 1024"] + +LABEL net.juniper.nodemgr.filter.name=contrail-vrouter-agent +LABEL net.juniper.contrail=agent +LABEL net.juniper.node=agent diff --git a/containers/agent/vrouter-dpdk/entrypoint.sh b/containers/agent/vrouter-dpdk/entrypoint.sh new file mode 100755 index 000000000..07e813262 --- /dev/null +++ b/containers/agent/vrouter-dpdk/entrypoint.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +source /common.sh + +echo "INFO: agent started in $AGENT_MODE mode" +IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac) +pci_address=$(get_pci_address_for_nic $phys_int) +echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address" + +VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic $phys_int`} +vrouter_cidr=$(get_cidr_for_nic $phys_int) +echo "INFO: $phys_int cidr $vrouter_cidr, gateway $VROUTER_GATEWAY" + +# TODO: consider to avoid taskset here and leave to manage by Docker +cmd="$@" +real_cmd=$cmd +if [[ -n "$CPU_CORE_MASK" ]] ; then + taskset_param="$CPU_CORE_MASK" + if [[ "${CPU_CORE_MASK}" =~ '[,-]' ]]; then + taskset_param="-c $CPU_CORE_MASK" + fi + if is_ubuntu ; then + real_cmd="/usr/bin/taskset $taskset_param $cmd" + else + real_cmd="/bin/taskset $taskset_param $cmd" + fi +fi + +mkdir -p -m 777 /var/crashes + +# remove rte configuration file (for case if vRouter has crashed) +# TODO: most probably not needed.. since crash means container re-created +rm -f '/run/.rte_config' + +# set maximum socket buffer size to (max hold flows entries * 9160 bytes) +sysctl -w net.core.wmem_max=9160000 + +# bind iface to dpdk uio driver before start dpdk agent +bind_dev_to_driver $DPDK_UIO_DRIVER $phys_int + +function background_init() { + wait_dpdk_agent_start + if [[ -n $cmd ]] ; then + local pname=`echo $cmd | cut -d ' ' -f 1` + enable_hugepages_to_coredump "$pname" + fi + create_vhost0_dpdk $phys_int $phys_int_mac $vrouter_cidr $VROUTER_GATEWAY +} + +# TODO: tart backgound job to enable hugepase to coredump +# and initialize vhost0 +background_init & + +echo "INFO: exec '$real_cmd'" +exec $real_cmd diff --git a/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos b/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos new file mode 100644 index 000000000..49c2109fe --- /dev/null +++ b/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.centos @@ -0,0 +1,13 @@ +ARG CONTRAIL_REGISTRY=localhost:5000 +ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton +FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} + +RUN yum clean all -y && \ + rm -rf /var/cache/yum + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +LABEL net.juniper.contrail=vrouter-init-kernel +LABEL net.juniper.node=agent diff --git a/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.ubuntu b/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.ubuntu new file mode 100644 index 000000000..39dceea31 --- /dev/null +++ b/containers/agent/vrouter-init-kernel-dpdk/Dockerfile.ubuntu @@ -0,0 +1,16 @@ +ARG CONTRAIL_REGISTRY=localhost:5000 +ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton +FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get install -y kmod && \ + apt-get autoremove && \ + apt-get clean && \ + rm -rf /var/cache/apt + +COPY entrypoint.sh / + +ENTRYPOINT ["/entrypoint.sh"] + +LABEL net.juniper.contrail=vrouter-init-kernel +LABEL net.juniper.node=agent diff --git a/containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh b/containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh new file mode 100755 index 000000000..bfa890a29 --- /dev/null +++ b/containers/agent/vrouter-init-kernel-dpdk/entrypoint.sh @@ -0,0 +1,67 @@ +#!/bin/bash -x + +source /common.sh + +HUGE_PAGES_DIR=${HUGE_PAGES_DIR:-'/dev/hugepages'} +if [[ ! -d "$HUGE_PAGES_DIR" ]] ; then + echo "WARNING: There is no $HUGE_PAGES_DIR mounted from host. Try to create and mount hugetlbfs." + if ! mkdir -p $HUGE_PAGES_DIR ; then + echo "ERROR: failed to create $HUGE_PAGES_DIR" + exit -1 + fi + if ! mount -t hugetlbfs hugetlbfs $HUGE_PAGES_DIR ; then + echo "ERROR: failed to mount hugetlbfs to $HUGE_PAGES_DIR" + exit -1 + fi +fi + +if [[ ! -d "$HUGE_PAGES_DIR" ]] ; then + echo "ERROR: There is no $HUGE_PAGES_DIR. Probably HugeTables are anuvailable on the host." + exit -1 +fi + +function set_ctl() { + local var=$1 + local value=$2 + if grep -q "^$var" /etc/sysctl.conf ; then + sed -i "s/^$var.*=.*/$var=$value/g" /etc/sysctl.conf + else + echo "$var=$value" >> /etc/sysctl.conf + fi +} + +set_ctl vm.nr_hugepages ${HUGE_PAGES} +set_ctl vm.max_map_count 128960 +set_ctl net.ipv4.tcp_keepalive_time 5 +set_ctl net.ipv4.tcp_keepalive_probes 5 +set_ctl net.ipv4.tcp_keepalive_intvl 1 +sysctl --system + +function load_kernel_module() { + local module=$1 + shift 1 + local opts=$@ + echo "INFO: load $module kernel module" + if ! modprobe -v "$module" $opts ; then + echo "ERROR: failed to load $module driver" + exit -1 + fi +} + +function unload_kernel_module() { + local module=$1 + echo "INFO: unload $module kernel module" + if ! rmmod $module ; then + echo "WARNING: Failed to unload $module driver" + fi +} + +load_kernel_module uio +load_kernel_module "$DPDK_UIO_DRIVER" +if ! is_ubuntu_xenial && ! is_centos; then + # multiple kthreads for port monitoring + # TODO: for centos if failes + load_kernel_module rte_kni kthread_mode=multiple +fi + +exec "$@" diff --git a/containers/agent/vrouter/Dockerfile.centos b/containers/agent/vrouter/Dockerfile.centos index 6913b0973..ca8893561 100644 --- a/containers/agent/vrouter/Dockerfile.centos +++ b/containers/agent/vrouter/Dockerfile.centos @@ -2,7 +2,7 @@ ARG CONTRAIL_REGISTRY=localhost:5000 ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-centos7-newton FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} -RUN yum install -y contrail-vrouter-agent contrail-vrouter-utils \ +RUN yum install -y ethtool contrail-vrouter-agent contrail-vrouter-utils \ sudo python-opencontrail-vrouter-netns python-contrail-vrouter-api && \ yum clean all && \ rm -rf /var/cache/yum && \ diff --git a/containers/agent/vrouter/Dockerfile.ubuntu b/containers/agent/vrouter/Dockerfile.ubuntu index b23a704eb..723e4e46a 100644 --- a/containers/agent/vrouter/Dockerfile.ubuntu +++ b/containers/agent/vrouter/Dockerfile.ubuntu @@ -3,7 +3,7 @@ ARG CONTRAIL_CONTAINER_TAG=4.1.0.0-6-ubuntu16-newton FROM ${CONTRAIL_REGISTRY}/contrail-base:${CONTRAIL_CONTAINER_TAG} RUN export DEBIAN_FRONTEND=noninteractive && \ - apt-get install -y contrail-vrouter-agent contrail-vrouter-utils && \ + apt-get install -y ethtool contrail-vrouter-agent contrail-vrouter-utils && \ apt-get autoremove && \ apt-get clean && \ rm -rf /var/cache/apt && \ diff --git a/containers/agent/vrouter/entrypoint.sh b/containers/agent/vrouter/entrypoint.sh index 95e41522b..5bbd74b52 100755 --- a/containers/agent/vrouter/entrypoint.sh +++ b/containers/agent/vrouter/entrypoint.sh @@ -5,26 +5,41 @@ source /common.sh HYPERVISOR_TYPE=${HYPERVISOR_TYPE:-'kvm'} VROUTER_HOSTNAME=${VROUTER_HOSTNAME:-${DEFAULT_HOSTNAME}} +echo "INFO: agent started in $AGENT_MODE mode" IFS=' ' read -r phys_int phys_int_mac <<< $(get_physical_nic_and_mac) -echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac" - -# It is expected that vhost0 is up and running here -VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic vhost0`} -vrouter_cidr=$(get_cidr_for_nic vhost0) -echo "INFO: vhost0 cidr $vrouter_cidr, gateway $VROUTER_GATEWAY" - -# It is expected that vhost0 is up and running here +pci_address=$(get_pci_address_for_nic $phys_int) +echo "INFO: Physical interface: $phys_int, mac=$phys_int_mac, pci=$pci_address" + +# For dpdk case is is expected vhost0 is down here +# but for regular case it shold be up +if is_dpdk ; then + nic_to_read_net_parameters=$phys_int +else + nic_to_read_net_parameters='vhost0' +fi +VROUTER_GATEWAY=${VROUTER_GATEWAY:-`get_default_gateway_for_nic $nic_to_read_net_parameters`} +vrouter_cidr=$(get_cidr_for_nic $nic_to_read_net_parameters) +echo "INFO: $nic_to_read_net_parameters cidr $vrouter_cidr, gateway $VROUTER_GATEWAY" if [[ -z "$vrouter_cidr" ]] ; then - echo "ERROR: vhost0 interface is down or has no assigned IP" + echo "ERROR: $nic_to_read_net_parameters interface is down or has no assigned IP" exit -1 fi vrouter_ip=${vrouter_cidr%/*} - if [[ -z "$VROUTER_GATEWAY" ]] ; then - echo "ERROR: VROUTER_GATEWAY is empty or there is no default route for vhost0" + echo "ERROR: VROUTER_GATEWAY is empty or there is no default route for $nic_to_read_net_parameters" exit -1 fi +agent_mode_options="physical_interface_mac = $phys_int_mac" +if is_dpdk ; then + read -r -d '' agent_mode_options << EOM +platform=${AGENT_MODE} +physical_interface_mac = $phys_int_mac +physical_interface_address = $pci_address +physical_uio_driver=${DPDK_UIO_DRIVER} +EOM +fi + mkdir -p -m 777 /var/crashes echo "INFO: Preparing /etc/contrail/contrail-vrouter-agent.conf" @@ -44,7 +59,7 @@ xmpp_server_cert=${XMPP_SERVER_CERT} xmpp_server_key=${XMPP_SERVER_KEY} xmpp_ca_cert=${XMPP_SERVER_CA_CERT} -physical_interface_mac = $phys_int_mac +$agent_mode_options $sandesh_client_config @@ -76,11 +91,11 @@ cat /etc/contrail/contrail-vrouter-agent.conf set_vnc_api_lib_ini -# Prepare default_pmac -echo $phys_int_mac > /etc/contrail/default_pmac - -wait_for_contrail_api +function provision_node_background() { + wait_for_contrail_api + provision_node provision_vrouter.py $vrouter_ip $VROUTER_HOSTNAME +} -provision_node provision_vrouter.py $vrouter_ip $VROUTER_HOSTNAME +provision_node_background & exec "$@" diff --git a/containers/base/common.sh b/containers/base/common.sh index 4459a9fca..011eed9f6 100644 --- a/containers/base/common.sh +++ b/containers/base/common.sh @@ -159,3 +159,7 @@ sandesh_certfile=${SANDESH_CERTFILE} sandesh_ca_cert=${SANDESH_CA_CERT} EOM +AGENT_MODE=${AGENT_MODE:-'vrouter'} +DPDK_UIO_DRIVER=${DPDK_UIO_DRIVER:-'uio_pci_generic'} +CPU_CORE_MASK=${CPU_CORE_MASK:-'0x01'} +HUGE_PAGES=${HUGE_PAGES:-1024} diff --git a/containers/general-base/functions.sh b/containers/general-base/functions.sh index 6c80b646a..edb7d4562 100644 --- a/containers/general-base/functions.sh +++ b/containers/general-base/functions.sh @@ -48,6 +48,11 @@ function get_server_list() { echo ${extended_list} } +function get_pci_address_for_nic() { + local nic=$1 + ethtool -i ${nic} | grep bus-info | awk '{print $2}' | tr -d ' ' +} + function get_default_nic() { ip route get 1 | grep -o "dev.*" | awk '{print $2}' } @@ -182,4 +187,163 @@ function get_physical_nic_and_mac() exit -1 fi echo $nic $mac +} + +function is_dpdk() { + test "$AGENT_MODE" == 'dpdk' +} + +function enable_hugepages_to_coredump() { + local name=$1 + local pid=$(pidof $name) + echo "INFO: enable hugepages to coredump for $name with pid=$pid" + local coredump_filter="/proc/$pid/coredump_filter" + local cdump_filter=0x73 + if [[ -f "$coredump_filter" ]] ; then + cdump_filter=`cat "$coredump_filter"` + cdump_filter=$((0x40 | 0x$cdump_filter)) + fi + echo $cdump_filter > "$coredump_filter" +} + +function wait_nic () { + local nic=$1 + local i=0 + for i in {1..60} ; do + echo "INFO: Waiting for ${nic}... $i" + if [[ -L /sys/class/net/${nic} ]] ; then + return 0 + fi + sleep 2 + done + return 1 +} + +function wait_device_for_driver () { + local driver=$1 + local pci_address=$2 + local i=0 + for i in {1..60} ; do + echo "INFO: Waiting device $pci_address for driver ${driver} ... $i" + if [[ -L /sys/bus/pci/drivers/${driver}/${pci_address} ]] ; then + return 0 + fi + sleep 2 + done + return 1 +} + +# TODO: move to agent specific file +function is_dpdk_agent_running() { + lsof -ni:20914 > /dev/null 2>&1 +} + +function wait_dpdk_agent_start() { + local i=0 + for i in {1..60} ; do + echo "INFO: wait DPDK agent to run... $i" + if is_dpdk_agent_running ; then + return + fi + sleep 5 + done + echo "ERROR: DPDK agent is not started during timeout" + exit -1 +} + +function create_vhost0_dpdk() { + local nic='vhost0' + local phys_int=$1 + local phys_int_mac=$2 + local vrouter_cidr=$3 + local gateway=$4 + # Wait nic to be configured by agent + if ! wait_nic $nic ; then + echo "WARNING: interface $nic does not exist.. try tro create" + # vhost0 is not present, so create vhost0 and $dev + echo "INFO: Creating ${phys_int} interface with mac $phys_int_mac via vif utility..." + if ! vif --add 0 --mac ${phys_int_mac} --vrf 0 --vhost-phys --type physical --pmd --id 0 ; then + echo "ERROR: Failed to adding ${phys_int} interface" + exit -1 + fi + echo "INFO: Adding ${nic} interface with vif utility..." + # TODO: vif --xconnect seems does not work without --id parameter? + if ! vif --add ${nic} --mac ${phys_int_mac} --vrf 0 --type vhost --xconnect 0 --pmd --id 1 ; then + echo "ERROR: Failed to add ${nic} interface" + exit -1 + fi + fi + + ip link set dev $nic address $phys_int_mac + ip link set dev $nic up + ip address add $vrouter_cidr dev $nic + if [[ -n "$gateway" ]] ; then + ip route add default via $gateway dev $nic + fi +} + +function save_pci_info() { + local pci_address=$1 + local binding_data_dir='/var/run/vrouter' + mkdir -p ${binding_data_dir} + local binding_data_file="${binding_data_dir}/${pci_address}" + if [[ ! -e "$binding_data_file" ]] ; then + local pci_data=`lspci -vmmks ${pci_address}` + echo "INFO: Add lspci data to ${binding_data_file}" + echo "$pci_data" + echo "$pci_data" > ${binding_data_file} + else + echo "INFO: lspci data for $pci_address already exists" + fi +} + +function bind_dev_to_driver() { + local driver=$1 + shift 1 + local nics=( $@ ) + # bind physical device(s) to DPDK driver + local ret=0 + local n='' + for n in ${nics[@]} ; do + echo "INFO: Binding device $n to driver $driver ..." + local pci_address=`get_pci_address_for_nic $n` + save_pci_info $pci_address + if ! /opt/contrail/bin/dpdk_nic_bind.py --force --bind="$driver" $n ; then + echo "ERROR: Failed to bind $n to driver $driver" + exit -1 + fi + if ! wait_device_for_driver $driver $pci_address ; then + echo "ERROR: Failed to wait device $n ($pci_address) to appears for driver $driver" + exit -1 + fi + done +} + +function restore_bindinds() { + local binding_data_dir='/var/run/vrouter' + if [[ ! -d "$binding_data_dir" ]] ; then + ehoc "INFO: there is no local data with devs bound to dpdk uio" + return 0 + fi + local dev='' + for dev in `ls "$binding_data_dir" | grep '^[0-9]\{4\}:[0-9]\{2\}'` ; do + local driver=`awk '/Driver:/ {print($2)}' "$binding_data_dir/$dev"` + echo "INFO: Binding device $dev to default driver $driver..." + if ! /opt/contrail/bin/dpdk_nic_bind.py --force --bind=$driver $dev ; then + echo "WARNING: Failed to bind $dev to driver $driver. Probable it already bind." + fi + # remove binding data + rm -f $binding_data_dir/$dev + local nic=`ls "/sys/bus/pci/drivers/$driver/$dev/net"` + if [[ -z "$nic" ]] ; then + echo "WARNING: there is no path /sys/bus/pci/drivers/$driver/$dev/net, skip re-init device." + continue + fi + if ! wait_nic $nic ; then + echo "WARNING: there is no $nic device, skip re-init device." + continue + fi + ip link set dev $nic down || echo "INFO: $nic is already down" + ip link set dev $nic up || echo "WARNING: failed to up interface $nic" + done } \ No newline at end of file diff --git a/docker-compose/docker-compose-dpdk.yaml b/docker-compose/docker-compose-dpdk.yaml new file mode 100644 index 000000000..25e97aba5 --- /dev/null +++ b/docker-compose/docker-compose-dpdk.yaml @@ -0,0 +1,348 @@ +# Docker-compose Contrail application. +# +# The following environment variables are used in this file and +# they are expected to be provided: +# - CONTRAIL_REGISTRY (docker registry where contrail containers are) +# - CONTRAIL_VERSION (Contrail release version, e.g. 4.1.0.0-6) +# - OPENSTACK_VERSION (Openstack version, e.g. ocata) +# - ENV_FILE (a file with env variables for Contrail services) +# - CONFIGDB_NODES (comma separated list of IPs for config DB nodes) +# - ANALYTICSGDB_NODES (comma separated list of IPs for config DB nodes) +# - ZOOKEEPER_NODES (comma separated list of IPs for zookeeper nodes) +# - RABBITMQ_NODES (comma separated list of IPs for rabbitmq nodes) +# (all other params for Contrail are to be provided via env file) +# In case of using helping scripts (up.sh, down.sh, ..) from this directory +# all these variables are defined automatically based on a provided info in +# the common.env in root project folder. Full list of supported env variables +# could be found in common.env.sample in the root project folder. +--- +version: '2.1' +services: + network: + image: docker.io/kubernetes/pause + network_mode: "host" + volumes: + - /etc/hosts:/etc/hosts + + # === + # common services + zookeeper: + image: "${CONTRAIL_REGISTRY}/contrail-external-zookeeper:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - ZOOKEEPER_NODES + network_mode: "service:network" + depends_on: + - network + restart: on-failure + rabbitmq: + image: "${CONTRAIL_REGISTRY}/contrail-external-rabbitmq:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - RABBITMQ_NODES + - RABBITMQ_ERLANG_COOKIE=${RABBITMQ_ERLANG_COOKIE:-'abcdefg'} + network_mode: "service:network" + depends_on: + - network + restart: on-failure + redis: + image: "redis:4.0.2" + env_file: ${ENV_FILE} + network_mode: "service:network" + depends_on: + - network + restart: on-failure + + # === + # config db services + configdb-cassandra: + image: "${CONTRAIL_REGISTRY}/contrail-external-cassandra:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - CASSANDRA_SEEDS=${CONFIGDB_NODES} + - CASSANDRA_LISTEN_ADDRESS=auto + - CASSANDRA_START_RPC=true + - CASSANDRA_CLUSTER_NAME=contrail_database + - CASSANDRA_PORT=9161 + - CASSANDRA_CQL_PORT=9041 + - CASSANDRA_SSL_STORAGE_PORT=7011 + - CASSANDRA_STORAGE_PORT=7010 + - CASSANDRA_JMX_LOCAL_PORT=7201 + network_mode: "service:network" + depends_on: + - network + restart: on-failure + + # === + # config services + config-nodemgr: + image: "${CONTRAIL_REGISTRY}/contrail-nodemgr:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - NODE_TYPE=config + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - network + restart: on-failure + api: + image: "${CONTRAIL_REGISTRY}/contrail-controller-config-api:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + svcmonitor: + image: "${CONTRAIL_REGISTRY}/contrail-controller-config-svcmonitor:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + schema: + image: "${CONTRAIL_REGISTRY}/contrail-controller-config-schema:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + devicemgr: + image: "${CONTRAIL_REGISTRY}/contrail-controller-config-devicemgr:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + + # === + # control services + control-nodemgr: + image: "${CONTRAIL_REGISTRY}/contrail-nodemgr:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - NODE_TYPE=control + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - network + restart: on-failure + control: + image: "${CONTRAIL_REGISTRY}/contrail-controller-control-control:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + control-named: + image: "${CONTRAIL_REGISTRY}/contrail-controller-control-named:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + privileged: true + volumes: + - /var/log/contrail:/var/log/contrail + volumes_from: + - control-dns + depends_on: + - network + - control-dns + restart: on-failure + control-dns: + image: "${CONTRAIL_REGISTRY}/contrail-controller-control-dns:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + - /etc/contrail + depends_on: + - network + restart: on-failure + + # === + # analytics db services + analyticsdb-kafka: + image: "${CONTRAIL_REGISTRY}/contrail-external-kafka:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + depends_on: + - network + restart: on-failure + analyticsdb-cassandra: + image: "${CONTRAIL_REGISTRY}/contrail-external-cassandra:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - CASSANDRA_SEEDS=${ANALYTICSDB_NODES} + - CASSANDRA_LISTEN_ADDRESS=auto + - CASSANDRA_START_RPC=true + - CASSANDRA_CLUSTER_NAME=contrail_analytics + - CASSANDRA_PORT=9160 + - CASSANDRA_CQL_PORT=9042 + - CASSANDRA_SSL_STORAGE_PORT=7001 + - CASSANDRA_STORAGE_PORT=7000 + - CASSANDRA_JMX_LOCAL_PORT=7200 + network_mode: "service:network" + depends_on: + - network + restart: on-failure + + # === + # analytics services + analytics-nodemgr: + image: "${CONTRAIL_REGISTRY}/contrail-nodemgr:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - NODE_TYPE=analytics + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + - /var/run/docker.sock:/var/run/docker.sock + depends_on: + - network + restart: on-failure + analytics-api: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-api:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-collector: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-collector:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-query-engine: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-query-engine:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-alarm-gen: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-alarm-gen:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-snmp-collector: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-snmp-collector:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-topology: + image: "${CONTRAIL_REGISTRY}/contrail-analytics-topology:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + analytics-redis: + env_file: ${ENV_FILE} + image: "redis:4.0.2" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + + # === + # contrail web ui services + contrail-webui: + image: "${CONTRAIL_REGISTRY}/contrail-controller-webui-web:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + contrail-webui-job: + image: "${CONTRAIL_REGISTRY}/contrail-controller-webui-job:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: "service:network" + volumes: + - /var/log/contrail:/var/log/contrail + depends_on: + - network + restart: on-failure + + # === + # vrouter services + vrouter-nodemgr: + image: "${CONTRAIL_REGISTRY}/contrail-nodemgr:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + environment: + - NODE_TYPE=vrouter + network_mode: host + volumes: + - /var/log/contrail:/var/log/contrail + - /var/run/docker.sock:/var/run/docker.sock + restart: on-failure + vrouter-init-kernel: + image: "${CONTRAIL_REGISTRY}/contrail-agent-vrouter-init-kernel-dpdk:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: host + privileged: true + volumes: + - /etc:/etc + - /dev:/dev + - /lib/modules:/lib/modules + - /var/log/contrail:/var/log/contrail + vrouter-agent: + image: "${CONTRAIL_REGISTRY}/contrail-agent-vrouter:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: host + privileged: true + volumes: + - /dev:/dev + - /lib/modules:/lib/modules + - /var/run:/var/run + - /var/log/contrail:/var/log/contrail + restart: on-failure + depends_on: + - vrouter-init-kernel + vrouter-agent-dpdk: + image: "${CONTRAIL_REGISTRY}/contrail-agent-vrouter-dpdk:${CONTRAIL_CONTAINER_TAG}" + env_file: ${ENV_FILE} + network_mode: host + pid: host + privileged: true + volumes: + - /dev:/dev + - /lib/modules:/lib/modules + - /var/run:/var/run + - /var/log/contrail:/var/log/contrail + restart: on-failure + depends_on: + - vrouter-init-kernel + - vrouter-agent diff --git a/kubernetes/manifests/contrail-template-dpdk.yaml b/kubernetes/manifests/contrail-template-dpdk.yaml new file mode 100644 index 000000000..100914924 --- /dev/null +++ b/kubernetes/manifests/contrail-template-dpdk.yaml @@ -0,0 +1,966 @@ +# Configs section +# TODOs for new kubernetes: +# 1) Rework the way of providing docker socket in nodemgr container. +# Use new volume type Socket after switching to new kubernetes: +# hostPath: +# path: /var/run/docker.sock +# type: Socket +# 2) Avoid using ".." for numberic types (ports, etc), because in v1 there is a bug +# which leads to an error +# "..error unmarshaling JSON: json: cannot unmarshal number into Go value of type string.." +# (https://github.com/kubernetes/kubernetes/issues/2763) +# 3) Use new type HugePages instead of mapping it from host folder +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: env + namespace: kube-system +data: + AAA_MODE: {{ AAA_MODE }} + AGENT_MODE: dpdk + ANALYTICS_API_VIP: {{ ANALYTICS_API_VIP }} + ANALYTICS_NODES: {{ ANALYTICS_NODES }} + ANALYTICSDB_NODES: {{ ANALYTICSDB_NODES }} + AUTH_MODE: {{ AUTH_MODE }} + CLOUD_ORCHESTRATOR: {{ CLOUD_ORCHESTRATOR }} + CONFIG_API_VIP: {{ CONFIG_API_VIP }} + CONFIG_NODES: {{ CONFIG_NODES }} + CONFIGDB_NODES: {{ CONFIGDB_NODES }} + CONTROL_NODES: {{ CONTROL_NODES }} + CONTROLLER_NODES: {{ CONTROLLER_NODES }} + CPU_CORE_MASK: "{{ CPU_CORE_MASK }}" + DPDK_UIO_DRIVER: {{ DPDK_UIO_DRIVER }} + HUGE_PAGES: "{{ HUGE_PAGES }}" + KAFKA_NODES: {{ KAFKA_NODES }} + LOG_LEVEL: {{ LOG_LEVEL }} + METADATA_PROXY_SECRET: {{ METADATA_PROXY_SECRET }} + PHYSICAL_INTERFACE: {{ PHYSICAL_INTERFACE }} + RABBITMQ_NODES: {{ RABBITMQ_NODES }} + RABBITMQ_NODE_PORT: "{{ RABBITMQ_NODE_PORT }}" + REDIS_NODES: {{ REDIS_NODES }} + VROUTER_GATEWAY: {{ VROUTER_GATEWAY }} + WEBUI_NODES: {{ WEBUI_NODES }} + WEBUI_VIP: {{ WEBUI_VIP }} + ZOOKEEPER_ANALYTICS_PORT: "{{ ZOOKEEPER_ANALYTICS_PORT }}" + ZOOKEEPER_NODES: {{ ZOOKEEPER_NODES }} + ZOOKEEPER_PORT: "{{ ZOOKEEPER_PORT }}" + ZOOKEEPER_PORTS: "{{ ZOOKEEPER_PORTS }}" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nodemgr-config + namespace: kube-system +data: + DOCKER_HOST: "unix://mnt/docker.sock" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: contrail-analyticsdb-config + namespace: kube-system +data: + CASSANDRA_SEEDS: {{ ANALYTICSDB_NODES }} + CASSANDRA_CLUSTER_NAME: Contrail + CASSANDRA_START_RPC: "true" + CASSANDRA_LISTEN_ADDRESS: auto + CASSANDRA_PORT: "9160" + CASSANDRA_CQL_PORT: "9042" + CASSANDRA_SSL_STORAGE_PORT: "7001" + CASSANDRA_STORAGE_PORT: "7000" + CASSANDRA_JMX_LOCAL_PORT: "7200" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: contrail-configdb-config + namespace: kube-system +data: + CASSANDRA_SEEDS: {{ CONFIGDB_NODES }} + CASSANDRA_CLUSTER_NAME: ContrailConfigDB + CASSANDRA_START_RPC: "true" + CASSANDRA_LISTEN_ADDRESS: auto + CASSANDRA_PORT: "9161" + CASSANDRA_CQL_PORT: "9041" + CASSANDRA_SSL_STORAGE_PORT: "7011" + CASSANDRA_STORAGE_PORT: "7010" + CASSANDRA_JMX_LOCAL_PORT: "7201" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-config + namespace: kube-system +data: + RABBITMQ_ERLANG_COOKIE: "47EFF3BB-4786-46E0-A5BB-58455B3C2CB4" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-manager-config + namespace: kube-system +data: + KUBERNETES_API_SERVER: {{ KUBERNETES_API_SERVER }} + K8S_TOKEN_FILE: "/tmp/serviceaccount/token" +# Containers section +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: zookeeper + namespace: kube-system + labels: + app: zookeeper +spec: + template: + metadata: + labels: + app: zookeeper + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/zookeeper" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: zookeeper + image: "{{CONTRAIL_REGISTRY}}/contrail-external-zookeeper:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/lib/zookeeper + name: zookeeper-data + - mountPath: /var/log/zookeeper + name: zookeeper-logs + volumes: + - name: zookeeper-data + hostPath: + path: /var/lib/contrail/zookeeper + - name: zookeeper-logs + hostPath: + path: /var/log/contrail/zookeeper +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kafka + namespace: kube-system + labels: + app: kafka +spec: + template: + metadata: + labels: + app: kafka + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/kafka" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: kafka + image: "{{CONTRAIL_REGISTRY}}/contrail-external-kafka:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /tmp/kafka-logs + name: kafka-logs + volumes: + - name: kafka-logs + hostPath: + path: /var/lib/contrail/kafka-logs +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-analyticsdb + namespace: kube-system + labels: + app: contrail-analyticsdb +spec: + template: + metadata: + labels: + app: contrail-analyticsdb + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/analyticsdb" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-analyticsdb + image: "{{CONTRAIL_REGISTRY}}/contrail-external-cassandra:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: contrail-analyticsdb-config + volumeMounts: + - mountPath: /var/lib/cassandra + name: analyticsdb-data + - mountPath: /var/log/cassandra + name: analyticsdb-log + volumes: + - name: analyticsdb-data + hostPath: + path: /var/lib/contrail/analyticsdb + - name: analyticsdb-log + hostPath: + path: /var/log/contrail/analyticsdb +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-configdb + namespace: kube-system + labels: + app: contrail-configdb +spec: + template: + metadata: + labels: + app: contrail-configdb + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/configdb" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-configdb + image: "{{CONTRAIL_REGISTRY}}/contrail-external-cassandra:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: contrail-configdb-config + volumeMounts: + - mountPath: /var/lib/cassandra + name: configdb-data + - mountPath: /var/log/cassandra + name: configdb-log + volumes: + - name: configdb-data + hostPath: + path: /var/lib/contrail/configdb + - name: configdb-log + hostPath: + path: /var/log/contrail/configdb +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-analytics + namespace: kube-system + labels: + app: contrail-analytics +spec: + template: + metadata: + labels: + app: contrail-analytics + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/analytics" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-analytics-api + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-api:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-collector + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-collector:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-alarm-gen + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-alarm-gen:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-query-engine + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-query-engine:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-snmp-collector + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-snmp-collector:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-topology + image: "{{CONTRAIL_REGISTRY}}/contrail-analytics-topology:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - name: contrail-analytics-nodemgr + image: "{{CONTRAIL_REGISTRY}}/contrail-nodemgr:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: nodemgr-config + env: + - name: NODE_TYPE + value: analytics + volumeMounts: + - mountPath: /var/log/contrail + name: analytics-logs + - mountPath: /mnt + name: docker-unix-socket + volumes: + - name: analytics-logs + hostPath: + path: /var/log/contrail/analytics + - name: docker-unix-socket + hostPath: + path: /var/run +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-controller-control + namespace: kube-system + labels: + app: contrail-controller-control +spec: + template: + metadata: + labels: + app: contrail-controller-control + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/control" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-controller-control + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-control-control:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: control-logs + - name: contrail-controller-control-dns + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-control-dns:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /etc/contrail + name: dns-config + - mountPath: /var/log/contrail + name: control-logs + - name: contrail-controller-control-named + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-control-named:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/contrail + name: dns-config + - mountPath: /var/log/contrail + name: control-logs + - name: contrail-controller-nodemgr + image: "{{CONTRAIL_REGISTRY}}/contrail-nodemgr:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: nodemgr-config + env: + - name: NODE_TYPE + value: control + volumeMounts: + - mountPath: /var/log/contrail + name: control-logs + - mountPath: /mnt + name: docker-unix-socket + volumes: + - name: control-logs + hostPath: + path: /var/log/contrail/control + - name: docker-unix-socket + hostPath: + path: /var/run + - name: dns-config + emptyDir: {} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-controller-config + namespace: kube-system + labels: + app: contrail-controller-config +spec: + template: + metadata: + labels: + app: contrail-controller-config + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/config" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-controller-config-api + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-config-api:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: config-logs + - name: contrail-controller-config-devicemgr + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-config-devicemgr:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: config-logs + - name: contrail-controller-config-schema + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-config-schema:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: config-logs + - name: contrail-controller-config-svcmonitor + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-config-svcmonitor:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: config-logs + - name: contrail-controller-config-nodemgr + image: "{{CONTRAIL_REGISTRY}}/contrail-nodemgr:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: nodemgr-config + env: + - name: NODE_TYPE + value: config + volumeMounts: + - mountPath: /var/log/contrail + name: config-logs + - mountPath: /mnt + name: docker-unix-socket + volumes: + - name: config-logs + hostPath: + path: /var/log/contrail/config + - name: docker-unix-socket + hostPath: + path: /var/run +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-controller-webui + namespace: kube-system + labels: + app: contrail-controller-webui +spec: + template: + metadata: + labels: + app: contrail-controller-webui + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/webui" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: contrail-controller-webui-job + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-webui-job:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: webui-logs + - name: contrail-controller-webui-web + image: "{{CONTRAIL_REGISTRY}}/contrail-controller-webui-web:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/log/contrail + name: webui-logs + volumes: + - name: webui-logs + hostPath: + path: /var/log/contrail/webui +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: redis + namespace: kube-system + labels: + app: redis +spec: + template: + metadata: + labels: + app: redis + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/analytics" + operator: Exists + - matchExpressions: + - key: "node-role.opencontrail.org/webui" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: redis + image: "redis:4.0.2" + imagePullPolicy: "" + volumeMounts: + - mountPath: /var/lib/redis + name: redis-data + - mountPath: /var/log/redis + name: redis-logs + volumes: + - name: redis-data + hostPath: + path: /var/lib/contrail/redis + - name: redis-logs + hostPath: + path: /var/log/contrail/redis +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: rabbitmq + namespace: kube-system + labels: + app: rabbitmq +spec: + template: + metadata: + labels: + app: rabbitmq + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/rabbitmq" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + containers: + - name: rabbitmq + image: "{{CONTRAIL_REGISTRY}}/contrail-external-rabbitmq:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: rabbitmq-config + volumeMounts: + - mountPath: /var/lib/rabbitmq + name: rabbitmq-data + - mountPath: /var/log/rabbitmq + name: rabbitmq-logs + volumes: + - name: rabbitmq-data + hostPath: + path: /var/lib/contrail/rabbitmq + - name: rabbitmq-logs + hostPath: + path: /var/log/contrail/rabbitmq +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-kube-manager + namespace: kube-system + labels: + app: contrail-kube-manager +spec: + template: + metadata: + labels: + app: contrail-kube-manager + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + automountServiceAccountToken: false + hostNetwork: true + containers: + - name: contrail-kube-manager + image: "{{CONTRAIL_REGISTRY}}/contrail-kubernetes-kube-manager:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: kube-manager-config + volumeMounts: + - mountPath: /var/log/contrail + name: kube-manager-logs + - mountPath: /tmp/serviceaccount + name: pod-secret + volumes: + - name: kube-manager-logs + hostPath: + path: /var/log/contrail/kube-manager + - name: pod-secret + secret: + secretName: contrail-kube-manager-token +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: contrail-agent + namespace: kube-system + labels: + app: contrail-agent +spec: + template: + metadata: + labels: + app: contrail-agent + spec: + #Disable affinity for single node setup + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.opencontrail.org/agent" + operator: Exists + #Enable tolerations for single node setup + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + automountServiceAccountToken: false + hostNetwork: true + initContainers: + - name: contrail-agent-vrouter-init-kernel-dpdk + image: "{{CONTRAIL_REGISTRY}}/contrail-agent-vrouter-init-kernel-dpdk:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + securityContext: + privileged: true + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /dev + name: dev + # TODO: to access /etc/sysctl.conf , ifcfg & alike file on host, /etc/os-release + - mountPath: /etc + name: etc + - mountPath: /lib/modules + name: lib-modules + - name: contrail-kubernetes-vrouter-init + image: "{{CONTRAIL_REGISTRY}}/contrail-kubernetes-vrouter-init:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + volumeMounts: + - mountPath: /var/lib/contrail + name: var-lib-contrail + - mountPath: /host/etc_cni + name: etc-cni + - mountPath: /host/opt_cni_bin + name: opt-cni-bin + - mountPath: /host/log_cni + name: var-log-contrail-cni + - mountPath: /var/log/contrail + name: agent-logs + containers: + - name: contrail-agent + image: "{{CONTRAIL_REGISTRY}}/contrail-agent-vrouter:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + # TODO: Priveleged mode is requied because w/o it the device /dev/net/tun + # is not present in the container. The mounting it into container + # doesnt help because of permissions are not enough syscalls, + # e.g. https://github.com/Juniper/contrail-controller/blob/master/src/vnsw/agent/contrail/linux/pkt0_interface.cc: 48. + securityContext: + privileged: true + envFrom: + - configMapRef: + name: env + volumeMounts: + # TODO: need for hugepages & communication with dpdk agent + - mountPath: /dev + name: dev + - mountPath: /lib/modules + name: lib-modules + # TODO: looks like a shared storage for agent and dpdk agent + # there are sockets and pci dev info + - mountPath: /var/run + name: var-run + - mountPath: /usr/src + name: usr-src + - mountPath: /var/log/contrail + name: agent-logs + - mountPath: /var/lib/contrail + name: var-lib-contrail + - mountPath: /var/crashes + name: var-crashes + - mountPath: /tmp/serviceaccount + name: pod-secret + - name: contrail-agent-dpdk + image: "{{CONTRAIL_REGISTRY}}/contrail-agent-vrouter-dpdk:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + # TODO: Priveleged mode is requied because w/o it the device /dev/net/tun + # is not present in the container. The mounting it into container + # doesnt help because of permissions are not enough syscalls, + # e.g. https://github.com/Juniper/contrail-controller/blob/master/src/vnsw/agent/contrail/linux/pkt0_interface.cc: 48. + securityContext: + privileged: true + envFrom: + - configMapRef: + name: env + volumeMounts: + # TODO: need for hugepages & communication with agent + - mountPath: /dev + name: dev + # TODO: it is for lspci -vmmks works, since it uses libkmod + - mountPath: /lib/modules + name: lib-modules + # TODO: looks like a shared storage for agent and dpdk agent + # there are sockets and pci dev info + - mountPath: /var/run + name: var-run + - mountPath: /var/log/contrail + name: agent-logs + - mountPath: /var/lib/contrail + name: var-lib-contrail + - mountPath: /var/crashes + name: var-crashes + - mountPath: /tmp/serviceaccount + name: pod-secret + - name: contrail-agent-nodemgr + image: "{{CONTRAIL_REGISTRY}}/contrail-nodemgr:{{CONTRAIL_CONTAINER_TAG}}" + imagePullPolicy: "" + envFrom: + - configMapRef: + name: env + - configMapRef: + name: nodemgr-config + env: + - name: NODE_TYPE + value: vrouter + volumeMounts: + - mountPath: /var/log/contrail + name: agent-logs + - mountPath: /mnt + name: var-run + volumes: + - name: dev + hostPath: + path: /dev + - name: etc + hostPath: + path: /etc + - name: var-run + hostPath: + path: /var/run + - name: pod-secret + secret: + secretName: contrail-kube-manager-token + - name: usr-src + hostPath: + path: /usr/src + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-lib-contrail + hostPath: + path: /var/lib/contrail + - name: var-crashes + hostPath: + path: /var/contrail/crashes + - name: etc-cni + hostPath: + path: /etc/cni + - name: opt-cni-bin + hostPath: + path: /opt/cni/bin + - name: var-log-contrail-cni + hostPath: + path: /var/log/contrail/cni + - name: agent-logs + hostPath: + path: /var/log/contrail/agent + +# Meta information section +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: contrail-kube-manager + namespace: kube-system +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contrail-kube-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: contrail-kube-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: contrail-kube-manager +subjects: +- kind: ServiceAccount + name: contrail-kube-manager + namespace: kube-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: contrail-kube-manager-token + namespace: kube-system + annotations: + kubernetes.io/service-account.name: contrail-kube-manager +type: kubernetes.io/service-account-token + diff --git a/parse-env.sh b/parse-env.sh index a3010e5e8..97b7ff6ec 100644 --- a/parse-env.sh +++ b/parse-env.sh @@ -116,3 +116,7 @@ export AUTH_MODE=${AUTH_MODE:-'noauth'} export CLOUD_ORCHESTRATOR=${CLOUD_ORCHESTRATOR:-'none'} export KUBERNETES_API_SERVER=${KUBERNETES_API_SERVER:-$HOST_IP} + +export DPDK_UIO_DRIVER=${DPDK_UIO_DRIVER:-'uio_pci_generic'} +export CPU_CORE_MASK=${CPU_CORE_MASK:-'0x01'} +export HUGE_PAGES=${HUGE_PAGES:-1024}