From 59090561c48ae6f1766a5aaac20ba50005199ccd Mon Sep 17 00:00:00 2001 From: Madhukar Nayakbomman Date: Mon, 18 Jun 2018 16:46:26 -0700 Subject: [PATCH] SRIOV support for helm charts Change-Id: I5a1cba46f3d161824d618e8d0bf01e9e8315b541 Depends-On: Id6dc555ba91531f49ba69f7019093a8ce515158e Depends-On: Idcabbbcf4b9eecb21ab24d93f3ca84137c5a12b6 Closes-bug: #1767470 --- contrail-vrouter/templates/configmap-env.yaml | 22 ++ .../daemonset-vrouter-agent-kernel.yaml | 2 +- .../daemonset-vrouter-agent-sriov.yaml | 277 ++++++++++++++++++ contrail-vrouter/values.yaml | 21 ++ doc/contrail-osh-aio-install.md | 40 ++- doc/contrail-osh-multinode-install.md | 107 ++++--- .../files/helm-deploy.sh | 5 +- 7 files changed, 415 insertions(+), 59 deletions(-) create mode 100644 contrail-vrouter/templates/daemonset-vrouter-agent-sriov.yaml diff --git a/contrail-vrouter/templates/configmap-env.yaml b/contrail-vrouter/templates/configmap-env.yaml index 2608323..7b8813a 100644 --- a/contrail-vrouter/templates/configmap-env.yaml +++ b/contrail-vrouter/templates/configmap-env.yaml @@ -39,6 +39,28 @@ data: {{- end }} {{- end }} +{{- if .Values.manifests.configmap_vrouter_sriov }} +{{- $_ := required ".Values.global.contrail_env_vrouter_sriov.per_compute_info must be specified" .Values.global.contrail_env_vrouter_sriov.per_compute_info }} +{{- $common_sriov_vars := .Values.global.contrail_env_vrouter_sriov }} +{{- $per_compute_info := .Values.global.contrail_env_vrouter_sriov.per_compute_info }} +{{- range $_, $compute_vars := $per_compute_info }} +{{- $compute_name := index $compute_vars "node_name" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: contrail-vrouter-sriov-{{ $compute_name }} +data: + {{- range $key, $value := $compute_vars }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- range $key, $value := $common_sriov_vars }} + {{ if ne $key "per_compute_info" }}{{ $key }}: {{ $value | quote }}{{ end }} + {{- end }} +{{- end }} +{{- end }} + + {{- if .Values.manifests.configmap_vrouter_keystone }} --- apiVersion: v1 diff --git a/contrail-vrouter/templates/daemonset-vrouter-agent-kernel.yaml b/contrail-vrouter/templates/daemonset-vrouter-agent-kernel.yaml index 2cced32..5b5ecd4 100644 --- a/contrail-vrouter/templates/daemonset-vrouter-agent-kernel.yaml +++ b/contrail-vrouter/templates/daemonset-vrouter-agent-kernel.yaml @@ -71,7 +71,7 @@ spec: name: lib-modules - mountPath: /host/bin name: host-bin -{{- else if $host_os "centos" }} +{{- else if eq $host_os "centos" }} - name: contrail-vrouter-init-kernel image: {{ .Values.global.images.tags.vrouter_init_kernel | quote }} imagePullPolicy: {{ default "" .Values.global.images.imagePullPolicy | quote }} diff --git a/contrail-vrouter/templates/daemonset-vrouter-agent-sriov.yaml b/contrail-vrouter/templates/daemonset-vrouter-agent-sriov.yaml new file mode 100644 index 0000000..76444ab --- /dev/null +++ b/contrail-vrouter/templates/daemonset-vrouter-agent-sriov.yaml @@ -0,0 +1,277 @@ +{{- define "contrail.sriov.daemonset" }} +{{- $daemonset_name := index . 0 }} +{{- $config_map_name := index . 1 }} +{{- $compute_name := index . 2 }} +{{- $sriov_env := index . 3 }} +{{- $context := index . 4 }} +{{- $agent_mode := $sriov_env.AGENT_MODE | default "kernel" }} + +{{- $_ := set $context.Values "images" $context.Values.global.images }} +# host_os is a mandatory field +{{- $_ := required ".Values.global.node.host_os must be specified, valid values are ubuntu, centos" $context.Values.global.node.host_os }} +{{- $host_os := $context.Values.global.node.host_os }} +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: {{ $daemonset_name }} +spec: +{{ tuple $context "vrouter_agent_sriov" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $context "opencontrail" "contrail-vrouter-agent-sriov" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-hash: {{ tuple "configmap-env.yaml" $context | include "helm-toolkit.utils.hash" }} + spec: + #Disable affinity for single node setup + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ $context.Values.labels.vrouter_agent_sriov.node_selector_key }} + operator: In + values: + - {{ $context.Values.labels.vrouter_agent_sriov.node_selector_value }} + - key: "kubernetes.io/hostname" + operator: In + values: + - {{ $compute_name }} + #Enable tolerations for single node setup + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + imagePullSecrets: + - name: contrail-vrouter-docker-secret + initContainers: +{{ tuple $context $context.Values.dependencies.vrouter_agent_sriov nil | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: contrail-node-init + image: {{ $context.Values.global.images.tags.node_init | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + env: + - name: CONTRAIL_STATUS_IMAGE + value: {{ $context.Values.global.images.tags.contrail_status | quote }} + envFrom: + - configMapRef: + name: {{ $config_map_name }} + volumeMounts: + - mountPath: /host/usr/bin + name: host-usr-bin +{{- if and $agent_mode ( eq $agent_mode "kernel" ) }} +{{- if eq $host_os "ubuntu"}} + - name: contrail-vrouter-init-kernel + image: {{ $context.Values.global.images.tags.build_driver_init | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + envFrom: + - configMapRef: + name: contrail-vrouter-kernel + - configMapRef: + name: contrail-vrouter-keystone + volumeMounts: + - mountPath: /usr/src + name: usr-src + - mountPath: /lib/modules + name: lib-modules + - mountPath: /host/bin + name: host-bin +{{- else if $host_os "centos" }} + - name: contrail-vrouter-init-kernel + image: {{ $context.Values.global.images.tags.vrouter_init_kernel | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + envFrom: + - configMapRef: + name: configmap-vrouter + - configMapRef: + name: contrail-vrouter-keystone + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + - mountPath: /etc/sysconfig/network-scripts + name: network-scripts + - mountPath: /host/bin + name: host-bin +{{- end }} +{{- else if and $agent_mode ( eq $agent_mode "dpdk" ) }} + - name: contrail-vrouter-init-dpdk + image: {{ $context.Values.global.images.tags.vrouter_init_dpdk | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + envFrom: + - configMapRef: + name: contrail-vrouter-dpdk + - configMapRef: + name: contrail-vrouter-keystone + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + - mountPath: /var/run + name: var-run + - mountPath: /etc/os-release + name: os-release + - mountPath: /etc/lsb-release + name: lsb-release + - mountPath: /dev/hugepages + name: hugepages +{{- end }} + containers: +{{- if and $agent_mode ( eq $agent_mode "dpdk" ) }} + - name: contrail-vrouter-dpdk + image: {{ $context.Values.global.images.tags.vrouter_dpdk | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + envFrom: + - configMapRef: + name: contrail-vrouter-dpdk + - configMapRef: + name: contrail-vrouter-keystone + volumeMounts: + - mountPath: /etc/ + name: mount-etc + - mountPath: /lib/modules + name: lib-modules + - mountPath: /var/run + name: var-run + - mountPath: /var/lib/contrail + name: var-lib-contrail + - mountPath: /var/log/contrail + name: contrail-log + - mountPath: /etc/os-release + name: os-release + - mountPath: /etc/lsb-release + name: lsb-release + - mountPath: /dev + name: host-dev + - mountPath: /var/crashes + name: var-crashes-vrouter +{{- end }} + - name: contrail-vrouter-agent + image: {{ $context.Values.global.images.tags.vrouter_agent | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + securityContext: + privileged: true + envFrom: + - configMapRef: + name: {{ $config_map_name }} + - configMapRef: + name: contrail-vrouter-keystone + volumeMounts: +{{- if eq $host_os "centos"}} + - mountPath: /etc/sysconfig/network-scripts + name: network-scripts +{{- end }} + - mountPath: /lib/modules + name: lib-modules + - mountPath: /var/lib/contrail/ + name: var-lib-contrail + - mountPath: /var/log/contrail/ + name: contrail-log + - mountPath: /var/crashes + name: var-crashes-vrouter + - mountPath: /host/bin + name: host-bin + - mountPath: /dev/log + name: host-dev-log + - mountPath: /var/run/libvirt + name: var-run-libvirt + - mountPath: /etc/contrail/ssl + name: ssl-dir + - name: contrail-vrouter-nodemgr + image: {{ $context.Values.global.images.tags.nodemgr | quote }} + imagePullPolicy: {{ default "" $context.Values.global.images.imagePullPolicy | quote }} + envFrom: + - configMapRef: + name: {{ $config_map_name }} + - configMapRef: + name: contrail-vrouter-keystone + env: + - name: NODE_TYPE + value: vrouter + - name: DOCKER_HOST + value: "unix://mnt/docker.sock" + volumeMounts: + - mountPath: /mnt/docker.sock + name: docker-unix-socket + - mountPath: /var/log/contrail/ + name: contrail-log + - mountPath: /var/crashes + name: var-crashes-vrouter + - mountPath: /etc/contrail/ssl + name: ssl-dir + volumes: +{{- if eq $host_os "ubuntu"}} + - name: usr-src + hostPath: + path: /usr/src +{{- else if $host_os "centos" }} + - name: network-scripts + hostPath: + path: /etc/sysconfig/network-scripts +{{- end }} + - name: lib-modules + hostPath: + path: /lib/modules + - name: docker-unix-socket + hostPath: + path: /var/run/docker.sock + type: Socket + - name: os-release + hostPath: + path: /usr/lib/os-release + - name: lsb-release + hostPath: + path: /etc/lsb-release + - name: hugepages + hostPath: + path: /dev/hugepages + - name: var-run + hostPath: + path: /var/run/ + - name: var-lib-contrail + hostPath: + path: /var/lib/contrail/ + - name: contrail-log + hostPath: + path: /var/log/contrail + - name: var-crashes-vrouter + hostPath: + path: /var/crashes + - name: host-bin + hostPath: + path: /bin + - name: var-run-libvirt + hostPath: + path: /var/run/libvirt + - name: host-usr-bin + hostPath: + path: /usr/bin + - name: host-dev-log + hostPath: + path: /dev/log + - name: ssl-dir + hostPath: + path: /etc/contrail/ssl +{{- end }} + +{{- if .Values.manifests.daemonset_vrouter_sriov }} +{{- $_ := required ".Values.global.contrail_env_vrouter_sriov.per_compute_info must be specified" .Values.global.contrail_env_vrouter_sriov.per_compute_info }} +{{- $context := . }} +{{- range $_, $compute_vars := .Values.global.contrail_env_vrouter_sriov.per_compute_info }} +{{- $compute_name := index $compute_vars "node_name" }} +{{- $sriov_env := omit $compute_vars "node_name" }} +{{- $daemonset_name := printf "contrail-vrouter-agent-sriov-%s" $compute_name }} +{{- $config_map_name := printf "contrail-vrouter-sriov-%s" $compute_name }} +{{- list $daemonset_name $config_map_name $compute_name $sriov_env $context | include "contrail.sriov.daemonset" }} +{{- end }} +{{- end }} diff --git a/contrail-vrouter/values.yaml b/contrail-vrouter/values.yaml index 3aba072..b051b21 100644 --- a/contrail-vrouter/values.yaml +++ b/contrail-vrouter/values.yaml @@ -37,6 +37,14 @@ global: contrail_env_vrouter_dpdk: AGENT_MODE: dpdk + contrail_env_vrouter_sriov: + SRIOV: true + #per_compute_info: + # - node_name: k8snode1 + # SRIOV_VF: 10 + # SRIOV_PHYSICAL_INTERFACE: enp129s0f1 + # SRIOV_PHYS_NET: physnet1 + node: host_os: ubuntu @@ -47,6 +55,9 @@ labels: vrouter_agent_dpdk: node_selector_key: "opencontrail.org/vrouter-dpdk" node_selector_value: "enabled" + vrouter_agent_sriov: + node_selector_key: "vrouter-sriov" + node_selector_value: "enabled" dependencies: vrouter_agent_kernel: @@ -57,6 +68,10 @@ dependencies: daemonset: - contrail-config - contrail-control + vrouter_agent_sriov: + daemonset: + - contrail-config + - contrail-control # typically overriden by environmental # values, but should include all endpoints @@ -99,10 +114,16 @@ pod: enabled: true min_ready_seconds: 0 max_unavailable: 1 + vrouter_agent_sriov: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 manifests: configmap_vrouter_kernel: true configmap_vrouter_keystone: true configmap_vrouter_dpdk: false + configmap_vrouter_sriov: false daemonset_kernel: true daemonset_dpdk: false + daemonset_vrouter_sriov: false diff --git a/doc/contrail-osh-aio-install.md b/doc/contrail-osh-aio-install.md index e6c9015..3bfb4f4 100644 --- a/doc/contrail-osh-aio-install.md +++ b/doc/contrail-osh-aio-install.md @@ -8,7 +8,7 @@ Using below step you can bring an all-in-one cluster with openstack and contrail 2. Kernel: 4.4.0-87-generic 3. docker: 1.13.1 4. helm: v2.7.2 -5. kubernetes: v1.8.3 +5. kubernetes: v1.9.3 6. openstack: ocata ### Resource spec (used for internal validation) @@ -53,7 +53,7 @@ Edit `${OSH_INFRA_PATH}/tools/gate/devel/local-vars.yaml` if you would want to i Sample `${OSH_INFRA_PATH}/tools/gate/devel/local-vars.yaml` file ```yaml version: - kubernetes: v1.8.3 + kubernetes: v1.9.3 helm: v2.7.2 cni: v0.6.0 @@ -76,6 +76,29 @@ Edit `${OSH_INFRA_PATH}/tools/gate/devel/local-vars.yaml` if you would want to i cni: calico pod_subnet: 192.168.0.0/16 domain: cluster.local + nodes: + labels: + all: + - name: openstack-control-plane + value: enabled + - name: openstack-compute-node + value: enabled + - name: linuxbridge + value: enabled + - name: ceph-mon + value: enabled + - name: ceph-osd + value: enabled + - name: ceph-mds + value: enabled + - name: ceph-rgw + value: enabled + - name: ceph-mgr + value: enabled + - name: opencontrail.org/controller + value: enabled + - name: opencontrail.org/vrouter-kernel + value: enabled ``` ```bash @@ -128,9 +151,6 @@ Edit `${OSH_INFRA_PATH}/tools/gate/devel/local-vars.yaml` if you would want to i export CONTRAIL_REG_USERNAME="abc@abc.com" export CONTRAIL_REG_PASSWORD="password" - kubectl label node opencontrail.org/controller=enabled --all - kubectl label node opencontrail.org/vrouter-kernel=enabled --all - kubectl replace -f ${CHD_PATH}/rbac/cluster-admin.yaml tee /tmp/contrail.yaml << EOF @@ -186,10 +206,12 @@ EOF export CONTRAIL_REGISTRY_ARG="--values=/tmp/contrail-registry-auth.yaml " ``` - helm install --name contrail ${CHD_PATH}/contrail \ - --namespace=contrail --values=/tmp/contrail.yaml \ - ${CONTRAIL_REGISTRY_ARG} - ``` +Deploying contrail charts +```bash +helm install --name contrail ${CHD_PATH}/contrail \ +--namespace=contrail --values=/tmp/contrail.yaml \ +${CONTRAIL_REGISTRY_ARG} +``` 6. Deploy heat charts diff --git a/doc/contrail-osh-multinode-install.md b/doc/contrail-osh-multinode-install.md index a4841d5..b09983f 100644 --- a/doc/contrail-osh-multinode-install.md +++ b/doc/contrail-osh-multinode-install.md @@ -66,10 +66,11 @@ This installation procedure will use Juniper OpenStack Helm infra and OpenStack (k8s-slave)> sudo apt-get install --no-install-recommends -y git ``` -6. Create an inventory file on the master node for ansible base provisoning, please note in below output 10.13.82.43/.44/.45 are nodes IP addresses and will use SSK-key generated in step 1 +6. Create an inventory file on the master node for ansible base provisoning, please note in below output 10.13.82.43/.44/.45 are nodes IP addresses and will use SSH-key generated in step 1. Refer to `${OSH_INFRA_PATH}/tools/gate/devel/sample-contrail-multinode-inventory.yaml` for much more options - ```bash - #!/bin/bash +Sample `multinode-inventory.yaml` + +```bash (k8s-master)> set -xe (k8s-master)> cat > /opt/openstack-helm-infra/tools/gate/devel/multinode-inventory.yaml < cat > /opt/openstack-helm-infra/tools/gate/devel/multinode-vars.yaml < kubectl get nodes -o wide -l openstack-control-plane=enabled -(k8s-master)> kubectl get nodes -o wide -l openstack-compute-node=enabled -``` - -**Note**: If requried please disable openstack labels using following commands to restrict OSH pods creation on specific nodes. In following example "openstack-compute-node" lable is disabled on "ubuntu-contrail-9" node. +Use below commands to verify labelling of nodes ```bash -(k8s-master)> kubectl label node ubuntu-contrail-9 --overwrite openstack-compute-node=disabled +(k8s-master)> kubectl get nodes -l openstack-compute-node=enabled +(k8s-master)> kubectl get nodes -l openstack-control-plane=enabled +(k8s-master)> kubectl get nodes -l opencontrail.org/controller=enabled +(k8s-master)> kubectl get nodes -l opencontrail.org/vrouter-kernel=enabled ``` 1. Deploy OpenStack Helm charts using following commands. @@ -200,28 +229,14 @@ All nodes by default labeled with "openstack-control-plane" and "openstack-compu #### Installation of Contrail Helm charts -1. All contrail pods will be deployed in Namespace "contrail". Label Contrail Nodes using below command and following labels are used by Contrail - -* Control Nodes: opencontrail.org/controller -* vRouter Kernel: opencontrail.org/vrouter-kernel -* vRouter DPDK: opencontrail.org/vrouter-dpdk - -In following example "ubuntu-contrail-11" and "ubuntu-contrail-10" are dpdk vrouter and kernel vrouter compute node respectively. Whereas, ubuntu-contrail-7 ubuntu-contrail-8 ubuntu-contrail-9 are contrail controller nodes - - ```bash -(k8s-master)> kubectl label node ubuntu-contrail-11 opencontrail.org/vrouter-dpdk=enabled -(k8s-master)> kubectl label node ubuntu-contrail-10 opencontrail.org/vrouter-kernel=enabled -(k8s-master)> kubectl label nodes ubuntu-contrail-7 ubuntu-contrail-8 ubuntu-contrail-9 opencontrail.org/controller=enabled - ``` - -2. K8s clusterrolebinding for contrail +1. K8s clusterrolebinding for contrail ```bash (k8s-master)> cd $CHD_PATH (k8s-master)> kubectl replace -f ${CHD_PATH}/rbac/cluster-admin.yaml ``` -3. Now deploy opencontrail charts +2. Now deploy opencontrail charts ```bash (k8s-master)> cd $CHD_PATH @@ -325,7 +340,7 @@ export CONTRAIL_REGISTRY_ARG="--values=/tmp/contrail-registry-auth.yaml " ${CONTRAIL_REGISTRY_ARG} ``` -4. Once Contrail PODs are up and running deploy OpenStack Heat chart using following command. +3. Once Contrail PODs are up and running deploy OpenStack Heat chart using following command. ```bash # Edit ${OSH_PATH}/tools/overrides/backends/opencontrail/nova.yaml and @@ -334,7 +349,7 @@ export CONTRAIL_REGISTRY_ARG="--values=/tmp/contrail-registry-auth.yaml " (k8s-master)> ./tools/deployment/multinode/151-heat-opencontrail.sh ``` -5. Run compute kit test using following command at the end. +4. Run compute kit test using following command at the end. ```bash (k8s-master)> ./tools/deployment/multinode/143-compute-kit-opencontrail-test.sh diff --git a/roles/contrail-helm-deployer/files/helm-deploy.sh b/roles/contrail-helm-deployer/files/helm-deploy.sh index 9e6e3dd..210a533 100644 --- a/roles/contrail-helm-deployer/files/helm-deploy.sh +++ b/roles/contrail-helm-deployer/files/helm-deploy.sh @@ -40,9 +40,8 @@ export OSH_EXTRA_HELM_ARGS_NEUTRON="--set images.tags.opencontrail_neutron_init= cd $CHD_PATH make -#Label nodes with contrail specific labels -kubectl label node opencontrail.org/controller=enabled --all -kubectl label node opencontrail.org/vrouter-kernel=enabled --all +kubectl label node opencontrail.org/controller=enabled --all --overwrite=true +kubectl label node opencontrail.org/vrouter-kernel=enabled --all --overwrite=true #Give cluster-admin permission for the user to create contrail pods kubectl replace -f rbac/cluster-admin.yaml