[agilio-vrouter-enablement] Server-manager integration

Integrate agilio-vrouter deployment with the existing ansible infrastructure.
The assumption for the deployment to work is that the netronome repo is set
up on the server-manager node.

Change-Id: I627e5d05162148a2f66c59cea1481a0bd78e79b3
Closes-Bug: #1767110
Signed-off-by: Pieter Malan <pieter.malan@netronome.com>

playbooks/roles/common/tasks/host/Debian.yml

@@ -9,6 +9,14 @@
   when: contrail_apt_repo is defined and contrail_apt_repo != ""
   ignore_errors: yes
 
+- name: Add netronome apt repository
+  apt_repository:
+    repo: "deb {{ netronome_apt_repo }} ./"
+    state: present
+  register: apt_repo_added
+  when: ns_hw_acceleration is defined and ns_hw_acceleration == True
+  ignore_errors: yes
+
 - name: Allow unauthenticated packages for contrail packages to work
   copy: content='APT::Get::AllowUnauthenticated "true";' dest=/etc/apt/apt.conf.d/99allowunauth
 

playbooks/roles/contrail/bare_metal_agent/defaults/main.yml

@@ -8,6 +8,10 @@ default_kernel_compute_packages:
   - contrail-vrouter-dkms
   - contrail-vrouter-common
 
+netronome_vrouter_package:
+  - ns-agilio-vrouter
+  - virtio-forwarder
+
 upgrade_kernel_vrouter_packages:
   - "contrail-vrouter-{{ kernel_version }}-generic"
   - contrail-vrouter-common
@@ -89,7 +93,9 @@ compute_packages_redhat:
  - contrail-vrouter
  - contrail-vrouter-init
 compute_packages_default: "{{ compute_packages_to_install + lbaas_packages if enable_lbaas else compute_packages_to_install }}"
-compute_packages: "{{ compute_packages_redhat if ansible_os_family == 'RedHat' else compute_packages_default }}"
+enable_netronome: "{{ ns_hw_acceleration if ns_hw_acceleration is defined else 'False' }}"
+compute_packages_netronome: "{{ compute_packages_to_install + netronome_vrouter_package if enable_netronome == True else compute_packages_default }}"
+compute_packages: "{{ compute_packages_redhat if ansible_os_family == 'RedHat' else compute_packages_netronome }}"
 
 #SRIOV defaults
 sriov: {}

playbooks/roles/contrail/bare_metal_agent/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: "update-grub"
+  command: "update-grub"

playbooks/roles/contrail/bare_metal_agent/tasks/provision/main.yml

@@ -190,3 +190,93 @@
   when:
     - (provision_type == 'ISSU')
     - (dpdk_mode or sriov_interface_list)
+
+- name: "Netronome: Enable KVM Hugepages"
+  lineinfile:
+    dest: /etc/default/qemu-kvm
+    regexp: '^KVM_HUGEPAGES=.*'
+    line: 'KVM_HUGEPAGES=1'
+    backrefs: yes
+    state: present
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+- name: "Netronome: Disable KSM-Enable"
+  lineinfile:
+    dest: /etc/default/qemu-kvm
+    regexp: '^KSM_ENABLED=.*'
+    line: 'KSM_ENABLED=0'
+    backrefs: yes
+    state: present
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+- name: "Netronome: Enable offloads in contrail-vrouter-agent.conf"
+  lineinfile:
+    dest: /etc/contrail/contrail-vrouter-agent.conf
+    insertafter: '\[DEFAULT\]'
+    line: 'offload=enabled'
+    state: present
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+- name: "Netronome: Update default grub for SR-IOV iommu"
+  lineinfile:
+    dest: /etc/default/grub
+    insertafter: 'GRUB_CMDLINE_LINUX_DEFAULT="[^"]*'
+    line: 'GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} intel_iommu=on iommu=pt"'
+    state: present
+  notify: update-grub
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+- name: "Netronome: Update default grub for virtioforwarder hugepages"
+  lineinfile:
+    dest: /etc/default/grub
+    insertafter: 'GRUB_CMDLINE_LINUX_DEFAULT="[^"]*'
+    line: 'GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} default_hugepagesz=2M hugepagesz=2M hugepages={{ns_virtioforwarder_hugepages}}"'
+    state: present
+  notify: update-grub
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+- name: "Netronome: Set virtio-forwarder socket owner"
+  lineinfile:
+    dest: /etc/default/virtioforwarder
+    regexp: '^VIRTIOFWD_SOCKET_OWNER=.*'
+    line: 'VIRTIOFWD_SOCKET_OWNER=libvirt-qemu'
+    state: present
+  when:
+    - ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+    - (ns_virtioforwarder_hugepages is defined)
+
+- name: "Netronome: Set virtio-forwarder socket group"
+  lineinfile:
+    dest: /etc/default/virtioforwarder
+    regexp: '^VIRTIOFWD_SOCKET_GROUP=.*'
+    line: 'VIRTIOFWD_SOCKET_GROUP=kvm'
+    state: present
+  when:
+    - ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+    - (ns_virtioforwarder_hugepages is defined)
+
+- name: "Netronome: Set virtio-forwarder dynamic sockets"
+  lineinfile:
+    dest: /etc/default/virtioforwarder
+    regexp: '^VIRTIOFWD_DYNAMIC_SOCKETS=.*'
+    line: 'VIRTIOFWD_DYNAMIC_SOCKETS=y'
+    state: present
+  when:
+    - ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+    - (ns_virtioforwarder_hugepages is defined)
+
+- name: "Netronome: Set virtio-forwarder socket dir"
+  lineinfile:
+    dest: /etc/default/virtioforwarder
+    regexp: '^VIRTIOFWD_SOCKET_DIR=.*'
+    line: 'VIRTIOFWD_SOCKET_DIR=/var/run/vrouter'
+    state: present
+  when:
+    - ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+    - (ns_virtioforwarder_hugepages is defined)
+
+- name: "Netronome: Enable virtio-forwarder"
+  shell: systemctl enable virtio-forwarder.service
+  when:
+    - ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+    - (ns_virtioforwarder_hugepages is defined)

playbooks/roles/openstack/compute/tasks/configure.yml

@@ -111,6 +111,15 @@
   notify: restart libvirt
   ignore_errors: yes
 
+- name: Configure apparmor for sockets
+  blockinfile:
+    dest: /etc/apparmor.d/abstractions/libvirt-qemu
+    block: |
+      # for access to create sockets in /var/run/vrouter
+      /{var/,}run/vrouter/** rwmix,
+  notify: restart libvirt
+  when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
 - name: Configure qemu
   blockinfile:
     dest: /etc/libvirt/qemu.conf
@@ -119,7 +128,7 @@
        "/dev/null", "/dev/full", "/dev/zero",
         "/dev/random", "/dev/urandom",
         "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
-        "/dev/rtc", "/dev/hpet","/dev/net/tun",
+        "/dev/rtc", "/dev/hpet","/dev/net/tun","/dev/vfio/vfio",
       ]
   notify: restart libvirt
 

playbooks/site.yml

@@ -97,6 +97,34 @@
       tags: [neutron_plugin]
       when: cloud_orchestrator == "openstack"
 
+- name: "Netronome: update initramfs"
+  hosts: contrail-compute
+  pre_tasks:
+    - name: "Netronome: Stop contrail-vrouter-agent"
+      service: name=contrail-vrouter-agent state=stopped
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+    - name: "Netronome: Unload vRouter Module"
+      modprobe: name=vrouter state=absent
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+    - name: "Netronome: Start Accelerated vRouter"
+      shell: /opt/netronome/bin/ns-vrouter-ctl start
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+    - name: "Netronome: Add whitelist pci addr to nova.conf"
+      shell: /opt/netronome/libexec/add_nova_whitelist.py
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+    - name: "Netronome: Start contrail-vrouter-agent"
+      service: name=contrail-vrouter-agent state=started
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
+  post_tasks:
+    - name: "Netronome: Update initramfs"
+      shell: update-initramfs -u
+      when: ((ns_hw_acceleration is defined) and (ns_hw_acceleration|bool == true))
+
 - name: Reboot compute nodes
   hosts: contrail-compute
   pre_tasks:
@@ -132,6 +160,8 @@
         reboot_required: >-
           {%- if vrouter_loaded == "no" or vhost_loaded == "no" or vhost0_exists == false  or vrouter_kmod_hugepage == true -%}
             true
+          {%- elif ns_hw_acceleration is defined and ns_hw_acceleration|bool == true -%}
+            true
           {%- else %}
             false
           {%- endif -%}