Add plumbing for hardware offloads

* This change expands the VNIC type support for the vrouter VIF type by
  adding 'direct' and 'virtio-forwarder' plugging support.

* After this change, the vrouter VIF type will support the following three
  modes:
  * 'normal': the 'classic' tap-style VNIC plugged into the instance,
  * 'direct': a PCI Virtual Function is passed through to the instance,
  * 'virtio-forwarder': a PCI Virtual Function is proxied to the
    instance via a vhost-user virtio forwarder.

* The libvirt xml generation code was updated to support the modes.

* The Nova legacy VIF plugging code was updated to support the modes.

* The corresponding review on OpenContrail is currently at:
  https://review.opencontrail.org/#/c/42850/ for master
  https://review.opencontrail.org/#/c/42478/ for 4.1

Change-Id: I73908368d48b0c6f8780b8dabe30b83875efa0b8
Closes-Bug: #1772612
Signed-off-by: Jan Gutter <jan.gutter@netronome.com>

nova/network/model.py

@@ -97,12 +97,13 @@ def ensure_string_keys(d):
 VNIC_TYPE_MACVTAP = 'macvtap'
 VNIC_TYPE_DIRECT_PHYSICAL = 'direct-physical'
 VNIC_TYPE_BAREMETAL = 'baremetal'
+VNIC_TYPE_VIRTIO_FORWARDER = 'virtio-forwarder'
 
 # Define list of ports which needs pci request.
 # Note: The macvtap port needs a PCI request as it is a tap interface
 # with VF as the lower physical interface.
 VNIC_TYPES_SRIOV = (VNIC_TYPE_DIRECT, VNIC_TYPE_MACVTAP,
-                    VNIC_TYPE_DIRECT_PHYSICAL)
+                    VNIC_TYPE_DIRECT_PHYSICAL, VNIC_TYPE_VIRTIO_FORWARDER)
 
 # Define list of ports which are passthrough to the guest
 # and need a special treatment on snapshot and suspend/resume

nova/virt/libvirt/vif.py

@@ -454,7 +454,17 @@ def get_config_vrouter(self, instance, vif, image_meta,
                                     inst_type, virt_type, vif['vnic_type'])
         dev = self.get_vif_devname(vif)
 
-        if CONF.contrail.use_userspace_vhost:
+        vnic_type = vif.get('vnic_type', network_model.VNIC_TYPE_NORMAL)
+        if vnic_type == network_model.VNIC_TYPE_DIRECT:
+            conf.vlan = None
+            conf.source_mode = 'passthrough'
+            conf.source_dev = vif['profile']['pci_slot']
+            conf.net_type = 'hostdev'
+            conf.driver_name = 'vfio'
+        elif vnic_type == network_model.VNIC_TYPE_VIRTIO_FORWARDER:
+            return self.get_config_vhostuser(instance, vif, image_meta,
+                                             inst_type, virt_type, host)
+        elif CONF.contrail.use_userspace_vhost:
             dev = path.join(CONF.contrail.userspace_vhost_socket_dir,
                             'uvh_vif_' + dev)
             designer.set_vif_host_backend_vhostuser_config(conf, 'client', dev)
@@ -771,8 +781,20 @@ def plug_vrouter(self, instance, vif):
                     instance.project_id, ip_addr, ip6_addr,
                     instance.display_name, vif['address'],
                     vif['devname'], ptype, -1, -1))
+        vnic_type = vif.get('vnic_type', network_model.VNIC_TYPE_NORMAL)
+        if vnic_type == network_model.VNIC_TYPE_DIRECT:
+            pci_dev = vif['profile']['pci_slot']
+            cmd_args += (" --pci_dev=%s --vnic_type=%s" %
+                         (pci_dev, vnic_type))
+        if vnic_type == network_model.VNIC_TYPE_VIRTIO_FORWARDER:
+            pci_dev = vif['profile']['pci_slot']
+            mode, socket = self._get_vhostuser_settings(vif)
+            cmd_args += (" --pci_dev=%s --vnic_type=%s"
+                         " --vhostuser_mode=%s --vhostuser_socket=%s" %
+                         (pci_dev, vnic_type, mode, socket))
         try:
-            if not CONF.contrail.use_userspace_vhost:
+            if (vnic_type == network_model.VNIC_TYPE_NORMAL and
+               not CONF.contrail.use_userspace_vhost):
                 multiqueue = self._is_multiqueue_enabled(instance.image_meta,
                                                      instance.flavor)
                 linux_net.create_tap_dev(dev, multiqueue=multiqueue)
@@ -968,9 +990,21 @@ def unplug_vrouter(self, instance, vif):
         # made in nova_contrail_vif/vif_plug_vrouter/vrouter.py
         dev = self.get_vif_devname(vif)
         cmd_args = ("--oper=delete --uuid=%s" % (vif['id']))
+        vnic_type = vif.get('vnic_type', network_model.VNIC_TYPE_NORMAL)
+        if vnic_type == network_model.VNIC_TYPE_DIRECT:
+            pci_dev = vif['profile']['pci_slot']
+            cmd_args += (" --pci_dev=%s --vnic_type=%s" %
+                         (pci_dev, vnic_type))
+        if vnic_type == network_model.VNIC_TYPE_VIRTIO_FORWARDER:
+            pci_dev = vif['profile']['pci_slot']
+            mode, socket = self._get_vhostuser_settings(vif)
+            cmd_args += (" --pci_dev=%s --vnic_type=%s"
+                         " --vhostuser_mode=%s --vhostuser_socket=%s" %
+                         (pci_dev, vnic_type, mode, socket))
         try:
             utils.execute('vrouter-port-control', cmd_args, run_as_root=True)
-            if not CONF.contrail.use_userspace_vhost:
+            if (vnic_type == network_model.VNIC_TYPE_NORMAL and
+               not CONF.contrail.use_userspace_vhost):
                 linux_net.delete_net_dev(dev)
         except processutils.ProcessExecutionError:
             LOG.exception(