KVM: DPDK live migrations

api/src/main/java/com/cloud/agent/api/to/DPDKTO.java

@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+public class DPDKTO {
+
+    private String path;
+    private String port;
+    private String mode;
+
+    public DPDKTO() {
+    }
+
+    public DPDKTO(String path, String port, String mode) {
+        this.path = path;
+        this.port = port;
+        this.mode = mode;
+    }
+
+    public String getPath() {
+        return path;
+    }
+
+    public String getPort() {
+        return port;
+    }
+
+    public String getMode() {
+        return mode;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/NicTO.java

@@ -30,7 +30,7 @@ public class NicTO extends NetworkTO {
     String nicUuid;
     List<String> nicSecIps;
     Map<NetworkOffering.Detail, String> details;
-    boolean dpdkDisabled;
+    boolean dpdkEnabled;
 
     public NicTO() {
         super();
@@ -111,11 +111,11 @@ public void setDetails(final Map<NetworkOffering.Detail, String> details) {
         this.details = details;
     }
 
-    public boolean isDpdkDisabled() {
-        return dpdkDisabled;
+    public boolean isDpdkEnabled() {
+        return dpdkEnabled;
     }
 
-    public void setDpdkDisabled(boolean dpdkDisabled) {
-        this.dpdkDisabled = dpdkDisabled;
+    public void setDpdkEnabled(boolean dpdkEnabled) {
+        this.dpdkEnabled = dpdkEnabled;
     }
 }

core/src/main/java/com/cloud/agent/api/MigrateCommand.java

@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.agent.api.to.DPDKTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 
 public class MigrateCommand extends Command {
@@ -37,6 +38,15 @@ public class MigrateCommand extends Command {
     private VirtualMachineTO vmTO;
     private boolean executeInSequence = false;
     private List<MigrateDiskInfo> migrateDiskInfoList = new ArrayList<>();
+    private Map<String, DPDKTO> dpdkInterfaceMapping = new HashMap<>();
+
+    public Map<String, DPDKTO> getDpdkInterfaceMapping() {
+        return dpdkInterfaceMapping;
+    }
+
+    public void setDpdkInterfaceMapping(Map<String, DPDKTO> dpdkInterfaceMapping) {
+        this.dpdkInterfaceMapping = dpdkInterfaceMapping;
+    }
 
     protected MigrateCommand() {
     }

core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java

@@ -19,7 +19,15 @@
 
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.DPDKTO;
+
+import java.util.HashMap;
+import java.util.Map;
+
 public class PrepareForMigrationAnswer extends Answer {
+
+    private Map<String, DPDKTO> dpdkInterfaceMapping = new HashMap<>();
+
     protected PrepareForMigrationAnswer() {
     }
 
@@ -34,4 +42,12 @@ public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd, Exception ex) {
     public PrepareForMigrationAnswer(PrepareForMigrationCommand cmd) {
         super(cmd, true, null);
     }
+
+    public void setDpdkInterfaceMapping(Map<String, DPDKTO> mapping) {
+        this.dpdkInterfaceMapping = mapping;
+    }
+
+    public Map<String, DPDKTO> getDpdkInterfaceMapping() {
+        return this.dpdkInterfaceMapping;
+    }
 }

core/src/main/java/com/cloud/agent/api/StopCommand.java

@@ -19,6 +19,7 @@
 
 package com.cloud.agent.api;
 
+import com.cloud.agent.api.to.DPDKTO;
 import com.cloud.agent.api.to.GPUDeviceTO;
 import com.cloud.vm.VirtualMachine;
 
@@ -34,6 +35,15 @@ public class StopCommand extends RebootCommand {
     boolean checkBeforeCleanup = false;
     String controlIp = null;
     boolean forceStop = false;
+    private Map<String, DPDKTO> dpdkInterfaceMapping;
+
+    public Map<String, DPDKTO> getDpdkInterfaceMapping() {
+        return dpdkInterfaceMapping;
+    }
+
+    public void setDpdkInterfaceMapping(Map<String, DPDKTO> dpdkInterfaceMapping) {
+        this.dpdkInterfaceMapping = dpdkInterfaceMapping;
+    }
     /**
      * On KVM when using iSCSI-based managed storage, if the user shuts a VM down from the guest OS (as opposed to doing so from CloudStack),
      * we need to pass to the KVM agent a list of applicable iSCSI volumes that need to be disconnected.

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -39,7 +39,8 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import org.apache.cloudstack.api.ApiConstants;
+import com.cloud.agent.api.PrepareForMigrationAnswer;
+import com.cloud.agent.api.to.DPDKTO;
 import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao;
 import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd;
 import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin;
@@ -1118,8 +1119,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
 
                     vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
 
-                    addExtraConfig(vmTO);
-
                     work = _workDao.findById(work.getId());
                     if (work == null || work.getStep() != Step.Prepare) {
                         throw new ConcurrentOperationException("Work steps have been changed: " + work);
@@ -1284,15 +1283,6 @@ public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfil
         }
     }
 
-    private void addExtraConfig(VirtualMachineTO vmTO) {
-        Map<String, String> details = vmTO.getDetails();
-        for (String key : details.keySet()) {
-            if (key.startsWith(ApiConstants.EXTRA_CONFIG)) {
-                vmTO.addExtraConfig(key, details.get(key));
-            }
-        }
-    }
-
     // for managed storage on KVM, need to make sure the path field of the volume in question is populated with the IQN
     private void handlePath(final DiskTO[] disks, final HypervisorType hypervisorType) {
         if (hypervisorType != HypervisorType.KVM) {
@@ -2362,6 +2352,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
         }
 
         boolean migrated = false;
+        Map<String, DPDKTO> dpdkInterfaceMapping = null;
         try {
             final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows");
             final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType()));
@@ -2370,6 +2361,11 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
             mc.setAutoConvergence(kvmAutoConvergence);
             mc.setHostGuid(dest.getHost().getGuid());
 
+            dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping();
+            if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
+                mc.setDpdkInterfaceMapping(dpdkInterfaceMapping);
+            }
+
             try {
                 final Answer ma = _agentMgr.send(vm.getLastHostId(), mc);
                 if (ma == null || !ma.getResult()) {
@@ -2396,7 +2392,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                 if (!checkVmOnHost(vm, dstHostId)) {
                     s_logger.error("Unable to complete migration for " + vm);
                     try {
-                        _agentMgr.send(srcHostId, new Commands(cleanup(vm)), null);
+                        _agentMgr.send(srcHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                     } catch (final AgentUnavailableException e) {
                         s_logger.error("AgentUnavailableException while cleanup on source host: " + srcHostId);
                     }
@@ -2417,7 +2413,7 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy
                         "Unable to migrate vm " + vm.getInstanceName() + " from host " + fromHost.getName() + " in zone " + dest.getDataCenter().getName() + " and pod " +
                                 dest.getPod().getName(), "Migrate Command failed.  Please check logs.");
                 try {
-                    _agentMgr.send(dstHostId, new Commands(cleanup(vm)), null);
+                    _agentMgr.send(dstHostId, new Commands(cleanup(vm, dpdkInterfaceMapping)), null);
                 } catch (final AgentUnavailableException ae) {
                     s_logger.info("Looks like the destination Host is unavailable for cleanup");
                 }
@@ -3106,9 +3102,12 @@ private void orchestrateReboot(final String vmUuid, final Map<VirtualMachineProf
         }
     }
 
-    public Command cleanup(final VirtualMachine vm) {
+    public Command cleanup(final VirtualMachine vm, Map<String, DPDKTO> dpdkInterfaceMapping) {
         StopCommand cmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
         cmd.setControlIp(getControlNicIpForVM(vm));
+        if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) {
+            cmd.setDpdkInterfaceMapping(dpdkInterfaceMapping);
+        }
         return cmd;
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -2706,7 +2706,10 @@ public StartupCommand[] initialize() {
 
         final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem);
 
-        final String capabilities = String.join(",", info.getCapabilities());
+        String capabilities = String.join(",", info.getCapabilities());
+        if (dpdkSupport) {
+            capabilities += ",dpdk";
+        }
 
         final StartupRoutingCommand cmd =
                 new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java

@@ -224,9 +224,13 @@ public boolean parseDomainXML(String domXML) {
                     def.defEthernet(dev, mac, NicModel.valueOf(model.toUpperCase()), scriptPath, networkRateKBps);
                 } else if (type.equals("vhostuser")) {
                     String sourcePort = getAttrValue("source", "path", nic);
-                    String[] sourcePathParts = sourcePort.split("/");
-                    String port = sourcePathParts[sourcePathParts.length - 1];
+                    String mode = getAttrValue("source", "mode", nic);
+                    int lastSlashIndex = sourcePort.lastIndexOf("/");
+                    String ovsPath = sourcePort.substring(0,lastSlashIndex);
+                    String port = sourcePort.substring(lastSlashIndex + 1);
                     def.setDpdkSourcePort(port);
+                    def.setDpdkOvsPath(ovsPath);
+                    def.setInterfaceMode(mode);
                 }
 
                 if (StringUtils.isNotBlank(slot)) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -963,7 +963,7 @@ public String toString() {
     }
 
     public static class InterfaceDef {
-        enum GuestNetType {
+        public enum GuestNetType {
             BRIDGE("bridge"), DIRECT("direct"), NETWORK("network"), USER("user"), ETHERNET("ethernet"), INTERNAL("internal"), VHOSTUSER("vhostuser");
             String _type;
 
@@ -1176,10 +1176,24 @@ public void setDpdkSourcePort(String port) {
             _dpdkSourcePort = port;
         }
 
-        @Override
-        public String toString() {
+        public String getDpdkOvsPath() {
+            return _dpdkSourcePath;
+        }
+
+        public void setDpdkOvsPath(String path) {
+            _dpdkSourcePath = path;
+        }
+
+        public String getInterfaceMode() {
+            return _interfaceMode;
+        }
+
+        public void setInterfaceMode(String mode) {
+            _interfaceMode = mode;
+        }
+
+        public String getContent() {
             StringBuilder netBuilder = new StringBuilder();
-            netBuilder.append("<interface type='" + _netType + "'>\n");
             if (_netType == GuestNetType.BRIDGE) {
                 netBuilder.append("<source bridge='" + _sourceName + "'/>\n");
             } else if (_netType == GuestNetType.NETWORK) {
@@ -1233,6 +1247,14 @@ public String toString() {
             if (_slot  != null) {
                 netBuilder.append(String.format("<address type='pci' domain='0x0000' bus='0x00' slot='0x%02x' function='0x0'/>\n", _slot));
             }
+            return netBuilder.toString();
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder netBuilder = new StringBuilder();
+            netBuilder.append("<interface type='" + _netType + "'>\n");
+            netBuilder.append(getContent());
             netBuilder.append("</interface>\n");
             return netBuilder.toString();
         }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java

@@ -87,12 +87,34 @@ public void getPifs() {
         s_logger.debug("done looking for pifs, no more bridges");
     }
 
+    /**
+     * Plug interface with DPDK support:
+     *      - Create a new port with DPDK support for the interface
+     *      - Set the 'intf' path to the new port
+     */
+    protected void plugDPDKInterface(InterfaceDef intf, String trafficLabel, Map<String, String> extraConfig,
+                                     String vlanId, String guestOsType, NicTO nic, String nicAdapter) {
+        s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
+        String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
+        if (StringUtils.isBlank(dpdkOvsPath)) {
+            throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
+        }
+        String port = dpdkDriver.getNextDpdkPort();
+        DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
+        dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
+        String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
+        intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
+                getGuestNicModel(guestOsType, nicAdapter), 0,
+                dpdkDriver.getExtraDpdkProperties(extraConfig),
+                interfaceMode);
+    }
+
     @Override
     public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
         s_logger.debug("plugging nic=" + nic);
 
         LibvirtVMDef.InterfaceDef intf = new LibvirtVMDef.InterfaceDef();
-        if (!_libvirtComputingResource.dpdkSupport || nic.isDpdkDisabled()) {
+        if (!_libvirtComputingResource.dpdkSupport || !nic.isDpdkEnabled()) {
             // Let libvirt handle OVS ports creation when DPDK property is disabled or when it is enabled but disabled for the nic
             // For DPDK support, libvirt does not handle ports creation, invoke 'addDpdkPort' method
             intf.setVirtualPortType("openvswitch");
@@ -114,20 +136,8 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
             if ((nic.getBroadcastType() == Networks.BroadcastDomainType.Vlan || nic.getBroadcastType() == Networks.BroadcastDomainType.Pvlan) &&
                     !vlanId.equalsIgnoreCase("untagged")) {
                 if (trafficLabel != null && !trafficLabel.isEmpty()) {
-                    if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
-                        s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
-                        String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
-                        if (StringUtils.isBlank(dpdkOvsPath)) {
-                            throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
-                        }
-                        String port = dpdkDriver.getNextDpdkPort();
-                        DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
-                        dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
-                        String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
-                        intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
-                                getGuestNicModel(guestOsType, nicAdapter), 0,
-                                dpdkDriver.getExtraDpdkProperties(extraConfig),
-                                interfaceMode);
+                    if (_libvirtComputingResource.dpdkSupport && nic.isDpdkEnabled()) {
+                        plugDPDKInterface(intf, trafficLabel, extraConfig, vlanId, guestOsType, nic, nicAdapter);
                     } else {
                         s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
                         intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);
@@ -180,7 +190,7 @@ public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<S
     @Override
     public void unplug(InterfaceDef iface) {
         // Libvirt apparently takes care of this, see BridgeVifDriver unplug
-        if (_libvirtComputingResource.dpdkSupport) {
+        if (_libvirtComputingResource.dpdkSupport && StringUtils.isNotBlank(iface.getDpdkSourcePort())) {
             // If DPDK is enabled, we'll need to cleanup the port as libvirt won't
             String dpdkPort = iface.getDpdkSourcePort();
             String cmd = String.format("ovs-vsctl del-port %s", dpdkPort);