Numa migration

lib/vdsm/api/vdsm-api.yml

@@ -3606,6 +3606,16 @@ types:
             -   string
             added: '4.5'
 
+        -   defaultvalue: null
+            description: A list of NUMA nodesets (e.g. '1,3-5') for all NUMA
+              nodes. Items of the list correspond to vNUMA cells and, if
+              specified, length of the list must match number of vNUMA cells
+              in the VM.
+            name: numaNodesets
+            type:
+            -   string
+            added: '4.5'
+
         type: object
 
     MigrationStatus: &MigrationStatus

lib/vdsm/virt/domain_descriptor.py

@@ -175,6 +175,17 @@ def pinned_cpus(self):
                     pinning[int(vcpu)] = cpus
         return pinning
 
+    @property
+    def vnuma_count(self):
+        """
+        :return: Number of vNUMA cells defined in VM. Zero is returned when
+          NUMA is not defined.
+        """
+        numa = vmxml.find_first(self._dom, 'cpu/numa', None)
+        if numa is None:
+            return 0
+        return len(list(vmxml.find_all(numa, 'cell')))
+
 
 class DomainDescriptor(MutableDomainDescriptor):
 

lib/vdsm/virt/migration.py

@@ -29,6 +29,7 @@
 import threading
 import time
 import libvirt
+import xml.etree.ElementTree as ET
 
 from vdsm.common import concurrent
 from vdsm.common import conv
@@ -139,7 +140,7 @@ def __init__(self, vm, dst='', dstparams='',
                  tunneled=False, dstqemu='', abortOnError=False,
                  consoleAddress=None, compressed=False,
                  autoConverge=False, recovery=False, encrypted=False,
-                 cpusets=None, parallel=None, **kwargs):
+                 cpusets=None, parallel=None, numaNodesets=None, **kwargs):
         self.log = vm.log
         self._vm = vm
         self._dom = DomainAdapter(self._vm)
@@ -202,6 +203,7 @@ def __init__(self, vm, dst='', dstparams='',
         # the first extend of the disk during migration if finished.
         self._supports_disk_refresh = None
         self._destination_cpusets = cpusets
+        self._destination_numa_nodesets = numaNodesets
 
     def start(self):
         self._thread.start()
@@ -648,11 +650,37 @@ def _migration_params(self, muri):
                     continue
                 cputune.remove(vcpu)
         if self._destination_cpusets is not None:
+            # Reconfigure CPU pinning based on the call parameter
             if cputune is None:
                 cputune = xml.etree.ElementTree.Element('cputune')
                 dom.append(cputune)
-            for vcpupin in self._destination_cpusets:
+            # First modify existing elements
+            for vcpupin in vmxml.find_all(cputune, 'vcpupin'):
+                vcpu_id = int(vcpupin.get('vcpu'))
+                if vcpu_id >= 0 and vcpu_id < len(self._destination_cpusets):
+                    vcpupin.set('cpuset',
+                                str(self._destination_cpusets[vcpu_id]))
+                    self._destination_cpusets[vcpu_id] = None
+            # Now create elements for pinning that was not there before.
+            # This should happen only for pinning that was removed above. It
+            # should not happen for manual CPU pinning because it would render
+            # the value of manuallyPinedCPUs metadata invalid.
+            for vcpu_id, cpuset in enumerate(self._destination_cpusets):
+                if cpuset is None:
+                    continue
+                vcpupin = ET.Element('vcpupin')
+                vcpupin.set('vcpu', str(vcpu_id))
+                vcpupin.set('cpuset', str(cpuset))
                 cputune.append(vcpupin)
+        if self._destination_numa_nodesets is not None:
+            numatune = dom.find('numatune')
+            if numatune is not None:
+                for memnode in vmxml.find_all(numatune, 'memnode'):
+                    cellid = int(memnode.get('cellid'))
+                    if (cellid >= 0 and
+                            cellid < len(self._destination_numa_nodesets)):
+                        memnode.set('nodeset',
+                                    self._destination_numa_nodesets[cellid])
         xml = xmlutils.tostring(dom)
         self._vm.log.debug("Migrating domain XML: %s", xml)
         params[libvirt.VIR_MIGRATE_PARAM_DEST_XML] = xml

lib/vdsm/virt/vm.py

@@ -2102,8 +2102,9 @@ def hasTransientDisks(self):
                 return True
         return False
 
-    def _process_migration_cpusets(self, cpusets):
-        xml_cpusets = []
+    def _validate_migration_cpusets(self, cpusets):
+        if cpusets is None:
+            return
         if len(cpusets) != self.get_number_of_cpus():
             raise exception.InvalidParameter(
                 "length of cpusets (%d) must match"
@@ -2112,27 +2113,36 @@ def _process_migration_cpusets(self, cpusets):
         for vcpu, cpuset in enumerate(cpusets):
             if cpuset is None:
                 continue
-            vcpupin = ET.Element('vcpupin')
             # Try parsing the content to validate it
             try:
                 taskset.cpulist_parse(cpuset)
             except ValueError:
                 raise exception.InvalidParameter(
                     "One or more invalid cpuset list descriptions in: %r" %
                     cpusets)
-            vcpupin.set('vcpu', str(vcpu))
-            vcpupin.set('cpuset', str(cpuset))
-            xml_cpusets.append(vcpupin)
-        return xml_cpusets
+
+    def _validate_migration_numa_nodesets(self, numaNodesets):
+        if numaNodesets is None:
+            return
+        if len(numaNodesets) != self._domain.vnuma_count:
+            raise exception.InvalidParameter(
+                "length of numaNodesets (%d) must match"
+                " number of vNUMA nodes (%d)" % (
+                    len(numaNodesets), self._domain.vnuma_count))
+        for nodeset in numaNodesets:
+            try:
+                # Validate node set string just like cpu list
+                taskset.cpulist_parse(nodeset)
+            except ValueError:
+                raise exception.InvalidParameter(
+                    "One or more invalid node set descriptions in: %r" %
+                    numaNodesets)
 
     @api.guard(_not_migrating)
     @api.guard(_not_paused_on_io_error)
     def migrate(self, params):
-        if params.get('cpusets') is not None:
-            # Validate content and turn it into <vcpupin> elements here to
-            # avoid failures later during migration
-            params['cpusets'] = self._process_migration_cpusets(
-                params['cpusets'])
+        self._validate_migration_cpusets(params.get('cpusets'))
+        self._validate_migration_numa_nodesets(params.get('numaNodesets'))
         self._acquireCpuLockWithTimeout(flow='migrate')
         try:
             # It is unlikely, but we could receive migrate()

tests/virt/vmoperations_test.py

@@ -24,7 +24,6 @@
 
 import libvirt
 from six.moves import zip
-import xml.etree.ElementTree as ET
 
 from vdsm import numa
 from vdsm.common import define
@@ -465,25 +464,18 @@ def testAcpiRebootConnected(self):
             testvm._dom = fake.Domain(vmId='testvm')
             assert not response.is_error(testvm.acpiReboot())
 
-    def test_process_migration_cpusets(self):
-        with fake.VM() as testvm:
-            elements = testvm._process_migration_cpusets(['1,3-5'])
-            assert len(elements) == 1
-            assert isinstance(elements[0], ET.Element)
-            assert elements[0].tag == 'vcpupin'
-            assert sorted(elements[0].keys()) == ['cpuset', 'vcpu']
-            assert elements[0].get('vcpu') == '0'
-            assert elements[0].get('cpuset') == '1,3-5'
-
     @permutations([
+        # length should be 1
         [[], exception.InvalidParameter],
+        # length should be 1
         [['2', '4'], exception.InvalidParameter],
+        # cannot be arbitrary string
         [['abc'], exception.InvalidParameter],
     ])
     def test_process_migration_cpusets_invalid(self, cpusets, exception):
         with fake.VM() as testvm:
             with pytest.raises(exception):
-                testvm._process_migration_cpusets(cpusets)
+                testvm._validate_migration_cpusets(cpusets)
 
 
 def _mem_committed(mem_size_mb):