objects: Introduce 'pcpuset' field for InstanceNUMACell

Introduce the 'pcpuset' to 'InstanceNUMACell' object to track the
instance pinned CPUs. The 'InstanceNUMACell.cpuset' is switched to
keep the instance unpinned CPUs only. As a result, the vCPUs of a
dedicated instance is tracked in NUMA cell object's 'pcpuset', and
vCPUs of a shared instance is put into the 'cpuset' field.

This introduces some object data migration task for an existing instance
that is in the 'dedicated' CPU allocation policy with the fact that all
the CPUs are 1:1 pinned with host CPUs, and it requires to clear the
content of 'InstanceNUMACell.cpuset' and move it to
'InstanceNUMACell.pcpuset' field.

Part of blueprint use-pcpu-and-vcpu-in-one-instance

Change-Id: I901fbd7df00e45196395ff4c69e7b8aa3359edf6
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
Signed-off-by: Wang Huaqiang <huaqiang.wang@intel.com>

nova/api/openstack/compute/server_topology.py

@@ -53,7 +53,7 @@ def _get_numa_topology(self, context, instance, show_host_info):
 
         for cell_ in instance.numa_topology.cells:
             cell = {}
-            cell['vcpu_set'] = cell_.cpuset
+            cell['vcpu_set'] = cell_.total_cpus
             cell['siblings'] = cell_.siblings
             cell['memory_mb'] = cell_.memory
 

nova/objects/instance_numa.py

@@ -33,12 +33,23 @@ class InstanceNUMACell(base.NovaEphemeralObject,
     # Version 1.2: Add cpu_pinning_raw and topology fields
     # Version 1.3: Add cpu_policy and cpu_thread_policy fields
     # Version 1.4: Add cpuset_reserved field
-    VERSION = '1.4'
+    # Version 1.5: Add pcpuset field
+    VERSION = '1.5'
 
     def obj_make_compatible(self, primitive, target_version):
         super(InstanceNUMACell, self).obj_make_compatible(primitive,
-                                                        target_version)
+                                                          target_version)
         target_version = versionutils.convert_version_to_tuple(target_version)
+        # NOTE(huaqiang): Since version 1.5, 'cpuset' is modified to track the
+        # unpinned CPUs only, with pinned CPUs tracked via 'pcpuset' instead.
+        # For a backward compatibility, move the 'dedicated' instance CPU list
+        # from 'pcpuset' to 'cpuset'.
+        if target_version < (1, 5):
+            if (primitive['cpu_policy'] ==
+                    obj_fields.CPUAllocationPolicy.DEDICATED):
+                primitive['cpuset'] = primitive['pcpuset']
+            primitive.pop('pcpuset', None)
+
         if target_version < (1, 4):
             primitive.pop('cpuset_reserved', None)
 
@@ -49,6 +60,10 @@ def obj_make_compatible(self, primitive, target_version):
     fields = {
         'id': obj_fields.IntegerField(),
         'cpuset': obj_fields.SetOfIntegersField(),
+        'pcpuset': obj_fields.SetOfIntegersField(),
+        # These physical CPUs are reserved for use by the hypervisor
+        'cpuset_reserved': obj_fields.SetOfIntegersField(nullable=True,
+                                                         default=None),
         'memory': obj_fields.IntegerField(),
         'pagesize': obj_fields.IntegerField(nullable=True,
                                             default=None),
@@ -60,19 +75,20 @@ def obj_make_compatible(self, primitive, target_version):
                                                           default=None),
         'cpu_thread_policy': obj_fields.CPUThreadAllocationPolicyField(
             nullable=True, default=None),
-        # These physical CPUs are reserved for use by the hypervisor
-        'cpuset_reserved': obj_fields.SetOfIntegersField(nullable=True,
-                                                         default=None),
     }
 
     cpu_pinning = obj_fields.DictProxyField('cpu_pinning_raw')
 
     def __len__(self):
-        return len(self.cpuset)
+        return len(self.total_cpus)
+
+    @property
+    def total_cpus(self):
+        return self.cpuset | self.pcpuset
 
     @property
     def siblings(self):
-        cpu_list = sorted(list(self.cpuset))
+        cpu_list = sorted(list(self.total_cpus))
 
         threads = 0
         if ('cpu_topology' in self) and self.cpu_topology:
@@ -83,7 +99,7 @@ def siblings(self):
         return list(map(set, zip(*[iter(cpu_list)] * threads)))
 
     def pin(self, vcpu, pcpu):
-        if vcpu not in self.cpuset:
+        if vcpu not in self.pcpuset:
             return
         pinning_dict = self.cpu_pinning or {}
         pinning_dict[vcpu] = pcpu
@@ -115,7 +131,7 @@ class InstanceNUMATopology(base.NovaObject,
 
     def obj_make_compatible(self, primitive, target_version):
         super(InstanceNUMATopology, self).obj_make_compatible(primitive,
-                                                        target_version)
+                                                              target_version)
         target_version = versionutils.convert_version_to_tuple(target_version)
         if target_version < (1, 3):
             primitive.pop('emulator_threads_policy', None)
@@ -136,11 +152,43 @@ def obj_from_db_obj(cls, context, instance_uuid, db_obj):
 
         if 'nova_object.name' in primitive:
             obj = cls.obj_from_primitive(primitive)
+            cls._migrate_legacy_dedicated_instance_cpuset(
+                context, instance_uuid, obj)
         else:
             obj = cls._migrate_legacy_object(context, instance_uuid, primitive)
 
         return obj
 
+    # TODO(huaqiang): Remove after Wallaby once we are sure these objects have
+    # been loaded at least once.
+    @classmethod
+    def _migrate_legacy_dedicated_instance_cpuset(cls, context, instance_uuid,
+                                                  obj):
+        # NOTE(huaqiang): We may meet some topology object with the old version
+        # 'InstanceNUMACell' cells, in that case, the 'dedicated' CPU is kept
+        # in 'InstanceNUMACell.cpuset' field, but it should be kept in
+        # 'InstanceNUMACell.pcpuset' field since Victoria. Making an upgrade
+        # and persisting to database.
+        update_db = False
+        for cell in obj.cells:
+            if len(cell.cpuset) == 0:
+                continue
+
+            if cell.cpu_policy != obj_fields.CPUAllocationPolicy.DEDICATED:
+                continue
+
+            cell.pcpuset = cell.cpuset
+            cell.cpuset = set()
+            update_db = True
+
+        if update_db:
+            db_obj = jsonutils.dumps(obj.obj_to_primitive())
+            values = {
+                'numa_topology': db_obj,
+            }
+            db.instance_extra_update_by_uuid(context, instance_uuid,
+                                             values)
+
     # TODO(stephenfin): Remove in X or later, once this has bedded in
     @classmethod
     def _migrate_legacy_object(cls, context, instance_uuid, primitive):
@@ -161,6 +209,7 @@ def _migrate_legacy_object(cls, context, instance_uuid, primitive):
                 InstanceNUMACell(
                     id=cell.get('id'),
                     cpuset=hardware.parse_cpu_spec(cell.get('cpus', '')),
+                    pcpuset=set(),
                     memory=cell.get('mem', {}).get('total', 0),
                     pagesize=cell.get('pagesize'),
                 ) for cell in primitive.get('cells', [])

nova/tests/functional/api_sample_tests/test_server_topology.py

@@ -22,12 +22,14 @@ def fake_get_numa():
     cell_0 = numa.InstanceNUMACell(node=0, memory=1024, pagesize=4, id=0,
            cpu_topology=cpu_topology,
            cpu_pinning={0: 0, 1: 5},
-           cpuset=set([0, 1]))
+           cpuset=set(),
+           pcpuset=set([0, 1]))
 
     cell_1 = numa.InstanceNUMACell(node=1, memory=2048, pagesize=4, id=1,
            cpu_topology=cpu_topology,
            cpu_pinning={2: 1, 3: 8},
-           cpuset=set([2, 3]))
+           cpuset=set(),
+           pcpuset=set([2, 3]))
 
     return numa.InstanceNUMATopology(cells=[cell_0, cell_1])
 

nova/tests/unit/api/openstack/compute/test_server_topology.py

@@ -40,9 +40,8 @@ def setUp(self):
 
     def _fake_numa(self, cpu_pinning=None):
         ce0 = numa.InstanceNUMACell(node=0, memory=1024, pagesize=4, id=0,
-            cpu_topology=None,
-            cpu_pinning=cpu_pinning,
-            cpuset=set([0, 1]))
+            cpu_topology=None, cpu_pinning=cpu_pinning,
+            cpuset=set([0, 1]), pcpuset=set())
 
         return numa.InstanceNUMATopology(cells=[ce0])
 

nova/tests/unit/compute/test_claims.py

@@ -186,13 +186,14 @@ def test_pci_pass_no_requests(self, mock_pci_supports_requests):
     def test_numa_topology_no_limit(self):
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2]), memory=512)])
+                    id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
         self._claim(numa_topology=huge_instance)
 
     def test_numa_topology_fails(self):
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2, 3, 4, 5]), memory=2048)])
+                    id=1, cpuset=set([1, 2, 3, 4, 5]), pcpuset=set(),
+                    memory=2048)])
         limit_topo = objects.NUMATopologyLimits(
             cpu_allocation_ratio=1, ram_allocation_ratio=1)
         self.assertRaises(exception.ComputeResourcesUnavailable,
@@ -203,7 +204,7 @@ def test_numa_topology_fails(self):
     def test_numa_topology_passes(self):
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2]), memory=512)])
+                    id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
         limit_topo = objects.NUMATopologyLimits(
             cpu_allocation_ratio=1, ram_allocation_ratio=1)
         self._claim(limits={'numa_topology': limit_topo},
@@ -230,7 +231,7 @@ def test_numa_topology_with_pci(self, mock_get_by_instance):
 
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2]), memory=512)])
+                    id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
 
         self._claim(requests=requests, numa_topology=huge_instance)
 
@@ -265,7 +266,7 @@ def test_numa_topology_with_pci_fail(self, mock_get_by_instance):
 
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2]), memory=512)])
+                    id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
 
         self.assertRaises(exception.ComputeResourcesUnavailable,
                           self._claim,
@@ -294,7 +295,7 @@ def test_numa_topology_with_pci_no_numa_info(self, mock_get_by_instance):
 
         huge_instance = objects.InstanceNUMATopology(
                 cells=[objects.InstanceNUMACell(
-                    id=1, cpuset=set([1, 2]), memory=512)])
+                    id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512)])
 
         self._claim(requests=requests, numa_topology=huge_instance)
 
@@ -381,11 +382,12 @@ def test_live_migration_page_size(self):
         instance_type = self._fake_instance_type()
         instance = self._fake_instance()
         instance.numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2]),
-                                            memory=512, pagesize=2)])
+            cells=[objects.InstanceNUMACell(
+                id=1, cpuset=set([1, 2]),
+                pcpuset=set(), memory=512, pagesize=2)])
         claimed_numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2]),
-                                            memory=512, pagesize=1)])
+            cells=[objects.InstanceNUMACell(
+            id=1, cpuset=set([1, 2]), pcpuset=set(), memory=512, pagesize=1)])
         with mock.patch('nova.virt.hardware.numa_fit_instance_to_host',
                         return_value=claimed_numa_topology):
             self.assertRaisesRegex(
@@ -402,8 +404,9 @@ def test_claim_fails_page_size_not_called(self):
         # This topology cannot fit in self.compute_node
         # (see _fake_compute_node())
         numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=1, cpuset=set([1, 2, 3]),
-                                            memory=1024)])
+            cells=[objects.InstanceNUMACell(
+            id=1, cpuset=set([1, 2, 3]), pcpuset=set(),
+            memory=1024)])
         with test.nested(
             mock.patch('nova.virt.hardware.numa_get_constraints',
                         return_value=numa_topology),

nova/tests/unit/compute/test_compute.py

@@ -5682,7 +5682,8 @@ def test_confirm_resize_with_numa_topology_and_cpu_pinning(
         old_inst_topology = objects.InstanceNUMATopology(
             instance_uuid=instance.uuid, cells=[
                 objects.InstanceNUMACell(
-                    id=0, cpuset=set([1, 2]), memory=512, pagesize=2048,
+                    id=0, cpuset=set(), pcpuset=set([1, 2]), memory=512,
+                    pagesize=2048,
                     cpu_policy=obj_fields.CPUAllocationPolicy.DEDICATED,
                     cpu_pinning={'0': 1, '1': 2})
         ])
@@ -5691,7 +5692,8 @@ def test_confirm_resize_with_numa_topology_and_cpu_pinning(
         new_inst_topology = objects.InstanceNUMATopology(
             instance_uuid=instance.uuid, cells=[
                 objects.InstanceNUMACell(
-                    id=1, cpuset=set([3, 4]), memory=512, pagesize=2048,
+                    id=1, cpuset=set(), pcpuset=set([3, 4]), memory=512,
+                    pagesize=2048,
                     cpu_policy=obj_fields.CPUAllocationPolicy.DEDICATED,
                     cpu_pinning={'0': 3, '1': 4})
         ])
@@ -8694,9 +8696,10 @@ def test_create_with_deleted_image(self):
     def test_create_with_numa_topology(self, numa_constraints_mock):
         numa_topology = objects.InstanceNUMATopology(
             cells=[objects.InstanceNUMACell(
-                id=0, cpuset=set([1, 2]), memory=512),
+                id=0, cpuset=set([1, 2]), pcpuset=set(),
+                memory=512),
                    objects.InstanceNUMACell(
-                id=1, cpuset=set([3, 4]), memory=512)])
+                id=1, cpuset=set([3, 4]), pcpuset=set(), memory=512)])
         numa_topology.obj_reset_changes()
         numa_constraints_mock.return_value = numa_topology
 

nova/tests/unit/compute/test_compute_api.py

@@ -1692,7 +1692,8 @@ def _test_revert_resize(
         fake_reqspec.flavor = fake_inst.flavor
         fake_numa_topology = objects.InstanceNUMATopology(cells=[
             objects.InstanceNUMACell(
-                id=0, cpuset=set([0]), memory=512, pagesize=None,
+                id=0, cpuset=set([0]), pcpuset=set(), memory=512,
+                pagesize=None,
                 cpu_pinning_raw=None, cpuset_reserved=None, cpu_policy=None,
                 cpu_thread_policy=None)])
 
@@ -1852,7 +1853,8 @@ def _test_resize(self, mock_get_all_by_host,
         fake_inst = self._create_instance_obj(params=params)
         fake_numa_topology = objects.InstanceNUMATopology(cells=[
             objects.InstanceNUMACell(
-                id=0, cpuset=set([0]), memory=512, pagesize=None,
+                id=0, cpuset=set([0]), pcpuset=set(), memory=512,
+                pagesize=None,
                 cpu_pinning_raw=None, cpuset_reserved=None, cpu_policy=None,
                 cpu_thread_policy=None)])
 

nova/tests/unit/compute/test_compute_mgr.py

@@ -1206,14 +1206,18 @@ def _test__validate_pinning_configuration(self, supports_pcpus=True):
 
         numa_wo_pinning = test_instance_numa.get_fake_obj_numa_topology(
             self.context)
+        numa_wo_pinning.cells[0].pcpuset = set()
+        numa_wo_pinning.cells[1].pcpuset = set()
         instance_2.numa_topology = numa_wo_pinning
 
         numa_w_pinning = test_instance_numa.get_fake_obj_numa_topology(
             self.context)
         numa_w_pinning.cells[0].pin_vcpus((1, 10), (2, 11))
+        numa_w_pinning.cells[0].cpuset = set()
         numa_w_pinning.cells[0].cpu_policy = (
             fields.CPUAllocationPolicy.DEDICATED)
         numa_w_pinning.cells[1].pin_vcpus((3, 0), (4, 1))
+        numa_w_pinning.cells[1].cpuset = set()
         numa_w_pinning.cells[1].cpu_policy = (
             fields.CPUAllocationPolicy.DEDICATED)
         instance_3.numa_topology = numa_w_pinning

nova/tests/unit/compute/test_resource_tracker.py

@@ -153,9 +153,9 @@
 _INSTANCE_NUMA_TOPOLOGIES = {
     '2mb': objects.InstanceNUMATopology(cells=[
         objects.InstanceNUMACell(
-            id=0, cpuset=set([1]), memory=_2MB, pagesize=0),
+            id=0, cpuset=set([1]), pcpuset=set(), memory=_2MB, pagesize=0),
         objects.InstanceNUMACell(
-            id=1, cpuset=set([3]), memory=_2MB, pagesize=0)]),
+            id=1, cpuset=set([3]), pcpuset=set(), memory=_2MB, pagesize=0)]),
 }
 
 _NUMA_LIMIT_TOPOLOGIES = {

nova/tests/unit/conductor/tasks/test_live_migrate.py

@@ -214,9 +214,10 @@ def test_check_instance_has_no_numa_passes_no_numa(self, mock_get):
     @mock.patch.object(objects.ComputeNode, 'get_by_host_and_nodename')
     def test_check_instance_has_no_numa_passes_non_kvm(self, mock_get):
         self.flags(enable_numa_live_migration=False, group='workarounds')
-        self.task.instance.numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
-                                            memory=1024)])
+        self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
+            objects.InstanceNUMACell(
+                id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
+            ])
         mock_get.return_value = objects.ComputeNode(
             uuid=uuids.cn1, hypervisor_type='xen')
         self.task._check_instance_has_no_numa()
@@ -227,9 +228,10 @@ def test_check_instance_has_no_numa_passes_non_kvm(self, mock_get):
     def test_check_instance_has_no_numa_passes_workaround(
             self, mock_get_min_ver, mock_get):
         self.flags(enable_numa_live_migration=True, group='workarounds')
-        self.task.instance.numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
-                                            memory=1024)])
+        self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
+            objects.InstanceNUMACell(
+                id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
+            ])
         mock_get.return_value = objects.ComputeNode(
             uuid=uuids.cn1, hypervisor_type='qemu')
         self.task._check_instance_has_no_numa()
@@ -243,9 +245,10 @@ def test_check_instance_has_no_numa_fails(self, mock_get_min_ver,
         self.flags(enable_numa_live_migration=False, group='workarounds')
         mock_get.return_value = objects.ComputeNode(
             uuid=uuids.cn1, hypervisor_type='qemu')
-        self.task.instance.numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
-                                            memory=1024)])
+        self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
+            objects.InstanceNUMACell(
+                id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
+            ])
         self.assertRaises(exception.MigrationPreCheckError,
                           self.task._check_instance_has_no_numa)
         mock_get_min_ver.assert_called_once_with(self.context, 'nova-compute')
@@ -258,9 +261,10 @@ def test_check_instance_has_no_numa_new_svc_passes(self, mock_get_min_ver,
         self.flags(enable_numa_live_migration=False, group='workarounds')
         mock_get.return_value = objects.ComputeNode(
             uuid=uuids.cn1, hypervisor_type='qemu')
-        self.task.instance.numa_topology = objects.InstanceNUMATopology(
-            cells=[objects.InstanceNUMACell(id=0, cpuset=set([0]),
-                                            memory=1024)])
+        self.task.instance.numa_topology = objects.InstanceNUMATopology(cells=[
+            objects.InstanceNUMACell(
+                id=0, cpuset=set([0]), pcpuset=set(), memory=1024),
+            ])
         self.task._check_instance_has_no_numa()
         mock_get_min_ver.assert_called_once_with(self.context, 'nova-compute')