Add PCI stats

PCI stats abstract the pci device info on compute node for scheduler.
provide better performance than join load from pci device table.

PCI stats contain the available count info for PCI devices with same
spec, ie, the same product_id and verndor_id.and provide method to check
if the requests is supported.

bp:pci-passthrough-base

Change-Id: I8d77c685f774aae22da87be9e822bfd44f0b7a6d
Signed-off-by: Yunhong Jiang <yunhong.jiang@intel.com>
Signed-off-by: Yongli He <yongli.he@intel.com>

nova/exception.py

@@ -1388,3 +1388,14 @@ class PciDeviceInvalidOwner(NovaException):
     msg_fmt = _(
         "PCI Device %(compute_node_id)s:%(address)s is owned by %(owner)s "
         "instead of %(hopeowner)s")
+
+
+class PciDeviceRequestFailed(NovaException):
+    msg_fmt = _(
+        "PCI Device request (%requests)s failed")
+
+
+class PciDevicePoolEmpty(NovaException):
+    msg_fmt = _(
+        "Attempt to consume PCI Device %(compute_node_id)s:%(address)s "
+        "from empty pool")

nova/pci/pci_stats.py

@@ -0,0 +1,143 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2013 Intel, Inc.
+# Copyright (c) 2013 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+
+from nova import exception
+from nova.openstack.common import jsonutils
+from nova.openstack.common import log as logging
+from nova.pci import pci_utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PciDeviceStats(object):
+
+    """PCI devices summary information.
+
+    According to the PCI SR-IOV spec, a PCI physical function can have up to
+    256 PCI virtual functions, thus the number of assignable PCI functions in
+    a cloud can be big. The scheduler needs to know all device availability
+    information in order to determine which compute hosts can support a PCI
+    request. Passing individual virtual device information to the scheduler
+    does not scale, so we provide summary information.
+
+    Usually the virtual functions provided by a host PCI device have the same
+    value for most properties, like vendor_id, product_id and class type.
+    The PCI stats class summarizes this information for the scheduler.
+
+    The pci stats information is maintained exclusively by compute node
+    resource tracker and updated to database. The scheduler fetches the
+    information and selects the compute node accordingly. If a comptue
+    node is selected, the resource tracker allocates the devices to the
+    instance and updates the pci stats information.
+
+    This summary information will be helpful for cloud management also.
+    """
+
+    pool_keys = ['product_id', 'vendor_id', 'extra_info']
+
+    def __init__(self, stats=None):
+        super(PciDeviceStats, self).__init__()
+        self.pools = jsonutils.loads(stats) if stats else []
+
+    def _equal_properties(self, dev, entry):
+        return all(dev.get(prop) == entry.get(prop)
+                   for prop in self.pool_keys)
+
+    def _get_first_pool(self, dev):
+        """Return the first pool that matches dev."""
+        return next((pool for pool in self.pools
+                    if self._equal_properties(dev, pool)), None)
+
+    def add_device(self, dev):
+        """Add a device to the first matching pool."""
+        pool = self._get_first_pool(dev)
+        if not pool:
+            pool = dict((k, dev.get(k)) for k in self.pool_keys)
+            pool['count'] = 0
+            self.pools.append(pool)
+        pool['count'] += 1
+
+    @staticmethod
+    def _decrease_pool_count(pool_list, pool, count=1):
+        """Decrement pool's size by count.
+
+        If pool becomes empty, remove pool from pool_list.
+        """
+        if pool['count'] > count:
+            pool['count'] -= count
+            count = 0
+        else:
+            count -= pool['count']
+            pool_list.remove(pool)
+        return count
+
+    def consume_device(self, dev):
+        """Remove one device from the first pool that it matches."""
+        pool = self._get_first_pool(dev)
+        if not pool:
+            raise exception.PciDevicePoolEmpty(
+                compute_node_id=dev.compute_node_id, address=dev.address)
+        self._decrease_pool_count(self.pools, pool)
+
+    @staticmethod
+    def _filter_pools_for_spec(pools, request_specs):
+        return [pool for pool in pools
+                if pci_utils.pci_device_prop_match(pool, request_specs)]
+
+    def _apply_request(self, pools, request):
+        count = request['count']
+        matching_pools = self._filter_pools_for_spec(pools, request['spec'])
+        if sum([pool['count'] for pool in matching_pools]) < count:
+            return False
+        else:
+            for pool in matching_pools:
+                count = self._decrease_pool_count(pools, pool, count)
+                if not count:
+                    break
+        return True
+
+    def support_requests(self, requests):
+        """Check if the pci requests can be met.
+
+        Scheduler checks compute node's PCI stats to decide if an
+        instance can be scheduled into the node. Support does not
+        mean real allocation.
+        """
+        # note (yjiang5): this function has high possibility to fail,
+        # so no exception should be triggered for performance reason.
+        pools = copy.deepcopy(self.pools)
+        return all([self._apply_request(pools, r) for r in requests])
+
+    def apply_requests(self, requests):
+        """Apply PCI requests to the PCI stats.
+
+        This is used in multiple instance creation, when the scheduler has to
+        maintain how the resources are consumed by the instances.
+        """
+        if not all([self._apply_request(self.pools, r) for r in requests]):
+            raise exception.PciDeviceRequestFailed(requests=requests)
+
+    def __iter__(self):
+        return iter(self.pools)
+
+    def clear(self):
+        """Clear all the stats maintained."""
+        self.pools = []

nova/tests/pci/test_pci_stats.py

@@ -0,0 +1,120 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2012 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nova import exception
+from nova.objects import pci_device
+from nova.openstack.common import jsonutils
+from nova.pci import pci_stats as pci
+from nova import test
+
+fake_pci_1 = {
+    'compute_node_id': 1,
+    'address': '0000:00:00.1',
+    'product_id': 'p1',
+    'vendor_id': 'v1',
+    'status': 'available',
+    'extra_k1': 'v1',
+    }
+
+
+fake_pci_2 = dict(fake_pci_1, vendor_id='v2',
+                  product_id='p2',
+                  address='0000:00:00.2')
+
+
+fake_pci_3 = dict(fake_pci_1, address='0000:00:00.3')
+
+
+pci_requests = [{'count': 1,
+                 'spec': [{'vendor_id': 'v1'}]},
+                {'count': 1,
+                 'spec': [{'vendor_id': 'v2'}]}]
+
+
+pci_requests_multiple = [{'count': 1,
+                          'spec': [{'vendor_id': 'v1'}]},
+                         {'count': 3,
+                          'spec': [{'vendor_id': 'v2'}]}]
+
+
+class PciDeviceStatsTestCase(test.TestCase):
+    def _create_fake_devs(self):
+        self.fake_dev_1 = pci_device.PciDevice.create(fake_pci_1)
+        self.fake_dev_2 = pci_device.PciDevice.create(fake_pci_2)
+        self.fake_dev_3 = pci_device.PciDevice.create(fake_pci_3)
+
+        map(self.pci_stats.add_device,
+            [self.fake_dev_1, self.fake_dev_2, self.fake_dev_3])
+
+    def setUp(self):
+        super(PciDeviceStatsTestCase, self).setUp()
+        self.pci_stats = pci.PciDeviceStats()
+        self._create_fake_devs()
+
+    def test_add_device(self):
+        self.assertEqual(len(self.pci_stats.pools), 2)
+        self.assertEqual(set([d['vendor_id'] for d in self.pci_stats]),
+                         set(['v1', 'v2']))
+        self.assertEqual(set([d['count'] for d in self.pci_stats]),
+                         set([1, 2]))
+
+    def test_remove_device(self):
+        self.pci_stats.consume_device(self.fake_dev_2)
+        self.assertEqual(len(self.pci_stats.pools), 1)
+        self.assertEqual(self.pci_stats.pools[0]['count'], 2)
+        self.assertEqual(self.pci_stats.pools[0]['vendor_id'], 'v1')
+
+    def test_remove_device_exception(self):
+        self.pci_stats.consume_device(self.fake_dev_2)
+        self.assertRaises(exception.PciDevicePoolEmpty,
+                          self.pci_stats.consume_device,
+                          self.fake_dev_2)
+
+    def test_json_creat(self):
+        m = jsonutils.dumps(self.pci_stats)
+        new_stats = pci.PciDeviceStats(m)
+
+        self.assertEqual(len(new_stats.pools), 2)
+        self.assertEqual(set([d['count'] for d in new_stats]),
+                         set([1, 2]))
+        self.assertEqual(set([d['vendor_id'] for d in new_stats]),
+                         set(['v1', 'v2']))
+
+    def test_support_requests(self):
+        self.assertEqual(self.pci_stats.support_requests(pci_requests),
+                         True)
+        self.assertEqual(len(self.pci_stats.pools), 2)
+        self.assertEqual(set([d['count'] for d in self.pci_stats]),
+                         set((1, 2)))
+
+    def test_support_requests_failed(self):
+        self.assertEqual(
+            self.pci_stats.support_requests(pci_requests_multiple), False)
+        self.assertEqual(len(self.pci_stats.pools), 2)
+        self.assertEqual(set([d['count'] for d in self.pci_stats]),
+                         set([1, 2]))
+
+    def test_apply_requests(self):
+        self.pci_stats.apply_requests(pci_requests)
+        self.assertEqual(len(self.pci_stats.pools), 1)
+        self.assertEqual(self.pci_stats.pools[0]['vendor_id'], 'v1')
+        self.assertEqual(self.pci_stats.pools[0]['count'], 1)
+
+    def test_apply_requests_failed(self):
+        self.assertRaises(exception.PciDeviceRequestFailed,
+            self.pci_stats.apply_requests,
+            pci_requests_multiple)