Add ComputeNode.mapped field

This is an integer that defaults to zero. This can be used by API services
to efficiently filter out records that have already been mapped by a
HostMapping. This is an integer instead of a boolean in case we later need
to do some other sort of mapping of these records and need to distinguish
yet another level of mapped-ness.

Note this also removes some tests from TestNewtonBlocker which can no longer
work with the new ComputeNode model since they attempt to use it to create
compute nodes at a schema older than needed for this new field. The point of
the test was to verify, in Newton, that the blocker migration caught the thing
we needed to catch. It doesn't need to be in master anymore and certainly
is not worth the acrobatics that would be required to keep it working. (Note
that we haven't even had such tests for many of our blocker migrations)

Related to blueprint discover-hosts-faster
Change-Id: I902d75efb0bbe177680d7211c23235f42497e3fe

nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py

@@ -0,0 +1,24 @@
+#    Copyright 2017 Red Hat, Inc.
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import MetaData, Table, Column, Integer
+
+
+def upgrade(migrate_engine):
+    meta = MetaData(bind=migrate_engine)
+
+    for prefix in ('', 'shadow_'):
+        compute_nodes = Table('%scompute_nodes' % prefix, meta, autoload=True)
+        mapped = Column('mapped', Integer, default=0, nullable=True)
+        if not hasattr(compute_nodes.c, 'mapped'):
+            compute_nodes.create_column(mapped)

nova/db/sqlalchemy/models.py

@@ -179,6 +179,7 @@ class ComputeNode(BASE, NovaBase, models.SoftDeleteMixin):
     ram_allocation_ratio = Column(Float, nullable=True)
     cpu_allocation_ratio = Column(Float, nullable=True)
     disk_allocation_ratio = Column(Float, nullable=True)
+    mapped = Column(Integer, nullable=True, default=0)
 
 
 class Certificate(BASE, NovaBase, models.SoftDeleteMixin):

nova/objects/compute_node.py

@@ -48,7 +48,8 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject):
     # Version 1.14: Added cpu_allocation_ratio and ram_allocation_ratio
     # Version 1.15: Added uuid
     # Version 1.16: Added disk_allocation_ratio
-    VERSION = '1.16'
+    # Version 1.17: Added mapped
+    VERSION = '1.17'
 
     fields = {
         'id': fields.IntegerField(read_only=True),
@@ -89,11 +90,15 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject):
         'cpu_allocation_ratio': fields.FloatField(),
         'ram_allocation_ratio': fields.FloatField(),
         'disk_allocation_ratio': fields.FloatField(),
+        'mapped': fields.IntegerField(),
         }
 
     def obj_make_compatible(self, primitive, target_version):
         super(ComputeNode, self).obj_make_compatible(primitive, target_version)
         target_version = versionutils.convert_version_to_tuple(target_version)
+        if target_version < (1, 17):
+            if 'mapped' in primitive:
+                del primitive['mapped']
         if target_version < (1, 16):
             if 'disk_allocation_ratio' in primitive:
                 del primitive['disk_allocation_ratio']
@@ -199,6 +204,9 @@ def _from_db_object(context, compute, db_compute):
                     if value == 0.0 and key == 'disk_allocation_ratio':
                         # It's not specified either on the controller
                         value = 1.0
+            elif key == 'mapped':
+                value = 0 if value is None else value
+
             setattr(compute, key, value)
 
         stats = db_compute['stats']

nova/tests/unit/compute/test_compute.py

@@ -176,6 +176,7 @@ def fake_get_compute_nodes_in_db(self, context, **kwargs):
                                    'memory_mb': 131072,
                                    'current_workload': 0,
                                    'vcpus': 16,
+                                   'mapped': 1,
                                    'cpu_info': 'ppc64,powervm,3940',
                                    'running_vms': 0,
                                    'free_disk_gb': 259,

nova/tests/unit/db/test_db_api.py

@@ -7801,6 +7801,7 @@ def setUp(self):
                                  supported_instances='',
                                  pci_stats='',
                                  metrics='',
+                                 mapped=0,
                                  extra_resources='',
                                  cpu_allocation_ratio=16.0,
                                  ram_allocation_ratio=1.5,
@@ -7850,6 +7851,7 @@ def test_compute_node_get_all_by_pagination(self):
                                  supported_instances='',
                                  pci_stats='',
                                  metrics='',
+                                 mapped=0,
                                  extra_resources='',
                                  cpu_allocation_ratio=16.0,
                                  ram_allocation_ratio=1.5,

nova/tests/unit/db/test_migrations.py

@@ -952,6 +952,10 @@ def _check_359(self, engine, data):
         self.assertIndexMembers(engine, 'services', 'services_uuid_idx',
                                 ['uuid'])
 
+    def _check_360(self, engine, data):
+        self.assertColumnExists(engine, 'compute_nodes', 'mapped')
+        self.assertColumnExists(engine, 'shadow_compute_nodes', 'mapped')
+
 
 class TestNovaMigrationsSQLite(NovaMigrationsCheckers,
                                test_base.DbTestCase,

nova/tests/unit/db/test_sqlalchemy_migration.py

@@ -239,29 +239,6 @@ def setUp(self):
             '330_enforce_mitaka_online_migrations')
         self.engine = db_api.get_engine()
 
-    def test_all_migrated(self):
-        cn = objects.ComputeNode(context=self.context,
-                                 vcpus=1, memory_mb=512, local_gb=10,
-                                 vcpus_used=0, memory_mb_used=256,
-                                 local_gb_used=5, hypervisor_type='HyperDanVM',
-                                 hypervisor_version='34', cpu_info='foo')
-        cn.create()
-        objects.Aggregate(context=self.context,
-                          name='foo').create()
-        objects.PciDevice.create(self.context, {})
-        self.migration.upgrade(self.engine)
-
-    def test_cn_not_migrated(self):
-        cn = objects.ComputeNode(context=self.context,
-                                 vcpus=1, memory_mb=512, local_gb=10,
-                                 vcpus_used=0, memory_mb_used=256,
-                                 local_gb_used=5, hypervisor_type='HyperDanVM',
-                                 hypervisor_version='34', cpu_info='foo')
-        cn.create()
-        db_api.compute_node_update(self.context, cn.id, {'uuid': None})
-        self.assertRaises(exception.ValidationError,
-                          self.migration.upgrade, self.engine)
-
     def test_aggregate_not_migrated(self):
         agg = db_api.aggregate_create(self.context, {"name": "foobar"})
         db_api.aggregate_update(self.context, agg.id, {'uuid': None})
@@ -308,31 +285,6 @@ def test_pci_device_type_pci_not_migrated(self):
         # blocker should not block on type-PCI devices
         self.migration.upgrade(self.engine)
 
-    def test_deleted_not_migrated(self):
-        cn_values = dict(vcpus=1, memory_mb=512, local_gb=10,
-                         vcpus_used=0, memory_mb_used=256,
-                         local_gb_used=5, hypervisor_type='HyperDanVM',
-                         hypervisor_version='34', cpu_info='foo')
-        cn = db_api.compute_node_create(self.context, cn_values)
-        agg_values = dict(name='foo')
-        agg = db_api.aggregate_create(self.context, agg_values)
-        pd = db_api.pci_device_update(self.context, 1, 'foo:bar',
-                                 {'parent_addr': None,
-                                  'compute_node_id': 1,
-                                  'address': 'foo:bar',
-                                  'vendor_id': '123',
-                                  'product_id': '456',
-                                  'dev_type': 'foo',
-                                  'label': 'foobar',
-                                  'status': 'whatisthis?'})
-        db_api.compute_node_delete(self.context, cn['id'])
-        db_api.aggregate_delete(self.context, agg['id'])
-        db_api.pci_device_destroy(self.context, pd['compute_node_id'],
-                                  pd['address'])
-
-        # blocker should not block on soft-deleted records
-        self.migration.upgrade(self.engine)
-
 
 class TestOcataCheck(test.TestCase):
     def setUp(self):

nova/tests/unit/objects/test_compute_node.py

@@ -90,6 +90,7 @@
     'cpu_allocation_ratio': 16.0,
     'ram_allocation_ratio': 1.5,
     'disk_allocation_ratio': 1.0,
+    'mapped': 0,
     }
 # FIXME(sbauza) : For compatibility checking, to be removed once we are sure
 # that all computes are running latest DB version with host field in it.
@@ -163,6 +164,18 @@ def test_get_by_id(self, get_mock):
         self.assertNotIn('uuid', compute.obj_what_changed())
         get_mock.assert_called_once_with(self.context, 123)
 
+    @mock.patch.object(db, 'compute_node_get')
+    def test_get_without_mapped(self, get_mock):
+        fake_node = copy.copy(fake_compute_node)
+        fake_node['mapped'] = None
+        get_mock.return_value = fake_node
+        compute = compute_node.ComputeNode.get_by_id(self.context, 123)
+        self.compare_obj(compute, fake_compute_node,
+                         subs=self.subs(),
+                         comparators=self.comparators())
+        self.assertIn('mapped', compute)
+        self.assertEqual(0, compute.mapped)
+
     @mock.patch.object(objects.Service, 'get_by_id')
     @mock.patch.object(db, 'compute_node_get')
     def test_get_by_id_with_host_field_not_in_db(self, mock_cn_get,

nova/tests/unit/objects/test_objects.py

@@ -1072,7 +1072,7 @@ def obj_name(cls):
     'BuildRequestList': '1.0-cd95608eccb89fbc702c8b52f38ec738',
     'CellMapping': '1.0-7f1a7e85a22bbb7559fc730ab658b9bd',
     'CellMappingList': '1.0-4ee0d9efdfd681fed822da88376e04d2',
-    'ComputeNode': '1.16-2436e5b836fa0306a3c4e6d9e5ddacec',
+    'ComputeNode': '1.17-abc222ef5a707dcd3e3d32b48197c9e7',
     'ComputeNodeList': '1.16-40258d802a6ed045690a127a2088544b',
     'DNSDomain': '1.0-7b0b2dab778454b6a7b6c66afe163a1a',
     'DNSDomainList': '1.0-4ee0d9efdfd681fed822da88376e04d2',