From 0ce4dff41f8d38edf790a301ec8e7040b279d65a Mon Sep 17 00:00:00 2001 From: Matt Riedemann Date: Thu, 27 Apr 2017 19:44:51 -0400 Subject: [PATCH] Add ComputeNode.mapped field This is an integer that defaults to zero. This can be used by API services to efficiently filter out records that have already been mapped by a HostMapping. This is an integer instead of a boolean in case we later need to do some other sort of mapping of these records and need to distinguish yet another level of mapped-ness. Note this also removes some tests from TestNewtonBlocker which can no longer work with the new ComputeNode model since they attempt to use it to create compute nodes at a schema older than needed for this new field. The point of the test was to verify, in Newton, that the blocker migration caught the thing we needed to catch. It doesn't need to be in master anymore and certainly is not worth the acrobatics that would be required to keep it working. (Note that we haven't even had such tests for many of our blocker migrations) Related to blueprint discover-hosts-faster Change-Id: I902d75efb0bbe177680d7211c23235f42497e3fe --- .../versions/360_add_compute_node_mapped.py | 24 ++++++++++ nova/db/sqlalchemy/models.py | 1 + nova/objects/compute_node.py | 10 +++- nova/tests/unit/compute/test_compute.py | 1 + nova/tests/unit/db/test_db_api.py | 2 + nova/tests/unit/db/test_migrations.py | 4 ++ .../unit/db/test_sqlalchemy_migration.py | 48 ------------------- nova/tests/unit/objects/test_compute_node.py | 13 +++++ nova/tests/unit/objects/test_objects.py | 2 +- 9 files changed, 55 insertions(+), 50 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py b/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py new file mode 100644 index 00000000000..d5b2acaea65 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/360_add_compute_node_mapped.py @@ -0,0 +1,24 @@ +# Copyright 2017 Red Hat, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Column, Integer + + +def upgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + + for prefix in ('', 'shadow_'): + compute_nodes = Table('%scompute_nodes' % prefix, meta, autoload=True) + mapped = Column('mapped', Integer, default=0, nullable=True) + if not hasattr(compute_nodes.c, 'mapped'): + compute_nodes.create_column(mapped) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 0e8894267dc..766aa0bad5c 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -179,6 +179,7 @@ class ComputeNode(BASE, NovaBase, models.SoftDeleteMixin): ram_allocation_ratio = Column(Float, nullable=True) cpu_allocation_ratio = Column(Float, nullable=True) disk_allocation_ratio = Column(Float, nullable=True) + mapped = Column(Integer, nullable=True, default=0) class Certificate(BASE, NovaBase, models.SoftDeleteMixin): diff --git a/nova/objects/compute_node.py b/nova/objects/compute_node.py index 6f83b39a76d..8be7d6bc209 100644 --- a/nova/objects/compute_node.py +++ b/nova/objects/compute_node.py @@ -48,7 +48,8 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject): # Version 1.14: Added cpu_allocation_ratio and ram_allocation_ratio # Version 1.15: Added uuid # Version 1.16: Added disk_allocation_ratio - VERSION = '1.16' + # Version 1.17: Added mapped + VERSION = '1.17' fields = { 'id': fields.IntegerField(read_only=True), @@ -89,11 +90,15 @@ class ComputeNode(base.NovaPersistentObject, base.NovaObject): 'cpu_allocation_ratio': fields.FloatField(), 'ram_allocation_ratio': fields.FloatField(), 'disk_allocation_ratio': fields.FloatField(), + 'mapped': fields.IntegerField(), } def obj_make_compatible(self, primitive, target_version): super(ComputeNode, self).obj_make_compatible(primitive, target_version) target_version = versionutils.convert_version_to_tuple(target_version) + if target_version < (1, 17): + if 'mapped' in primitive: + del primitive['mapped'] if target_version < (1, 16): if 'disk_allocation_ratio' in primitive: del primitive['disk_allocation_ratio'] @@ -199,6 +204,9 @@ def _from_db_object(context, compute, db_compute): if value == 0.0 and key == 'disk_allocation_ratio': # It's not specified either on the controller value = 1.0 + elif key == 'mapped': + value = 0 if value is None else value + setattr(compute, key, value) stats = db_compute['stats'] diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index 2b12090e947..8cd31e2c61a 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -176,6 +176,7 @@ def fake_get_compute_nodes_in_db(self, context, **kwargs): 'memory_mb': 131072, 'current_workload': 0, 'vcpus': 16, + 'mapped': 1, 'cpu_info': 'ppc64,powervm,3940', 'running_vms': 0, 'free_disk_gb': 259, diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py index 46f4358f1c6..2391a677540 100644 --- a/nova/tests/unit/db/test_db_api.py +++ b/nova/tests/unit/db/test_db_api.py @@ -7801,6 +7801,7 @@ def setUp(self): supported_instances='', pci_stats='', metrics='', + mapped=0, extra_resources='', cpu_allocation_ratio=16.0, ram_allocation_ratio=1.5, @@ -7850,6 +7851,7 @@ def test_compute_node_get_all_by_pagination(self): supported_instances='', pci_stats='', metrics='', + mapped=0, extra_resources='', cpu_allocation_ratio=16.0, ram_allocation_ratio=1.5, diff --git a/nova/tests/unit/db/test_migrations.py b/nova/tests/unit/db/test_migrations.py index 5580df3419b..3887a8d286e 100644 --- a/nova/tests/unit/db/test_migrations.py +++ b/nova/tests/unit/db/test_migrations.py @@ -952,6 +952,10 @@ def _check_359(self, engine, data): self.assertIndexMembers(engine, 'services', 'services_uuid_idx', ['uuid']) + def _check_360(self, engine, data): + self.assertColumnExists(engine, 'compute_nodes', 'mapped') + self.assertColumnExists(engine, 'shadow_compute_nodes', 'mapped') + class TestNovaMigrationsSQLite(NovaMigrationsCheckers, test_base.DbTestCase, diff --git a/nova/tests/unit/db/test_sqlalchemy_migration.py b/nova/tests/unit/db/test_sqlalchemy_migration.py index e53f8d08705..f5dbab723cb 100644 --- a/nova/tests/unit/db/test_sqlalchemy_migration.py +++ b/nova/tests/unit/db/test_sqlalchemy_migration.py @@ -239,29 +239,6 @@ def setUp(self): '330_enforce_mitaka_online_migrations') self.engine = db_api.get_engine() - def test_all_migrated(self): - cn = objects.ComputeNode(context=self.context, - vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn.create() - objects.Aggregate(context=self.context, - name='foo').create() - objects.PciDevice.create(self.context, {}) - self.migration.upgrade(self.engine) - - def test_cn_not_migrated(self): - cn = objects.ComputeNode(context=self.context, - vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn.create() - db_api.compute_node_update(self.context, cn.id, {'uuid': None}) - self.assertRaises(exception.ValidationError, - self.migration.upgrade, self.engine) - def test_aggregate_not_migrated(self): agg = db_api.aggregate_create(self.context, {"name": "foobar"}) db_api.aggregate_update(self.context, agg.id, {'uuid': None}) @@ -308,31 +285,6 @@ def test_pci_device_type_pci_not_migrated(self): # blocker should not block on type-PCI devices self.migration.upgrade(self.engine) - def test_deleted_not_migrated(self): - cn_values = dict(vcpus=1, memory_mb=512, local_gb=10, - vcpus_used=0, memory_mb_used=256, - local_gb_used=5, hypervisor_type='HyperDanVM', - hypervisor_version='34', cpu_info='foo') - cn = db_api.compute_node_create(self.context, cn_values) - agg_values = dict(name='foo') - agg = db_api.aggregate_create(self.context, agg_values) - pd = db_api.pci_device_update(self.context, 1, 'foo:bar', - {'parent_addr': None, - 'compute_node_id': 1, - 'address': 'foo:bar', - 'vendor_id': '123', - 'product_id': '456', - 'dev_type': 'foo', - 'label': 'foobar', - 'status': 'whatisthis?'}) - db_api.compute_node_delete(self.context, cn['id']) - db_api.aggregate_delete(self.context, agg['id']) - db_api.pci_device_destroy(self.context, pd['compute_node_id'], - pd['address']) - - # blocker should not block on soft-deleted records - self.migration.upgrade(self.engine) - class TestOcataCheck(test.TestCase): def setUp(self): diff --git a/nova/tests/unit/objects/test_compute_node.py b/nova/tests/unit/objects/test_compute_node.py index 95373d382a0..07f20a196a3 100644 --- a/nova/tests/unit/objects/test_compute_node.py +++ b/nova/tests/unit/objects/test_compute_node.py @@ -90,6 +90,7 @@ 'cpu_allocation_ratio': 16.0, 'ram_allocation_ratio': 1.5, 'disk_allocation_ratio': 1.0, + 'mapped': 0, } # FIXME(sbauza) : For compatibility checking, to be removed once we are sure # that all computes are running latest DB version with host field in it. @@ -163,6 +164,18 @@ def test_get_by_id(self, get_mock): self.assertNotIn('uuid', compute.obj_what_changed()) get_mock.assert_called_once_with(self.context, 123) + @mock.patch.object(db, 'compute_node_get') + def test_get_without_mapped(self, get_mock): + fake_node = copy.copy(fake_compute_node) + fake_node['mapped'] = None + get_mock.return_value = fake_node + compute = compute_node.ComputeNode.get_by_id(self.context, 123) + self.compare_obj(compute, fake_compute_node, + subs=self.subs(), + comparators=self.comparators()) + self.assertIn('mapped', compute) + self.assertEqual(0, compute.mapped) + @mock.patch.object(objects.Service, 'get_by_id') @mock.patch.object(db, 'compute_node_get') def test_get_by_id_with_host_field_not_in_db(self, mock_cn_get, diff --git a/nova/tests/unit/objects/test_objects.py b/nova/tests/unit/objects/test_objects.py index 870e1428961..9ea4cb62b7e 100644 --- a/nova/tests/unit/objects/test_objects.py +++ b/nova/tests/unit/objects/test_objects.py @@ -1072,7 +1072,7 @@ def obj_name(cls): 'BuildRequestList': '1.0-cd95608eccb89fbc702c8b52f38ec738', 'CellMapping': '1.0-7f1a7e85a22bbb7559fc730ab658b9bd', 'CellMappingList': '1.0-4ee0d9efdfd681fed822da88376e04d2', - 'ComputeNode': '1.16-2436e5b836fa0306a3c4e6d9e5ddacec', + 'ComputeNode': '1.17-abc222ef5a707dcd3e3d32b48197c9e7', 'ComputeNodeList': '1.16-40258d802a6ed045690a127a2088544b', 'DNSDomain': '1.0-7b0b2dab778454b6a7b6c66afe163a1a', 'DNSDomainList': '1.0-4ee0d9efdfd681fed822da88376e04d2',