Adds Overcloud Upgrade class and entry points for major upgrade

This further refactors the update/upgrades cli and separates
the update and upgrade code. This adds the overcloud_upgrade.py
which defines the UpgradePrepare and UpgradeRun. The entry
points are now:

  openstack overcloud upgrade prepare --container-registry-file ...

For the no-op heat stack update to refresh stack outputs and

  openstack overcloud upgrade run --nodes foo --playbooks all

For running all the upgrade ansible playbooks. A corresponding
converge sub-command will be introduced in a subsequent patch.

Change-Id: I1880e8f546df8d509871ba3b4f02877e95c611c8

setup.cfg

@@ -91,7 +91,9 @@ openstack.tripleoclient.v1 =
     overcloud_support_report_collect = tripleoclient.v1.overcloud_support:ReportExecute
     overcloud_update_prepare= tripleoclient.v1.overcloud_update:UpdatePrepare
     overcloud_update_run = tripleoclient.v1.overcloud_update:UpdateRun
-    overcloud_upgrade_converge = tripleoclient.v1.overcloud_update:UpgradeConvergeOvercloud
+    overcloud_upgrade_prepare = tripleoclient.v1.overcloud_upgrade:UpgradePrepare
+    overcloud_upgrade_run = tripleoclient.v1.overcloud_upgrade:UpgradeRun
+    overcloud_upgrade_converge = tripleoclient.v1.overcloud_upgrade:UpgradeConvergeOvercloud
     overcloud_execute = tripleoclient.v1.overcloud_execute:RemoteExecute
     overcloud_generate_fencing = tripleoclient.v1.overcloud_parameters:GenerateFencingParameters
     undercloud_deploy = tripleoclient.v1.undercloud_deploy:DeployUndercloud

tripleoclient/constants.py

@@ -35,8 +35,13 @@
 PUPPET_BASE = "/etc/puppet/"
 # Update Queue
 UPDATE_QUEUE = 'update'
+UPGRADE_QUEUE = 'upgrade'
 STACK_TIMEOUT = 240
 
 # The default minor update ansible playbooks generated from heat stack output
 MINOR_UPDATE_PLAYBOOKS = ['update_steps_playbook.yaml',
                           'deploy_steps_playbook.yaml']
+# The default major upgrade ansible playbooks generated from heat stack output
+MAJOR_UPGRADE_PLAYBOOKS = ["upgrade_steps_playbook.yaml",
+                           "deploy_steps_playbook.yaml",
+                           "post_upgrade_steps_playbook.yaml"]

tripleoclient/tests/v1/overcloud_upgrade/fakes.py

@@ -0,0 +1,61 @@
+#   Copyright 2015 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import mock
+from osc_lib.tests import utils
+
+from tripleoclient.tests import fakes
+
+
+class FakeClientWrapper(object):
+
+    def __init__(self):
+        self._instance = mock.Mock()
+        self.object_store = FakeObjectClient()
+
+    def messaging_websocket(self):
+        return fakes.FakeWebSocket()
+
+
+class FakeObjectClient(object):
+
+    def __init__(self):
+        self._instance = mock.Mock()
+        self.put_object = mock.Mock()
+
+    def get_object(self, *args):
+        return
+
+
+class TestOvercloudUpgradePrepare(utils.TestCommand):
+
+    def setUp(self):
+        super(TestOvercloudUpgradePrepare, self).setUp()
+
+        self.app.client_manager.auth_ref = mock.Mock(auth_token="TOKEN")
+        self.app.client_manager.baremetal = mock.Mock()
+        self.app.client_manager.orchestration = mock.Mock()
+        self.app.client_manager.tripleoclient = FakeClientWrapper()
+        self.app.client_manager.workflow_engine = mock.Mock()
+
+
+class TestOvercloudUpgradeRun(utils.TestCommand):
+
+    def setUp(self):
+        super(TestOvercloudUpgradeRun, self).setUp()
+
+        self.app.client_manager.auth_ref = mock.Mock(auth_token="TOKEN")
+        self.app.client_manager.tripleoclient = FakeClientWrapper()
+        self.app.client_manager.workflow_engine = mock.Mock()

tripleoclient/tests/v1/overcloud_upgrade/test_overcloud_upgrade.py

@@ -0,0 +1,188 @@
+#   Copyright 2018 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+import mock
+
+from osc_lib.tests.utils import ParserException
+from tripleoclient import constants
+from tripleoclient import exceptions
+from tripleoclient.tests.v1.overcloud_upgrade import fakes
+from tripleoclient.v1 import overcloud_upgrade
+
+
+class TestOvercloudUpgradePrepare(fakes.TestOvercloudUpgradePrepare):
+
+    def setUp(self):
+        super(TestOvercloudUpgradePrepare, self).setUp()
+
+        # Get the command object to test
+        app_args = mock.Mock()
+        app_args.verbose_level = 1
+        self.cmd = overcloud_upgrade.UpgradePrepare(self.app, app_args)
+
+        uuid4_patcher = mock.patch('uuid.uuid4', return_value="UUID4")
+        self.mock_uuid4 = uuid4_patcher.start()
+        self.addCleanup(self.mock_uuid4.stop)
+
+    @mock.patch('tripleoclient.utils.get_stack',
+                autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_upgrade.UpgradePrepare.log',
+                autospec=True)
+    @mock.patch('tripleoclient.workflows.package_update.update',
+                autospec=True)
+    @mock.patch('os.path.abspath')
+    @mock.patch('yaml.load')
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_deploy_tripleo_heat_templates', autospec=True)
+    def test_upgrade_out(self, mock_deploy, mock_open, mock_copy, mock_yaml,
+                         mock_abspath, mock_upgrade, mock_logger,
+                         mock_get_stack):
+        mock_stack = mock.Mock()
+        mock_stack.stack_name = 'mystack'
+        mock_get_stack.return_value = mock_stack
+        mock_abspath.return_value = '/home/fake/my-fake-registry.yaml'
+        mock_yaml.return_value = {'fake_container': 'fake_value'}
+
+        argslist = ['--stack', 'overcloud', '--templates',
+                    '--container-registry-file', 'my-fake-registry.yaml']
+        verifylist = [
+            ('stack', 'overcloud'),
+            ('templates', constants.TRIPLEO_HEAT_TEMPLATES),
+            ('container_registry_file', 'my-fake-registry.yaml')
+        ]
+
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        self.cmd.take_action(parsed_args)
+        mock_upgrade.assert_called_once_with(
+            self.app.client_manager,
+            container='mystack',
+            container_registry={'fake_container': 'fake_value'},
+            ceph_ansible_playbook='/usr/share/ceph-ansible'
+                                  '/site-docker.yml.sample'
+        )
+
+    @mock.patch('tripleoclient.workflows.package_update.update',
+                autospec=True)
+    @mock.patch('six.moves.builtins.open')
+    @mock.patch('os.path.abspath')
+    @mock.patch('yaml.load')
+    @mock.patch('shutil.copytree', autospec=True)
+    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
+                '_deploy_tripleo_heat_templates', autospec=True)
+    def test_upgrade_failed(self, mock_deploy, mock_copy, mock_yaml,
+                            mock_abspath, mock_open, mock_upgrade):
+        mock_upgrade.side_effect = exceptions.DeploymentError()
+        mock_abspath.return_value = '/home/fake/my-fake-registry.yaml'
+        mock_yaml.return_value = {'fake_container': 'fake_value'}
+        argslist = ['--stack', 'overcloud', '--templates',
+                    '--container-registry-file', 'my-fake-registry.yaml']
+        verifylist = [
+            ('stack', 'overcloud'),
+            ('templates', constants.TRIPLEO_HEAT_TEMPLATES),
+            ('container_registry_file', 'my-fake-registry.yaml')
+        ]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+
+        self.assertRaises(exceptions.DeploymentError,
+                          self.cmd.take_action, parsed_args)
+
+
+class TestOvercloudUpgradeRun(fakes.TestOvercloudUpgradeRun):
+
+    def setUp(self):
+        super(TestOvercloudUpgradeRun, self).setUp()
+
+        # Get the command object to test
+        app_args = mock.Mock()
+        app_args.verbose_level = 1
+        self.cmd = overcloud_upgrade.UpgradeRun(self.app, app_args)
+
+        uuid4_patcher = mock.patch('uuid.uuid4', return_value="UUID4")
+        self.mock_uuid4 = uuid4_patcher.start()
+        self.addCleanup(self.mock_uuid4.stop)
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    def test_upgrade_with_playbook(self, mock_open, mock_execute,
+                                   mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'Compute', '--playbook',
+                    'fake-playbook.yaml']
+        verifylist = [
+            ('nodes', 'Compute'),
+            ('static_inventory', None),
+            ('playbook', 'fake-playbook.yaml')
+        ]
+
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.cmd.take_action(parsed_args)
+            upgrade_ansible.assert_called_once_with(
+                self.app.client_manager,
+                nodes='Compute',
+                inventory_file=mock_open().read(),
+                playbook='fake-playbook.yaml',
+                ansible_queue_name=constants.UPGRADE_QUEUE
+            )
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    def test_upgrade_with_all_playbooks(self, mock_open, mock_execute,
+                                        mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = ['--nodes', 'Compute', '--playbook', 'all']
+        verifylist = [
+            ('nodes', 'Compute'),
+            ('static_inventory', None),
+            ('playbook', 'all')
+        ]
+
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        with mock.patch('os.path.exists') as mock_exists:
+            mock_exists.return_value = True
+            self.cmd.take_action(parsed_args)
+            for book in constants.MAJOR_UPGRADE_PLAYBOOKS:
+                upgrade_ansible.assert_any_call(
+                    self.app.client_manager,
+                    nodes='Compute',
+                    inventory_file=mock_open().read(),
+                    playbook=book,
+                    ansible_queue_name=constants.UPGRADE_QUEUE
+                )
+
+    @mock.patch('tripleoclient.workflows.package_update.update_ansible',
+                autospec=True)
+    @mock.patch('os.path.expanduser')
+    @mock.patch('oslo_concurrency.processutils.execute')
+    @mock.patch('six.moves.builtins.open')
+    def test_upgrade_with_no_nodes(self, mock_open, mock_execute,
+                                   mock_expanduser, upgrade_ansible):
+        mock_expanduser.return_value = '/home/fake/'
+        argslist = []
+        verifylist = [
+            ('static_inventory', None),
+            ('playbook', 'all')
+        ]
+        self.assertRaises(ParserException, lambda: self.check_parser(
+            self.cmd, argslist, verifylist))

tripleoclient/utils.py

@@ -33,6 +33,7 @@
 from heatclient.common import event_utils
 from heatclient.exc import HTTPNotFound
 from osc_lib.i18n import _
+from oslo_concurrency import processutils
 from six.moves import configparser
 
 from tripleoclient import exceptions
@@ -810,3 +811,22 @@ def load_environment_directories(directories):
                 if os.path.isfile(f):
                     environments.append(f)
     return environments
+
+
+def get_tripleo_ansible_inventory(inventory_file=''):
+    if not inventory_file:
+        inventory_file = '%s/%s' % (os.path.expanduser('~'),
+                                    'tripleo-ansible-inventory.yaml')
+        try:
+            processutils.execute(
+                '/usr/bin/tripleo-ansible-inventory',
+                '--static-yaml-inventory', inventory_file)
+        except processutils.ProcessExecutionError as e:
+                message = "Failed to generate inventory: %s" % str(e)
+                raise exceptions.InvalidConfiguration(message)
+    if os.path.exists(inventory_file):
+        inventory = open(inventory_file, 'r').read()
+        return inventory
+    else:
+        raise exceptions.InvalidConfiguration(
+            "Inventory file %s can not be found." % inventory_file)

tripleoclient/v1/overcloud_update.py

@@ -18,11 +18,9 @@
 import yaml
 
 from osc_lib.i18n import _
-from oslo_concurrency import processutils
 
 from tripleoclient import command
 from tripleoclient import constants
-from tripleoclient import exceptions
 from tripleoclient import utils as oooutils
 from tripleoclient.v1.overcloud_deploy import DeployOvercloud
 from tripleoclient.workflows import package_update
@@ -155,22 +153,8 @@ def take_action(self, parsed_args):
             # unset this, the ansible action deals with unset 'limithosts'
             nodes = None
         playbook = parsed_args.playbook
-        inventory_file = parsed_args.static_inventory
-        if inventory_file is None:
-            inventory_file = '%s/%s' % (os.path.expanduser('~'),
-                                        'tripleo-ansible-inventory.yaml')
-            try:
-                processutils.execute(
-                    '/usr/bin/tripleo-ansible-inventory',
-                    '--static-yaml-inventory', inventory_file)
-            except processutils.ProcessExecutionError as e:
-                message = "Failed to generate inventory: %s" % str(e)
-                raise exceptions.InvalidConfiguration(message)
-        if os.path.exists(inventory_file):
-            inventory = open(inventory_file, 'r').read()
-        else:
-            raise exceptions.InvalidConfiguration(
-                "Inventory file %s can not be found." % inventory_file)
+        inventory = oooutils.get_tripleo_ansible_inventory(
+            parsed_args.static_inventory)
         update_playbooks = [playbook]
         if playbook == "all":
             update_playbooks = constants.MINOR_UPDATE_PLAYBOOKS
@@ -181,26 +165,3 @@ def take_action(self, parsed_args):
                 inventory_file=inventory,
                 playbook=book,
                 ansible_queue_name=constants.UPDATE_QUEUE)
-
-
-class UpgradeConvergeOvercloud(DeployOvercloud):
-    """Converge the upgrade on Overcloud Nodes"""
-
-    log = logging.getLogger(__name__ + ".UpgradeConvergeOvercloud")
-
-    def get_parser(self, prog_name):
-        parser = super(UpgradeConvergeOvercloud, self).get_parser(prog_name)
-        return parser
-
-    def take_action(self, parsed_args):
-        self.log.debug("take_action(%s)" % parsed_args)
-        clients = self.app.client_manager
-
-        stack = oooutils.get_stack(clients.orchestration,
-                                   parsed_args.stack)
-        stack_name = stack.stack_name
-
-        parsed_args.update_plan_only = True
-        super(UpgradeConvergeOvercloud, self).take_action(parsed_args)
-        # Run converge steps
-        package_update.converge_nodes(clients, container=stack_name)