Add openstack overcloud export

Add a new command to export the data from a control plane stack for
multi-stack. The created export file can be used as input into a compute
stack for the multi-stack feature.

Also refactors the overcloud cell export command to use the same common
code.

Closes-Bug: #1850636
Change-Id: I6e145d7a54dcd306c4633ebd4d7471a19a967a86

releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - A new command "openstack overcloud export" is added. The command is used to
+    export the data from a control stack for use in a compute stack for the
+    multi-stack feature.

setup.cfg

@@ -62,6 +62,7 @@ openstack.tripleoclient.v1 =
     overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud
     overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
     overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud
+    overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud
     overcloud_failures = tripleoclient.v1.overcloud_deploy:GetDeploymentFailures
     overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus
     overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage

tripleoclient/constants.py

@@ -135,3 +135,7 @@
 UNUSED_PARAMETER_EXCLUDES_RE = ['^(Docker|Container).*Image$',
                                 '^SwiftFetchDir(Get|Put)Tempurl$',
                                 '^PythonInterpreter$']
+
+EXPORT_PASSWORD_EXCLUDE_PATTERNS = [
+    'ceph.*'
+]

tripleoclient/export.py

@@ -0,0 +1,122 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+import json
+import logging
+import os
+import re
+import sys
+import yaml
+
+from osc_lib.i18n import _
+
+from tripleoclient import constants
+from tripleoclient import utils as oooutils
+
+
+LOG = logging.getLogger(__name__ + ".utils")
+
+
+def export_passwords(swift, stack, excludes=True):
+    # Export the passwords from swift
+    obj = 'plan-environment.yaml'
+    container = stack
+    try:
+        resp_headers, content = swift.get_object(container, obj)
+    except Exception as e:
+        LOG.error("An error happened while exporting the password "
+                  "file from swift: %s", str(e))
+        sys.exit(1)
+
+    data = yaml.load(content)["passwords"]
+    if excludes:
+        excluded_passwords = []
+        for k in data:
+            for pattern in constants.EXPORT_PASSWORD_EXCLUDE_PATTERNS:
+                if re.match(pattern, k, re.I):
+                    excluded_passwords.append(k)
+        [data.pop(e) for e in excluded_passwords]
+    return data
+
+
+def export_stack(heat, stack, should_filter=False,
+                 config_download_dir='/var/lib/mistral/overcloud'):
+
+    # data to export
+    # parameter: Parameter to be exported
+    # file:   IF file specified it is taken as source instead of heat
+    #         output.File is relative to <config-download-dir>/stack.
+    # filter: in case only specific settings should be
+    #         exported from parameter data.
+    export_data = {
+        "EndpointMap": {
+            "parameter": "EndpointMapOverride",
+        },
+        "HostsEntry": {
+            "parameter": "ExtraHostFileEntries",
+        },
+        "GlobalConfig": {
+            "parameter": "GlobalConfigExtraMapData",
+        },
+        "AllNodesConfig": {
+            "file": "group_vars/overcloud.json",
+            "parameter": "AllNodesExtraMapData",
+            "filter": ["oslo_messaging_notify_short_bootstrap_node_name",
+                       "oslo_messaging_notify_node_names",
+                       "oslo_messaging_rpc_node_names",
+                       "memcached_node_ips",
+                       "ovn_dbs_vip",
+                       "redis_vip"]},
+    }
+
+    data = {}
+    heat_stack = oooutils.get_stack(heat, stack)
+
+    for export_key, export_param in export_data.items():
+        param = export_param["parameter"]
+        if "file" in export_param:
+            # get file data
+            file = os.path.join(config_download_dir,
+                                export_param["file"])
+            with open(file, 'r') as ff:
+                try:
+                    export_data = json.load(ff)
+                except Exception as e:
+                    LOG.error(
+                        _('Could not read file %s') % file)
+                    LOG.error(e)
+
+        else:
+            # get stack data
+            export_data = oooutils.get_stack_output_item(
+                            heat_stack, export_key)
+
+        if export_data:
+            # When we export information from a cell controller stack
+            # we don't want to filter.
+            if "filter" in export_param and should_filter:
+                for filter_key in export_param["filter"]:
+                    if filter_key in export_data:
+                        element = {filter_key: export_data[filter_key]}
+                        data.setdefault(param, {}).update(element)
+            else:
+                data[param] = export_data
+
+        else:
+            raise Exception(
+                "No data returned to export %s from." % param)
+
+    return data

tripleoclient/tests/test_export.py

@@ -0,0 +1,139 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+from io import StringIO
+import mock
+import six
+from unittest import TestCase
+import yaml
+
+from tripleoclient import export
+
+
+class TestExport(TestCase):
+    def setUp(self):
+        self.unlink_patch = mock.patch('os.unlink')
+        self.addCleanup(self.unlink_patch.stop)
+        self.unlink_patch.start()
+        self.mock_log = mock.Mock('logging.getLogger')
+
+        outputs = [
+            {'output_key': 'EndpointMap',
+             'output_value': dict(em_key='em_value')},
+            {'output_key': 'HostsEntry',
+             'output_value': 'hosts entry'},
+            {'output_key': 'GlobalConfig',
+             'output_value': dict(gc_key='gc_value')},
+        ]
+        self.mock_stack = mock.Mock()
+        self.mock_stack.to_dict.return_value = dict(outputs=outputs)
+        self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            data = export.export_stack(heat, "overcloud")
+
+        expected = \
+            {'AllNodesExtraMapData': {u'an_key': u'an_value'},
+             'EndpointMapOverride': {'em_key': 'em_value'},
+             'ExtraHostFileEntries': 'hosts entry',
+             'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
+
+        self.assertEqual(expected, data)
+        self.mock_open.assert_called_once_with(
+            '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_should_filter(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        self.mock_open = mock.mock_open(
+            read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}')
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            data = export.export_stack(heat, "overcloud", should_filter=True)
+
+        expected = \
+            {'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'},
+             'EndpointMapOverride': {'em_key': 'em_value'},
+             'ExtraHostFileEntries': 'hosts entry',
+             'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
+
+        self.assertEqual(expected, data)
+        self.mock_open.assert_called_once_with(
+            '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_cd_dir(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            export.export_stack(heat, "overcloud",
+                                config_download_dir='/foo/overcloud')
+        self.mock_open.assert_called_once_with(
+            '/foo/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_stack_name(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            export.export_stack(heat, "control")
+        mock_get_stack.assert_called_once_with(heat, 'control')
+
+    def test_export_passwords(self):
+        swift = mock.Mock()
+        mock_passwords = {
+            'passwords': {
+                'a': 'A',
+                'b': 'B'
+            }
+        }
+        sio = StringIO()
+        sio.write(six.text_type(yaml.dump(mock_passwords)))
+        sio.seek(0)
+        swift.get_object.return_value = ("", sio)
+        data = export.export_passwords(swift, 'overcloud')
+
+        swift.get_object.assert_called_once_with(
+            'overcloud', 'plan-environment.yaml')
+
+        self.assertEqual(mock_passwords['passwords'], data)
+
+    def test_export_passwords_excludes(self):
+        swift = mock.Mock()
+        mock_passwords = {
+            'passwords': {
+                'a': 'A',
+                'b': 'B',
+                'Cephkey': 'cephkey',
+                'cephkey': 'cephkey',
+                'CEPH': 'cephkey'
+            }
+        }
+        sio = StringIO()
+        sio.write(six.text_type(yaml.dump(mock_passwords)))
+        sio.seek(0)
+        swift.get_object.return_value = ("", sio)
+        data = export.export_passwords(swift, 'overcloud')
+
+        mock_passwords['passwords'].pop('Cephkey')
+        mock_passwords['passwords'].pop('cephkey')
+        mock_passwords['passwords'].pop('CEPH')
+
+        self.assertEqual(mock_passwords['passwords'], data)