diff --git a/openstack_virtual_baremetal/deploy.py b/openstack_virtual_baremetal/deploy.py index 2b37a45..27d6f8f 100755 --- a/openstack_virtual_baremetal/deploy.py +++ b/openstack_virtual_baremetal/deploy.py @@ -17,6 +17,7 @@ import os import random import sys +import time import yaml from heatclient import client as heat_client @@ -45,11 +46,19 @@ def _parse_args(): 'development.', action='store_true', default=False) + parser.add_argument('--role', + help='Additional environment file describing a ' + 'secondary role to be deployed alongside the ' + 'primary one described in the main environment.', + action='append', + default=[]) return parser.parse_args() def _process_args(args): if args.id and not args.quintupleo: raise RuntimeError('--id requires --quintupleo') + if args.role and not args.quintupleo: + raise RuntimeError('--role requires --quintupleo') env_path = args.env if args.name: @@ -132,7 +141,7 @@ def _get_heat_client(): return heat_client.Client('1', endpoint=heat_endpoint, token=token_id) -def _deploy(stack_name, stack_template, env_path): +def _deploy(stack_name, stack_template, env_path, poll): hclient = _get_heat_client() template_files, template = template_utils.get_template_contents( @@ -149,6 +158,90 @@ def _deploy(stack_name, stack_template, env_path): files=all_files) print 'Deployment of stack "%s" started.' % stack_name + if poll: + _poll_stack(stack_name, hclient) + +def _poll_stack(stack_name, hclient): + """Poll status for stack_name until it completes or fails""" + print 'Waiting for stack to complete', + done = False + while not done: + print '.', + stack = hclient.stacks.get(stack_name, resolve_outputs=False) + sys.stdout.flush() + if stack.status == 'COMPLETE': + print 'Stack %s created successfully' % stack_name + done = True + elif stack.status == 'FAILED': + raise RuntimeError('Failed to create stack %s' % stack_name) + else: + time.sleep(10) + +# Abstract out the role file interactions for easier unit testing +def _load_role_data(base_env, role_file, args): + with open(base_env) as f: + base_data = yaml.safe_load(f) + with open(role_file) as f: + role_data = yaml.safe_load(f) + with open(args.env) as f: + orig_data = yaml.safe_load(f) + return base_data, role_data, orig_data + +def _write_role_file(role_env, role_file): + with open(role_file, 'w') as f: + yaml.safe_dump(role_env, f, default_flow_style=False) + +def _process_role(role_file, base_env, stack_name, args): + """Merge a partial role env with the base env + + :param role: Filename of an environment file containing the definition + of the role. + :param base_env: Filename of the environment file used to deploy the + stack containing shared resources such as the undercloud and + networks. + :param stack_name: Name of the stack deployed using base_env. + :param args: The command-line arguments object from argparse. + """ + base_data, role_data, orig_data = _load_role_data(base_env, role_file, + args) + inherited_keys = ['baremetal_image', 'bmc_flavor', 'bmc_image', + 'external_net', 'key_name', 'os_auth_url', + 'os_password', 'os_tenant', 'os_user', + 'private_net', 'provision_net', 'public_net', + 'overcloud_internal_net', 'overcloud_storage_mgmt_net', + 'overcloud_storage_net','overcloud_tenant_net', + ] + allowed_registry_keys = ['OS::OVB::BaremetalPorts'] + role_env = role_data + # resource_registry is intentionally omitted as it should not be inherited + for section in ['parameters', 'parameter_defaults']: + role_env[section].update({ + k: v for k, v in base_data.get(section, {}).items() + if k in inherited_keys}) + # Most of the resource_registry should not be included in role envs. + # Only allow specific entries that may be needed. + role_env['resource_registry'] = { + k: v for k, v in role_env.get('resource_registry', {}).items() + if k in allowed_registry_keys} + # We need to start with the unmodified prefix + base_prefix = orig_data['parameters']['baremetal_prefix'] + # But we do need to add the id if one is in use + if args.id: + base_prefix += '-%s' % args.id + bmc_prefix = base_data['parameters']['bmc_prefix'] + role = role_data['parameter_defaults']['role'] + role_env['parameters']['baremetal_prefix'] = '%s-%s' % (base_prefix, role) + role_env['parameters']['bmc_prefix'] = '%s-%s' % (bmc_prefix, role) + role_file = 'env-%s-%s.yaml' % (stack_name, role) + _write_role_file(role_env, role_file) + return role_file, role + +def _deploy_roles(stack_name, args, env_path): + for r in args.role: + role_env, role_name = _process_role(r, env_path, stack_name, args) + _deploy(stack_name + '-%s' % role_name, + 'templates/virtual-baremetal.yaml', + role_env, poll=True) if __name__ == '__main__': args = _parse_args() @@ -156,4 +249,8 @@ def _deploy(stack_name, stack_template, env_path): stack_name, stack_template = _process_args(args) if args.id: env_path = _generate_id_env(args) - _deploy(stack_name, stack_template, env_path) + poll = False + if args.role: + poll = True + _deploy(stack_name, stack_template, env_path, poll=poll) + _deploy_roles(stack_name, args, env_path) diff --git a/openstack_virtual_baremetal/tests/test_build_nodes_json.py b/openstack_virtual_baremetal/tests/test_build_nodes_json.py index b3639e8..a88ea07 100644 --- a/openstack_virtual_baremetal/tests/test_build_nodes_json.py +++ b/openstack_virtual_baremetal/tests/test_build_nodes_json.py @@ -83,7 +83,8 @@ def test_get_names_env(self, mock_load, mock_open): {'bmc_prefix': 'bmc-foo', 'baremetal_prefix': 'baremetal-foo', 'provision_net': 'provision-foo' - } + }, + 'parameter_defaults': {} } mock_load.return_value = mock_env bmc_base, baremetal_base, provision_net = build_nodes_json._get_names( @@ -191,7 +192,8 @@ def test_build_nodes(self): nova.flavors.get.return_value = mock_flavor nodes, bmc_bm_pairs = build_nodes_json._build_nodes(nova, bmc_ports, bm_ports, - 'provision') + 'provision', + 'bm') expected_nodes = TEST_NODES self.assertEqual(expected_nodes, nodes) self.assertEqual([('1.1.1.1', 'bm_0'), ('1.1.1.2', 'bm_1')], @@ -256,7 +258,7 @@ def test_main(self, mock_parse_args, mock_get_names, mock_get_clients, mock_get_ports.assert_called_once_with(neutron, bmc_base, baremetal_base) mock_build_nodes.assert_called_once_with(nova, bmc_ports, bm_ports, - provision_net) + provision_net, baremetal_base) mock_write_nodes.assert_called_once_with(nodes, args) mock_write_pairs.assert_called_once_with(pairs) diff --git a/openstack_virtual_baremetal/tests/test_deploy.py b/openstack_virtual_baremetal/tests/test_deploy.py index e6a53ad..71baca2 100755 --- a/openstack_virtual_baremetal/tests/test_deploy.py +++ b/openstack_virtual_baremetal/tests/test_deploy.py @@ -26,6 +26,7 @@ def test_basic(self): mock_args.name = None mock_args.quintupleo = False mock_args.id = None + mock_args.role = [] name, template = deploy._process_args(mock_args) self.assertEqual('baremetal', name) self.assertEqual('templates/virtual-baremetal.yaml', template) @@ -35,6 +36,7 @@ def test_name(self): mock_args.name = 'foo' mock_args.quintupleo = False mock_args.id = None + mock_args.role = [] name, template = deploy._process_args(mock_args) self.assertEqual('foo', name) self.assertEqual('templates/virtual-baremetal.yaml', template) @@ -123,10 +125,92 @@ def test_generate_undercloud_name(self, mock_safe_dump): mock_safe_dump.assert_called_once_with(env_output, mock.ANY, default_flow_style=False) +# _process_role test data +role_base_data = { + 'parameter_defaults': { + 'overcloud_storage_mgmt_net': 'storage_mgmt-foo', + 'overcloud_internal_net': 'internal-foo', + 'overcloud_storage_net': 'storage-foo', + 'role': 'control', + 'overcloud_tenant_net': 'tenant-foo' + }, + 'parameters': { + 'os_user': 'admin', + 'key_name': 'default', + 'undercloud_name': 'undercloud-foo', + 'bmc_image': 'bmc-base', + 'baremetal_flavor': 'baremetal', + 'os_auth_url': 'http://1.1.1.1:5000/v2.0', + 'provision_net': 'provision-foo', + 'os_password': 'password', + 'os_tenant': 'admin', + 'bmc_prefix': 'bmc-foo', + 'public_net': 'public-foo', + 'undercloud_image': 'centos7-base', + 'baremetal_image': 'ipxe-boot', + 'external_net': 'external', + 'private_net': 'private', + 'baremetal_prefix': 'baremetal-foo-control', + 'undercloud_flavor': 'undercloud-16', + 'node_count': 3, + 'bmc_flavor': 'bmc' + }, + 'resource_registry': { + 'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml', + 'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-public-bond.yaml' + } + } +role_specific_data = { + 'parameter_defaults': { + 'role': 'compute', + }, + 'parameters': { + 'key_name': 'default', + 'baremetal_flavor': 'baremetal', + 'bmc_image': 'bmc-base', + 'bmc_prefix': 'bmc', + 'node_count': 2, + 'bmc_flavor': 'bmc' + }, + 'resource_registry': { + 'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml', + 'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-all.yaml' + } + } +role_original_data = { + 'parameter_defaults': { + 'role': 'control', + }, + 'parameters': { + 'os_user': 'admin', + 'key_name': 'default', + 'undercloud_name': 'undercloud', + 'baremetal_flavor': 'baremetal', + 'os_auth_url': 'http://1.1.1.1:5000/v2.0', + 'provision_net': 'provision', + 'bmc_image': 'bmc-base', + 'os_tenant': 'admin', + 'bmc_prefix': 'bmc', + 'public_net': 'public', + 'undercloud_image': 'centos7-base', + 'baremetal_image': 'ipxe-boot', + 'external_net': 'external', + 'os_password': 'password', + 'private_net': 'private', + 'baremetal_prefix': 'baremetal', + 'undercloud_flavor': 'undercloud-16', + 'node_count': 3, + 'bmc_flavor': 'bmc' + }, + 'resource_registry': { + 'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml', + 'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-public-bond.yaml' + } + } +# end _process_role test data + class TestDeploy(unittest.TestCase): - @mock.patch('deploy.template_utils') - @mock.patch('deploy._get_heat_client') - def test_deploy(self, mock_ghc, mock_tu): + def _test_deploy(self, mock_ghc, mock_tu, mock_poll, poll=False): mock_client = mock.Mock() mock_ghc.return_value = mock_client template_files = {'template.yaml': {'foo': 'bar'}} @@ -143,7 +227,7 @@ def test_deploy(self, mock_ghc, mock_tu): all_files = {} all_files.update(template_files) all_files.update(env_files) - deploy._deploy('test', 'template.yaml', 'env.yaml') + deploy._deploy('test', 'template.yaml', 'env.yaml', poll) mock_tu.get_template_contents.assert_called_once_with('template.yaml') process = mock_tu.process_multiple_environments_and_files process.assert_called_once_with(['templates/resource-registry.yaml', @@ -152,6 +236,96 @@ def test_deploy(self, mock_ghc, mock_tu): template=template, environment=env, files=all_files) + if not poll: + mock_poll.assert_not_called() + else: + mock_poll.assert_called_once_with('test', mock_client) + + @mock.patch('deploy._poll_stack') + @mock.patch('deploy.template_utils') + @mock.patch('deploy._get_heat_client') + def test_deploy(self, mock_ghc, mock_tu, mock_poll): + self._test_deploy(mock_ghc, mock_tu, mock_poll) + + @mock.patch('deploy._poll_stack') + @mock.patch('deploy.template_utils') + @mock.patch('deploy._get_heat_client') + def test_deploy_poll(self, mock_ghc, mock_tu, mock_poll): + self._test_deploy(mock_ghc, mock_tu, mock_poll, True) + + @mock.patch('time.sleep') + def test_poll(self, mock_sleep): + hclient = mock.Mock() + stacks = [mock.Mock(), mock.Mock()] + stacks[0].status = 'IN_PROGRESS' + stacks[1].status = 'COMPLETE' + hclient.stacks.get.side_effect = stacks + deploy._poll_stack('foo', hclient) + self.assertEqual([mock.call('foo', resolve_outputs=False), + mock.call('foo', resolve_outputs=False)], + hclient.stacks.get.mock_calls) + + @mock.patch('time.sleep') + def test_poll_fail(self, mock_sleep): + hclient = mock.Mock() + stacks = [mock.Mock(), mock.Mock()] + stacks[0].status = 'IN_PROGRESS' + stacks[1].status = 'FAILED' + hclient.stacks.get.side_effect = stacks + self.assertRaises(RuntimeError, deploy._poll_stack, 'foo', hclient) + self.assertEqual([mock.call('foo', resolve_outputs=False), + mock.call('foo', resolve_outputs=False)], + hclient.stacks.get.mock_calls) + + @mock.patch('deploy._write_role_file') + @mock.patch('deploy._load_role_data') + def test_process_role(self, mock_load, mock_write): + mock_load.return_value = (role_base_data, role_specific_data, + role_original_data) + args = mock.Mock() + args.id = 'foo' + role_file, role = deploy._process_role('foo-compute.yaml', 'foo.yaml', + 'foo', args) + mock_load.assert_called_once_with('foo.yaml', 'foo-compute.yaml', args) + self.assertEqual('env-foo-compute.yaml', role_file) + self.assertEqual('compute', role) + output = mock_write.call_args[0][0] + # These values are computed in _process_role + self.assertEqual('baremetal-foo-compute', + output['parameters']['baremetal_prefix']) + self.assertEqual('bmc-foo-compute', + output['parameters']['bmc_prefix']) + # These should be inherited + self.assertEqual('ipxe-boot', output['parameters']['baremetal_image']) + self.assertEqual('tenant-foo', + output['parameter_defaults']['overcloud_tenant_net']) + # This should not be present in a role env, even if set in the file + self.assertNotIn('OS::OVB::BaremetalNetworks', + output['resource_registry']) + # This should be the value set in the role env, not the base one + self.assertEqual('templates/baremetal-ports-all.yaml', + output['resource_registry']['OS::OVB::BaremetalPorts']) + + @mock.patch('deploy._deploy') + @mock.patch('deploy._process_role') + def test_deploy_roles(self, mock_process, mock_deploy): + args = mock.Mock() + args.role = ['foo-compute.yaml'] + mock_process.return_value = ('env-foo-compute.yaml', 'compute') + deploy._deploy_roles('foo', args, 'foo.yaml') + mock_process.assert_called_once_with('foo-compute.yaml', 'foo.yaml', + 'foo', args) + mock_deploy.assert_called_once_with('foo-compute', + 'templates/virtual-baremetal.yaml', + 'env-foo-compute.yaml', + poll=True) + + @mock.patch('deploy._process_role') + def test_deploy_roles_empty(self, mock_process): + args = mock.Mock() + args.role = [] + deploy._deploy_roles('foo', args, 'foo.yaml') + mock_process.assert_not_called() if __name__ == '__main__': unittest.main()