Skip to content

Commit

Permalink
Add support for deploying multiple roles
Browse files Browse the repository at this point in the history 
Allow specifying additional role environment files on the deploy
command-line.  These additional env files will be used to deploy
stacks that are wired into the base stack's resources in such a
way as to allow the instances from all the stacks to be deployed
as a single virtual baremetal environment.

Note that this feature requires use of --quintupleo.
  • Loading branch information
@cybertron
cybertron committed Nov 29, 2016
1 parent 06caa8e commit 5fc1b2c
Show file tree
Hide file tree
Showing 3 changed files with 282 additions and 9 deletions.
101 changes: 99 additions & 2 deletions openstack_virtual_baremetal/deploy.py
View file
Expand Up @@ -17,6 +17,7 @@
import os
import random
import sys
import time
import yaml

from heatclient import client as heat_client
Expand Down Expand Up @@ -45,11 +46,19 @@ def _parse_args():
'development.',
action='store_true',
default=False)
parser.add_argument('--role',
help='Additional environment file describing a '
'secondary role to be deployed alongside the '
'primary one described in the main environment.',
action='append',
default=[])
return parser.parse_args()

def _process_args(args):
if args.id and not args.quintupleo:
raise RuntimeError('--id requires --quintupleo')
if args.role and not args.quintupleo:
raise RuntimeError('--role requires --quintupleo')

env_path = args.env
if args.name:
Expand Down Expand Up @@ -132,7 +141,7 @@ def _get_heat_client():

return heat_client.Client('1', endpoint=heat_endpoint, token=token_id)

def _deploy(stack_name, stack_template, env_path):
def _deploy(stack_name, stack_template, env_path, poll):
hclient = _get_heat_client()

template_files, template = template_utils.get_template_contents(
Expand All @@ -149,11 +158,99 @@ def _deploy(stack_name, stack_template, env_path):
files=all_files)

print 'Deployment of stack "%s" started.' % stack_name
if poll:
_poll_stack(stack_name, hclient)

def _poll_stack(stack_name, hclient):
"""Poll status for stack_name until it completes or fails"""
print 'Waiting for stack to complete',
done = False
while not done:
print '.',
stack = hclient.stacks.get(stack_name, resolve_outputs=False)
sys.stdout.flush()
if stack.status == 'COMPLETE':
print 'Stack %s created successfully' % stack_name
done = True
elif stack.status == 'FAILED':
raise RuntimeError('Failed to create stack %s' % stack_name)
else:
time.sleep(10)

# Abstract out the role file interactions for easier unit testing
def _load_role_data(base_env, role_file, args):
with open(base_env) as f:
base_data = yaml.safe_load(f)
with open(role_file) as f:
role_data = yaml.safe_load(f)
with open(args.env) as f:
orig_data = yaml.safe_load(f)
return base_data, role_data, orig_data

def _write_role_file(role_env, role_file):
with open(role_file, 'w') as f:
yaml.safe_dump(role_env, f, default_flow_style=False)

def _process_role(role_file, base_env, stack_name, args):
"""Merge a partial role env with the base env
:param role: Filename of an environment file containing the definition
of the role.
:param base_env: Filename of the environment file used to deploy the
stack containing shared resources such as the undercloud and
networks.
:param stack_name: Name of the stack deployed using base_env.
:param args: The command-line arguments object from argparse.
"""
base_data, role_data, orig_data = _load_role_data(base_env, role_file,
args)
inherited_keys = ['baremetal_image', 'bmc_flavor', 'bmc_image',
'external_net', 'key_name', 'os_auth_url',
'os_password', 'os_tenant', 'os_user',
'private_net', 'provision_net', 'public_net',
'overcloud_internal_net', 'overcloud_storage_mgmt_net',
'overcloud_storage_net','overcloud_tenant_net',
]
allowed_registry_keys = ['OS::OVB::BaremetalPorts']
role_env = role_data
# resource_registry is intentionally omitted as it should not be inherited
for section in ['parameters', 'parameter_defaults']:
role_env[section].update({
k: v for k, v in base_data.get(section, {}).items()
if k in inherited_keys})
# Most of the resource_registry should not be included in role envs.
# Only allow specific entries that may be needed.
role_env['resource_registry'] = {
k: v for k, v in role_env.get('resource_registry', {}).items()
if k in allowed_registry_keys}
# We need to start with the unmodified prefix
base_prefix = orig_data['parameters']['baremetal_prefix']
# But we do need to add the id if one is in use
if args.id:
base_prefix += '-%s' % args.id
bmc_prefix = base_data['parameters']['bmc_prefix']
role = role_data['parameter_defaults']['role']
role_env['parameters']['baremetal_prefix'] = '%s-%s' % (base_prefix, role)
role_env['parameters']['bmc_prefix'] = '%s-%s' % (bmc_prefix, role)
role_file = 'env-%s-%s.yaml' % (stack_name, role)
_write_role_file(role_env, role_file)
return role_file, role

def _deploy_roles(stack_name, args, env_path):
for r in args.role:
role_env, role_name = _process_role(r, env_path, stack_name, args)
_deploy(stack_name + '-%s' % role_name,
'templates/virtual-baremetal.yaml',
role_env, poll=True)

if __name__ == '__main__':
args = _parse_args()
env_path = args.env
stack_name, stack_template = _process_args(args)
if args.id:
env_path = _generate_id_env(args)
_deploy(stack_name, stack_template, env_path)
poll = False
if args.role:
poll = True
_deploy(stack_name, stack_template, env_path, poll=poll)
_deploy_roles(stack_name, args, env_path)
8 changes: 5 additions & 3 deletions openstack_virtual_baremetal/tests/test_build_nodes_json.py
View file
Expand Up @@ -83,7 +83,8 @@ def test_get_names_env(self, mock_load, mock_open):
{'bmc_prefix': 'bmc-foo',
'baremetal_prefix': 'baremetal-foo',
'provision_net': 'provision-foo'
}
},
'parameter_defaults': {}
}
mock_load.return_value = mock_env
bmc_base, baremetal_base, provision_net = build_nodes_json._get_names(
Expand Down Expand Up @@ -191,7 +192,8 @@ def test_build_nodes(self):
nova.flavors.get.return_value = mock_flavor
nodes, bmc_bm_pairs = build_nodes_json._build_nodes(nova, bmc_ports,
bm_ports,
'provision')
'provision',
'bm')
expected_nodes = TEST_NODES
self.assertEqual(expected_nodes, nodes)
self.assertEqual([('1.1.1.1', 'bm_0'), ('1.1.1.2', 'bm_1')],
Expand Down Expand Up @@ -256,7 +258,7 @@ def test_main(self, mock_parse_args, mock_get_names, mock_get_clients,
mock_get_ports.assert_called_once_with(neutron, bmc_base,
baremetal_base)
mock_build_nodes.assert_called_once_with(nova, bmc_ports, bm_ports,
provision_net)
provision_net, baremetal_base)
mock_write_nodes.assert_called_once_with(nodes, args)
mock_write_pairs.assert_called_once_with(pairs)

182 changes: 178 additions & 4 deletions openstack_virtual_baremetal/tests/test_deploy.py
View file
Expand Up @@ -26,6 +26,7 @@ def test_basic(self):
mock_args.name = None
mock_args.quintupleo = False
mock_args.id = None
mock_args.role = []
name, template = deploy._process_args(mock_args)
self.assertEqual('baremetal', name)
self.assertEqual('templates/virtual-baremetal.yaml', template)
Expand All @@ -35,6 +36,7 @@ def test_name(self):
mock_args.name = 'foo'
mock_args.quintupleo = False
mock_args.id = None
mock_args.role = []
name, template = deploy._process_args(mock_args)
self.assertEqual('foo', name)
self.assertEqual('templates/virtual-baremetal.yaml', template)
Expand Down Expand Up @@ -123,10 +125,92 @@ def test_generate_undercloud_name(self, mock_safe_dump):
mock_safe_dump.assert_called_once_with(env_output, mock.ANY,
default_flow_style=False)

# _process_role test data
role_base_data = {
'parameter_defaults': {
'overcloud_storage_mgmt_net': 'storage_mgmt-foo',
'overcloud_internal_net': 'internal-foo',
'overcloud_storage_net': 'storage-foo',
'role': 'control',
'overcloud_tenant_net': 'tenant-foo'
},
'parameters': {
'os_user': 'admin',
'key_name': 'default',
'undercloud_name': 'undercloud-foo',
'bmc_image': 'bmc-base',
'baremetal_flavor': 'baremetal',
'os_auth_url': 'http://1.1.1.1:5000/v2.0',
'provision_net': 'provision-foo',
'os_password': 'password',
'os_tenant': 'admin',
'bmc_prefix': 'bmc-foo',
'public_net': 'public-foo',
'undercloud_image': 'centos7-base',
'baremetal_image': 'ipxe-boot',
'external_net': 'external',
'private_net': 'private',
'baremetal_prefix': 'baremetal-foo-control',
'undercloud_flavor': 'undercloud-16',
'node_count': 3,
'bmc_flavor': 'bmc'
},
'resource_registry': {
'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml',
'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-public-bond.yaml'
}
}
role_specific_data = {
'parameter_defaults': {
'role': 'compute',
},
'parameters': {
'key_name': 'default',
'baremetal_flavor': 'baremetal',
'bmc_image': 'bmc-base',
'bmc_prefix': 'bmc',
'node_count': 2,
'bmc_flavor': 'bmc'
},
'resource_registry': {
'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml',
'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-all.yaml'
}
}
role_original_data = {
'parameter_defaults': {
'role': 'control',
},
'parameters': {
'os_user': 'admin',
'key_name': 'default',
'undercloud_name': 'undercloud',
'baremetal_flavor': 'baremetal',
'os_auth_url': 'http://1.1.1.1:5000/v2.0',
'provision_net': 'provision',
'bmc_image': 'bmc-base',
'os_tenant': 'admin',
'bmc_prefix': 'bmc',
'public_net': 'public',
'undercloud_image': 'centos7-base',
'baremetal_image': 'ipxe-boot',
'external_net': 'external',
'os_password': 'password',
'private_net': 'private',
'baremetal_prefix': 'baremetal',
'undercloud_flavor': 'undercloud-16',
'node_count': 3,
'bmc_flavor': 'bmc'
},
'resource_registry': {
'OS::OVB::BaremetalNetworks': 'templates/baremetal-networks-all.yaml',
'OS::OVB::BaremetalPorts': 'templates/baremetal-ports-public-bond.yaml'
}
}
# end _process_role test data

class TestDeploy(unittest.TestCase):
@mock.patch('deploy.template_utils')
@mock.patch('deploy._get_heat_client')
def test_deploy(self, mock_ghc, mock_tu):
def _test_deploy(self, mock_ghc, mock_tu, mock_poll, poll=False):
mock_client = mock.Mock()
mock_ghc.return_value = mock_client
template_files = {'template.yaml': {'foo': 'bar'}}
Expand All @@ -143,7 +227,7 @@ def test_deploy(self, mock_ghc, mock_tu):
all_files = {}
all_files.update(template_files)
all_files.update(env_files)
deploy._deploy('test', 'template.yaml', 'env.yaml')
deploy._deploy('test', 'template.yaml', 'env.yaml', poll)
mock_tu.get_template_contents.assert_called_once_with('template.yaml')
process = mock_tu.process_multiple_environments_and_files
process.assert_called_once_with(['templates/resource-registry.yaml',
Expand All @@ -152,6 +236,96 @@ def test_deploy(self, mock_ghc, mock_tu):
template=template,
environment=env,
files=all_files)
if not poll:
mock_poll.assert_not_called()
else:
mock_poll.assert_called_once_with('test', mock_client)

@mock.patch('deploy._poll_stack')
@mock.patch('deploy.template_utils')
@mock.patch('deploy._get_heat_client')
def test_deploy(self, mock_ghc, mock_tu, mock_poll):
self._test_deploy(mock_ghc, mock_tu, mock_poll)

@mock.patch('deploy._poll_stack')
@mock.patch('deploy.template_utils')
@mock.patch('deploy._get_heat_client')
def test_deploy_poll(self, mock_ghc, mock_tu, mock_poll):
self._test_deploy(mock_ghc, mock_tu, mock_poll, True)

@mock.patch('time.sleep')
def test_poll(self, mock_sleep):
hclient = mock.Mock()
stacks = [mock.Mock(), mock.Mock()]
stacks[0].status = 'IN_PROGRESS'
stacks[1].status = 'COMPLETE'
hclient.stacks.get.side_effect = stacks
deploy._poll_stack('foo', hclient)
self.assertEqual([mock.call('foo', resolve_outputs=False),
mock.call('foo', resolve_outputs=False)],
hclient.stacks.get.mock_calls)

@mock.patch('time.sleep')
def test_poll_fail(self, mock_sleep):
hclient = mock.Mock()
stacks = [mock.Mock(), mock.Mock()]
stacks[0].status = 'IN_PROGRESS'
stacks[1].status = 'FAILED'
hclient.stacks.get.side_effect = stacks
self.assertRaises(RuntimeError, deploy._poll_stack, 'foo', hclient)
self.assertEqual([mock.call('foo', resolve_outputs=False),
mock.call('foo', resolve_outputs=False)],
hclient.stacks.get.mock_calls)

@mock.patch('deploy._write_role_file')
@mock.patch('deploy._load_role_data')
def test_process_role(self, mock_load, mock_write):
mock_load.return_value = (role_base_data, role_specific_data,
role_original_data)
args = mock.Mock()
args.id = 'foo'
role_file, role = deploy._process_role('foo-compute.yaml', 'foo.yaml',
'foo', args)
mock_load.assert_called_once_with('foo.yaml', 'foo-compute.yaml', args)
self.assertEqual('env-foo-compute.yaml', role_file)
self.assertEqual('compute', role)
output = mock_write.call_args[0][0]
# These values are computed in _process_role
self.assertEqual('baremetal-foo-compute',
output['parameters']['baremetal_prefix'])
self.assertEqual('bmc-foo-compute',
output['parameters']['bmc_prefix'])
# These should be inherited
self.assertEqual('ipxe-boot', output['parameters']['baremetal_image'])
self.assertEqual('tenant-foo',
output['parameter_defaults']['overcloud_tenant_net'])
# This should not be present in a role env, even if set in the file
self.assertNotIn('OS::OVB::BaremetalNetworks',
output['resource_registry'])
# This should be the value set in the role env, not the base one
self.assertEqual('templates/baremetal-ports-all.yaml',
output['resource_registry']['OS::OVB::BaremetalPorts'])

@mock.patch('deploy._deploy')
@mock.patch('deploy._process_role')
def test_deploy_roles(self, mock_process, mock_deploy):
args = mock.Mock()
args.role = ['foo-compute.yaml']
mock_process.return_value = ('env-foo-compute.yaml', 'compute')
deploy._deploy_roles('foo', args, 'foo.yaml')
mock_process.assert_called_once_with('foo-compute.yaml', 'foo.yaml',
'foo', args)
mock_deploy.assert_called_once_with('foo-compute',
'templates/virtual-baremetal.yaml',
'env-foo-compute.yaml',
poll=True)

@mock.patch('deploy._process_role')
def test_deploy_roles_empty(self, mock_process):
args = mock.Mock()
args.role = []
deploy._deploy_roles('foo', args, 'foo.yaml')
mock_process.assert_not_called()

if __name__ == '__main__':
unittest.main()

0 comments on commit 5fc1b2c

Please sign in to comment.