diff --git a/README.rst b/README.rst index 77f9f7d..e7d1474 100644 --- a/README.rst +++ b/README.rst @@ -221,11 +221,58 @@ The ``ec2`` key configures the EC2 instances created by auto-scaling groups (ASG ToPort: 4506 SourceSecurityGroupId: { Ref: Salt } +:``cloud_config``: + Dictionary to be feed in via userdata to drive `cloud-init `_ to set up the initial configuration of the host upon creation. Using cloud-config you can run commands, install packages + + There doesn't appear to be a definitive list of the possible config options but the examples are quite exhaustive: + + - `http://bazaar.launchpad.net/~cloud-init-dev/cloud-init/trunk/files/head:/doc/examples/` + - `http://cloudinit.readthedocs.org/en/latest/topics/examples.html`_ (similar list but all on one page so easier to read) + +:``hostname_pattern``: + A python-style string format to set the hostname of the instance upon creation. + + The default is ``{instance_id}.{environment}.{application}``. To disable this entirely set this field explicitly to null/empty:: + + dev: + ec2: + hostname_pattern: + + For ``sudo`` to not misbehave initially (because it cannot look up its own hostname) you will likely want to set ``manage_etc_hosts`` to true in the cloud_config section so that it will regenerate ``/etc/hosts`` with the new hostname resolving to 127.0.0.1. + + Setting the hostname is achived by adding a boothook into the userdata that will interpolate the instance_id correctly on the machine very soon after boottime. + + The currently support interpolations are: + + ``instance_id`` + The amazon instance ID + ``environment`` + The enviroment currently selected (from the fab task) + ``application`` + The application name (taken from the fab task) + ``stack_name`` + The full stack name being created + ``tags`` + A value from a tag for this autoscailing group. For example use ``tags[Role]`` to access the value of the ``Role`` tag. + + For example given this incomplete config:: + + dev: + ec2: + # … + hostname_pattern: "{instance_id}.{tags[Role]}.{environment}.{application}" + tags: + Role: docker + cloud_config: + manage_etc_hosts: true + + an instance created with ``fab application:myproject … cfn_create`` would get a hostname something like ``i-f623cfb9.docker.dev.my-project``. + ELBs ++++ By default the ELBs will have a security group opening them to the world on 80 and 443. You can replace this default SG with your own (see example ``ELBSecGroup`` above). -If you set the protocol on an ELB to HTTPS you must include a key called `certificate_name` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under `ssl` (see example above). The `cert` and `key` are required and the `chain` is optional. +If you set the protocol on an ELB to HTTPS you must include a key called ``certificate_name`` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under ``ssl`` (see example above). The ``cert`` and ``key`` are required and the ``chain`` is optional. The certificate will be uploaded before the stack is created and removed after it is deleted. diff --git a/bootstrap_cfn/config.py b/bootstrap_cfn/config.py index 53bb807..31d045c 100644 --- a/bootstrap_cfn/config.py +++ b/bootstrap_cfn/config.py @@ -3,6 +3,7 @@ import os import sys +import textwrap from troposphere import Base64, FindInMap, GetAZs, GetAtt, Join, Output, Ref, Tags, Template from troposphere.autoscaling import AutoScalingGroup, BlockDeviceMapping, \ @@ -19,7 +20,7 @@ import yaml -from bootstrap_cfn import errors, utils +from bootstrap_cfn import errors, mime_packer, utils class ProjectConfig: @@ -46,10 +47,14 @@ class ConfigParser: config = {} - def __init__(self, data, stack_name): + def __init__(self, data, stack_name, environment=None, application=None): self.stack_name = stack_name self.data = data + # Some things possibly used in user data templates + self.environment = environment + self.application = application + def process(self): template = self.base_template() @@ -529,6 +534,64 @@ def ref_fixup(x): return x return dict([(k, ref_fixup(v)) for k, v in o.items()]) + def get_ec2_userdata(self): + data = self.data['ec2'] + + parts = [] + + boothook = self.get_hostname_boothook(data) + + if boothook: + parts.append(boothook) + + if "cloud_config" in data: + parts.append({ + 'content': yaml.dump(data['cloud_config']), + 'mime_type': 'text/cloud-config' + }) + + if len(parts): + return mime_packer.pack(parts) + + HOSTNAME_BOOTHOOK_TEMPLATE = textwrap.dedent("""\ + #!/bin/sh + [ -e /etc/cloud/cloud.cfg.d/99_hostname.cfg ] || echo "hostname: {hostname}" > /etc/cloud/cloud.cfg.d/99_hostname.cfg + """) + + DEFAULT_HOSTNAME_PATTERN = "{instance_id}.{environment}.{application}" + + def get_hostname_boothook(self, data): + """ + Return a boothook part that will set the hostname of instances on boot. + + The pattern comes from the ``hostname_pattern`` pattern of data dict, + with a default of "{instance_id}.{environment}.{application}". To + disable this functionality explicitly pass None in this field. + """ + hostname_pattern = data.get('hostname_pattern', self.DEFAULT_HOSTNAME_PATTERN) + if hostname_pattern is None: + return None + + interploations = { + # This gets interploated by cloud-init at run time. + 'instance_id': '${INSTANCE_ID}', + 'tags': data['tags'], + 'environment': self.environment, + 'application': self.application, + 'stack_name': self.stack_name, + } + try: + hostname = hostname_pattern.format(**interploations) + except KeyError as e: + raise errors.CfnHostnamePatternError("Error interpolating hostname_pattern '{pattern}' - {key} is not a valid interpolation".format( + pattern=hostname_pattern, + key=e.args[0])) + + return { + 'mime_type': 'text/cloud-boothook', + 'content': self.HOSTNAME_BOOTHOOK_TEMPLATE.format(hostname=hostname) + } + def ec2(self): # LOAD STACK TEMPLATE data = self.data['ec2'] @@ -590,11 +653,11 @@ def ec2(self): IamInstanceProfile=Ref("InstanceProfile"), ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "AMI"), BlockDeviceMappings=devices, - UserData=Base64(Join("", [ - "#!/bin/bash -xe\n", - "#do nothing for now", - ])), ) + user_data = self.get_ec2_userdata() + if user_data: + launch_config.UserData = Base64(user_data) + resources.append(launch_config) # Allow deprecation of tags diff --git a/bootstrap_cfn/errors.py b/bootstrap_cfn/errors.py index b7ea8ab..474bcd2 100644 --- a/bootstrap_cfn/errors.py +++ b/bootstrap_cfn/errors.py @@ -1,7 +1,9 @@ import sys + class BootstrapCfnError(Exception): def __init__(self, msg): + super(BootstrapCfnError, self).__init__(msg) print >> sys.stderr, "[ERROR] {0}: {1}".format(self.__class__.__name__, msg) @@ -13,6 +15,10 @@ class CfnTimeoutError(BootstrapCfnError): pass +class CfnHostnamePatternError(BootstrapCfnError): + pass + + class NoCredentialsError(BootstrapCfnError): def __init__(self): super(NoCredentialsError, self).__init__( diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index 5396404..2552206 100755 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -271,7 +271,7 @@ def get_config(): env.environment, passwords=env.stack_passwords) - cfn_config = ConfigParser(project_config.config, get_stack_name()) + cfn_config = ConfigParser(project_config.config, get_stack_name(), environment=env.environment, application=env.application) return cfn_config diff --git a/bootstrap_cfn/mime_packer.py b/bootstrap_cfn/mime_packer.py new file mode 100644 index 0000000..7e44a55 --- /dev/null +++ b/bootstrap_cfn/mime_packer.py @@ -0,0 +1,80 @@ +import gzip + +from StringIO import StringIO +from contextlib import closing +from email import encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + + +STARTS_WITH_MAPPINGS = { + '#include': 'text/x-include-url', + '#include-once': 'text/x-include-once-url', + '#!': 'text/x-shellscript', + '#cloud-config': 'text/cloud-config', + '#cloud-config-archive': 'text/cloud-config-archive', + '#upstart-job': 'text/upstart-job', + '#part-handler': 'text/part-handler', + '#cloud-boothook': 'text/cloud-boothook' +} + + +def try_decode(data): + try: + return (True, data.decode()) + except UnicodeDecodeError: + return (False, data) + + +def get_type(content, deftype): + rtype = deftype + + (can_be_decoded, content) = try_decode(content) + + if can_be_decoded: + # slist is sorted longest first + slist = sorted(list(STARTS_WITH_MAPPINGS.keys()), key=lambda e: 0 - len(e)) + for sstr in slist: + if content.startswith(sstr): + rtype = STARTS_WITH_MAPPINGS[sstr] + break + else: + rtype = 'application/octet-stream' + + return(rtype) + + +def pack(parts, opts={}): + outer = MIMEMultipart() + + for arg in parts: + if isinstance(arg, basestring): + arg = {'content': arg} + + if 'mime_type' in arg: + mtype = arg['mime_type'] + else: + mtype = get_type(arg['content'], opts.get('deftype', "text/plain")) + + maintype, subtype = mtype.split('/', 1) + if maintype == 'text': + # Note: we should handle calculating the charset + msg = MIMEText(arg['content'], _subtype=subtype) + else: + msg = MIMEBase(maintype, subtype) + msg.set_payload(arg['content']) + # Encode the payload using Base64 + encoders.encode_base64(msg) + + outer.attach(msg) + + with closing(StringIO()) as buff: + if opts.get('compress', False): + gfile = gzip.GzipFile(fileobj=buff) + gfile.write(outer.as_string().encode()) + gfile.close() + else: + buff.write(outer.as_string().encode()) + + return buff.getvalue() diff --git a/docs/sample-project.yaml b/docs/sample-project.yaml index 73eeb3c..41a0c2c 100644 --- a/docs/sample-project.yaml +++ b/docs/sample-project.yaml @@ -12,7 +12,9 @@ dev: tags: Role: docker Apps: test - Env: dev + hostname_pattern: '{instance_id}.{tags[Role]}.{environment}.{application}' + cloud_config: + manage_etc_hosts: true parameters: KeyName: default InstanceType: t2.micro diff --git a/tests/tests.py b/tests/tests.py index b7d06e5..5ea43dd 100755 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,10 @@ #!/usr/bin/env python +import email import json import unittest +from StringIO import StringIO + +from mock import patch from testfixtures import compare @@ -11,6 +15,7 @@ from troposphere.iam import PolicyType from troposphere.route53 import RecordSetGroup +import yaml from bootstrap_cfn import errors from bootstrap_cfn.config import ConfigParser, ProjectConfig @@ -976,7 +981,6 @@ def test_ec2(self): BaseHostLaunchConfig = LaunchConfiguration( "BaseHostLaunchConfig", - UserData=Base64(Join("", ["#!/bin/bash -xe\n", "#do nothing for now"])), ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "AMI"), BlockDeviceMappings=[ { @@ -1013,11 +1017,133 @@ def test_ec2(self): ProjectConfig( 'tests/sample-project.yaml', 'dev').config, 'my-stack-name') - ec2_json = self._resources_to_dict(config.ec2()) - # compare(, ec2_json) + + with patch.object(config, 'get_ec2_userdata', return_value=None): + ec2_json = self._resources_to_dict(config.ec2()) compare(self._resources_to_dict(known), ec2_json) + # We just want to test that when we have userdata we return the right LaunchConfig. + def test_launchconfig_userdata(self): + config = ConfigParser( + ProjectConfig('tests/sample-project.yaml', 'dev').config, + 'my-stack-name') + + BaseHostLaunchConfig = LaunchConfiguration( + "BaseHostLaunchConfig", + ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "AMI"), + BlockDeviceMappings=[ + { + "DeviceName": "/dev/sda1", + "Ebs": {"VolumeSize": 10} + }, + { + "DeviceName": "/dev/sdf", + "Ebs": {"VolumeSize": 10} + } + ], + KeyName="default", + SecurityGroups=[Ref("BaseHostSG"), Ref("AnotherSG")], + IamInstanceProfile=Ref("InstanceProfile"), + InstanceType="t2.micro", + AssociatePublicIpAddress="true", + UserData=Base64("Mock Userdata String"), + ) + + with patch.object(config, "get_ec2_userdata", return_value="Mock Userdata String"): + ec2_json = self._resources_to_dict(config.ec2()) + expected = self._resources_to_dict([BaseHostLaunchConfig]) + compare(ec2_json['BaseHostLaunchConfig'], expected['BaseHostLaunchConfig']) + pass + + def test_get_ec2_userdata(self): + data = { + 'ec2': { + 'cloud_config': {'some': 'dict'} + } + } + config = ConfigParser(data, environment="env", application="test", stack_name="my-stack") + + with patch.object(config, 'get_hostname_boothook', return_value={"content": "sentinel"}) as mock_boothook: + # This test is slightly silly as we are testing the decoding with + # the same code as the generator... but it's that or we test it + # with regexp. + mime_text = config.get_ec2_userdata() + + mock_boothook.assert_called_once_with(data['ec2']) + + parts = [part for part in email.message_from_string(mime_text).walk()] + + compare( + [part.get_content_type() for part in parts], + ["multipart/mixed", "text/plain", "text/cloud-config"], + prefix="Userdata parts are in expected order") + + compare(parts[1].get_payload(), "sentinel") + compare(yaml.load(parts[2].get_payload()), data['ec2']['cloud_config']) + + def test_get_hostname_boothook(self): + config = ConfigParser({}, environment="env", application="test", stack_name="my-stack") + + cfg = { + # Longer than people would use but tests all interpolations. + 'hostname_pattern': '{instance_id}.{tags[Role]}.{environment}.{application}.{stack_name}', + 'tags': {'Role': 'docker'}, + } + part = config.get_hostname_boothook(cfg) + expected = { + 'content': ('#!/bin/sh\n' + '[ -e /etc/cloud/cloud.cfg.d/99_hostname.cfg ] || ' + 'echo "hostname: ${INSTANCE_ID}.docker.env.test.my-stack" > /etc/cloud/cloud.cfg.d/99_hostname.cfg\n'), + 'mime_type': 'text/cloud-boothook', + } + compare(part, expected) + + def test_get_hostname_boothook_default(self): + config = ConfigParser({}, environment="env", application="test", stack_name="my-stack") + + cfg = { + 'tags': {'Role': 'docker'}, + } + part = config.get_hostname_boothook(cfg) + expected = { + 'content': ('#!/bin/sh\n' + '[ -e /etc/cloud/cloud.cfg.d/99_hostname.cfg ] || ' + 'echo "hostname: ${INSTANCE_ID}.env.test" > /etc/cloud/cloud.cfg.d/99_hostname.cfg\n'), + 'mime_type': 'text/cloud-boothook', + } + compare(part, expected) + + def test_get_hostname_boothook_nonoe(self): + config = ConfigParser({}, environment="env", application="test", stack_name="my-stack") + + cfg = { + 'hostname_pattern': None, + 'tags': {'Role': 'docker'}, + } + part = config.get_hostname_boothook(cfg) + expected = None + compare(part, expected) + + @patch('sys.stderr', StringIO()) + def test_get_hostname_boothook_error(self): + config = ConfigParser({}, environment="env", application="test", stack_name="my-stack") + + cfg = { + # Longer than people would use but tests all interpolations. + 'hostname_pattern': '{tags[Fake]}', + 'tags': {'Role': 'docker'}, + } + + with self.assertRaisesRegexp(errors.CfnHostnamePatternError, r"Error interpolating hostname_pattern .*\bFake"): + config.get_hostname_boothook(cfg) + self.fail() + + cfg['hostname_pattern'] = '{non_existent}' + with self.assertRaisesRegexp(errors.CfnHostnamePatternError, r"Error interpolating hostname_pattern .*\bnon_existent"): + config.get_hostname_boothook(cfg) + self.fail() + def test_ec2_with_no_block_device_specified(self): project_config = ProjectConfig('tests/sample-project.yaml', 'dev') project_config.config['ec2'].pop('block_devices')