Generate Userdata script to set hostname of instances based on template

i.e. If we set hostname_pattern to "{instance_id}.my_project" then we
drop a script in via userdata that [cloud-init](http://cloudinit.readthedocs.org/)
will instruct later phases of cloud-init to set the hostname. So that
sudo can resolve the hostname and and we don't get warnings/delays we
also want to be able to instruct cloud_config to manage /etc/hosts -
which we do via letting the user pass in a chunk of YAML for
[cloud_config](http://cloudinit.readthedocs.org/en/latest/topics/examples.html)

There are other types of userdata we could add to the Userdata "Mime
Multipart archive" but we don't need them right now so don't expose them
from the config YAML

README.rst

@@ -221,11 +221,48 @@ The ``ec2`` key configures the EC2 instances created by auto-scaling groups (ASG
               ToPort: 4506
               SourceSecurityGroupId: { Ref: Salt }
 
+:``cloud_config``:
+  Dictionary to be feed in via userdata to drive `cloud-init <http://cloudinit.readthedocs.org/en/latest/>`_ to set up the initial configuration of the host upon creation. Using cloud-config you can run commands, install packages
+
+  There doesn't appear to be a definitive list of the possible config options but the examples are quite exhaustive:
+
+  - `http://bazaar.launchpad.net/~cloud-init-dev/cloud-init/trunk/files/head:/doc/examples/`
+  - `http://cloudinit.readthedocs.org/en/latest/topics/examples.html`_ (similar list but all on one page so easier to read)
+
+:``hostname_pattern``:
+  A python-style string format to set the hostname of the instance upon creation.
+
+  For ``sudo`` to not misbehave initially (because it cannot look up its own hostname) you will likely want to set ``manage_etc_hosts`` to true in the cloud_config section so that it will regenerate ``/etc/hosts`` with the new hostname resolving to 127.0.0.1.
+
+  Setting the hostname is achived by adding a boothook into the userdata that will interpolate the instance_id correctly on the machine very soon after boottime.
+
+  The currently support interpolations are:
+
+  ``instance_id``
+    The amazon instance ID
+  ``environment``
+    The enviroment currently selected (from the fab task)
+  ``application``
+    The application name (taken from the fab task)
+  ``stack_name``
+    The full stack name being created
+  ``tags``
+    A value from a tag for this autoscailing group. For example use ``tags[Role]`` to access the value of the ``Role`` tag.
+
+  Example::
+
+    dev:
+      ec2:
+        # …
+        hostname_pattern: "{instance_id}.{environment}.myproject
+        cloud_config:
+          manage_etc_hosts: true
+
 ELBs
 ++++
 By default the ELBs will have a security group opening them to the world on 80 and 443. You can replace this default SG with your own (see example ``ELBSecGroup`` above).
 
-If you set the protocol on an ELB to HTTPS you must include a key called `certificate_name` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under `ssl` (see example above). The `cert` and `key` are required and the `chain` is optional.
+If you set the protocol on an ELB to HTTPS you must include a key called ``certificate_name`` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under ``ssl`` (see example above). The ``cert`` and ``key`` are required and the ``chain`` is optional.
 
 The certificate will be uploaded before the stack is created and removed after it is deleted.
 

bootstrap_cfn/config.py

@@ -3,6 +3,7 @@
 import os
 import sys
 
+import textwrap
 
 from troposphere import Base64, FindInMap, GetAZs, GetAtt, Join, Output, Ref, Tags, Template
 from troposphere.autoscaling import AutoScalingGroup, BlockDeviceMapping, \
@@ -19,7 +20,7 @@
 
 import yaml
 
-from bootstrap_cfn import errors, utils
+from bootstrap_cfn import errors, mime_packer, utils
 
 
 class ProjectConfig:
@@ -46,10 +47,14 @@ class ConfigParser:
 
     config = {}
 
-    def __init__(self, data, stack_name):
+    def __init__(self, data, stack_name, environment=None, application=None):
         self.stack_name = stack_name
         self.data = data
 
+        # Some things possibly used in user data templates
+        self.environment = environment
+        self.application = application
+
     def process(self):
         template = self.base_template()
 
@@ -529,6 +534,54 @@ def ref_fixup(x):
                 return x
         return dict([(k, ref_fixup(v)) for k, v in o.items()])
 
+    def get_ec2_userdata(self):
+        data = self.data['ec2']
+
+        parts = []
+
+        boothook = self.get_hostname_boothook(data)
+
+        if boothook:
+            parts.append(boothook)
+
+        if "cloud_config" in data:
+            parts.append({
+                'content': yaml.dump(data['cloud_config']),
+                'mime_type': 'text/cloud-config'
+            })
+
+        if len(parts):
+            return mime_packer.pack(parts)
+
+    HOSTNAME_BOOTHOOK_TEMPLATE = textwrap.dedent("""\
+    #!/bin/sh
+    [ -e /etc/cloud/cloud.cfg.d/99_hostname.cfg ] || echo "hostname: {hostname}" > /etc/cloud/cloud.cfg.d/99_hostname.cfg
+    """)
+
+    def get_hostname_boothook(self, data):
+        if "hostname_pattern" not in data:
+            return None
+
+        interploations = {
+            # This gets interploated by cloud-init at run time.
+            'instance_id': '${INSTANCE_ID}',
+            'tags': data['tags'],
+            'environment': self.environment,
+            'application': self.application,
+            'stack_name': self.stack_name,
+        }
+        try:
+            hostname = data['hostname_pattern'].format(**interploations)
+        except KeyError as e:
+            raise errors.CfnHostnamePatternError("Error interpolating hostname_pattern '{pattern}' - {key} is not a valid interpolation".format(
+                pattern=data['hostname_pattern'],
+                key=e.args[0]))
+
+        return {
+            'mime_type': 'text/cloud-boothook',
+            'content': self.HOSTNAME_BOOTHOOK_TEMPLATE.format(hostname=hostname)
+        }
+
     def ec2(self):
         # LOAD STACK TEMPLATE
         data = self.data['ec2']
@@ -590,11 +643,11 @@ def ec2(self):
             IamInstanceProfile=Ref("InstanceProfile"),
             ImageId=FindInMap("AWSRegion2AMI", Ref("AWS::Region"), "AMI"),
             BlockDeviceMappings=devices,
-            UserData=Base64(Join("", [
-                "#!/bin/bash -xe\n",
-                "#do nothing for now",
-            ])),
         )
+        user_data = self.get_ec2_userdata()
+        if user_data:
+            launch_config.UserData = Base64(user_data)
+
         resources.append(launch_config)
 
         # Allow deprecation of tags

bootstrap_cfn/errors.py

@@ -1,7 +1,9 @@
 import sys
 
+
 class BootstrapCfnError(Exception):
     def __init__(self, msg):
+        super(BootstrapCfnError, self).__init__(msg)
         print >> sys.stderr,  "[ERROR] {0}: {1}".format(self.__class__.__name__, msg)
 
 
@@ -13,6 +15,10 @@ class CfnTimeoutError(BootstrapCfnError):
     pass
 
 
+class CfnHostnamePatternError(BootstrapCfnError):
+    pass
+
+
 class NoCredentialsError(BootstrapCfnError):
     def __init__(self):
         super(NoCredentialsError, self).__init__(

bootstrap_cfn/fab_tasks.py

@@ -271,7 +271,7 @@ def get_config():
         env.environment,
         passwords=env.stack_passwords)
 
-    cfn_config = ConfigParser(project_config.config, get_stack_name())
+    cfn_config = ConfigParser(project_config.config, get_stack_name(), environment=env.environment, application=env.application)
     return cfn_config
 
 

bootstrap_cfn/mime_packer.py

@@ -0,0 +1,80 @@
+import gzip
+
+from StringIO import StringIO
+from contextlib import closing
+from email import encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+
+STARTS_WITH_MAPPINGS = {
+    '#include': 'text/x-include-url',
+    '#include-once': 'text/x-include-once-url',
+    '#!': 'text/x-shellscript',
+    '#cloud-config': 'text/cloud-config',
+    '#cloud-config-archive': 'text/cloud-config-archive',
+    '#upstart-job': 'text/upstart-job',
+    '#part-handler': 'text/part-handler',
+    '#cloud-boothook': 'text/cloud-boothook'
+}
+
+
+def try_decode(data):
+    try:
+        return (True, data.decode())
+    except UnicodeDecodeError:
+        return (False, data)
+
+
+def get_type(content, deftype):
+    rtype = deftype
+
+    (can_be_decoded, content) = try_decode(content)
+
+    if can_be_decoded:
+        # slist is sorted longest first
+        slist = sorted(list(STARTS_WITH_MAPPINGS.keys()), key=lambda e: 0 - len(e))
+        for sstr in slist:
+            if content.startswith(sstr):
+                rtype = STARTS_WITH_MAPPINGS[sstr]
+                break
+    else:
+        rtype = 'application/octet-stream'
+
+    return(rtype)
+
+
+def pack(parts, opts={}):
+    outer = MIMEMultipart()
+
+    for arg in parts:
+        if isinstance(arg, basestring):
+            arg = {'content': arg}
+
+        if 'mime_type' in arg:
+            mtype = arg['mime_type']
+        else:
+            mtype = get_type(arg['content'], opts.get('deftype', "text/plain"))
+
+        maintype, subtype = mtype.split('/', 1)
+        if maintype == 'text':
+            # Note: we should handle calculating the charset
+            msg = MIMEText(arg['content'], _subtype=subtype)
+        else:
+            msg = MIMEBase(maintype, subtype)
+            msg.set_payload(arg['content'])
+            # Encode the payload using Base64
+            encoders.encode_base64(msg)
+
+        outer.attach(msg)
+
+    with closing(StringIO()) as buff:
+        if opts.get('compress', False):
+            gfile = gzip.GzipFile(fileobj=buff)
+            gfile.write(outer.as_string().encode())
+            gfile.close()
+        else:
+            buff.write(outer.as_string().encode())
+
+        return buff.getvalue()

docs/sample-project.yaml

@@ -12,7 +12,9 @@ dev:
     tags:
       Role: docker
       Apps: test
-      Env: dev
+    hostname_pattern: '{instance_id}.{tags[Role]}.{environment}.{application}'
+    cloud_config:
+      manage_etc_hosts: true
     parameters:
       KeyName: default
       InstanceType: t2.micro