AWS wont let us update out certs when ELBs are using them so to allow

for dynamic updates we follow the logic,
- go through all the ELBs listeners looking for https connections
- save the current cert ssl name
- replace the cert ssl with a uniquesly timestamped
- optionally delete the previous ssl

Note that AWS can take time to propogate changes, so even when they
appear to be done, we need to apply retry loops and delays to make
sure they have been applied across all AWS infrastructure and not
exiting prematurely.

Features:
* Allow dynamically updating SSL certs robustly
* Drop ssl data argument from delete_certificate tests

Fixes:
* Cloudformation get resource now returns a dict

README.rst

@@ -284,8 +284,6 @@ By default the ELBs will have a security group opening them to the world on 80 a
 
 If you set the protocol on an ELB to HTTPS you must include a key called ``certificate_name`` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under ``ssl`` (see example above). The ``cert`` and ``key`` are required and the ``chain`` is optional.
 
-The certificate will be uploaded before the stack is created and removed after it is deleted.
-
 It is possilbe to define a custom health check for an ELB like follows::
 
     health_check:
@@ -295,6 +293,20 @@ It is possilbe to define a custom health check for an ELB like follows::
       Timeout: 5
       UnhealthyThreshold: 2
 
+ELB Certificates
+~~~~~~~~~~~~~~~~
+
+The SSL certificate will be uploaded before the stack is created and removed after it is deleted.
+To update the SSL certificate on ELB listeners run the fab task below, this uploads and updates the
+certificate on each HTTPS listener on your ELBs, by default the old certificate is deleted.
+
+.. code:: bash
+
+   fab load_env:<env_data> update_certs
+
+Note that some errors appear in the log due to the time taken for AWS changes to propogate across infrastructure
+elements, these are handled internally and are not neccessarily a sign of failure.
+
 ELB Policies
 ~~~~~~~~~~~~
 

bootstrap_cfn/cloudformation.py

@@ -61,7 +61,7 @@ def get_stack_load_balancers(self, stack_name_or_id):
                 load balancers for this stack
         """
         resource_type = 'AWS::ElasticLoadBalancing::LoadBalancer'
-        return self.get_resource_type(stack_name_or_id, resource_type)
+        return get_resource_type(stack_name_or_id, resource_type)
 
 
 def get_resource_type(stack_name_or_id,

bootstrap_cfn/elb.py

@@ -1,8 +1,15 @@
 import logging
 
+import time
+
 import boto.ec2.elb
 
+from boto.exception import BotoServerError
+
+import boto.iam
+
 from bootstrap_cfn import cloudformation, iam, utils
+
 from bootstrap_cfn.errors import BootstrapCfnError, CloudResourceNotFoundError
 
 
@@ -24,31 +31,35 @@ def __init__(self, aws_profile_name, aws_region_name='eu-west-1'):
             aws_profile_name, aws_region_name
         )
 
-    def set_ssl_certificates(self, ssl_config, stack_name):
+    def set_ssl_certificates(self, cert_names, stack_name, max_retries=1, retry_delay=10):
         """
         Look for SSL listeners on all the load balancers connected to
-        this stack, then set update the certificate to that of the config
+        this stack, then set update the certificate to that of the config.
+        We can retry with delay, default is to only try once.
 
         Args:
             ssl_config (dictionary): Certification names to corresponding data
             stack_name (string): Name of the stack
+            max_retries(int): The number of retries to carry out on the operation
+            retry_delay(int): The retry delay of the operation
 
         Returns:
-            list: The list of load balancers that were affected by the change
+            list: The list of the certificates that were replaced
 
         Raises:
             CloudResourceNotFoundError: Raised when the load balancer key in the cloud
                 config is not found
         """
-        updated_load_balancers = []
-        for cert_name in ssl_config.keys():
+        # List of all certificates replaced
+        replaced_certificates = []
+        for cert_name in cert_names:
             # Get the cert id and also its arn
             cert_id = "{0}-{1}".format(cert_name, stack_name)
             cert_arn = self.iam.get_arn_for_cert(cert_id)
 
             # Get all stack load balancers
             load_balancer_resources = self.cfn.get_stack_load_balancers(stack_name)
-            found_load_balancer_names = [lb.physical_resource_id for lb in load_balancer_resources]
+            found_load_balancer_names = [lb["PhysicalResourceId"] for lb in load_balancer_resources]
             # Use load balancer names to filter getting load balancer details
             load_balancers = []
             if len(found_load_balancer_names) > 0:
@@ -60,29 +71,53 @@ def set_ssl_certificates(self, ssl_config, stack_name):
                 for load_balancer in load_balancers:
                     for listener in load_balancer.listeners:
                         # Get protocol, if https, update cert
-                        # in_port = listener[0]
-                        out_port = listener[1]
+                        in_port = listener[0]
                         protocol = listener[2]
                         # If the protocol is HTTPS then set the cert on the listener
                         if protocol == "HTTPS":
                             logging.info("ELB::set_ssl_certificates: "
                                          "Found HTTPS protocol on '%s', "
                                          "updating SSL certificate with '%s'"
                                          % (load_balancer.name, cert_arn))
-                            self.conn_elb.set_lb_listener_SSL_certificate(load_balancer.name,
-                                                                          out_port,
-                                                                          cert_arn
-                                                                          )
-                            updated_load_balancers.append(load_balancer)
-
+                            # Get current listener certificate arn
+                            previous_cert_arn = None
+                            lb = self.conn_elb.get_all_load_balancers(load_balancer.name)[0]
+                            for listener in lb.listeners:
+                                # We're looking for a tuple of the form (443, 80, 'HTTPS', 'HTTP', <cert_arn>)
+                                if 'HTTPS' in listener.get_tuple():
+                                    previous_cert_arn = listener[4]
+                            # Set the current certificate on the listener to the new one
+                            retries = 0
+                            while retries < max_retries:
+                                retries += 1
+                                try:
+                                    self.conn_elb.set_lb_listener_SSL_certificate(load_balancer.name,
+                                                                                  in_port,
+                                                                                  cert_arn)
+                                    if previous_cert_arn:
+                                        previous_cert_name = previous_cert_arn.split('/')[1].split("-%s" % stack_name)[0]
+                                        replaced_certificates.append(previous_cert_name)
+
+                                    logging.info("update_certs:Successfully set ssl cert to '%s', "
+                                                 " replacing cert '%s'"
+                                                 % (cert_arn, previous_cert_name))
+
+                                    break
+                                except BotoServerError as e:
+                                    logging.warning("update_certs: Cannot set ssl certs, reason '%s', "
+                                                    "waiting %s seconds on retry %s/%s"
+                                                    % (e.error_message, retry_delay, retries, max_retries))
+                                    # Only sleep if we're going to try again
+                                    if retries < max_retries:
+                                        time.sleep(retry_delay)
             else:
                 # Throw key error. There being no load balancers to update is not
                 # necessarily a problem but since the caller expected there to be let
                 # it handle this situation
                 raise CloudResourceNotFoundError("ELB::set_ssl_certificates: "
                                                  "No load balancers found in stack,")
 
-        return updated_load_balancers
+        return replaced_certificates
 
     def list_domain_names(self, stack_name):
         """

bootstrap_cfn/fab_tasks.py

@@ -20,7 +20,6 @@
 from bootstrap_cfn.iam import IAM
 from bootstrap_cfn.r53 import R53
 from bootstrap_cfn.utils import tail
-from bootstrap_cfn.vpc import VPC
 
 
 # Default fab config. Set via the tasks below or --set
@@ -427,10 +426,6 @@ def cfn_delete(force=False, pre_delete_callbacks=None):
         for callback in pre_delete_callbacks:
             callback(stack_name=stack_name, config=cfn_config)
 
-    # Disable all vpc peering before deletion
-    print green("\nSTACK {0}: Disabling VPC peering before deletion...\n").format(stack_name)
-    disable_vpc_peering()
-
     print green("\nSTACK {0} DELETING...\n").format(stack_name)
 
     cfn.delete(stack_name)
@@ -507,42 +502,42 @@ def cfn_create(test=False):
 
 
 @task
-def update_certs():
+def update_certs(delete_replaced_certificates=True):
     """
     Update the ssl certificates
 
     This will read in the certificates from the config
     file, update them in AWS Iam, and then also handle
-    setting the certificates on ELB's
+    setting the certificates on ELB's. By default, replaced
+    SSL certs will be deleted.
+
+    Args:
+        delete_replaced_certificates: Delete the certificates we have replaced
     """
 
     stack_name = get_stack_name()
     cfn_config = get_config()
     # Upload any SSL certificates to our EC2 instances
-    updated_count = False
     if 'ssl' in cfn_config.data:
-        logging.info("Reloading SSL certificates...")
+        logging.info("update_certs: Updating SSL certificates...")
         iam = get_connection(IAM)
-        updated_count = iam.update_ssl_certificates(cfn_config.ssl(),
+        updated_certs = iam.update_ssl_certificates(cfn_config.ssl(),
                                                     stack_name)
     else:
-        logging.error("No ssl section found in cloud config file, aborting...")
+        logging.error("update_certs: No ssl section found in cloud config file, aborting...")
         sys.exit(1)
 
-    # Arbitrary wait to allow SSL upload to register with AWS
-    # Otherwise, we can get an ARN for the load balancer certificates
-    # without it being ready to assign
-    time.sleep(3)
-
     # Set the certificates on ELB's if we have any
-    if updated_count > 0:
-        if 'elb' in cfn_config.data:
-            logging.info("Setting load balancer certificates...")
-            elb = get_connection(ELB)
-            elb.set_ssl_certificates(cfn_config.ssl(), stack_name)
-    else:
+    if len(updated_certs) <= 0:
         logging.error("No certificates updated so skipping "
                       "ELB certificate update...")
+    if 'elb' in cfn_config.data:
+        logging.info("update_certs: Setting load balancer certificates...")
+        elb = get_connection(ELB)
+        replaced_certificates = elb.set_ssl_certificates(updated_certs, stack_name, max_retries=3)
+        if delete_replaced_certificates:
+            for replaced_certificate in replaced_certificates:
+                iam.delete_certificate(replaced_certificate, stack_name, max_retries=3)
 
 
 def get_cloudformation_tags():