diff --git a/README.rst b/README.rst index 42915e0..5cd2503 100644 --- a/README.rst +++ b/README.rst @@ -284,8 +284,6 @@ By default the ELBs will have a security group opening them to the world on 80 a If you set the protocol on an ELB to HTTPS you must include a key called ``certificate_name`` in the ELB block (as example above) and matching cert data in a key with the same name as the cert under ``ssl`` (see example above). The ``cert`` and ``key`` are required and the ``chain`` is optional. -The certificate will be uploaded before the stack is created and removed after it is deleted. - It is possilbe to define a custom health check for an ELB like follows:: health_check: @@ -295,6 +293,20 @@ It is possilbe to define a custom health check for an ELB like follows:: Timeout: 5 UnhealthyThreshold: 2 +ELB Certificates +~~~~~~~~~~~~~~~~ + +The SSL certificate will be uploaded before the stack is created and removed after it is deleted. +To update the SSL certificate on ELB listeners run the fab task below, this uploads and updates the +certificate on each HTTPS listener on your ELBs, by default the old certificate is deleted. + +.. code:: bash + + fab load_env: update_certs + +Note that some errors appear in the log due to the time taken for AWS changes to propogate across infrastructure +elements, these are handled internally and are not neccessarily a sign of failure. + ELB Policies ~~~~~~~~~~~~ diff --git a/bootstrap_cfn/cloudformation.py b/bootstrap_cfn/cloudformation.py index fa75a42..8bb51b8 100644 --- a/bootstrap_cfn/cloudformation.py +++ b/bootstrap_cfn/cloudformation.py @@ -61,7 +61,7 @@ def get_stack_load_balancers(self, stack_name_or_id): load balancers for this stack """ resource_type = 'AWS::ElasticLoadBalancing::LoadBalancer' - return self.get_resource_type(stack_name_or_id, resource_type) + return get_resource_type(stack_name_or_id, resource_type) def get_resource_type(stack_name_or_id, diff --git a/bootstrap_cfn/elb.py b/bootstrap_cfn/elb.py index 11648a2..6a3369f 100644 --- a/bootstrap_cfn/elb.py +++ b/bootstrap_cfn/elb.py @@ -1,8 +1,15 @@ import logging +import time + import boto.ec2.elb +from boto.exception import BotoServerError + +import boto.iam + from bootstrap_cfn import cloudformation, iam, utils + from bootstrap_cfn.errors import BootstrapCfnError, CloudResourceNotFoundError @@ -24,31 +31,35 @@ def __init__(self, aws_profile_name, aws_region_name='eu-west-1'): aws_profile_name, aws_region_name ) - def set_ssl_certificates(self, ssl_config, stack_name): + def set_ssl_certificates(self, cert_names, stack_name, max_retries=1, retry_delay=10): """ Look for SSL listeners on all the load balancers connected to - this stack, then set update the certificate to that of the config + this stack, then set update the certificate to that of the config. + We can retry with delay, default is to only try once. Args: ssl_config (dictionary): Certification names to corresponding data stack_name (string): Name of the stack + max_retries(int): The number of retries to carry out on the operation + retry_delay(int): The retry delay of the operation Returns: - list: The list of load balancers that were affected by the change + list: The list of the certificates that were replaced Raises: CloudResourceNotFoundError: Raised when the load balancer key in the cloud config is not found """ - updated_load_balancers = [] - for cert_name in ssl_config.keys(): + # List of all certificates replaced + replaced_certificates = [] + for cert_name in cert_names: # Get the cert id and also its arn cert_id = "{0}-{1}".format(cert_name, stack_name) cert_arn = self.iam.get_arn_for_cert(cert_id) # Get all stack load balancers load_balancer_resources = self.cfn.get_stack_load_balancers(stack_name) - found_load_balancer_names = [lb.physical_resource_id for lb in load_balancer_resources] + found_load_balancer_names = [lb["PhysicalResourceId"] for lb in load_balancer_resources] # Use load balancer names to filter getting load balancer details load_balancers = [] if len(found_load_balancer_names) > 0: @@ -60,8 +71,7 @@ def set_ssl_certificates(self, ssl_config, stack_name): for load_balancer in load_balancers: for listener in load_balancer.listeners: # Get protocol, if https, update cert - # in_port = listener[0] - out_port = listener[1] + in_port = listener[0] protocol = listener[2] # If the protocol is HTTPS then set the cert on the listener if protocol == "HTTPS": @@ -69,12 +79,37 @@ def set_ssl_certificates(self, ssl_config, stack_name): "Found HTTPS protocol on '%s', " "updating SSL certificate with '%s'" % (load_balancer.name, cert_arn)) - self.conn_elb.set_lb_listener_SSL_certificate(load_balancer.name, - out_port, - cert_arn - ) - updated_load_balancers.append(load_balancer) - + # Get current listener certificate arn + previous_cert_arn = None + lb = self.conn_elb.get_all_load_balancers(load_balancer.name)[0] + for listener in lb.listeners: + # We're looking for a tuple of the form (443, 80, 'HTTPS', 'HTTP', ) + if 'HTTPS' in listener.get_tuple(): + previous_cert_arn = listener[4] + # Set the current certificate on the listener to the new one + retries = 0 + while retries < max_retries: + retries += 1 + try: + self.conn_elb.set_lb_listener_SSL_certificate(load_balancer.name, + in_port, + cert_arn) + if previous_cert_arn: + previous_cert_name = previous_cert_arn.split('/')[1].split("-%s" % stack_name)[0] + replaced_certificates.append(previous_cert_name) + + logging.info("update_certs:Successfully set ssl cert to '%s', " + " replacing cert '%s'" + % (cert_arn, previous_cert_name)) + + break + except BotoServerError as e: + logging.warning("update_certs: Cannot set ssl certs, reason '%s', " + "waiting %s seconds on retry %s/%s" + % (e.error_message, retry_delay, retries, max_retries)) + # Only sleep if we're going to try again + if retries < max_retries: + time.sleep(retry_delay) else: # Throw key error. There being no load balancers to update is not # necessarily a problem but since the caller expected there to be let @@ -82,7 +117,7 @@ def set_ssl_certificates(self, ssl_config, stack_name): raise CloudResourceNotFoundError("ELB::set_ssl_certificates: " "No load balancers found in stack,") - return updated_load_balancers + return replaced_certificates def list_domain_names(self, stack_name): """ diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index 17e799f..4dea1bf 100755 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -20,7 +20,6 @@ from bootstrap_cfn.iam import IAM from bootstrap_cfn.r53 import R53 from bootstrap_cfn.utils import tail -from bootstrap_cfn.vpc import VPC # Default fab config. Set via the tasks below or --set @@ -427,10 +426,6 @@ def cfn_delete(force=False, pre_delete_callbacks=None): for callback in pre_delete_callbacks: callback(stack_name=stack_name, config=cfn_config) - # Disable all vpc peering before deletion - print green("\nSTACK {0}: Disabling VPC peering before deletion...\n").format(stack_name) - disable_vpc_peering() - print green("\nSTACK {0} DELETING...\n").format(stack_name) cfn.delete(stack_name) @@ -507,42 +502,42 @@ def cfn_create(test=False): @task -def update_certs(): +def update_certs(delete_replaced_certificates=True): """ Update the ssl certificates This will read in the certificates from the config file, update them in AWS Iam, and then also handle - setting the certificates on ELB's + setting the certificates on ELB's. By default, replaced + SSL certs will be deleted. + + Args: + delete_replaced_certificates: Delete the certificates we have replaced """ stack_name = get_stack_name() cfn_config = get_config() # Upload any SSL certificates to our EC2 instances - updated_count = False if 'ssl' in cfn_config.data: - logging.info("Reloading SSL certificates...") + logging.info("update_certs: Updating SSL certificates...") iam = get_connection(IAM) - updated_count = iam.update_ssl_certificates(cfn_config.ssl(), + updated_certs = iam.update_ssl_certificates(cfn_config.ssl(), stack_name) else: - logging.error("No ssl section found in cloud config file, aborting...") + logging.error("update_certs: No ssl section found in cloud config file, aborting...") sys.exit(1) - # Arbitrary wait to allow SSL upload to register with AWS - # Otherwise, we can get an ARN for the load balancer certificates - # without it being ready to assign - time.sleep(3) - # Set the certificates on ELB's if we have any - if updated_count > 0: - if 'elb' in cfn_config.data: - logging.info("Setting load balancer certificates...") - elb = get_connection(ELB) - elb.set_ssl_certificates(cfn_config.ssl(), stack_name) - else: + if len(updated_certs) <= 0: logging.error("No certificates updated so skipping " "ELB certificate update...") + if 'elb' in cfn_config.data: + logging.info("update_certs: Setting load balancer certificates...") + elb = get_connection(ELB) + replaced_certificates = elb.set_ssl_certificates(updated_certs, stack_name, max_retries=3) + if delete_replaced_certificates: + for replaced_certificate in replaced_certificates: + iam.delete_certificate(replaced_certificate, stack_name, max_retries=3) def get_cloudformation_tags(): diff --git a/bootstrap_cfn/iam.py b/bootstrap_cfn/iam.py index b02b06b..eaa5a98 100644 --- a/bootstrap_cfn/iam.py +++ b/bootstrap_cfn/iam.py @@ -1,10 +1,14 @@ import logging +import time + from boto.connection import AWSQueryConnection + +from boto.exception import BotoServerError + import boto.iam from bootstrap_cfn import utils -from bootstrap_cfn.errors import CloudResourceNotFoundError class IAM: @@ -36,8 +40,9 @@ def delete_ssl_certificate(self, ssl_config, stack_name): def update_ssl_certificates(self, ssl_config, stack_name): """ - Update all the ssl certificates in the identified stack. Raise an - exception if we try to update a non-existent certificate + Update all the ssl certificates in the identified stack. Note, + this creates a uniquely named ssl certificate and doesn't overwrite + the current ones. Args: ssl_config(dictionary): A dictionary of ssl configuration data @@ -51,34 +56,26 @@ def update_ssl_certificates(self, ssl_config, stack_name): updated_certificates = [] for cert_name, ssl_data in ssl_config.items(): try: - delete_success = self.delete_certificate(cert_name, - stack_name, - ssl_data) - if delete_success: - upload_success = self.upload_certificate(cert_name, - stack_name, - ssl_data, - force=True) - if upload_success: - updated_certificates.append(cert_name) - logging.info("IAM::update_ssl_certificates: " - "Updated certificate '%s': " - % (cert_name)) - else: - logging.warn("IAM::update_ssl_certificates: " - "Failed to update certificate '%s': " - % (cert_name)) - else: - msg = ("IAM::update_ssl_certificates: " - "Could not update certificate '%s': " - "Certificate does not exist remotely" - % (cert_name)) - raise CloudResourceNotFoundError(msg) - + # Generate uniquely timestamped certificate name and upload it + timestamped_cert_name = ("%s-%s" + % (cert_name, time.time())) + upload_success = self.upload_certificate(timestamped_cert_name, + stack_name, + ssl_data, + force=True) + if upload_success: + updated_certificates.append(timestamped_cert_name) + logging.info("IAM::update_ssl_certificates: " + "Uploaded certificate with key '%s' to '%s': " + % (cert_name, timestamped_cert_name)) + else: + logging.warn("IAM::update_ssl_certificates: " + "Failed to upload certificate '%s' as '%s': " + % (cert_name, timestamped_cert_name)) except AWSQueryConnection.ResponseError as error: logging.warn("IAM::update_ssl_certificates: " "Could not update certificate '%s': " - "Error %s - %s" % (cert_name, + "Error %s - %s" % (timestamped_cert_name, error.status, error.reason)) return updated_certificates @@ -100,7 +97,7 @@ def get_remote_certificate(self, cert_name, stack_name): try: cert_id = "{0}-{1}".format(cert_name, stack_name) logging.info("IAM::get_remote_certificate: " - "Found certificate '%s'.." + "Looking for certificate '%s'.." % (cert_id)) # Fetch the remote AWS certificate configuration data @@ -144,7 +141,7 @@ def compare_remote_certificate_data(self, cert_name, stack_name, ssl_data): try: cert_id = "{0}-{1}".format(cert_name, stack_name) logging.info("IAM::get_remote_certificate: " - "Found certificate '%s'.." + "Looking for certificate '%s'.." % (cert_id)) # Fetch the remote AWS certificate configuration data @@ -282,15 +279,16 @@ def upload_certificate(self, cert_name, stack_name, ssl_data, force=False): return False - def delete_certificate(self, cert_name, stack_name, ssl_data): + def delete_certificate(self, cert_name, stack_name, max_retries=1, retry_delay=10): """ - Delete a certificate from AWS + Delete a certificate from AWS, we can retry with delay, default is to only + try once. Args: cert_name(string): The name of the certificate entry to look up stack_name(string): The name of the stack - ssl_data(dictionary): The configuration data for this - certificate entry + max_retries(int): The number of retries to carry out on the operation + retry_delay(int): The retry delay of the operation Returns: success(bool): True if a certificate is deleted, False otherwise @@ -299,27 +297,35 @@ def delete_certificate(self, cert_name, stack_name, ssl_data): # Try to delete cert, but handle any problems on # individual deletes and # continue to delete other certs - try: - if self.get_remote_certificate(cert_name, - stack_name): - self.conn_iam.delete_server_cert(cert_id) - logging.info("IAM::delete_certificate: " - "Deleting certificate '%s'.." - % (cert_name)) - return True - else: - logging.info("IAM::delete_certificate: " - "Certificate '%s' does not exist, " - "not deleting." % (cert_name)) - return False - except AWSQueryConnection.ResponseError as error: - logging.warn("IAM::delete_certificate: " - "Could not find expected certificate '%s': " - "Error %s - %s" % (cert_id, - error.status, - error.reason)) - return False - + retries = 0 + while retries < max_retries: + retries += 1 + try: + if self.get_remote_certificate(cert_name, + stack_name): + try: + self.conn_iam.delete_server_cert(cert_id) + logging.info("IAM::delete_certificate: " + "Deleting certificate '%s'.." + % (cert_name)) + return True + except BotoServerError as e: + logging.warning("IAM::delete_certificate: Cannot delete ssl cert, reason '%s', " + "waiting %s seconds on retry %s/%s" + % (e.error_message, retry_delay, retries, max_retries)) + # Only sleep if we're going to try again + if retries < max_retries: + time.sleep(retry_delay) + else: + logging.info("IAM::delete_certificate: " + "Certificate '%s' does not exist, " + "not deleting." % (cert_name)) + except AWSQueryConnection.ResponseError as error: + logging.warn("IAM::delete_certificate: " + "Could not find expected certificate '%s': " + "Error %s - %s" % (cert_id, + error.status, + error.reason)) return False def get_arn_for_cert(self, cert_name): diff --git a/tests/test_iam.py b/tests/test_iam.py index 7ef963c..f7cae75 100644 --- a/tests/test_iam.py +++ b/tests/test_iam.py @@ -138,38 +138,6 @@ def test_update_ssl_certificates(self, "Should be able update certs" ) - @raises(CloudResourceNotFoundError) - @patch("boto.iam.IAMConnection.delete_server_cert") - @patch("boto.iam.IAMConnection.upload_server_cert") - @patch("bootstrap_cfn.iam.IAM.get_remote_certificate") - def test_update_ssl_certificates_not_exist(self, - mock_get_remote_certificate, - mock_upload_server_cert, - mock_delete_server_cert): - """ - Test we cause an exception trying update over - non existing certificates - """ - mock_get_remote_certificate.side_effect = [True, - False, - False, - None] - mock_upload_server_cert.side_effect = [self.successful_response, - self.unsuccessful_response, - None] - mock_delete_server_cert.side_effect = [self.successful_response, - self.unsuccessful_response, - None] - ssl_config = self.test_certs - stack_name = "test_stack" - update_count = self.mock_iam.update_ssl_certificates(ssl_config, - stack_name) - self.assertEqual(update_count, - 1, - "TestIAM::test_update_ssl_certificates_force: " - "Should only be able to update existing certificates " - ) - @patch("boto.iam.IAMConnection.upload_server_cert") @patch("bootstrap_cfn.iam.IAM.get_remote_certificate") def test_upload_certificate_not_exists(self, @@ -228,11 +196,9 @@ def test_delete_certificate_exists(self, mock_delete_server_cert.return_value = self.successful_response cert_name = "cert1" stack_name = "test_stack" - ssl_data = self.test_certs["test_cert_1"] success = self.mock_iam.delete_certificate(cert_name, - stack_name, - ssl_data) + stack_name) mock_get_remote_certificate.assert_called_once_with(cert_name, stack_name) self.assertTrue(success, @@ -252,11 +218,8 @@ def test_delete_certificate_not_exists(self, mock_delete_server_cert.return_value = self.unsuccessful_response cert_name = "cert1" stack_name = "test_stack" - ssl_data = self.test_certs["test_cert_1"] - success = self.mock_iam.delete_certificate(cert_name, - stack_name, - ssl_data) + stack_name) mock_get_remote_certificate.assert_called_once_with(cert_name, stack_name) self.assertFalse(success,