Merge pull request #234 from kwilczynski/feature/update-set_active_stack

Make sure that set_active_stack activates all ELBs.

bootstrap_cfn/fab_tasks.py

@@ -7,10 +7,12 @@
 import uuid
 
 import boto.exception
+
 import boto3
 
+
 from fabric.api import env, task
-from fabric.colors import green, red
+from fabric.colors import green, red, yellow
 from fabric.utils import abort
 
 from bootstrap_cfn.autoscale import Autoscale
@@ -24,7 +26,7 @@
                                   ZoneIDNotFoundError)
 from bootstrap_cfn.iam import IAM
 from bootstrap_cfn.r53 import R53
-from bootstrap_cfn.utils import tail
+from bootstrap_cfn.utils import strip_prefix, tail
 from bootstrap_cfn.vpc import VPC
 
 
@@ -569,47 +571,61 @@ def cfn_delete(force=False, pre_delete_callbacks=None):
 
     r53_conn = get_connection(R53)
 
-    elb = get_first_public_elb()
     stack_id = stack_name.split('-')[-1]
-    zone_name = get_zone_name()
+    stack_tag = 'active' if isactive() else get_env_tag()
+
     zone_id = get_zone_id()
+    zone_name = get_zone_name()
+
+    txt_tag_record = get_tag_record_name(stack_tag)
+
+    print green("\nDELETING DNS RECORDS...\n")
+    for elb in get_public_elbs():
+        logger.info("Deleting '{}-{}' from '{}' ({})...".format(elb, stack_id, zone_name, zone_id))
+        try:
+            r53_conn.delete_record(zone_name, zone_id, elb, stack_id, stack_tag, txt_tag_record)
+        except boto.route53.exception.DNSServerError:
+            pass
+
     if not isactive():
-        # delete inactive stack
-        stack_tag = get_env_tag()
-        logger.info("Deleting '%s' inactive stack '%s'...".format(stack_tag, stack_name))
         print green("\nSTACK {0} DELETING...\n").format(stack_name)
+        logger.info("Deleting inactive stack '{}' ({})...".format(stack_name, stack_tag))
+
+        try:
+            txt_arn_record = 'deployarn.{0}.{1}.{2}'.format(stack_tag, env.environment, env.application)
+
+            txt_record_name = '{}.{}'.format(txt_arn_record, zone_name)
+            txt_record_value = '"{}"'.format(r53_conn.get_record(zone_name, zone_id, txt_arn_record, 'TXT'))
 
-        # delete Alias and TXT records
-        txt_tag_record = get_tag_record_name(stack_tag)
-        r53_conn.delete_record(zone_name, zone_id, elb, stack_id, stack_tag, txt_tag_record)
-        # Wait for stacks to delete
-        print 'Waiting for stack to delete.'
+            logger.info("Deleting '{}' from '{}' ({}) ...".format(txt_arn_record, zone_name, zone_id))
+            r53_conn.delete_dns_record(zone_id, txt_record_name, 'TXT', txt_record_value)
+        except boto.route53.exception.DNSServerError:
+            pass
+
+        print "Waiting for stack '{}' to be deleted...".format(stack_name)
         cfn.delete(stack_name)
-        if not env.blocking:
-            print 'Running in non blocking mode. Exiting.'
-            sys.exit(0)
-        tail(cfn, stack_name)
+
+        if env.blocking:
+            try:
+                tail(cfn, stack_name)
+            except boto.exception.BotoServerError as e:
+                if e.code == 'ValidationError':
+                    pass
+                raise e
+        else:
+            print 'Running in an non-blocking mode.'
 
         if cfn.stack_missing(stack_name):
-            print green("Stack successfully deleted")
+            print green("Stack '{}' successfully deleted.".format(stack_name))
         else:
-            print red("Stack deletion was unsuccessful")
+            print red("Failed to delete stack '{}' successfully.".format(stack_name))
             return False
-        # cleanup ssl if exists
-        # currently we read ssl from configuration file instead of from AWS
-        # this can cause some mismatch when local config file has been changed.s
+
         try:
             iam = get_connection(IAM)
             iam.delete_ssl_certificate(cfn_config.ssl(), stack_name)
         except AttributeError, boto.exception:
-            print green("ssl did not exist")
-    else:
-        # delete active dns records
-
-        stack_tag = 'active'
-        print green("\nDELETING ACTIVE DNS RECORDS...\n")
-        txt_tag_record = get_tag_record_name(stack_tag)
-        r53_conn.delete_record(zone_name, zone_id, elb, stack_id, stack_tag, txt_tag_record)
+            print green("SSL certificate was already deleted.")
 
     return True
 
@@ -832,44 +848,47 @@ def set_active_stack(stack_tag, force=False):
     zone_id = get_zone_id()
 
     tag_record = get_tag_record_name(stack_tag)
-    try:
-        tag_stack_id = r53_conn.get_record(zone_name, zone_id, tag_record, 'TXT')
-    except Exception:
+
+    tag_stack_id = r53_conn.get_record(zone_name, zone_id, tag_record, 'TXT')
+    if not tag_stack_id:
         raise TagRecordNotFoundError(tag_record)
 
     if get_active_stack() and not force:
-        x = raw_input("Your stack is {}. Do you wanna change? (y/n)\n".format(tag_stack_id))
+        x = raw_input("Your stack is {}. Do you want to change? (y/n)\n".format(tag_stack_id))
         if x not in ['y', 'Y', 'Yes', 'yes']:
             sys.exit(1)
 
-    # update TXT record
     try:
         r53_conn.update_dns_record(zone_id, "{}.{}".format(active_record, get_zone_name()), 'TXT',
                                    '"{}"'.format(tag_stack_id))
-        logger.info("fab_tasks::set_active_stack: Successfully updated dns alias record")
+        logger.info("fab_tasks::set_active_stack: Successfully updated DNS "
+                    "alias record for stack: %s", tag_stack_id)
     except Exception:
         raise UpdateDNSRecordError
 
-    # get the first public facing elb
-    elb = get_first_public_elb()
-    main_record_name = "{}.{}".format(elb, zone_name)
-    record_name = "{}-{}".format(elb, tag_stack_id)
-    try:
-        record_object = r53_conn.get_full_record(zone_name, zone_id, record_name, 'A')
-        record_value = [record_object.alias_hosted_zone_id,
-                        record_object.alias_dns_name,
-                        record_object.alias_evaluate_target_health]
-        # point [helloworld.dsd.io] to [helloworld-12345.dsd.io]'s ELB
+    elbs = get_public_elbs()
+    logger.info('fab_tasks::set_active_stack: Found ELBs matching the stack: %s',
+                ', '.join(elbs))
+    for elb in elbs:
+        main_record_name = "{}.{}".format(elb, zone_name)
+        record_name = "{}-{}".format(elb, tag_stack_id)
         try:
-            r53_conn.update_dns_record(zone_id, main_record_name, 'A', record_value, is_alias=True)
-            logger.info("fab_tasks::set_active_stack: Successfully updated dns alias record")
+            record_object = r53_conn.get_full_record(zone_name, zone_id, record_name, 'A')
+            record_value = [record_object.alias_hosted_zone_id,
+                            record_object.alias_dns_name,
+                            record_object.alias_evaluate_target_health]
+            try:
+                r53_conn.update_dns_record(zone_id, main_record_name, 'A',
+                                           record_value, is_alias=True)
+                logger.info("fab_tasks::set_active_stack: Successfully "
+                            "updated DNS alias record for ELB: %s", elb)
+            except Exception:
+                raise UpdateDNSRecordError
         except Exception:
-            raise UpdateDNSRecordError
-    except Exception:
-        raise StackRecordNotFoundError(record_name)
+            raise StackRecordNotFoundError(record_name)
     try:
         set_active_deployarn(stack_tag)
-        print green("Active stack is switched to {}".format(tag_record))
+        print green("Active stack switched to '{}' ({}).".format(tag_record, tag_stack_id))
         return True
     except:
         raise UpdateDeployarnRecordError
@@ -918,51 +937,74 @@ def get_active_stack():
     """
     Returns stack id if active stack exists AND Alias record is set appropriately
     """
+
+    r53_conn = get_connection(R53)
+
+    prefix = r'dualstack.'
+    suffix = r'.+\.amazonaws.com\.?$'
+
     try:
-        active_record = get_tag_record_name('active')
-        r53_conn = get_connection(R53)
-        zone_name = get_zone_name()
         zone_id = get_zone_id()
+        zone_name = get_zone_name()
+        active_record = get_tag_record_name('active')
         active_stack_id = r53_conn.get_record(zone_name, zone_id, active_record, 'TXT')
-        elb = get_first_public_elb()
-        dns_record_name = '{}-{}'.format(elb, active_stack_id)
-        dns_record_value = r53_conn.get_record(zone_name, zone_id, dns_record_name, 'A')
-        main_record_value = r53_conn.get_record(zone_name, zone_id, elb, 'A')
+
+        records = []
+        for elb in get_public_elbs():
+            dns_record_name = '{}-{}'.format(elb, active_stack_id)
+
+            main_record_value = r53_conn.get_record(zone_name, zone_id, elb, 'A')
+            dns_record_value = r53_conn.get_record(zone_name, zone_id, dns_record_name, 'A')
+
+            if re.match(suffix, main_record_value):
+                main_record_value = strip_prefix(main_record_value, prefix)
+
+            if re.match(suffix, dns_record_value):
+                dns_record_value = strip_prefix(dns_record_value, prefix)
+
+            records += [dns_record_value, main_record_value == dns_record_value]
+
     except Exception:
-        print green("No active stack exists.")
+        print yellow("No active stack exists.")
         return
-    if active_stack_id and dns_record_value and dns_record_value == main_record_value:
-        logger.info("fab_tasks::get_active_stack: "
-                    "Active stack id is: '%s'", active_stack_id)
+
+    if active_stack_id and all(records):
         print green("Active stack id is: {}".format(active_stack_id))
         return active_stack_id
     else:
-        print green("No active stack exists.")
-        return None
+        print yellow("No active stack exists.")
 
 
-def get_all_elbs():
+def get_all_elbs(f=None):
     """
-    Returns all internet-facing elbs from cloudformation configuration
+    Returns a list of internet-facing ELBs from the CloudFormation
+    configuration containing items for which the filter function f
+    returns True, or everything.
     """
     cfn_config = get_config()
     elbs = [x.get('name') for x in cfn_config.data.get('elb', {}) if x.get('scheme') == 'internet-facing']
+    return filter(f, elbs) if f else elbs
+
+
+def get_public_elbs(f=None):
+    """
+    Returns a list of internet-facing ELBs from the CloudFormation
+    configuration containing items for which the filter function f
+    returns True, or everything. Raises an error if the list is empty.
+    """
+    elbs = get_all_elbs(f)
+    if len(elbs) < 1:
+        raise PublicELBNotFoundError
     return elbs
 
 
 def get_first_public_elb():
     """
-    Returns the first elb if exists.
+    Returns the first public ELB if exists, or raise an error.
     """
     elbs = get_all_elbs()
     if len(elbs) < 1:
         raise PublicELBNotFoundError
-    elif len(elbs) == 1:
-        logger.info("fab_tasks::set_active_stack: Found one ELB '%s', "
-                    "using it for public ELB... ", elbs[0])
-    else:
-        logger.info("fab_tasks::set_active_stack: Found multiple ELBs,"
-                    "using the first one '%s' as public ELB", elbs[0])
     return elbs[0]
 
 

bootstrap_cfn/utils.py

@@ -151,3 +151,9 @@ def sleep_countdown(sleep_time):
         sys.stdout.flush()
         time.sleep(1)
         sleep_time -= 1
+
+
+def strip_prefix(string, prefix):
+    if string.startswith(prefix):
+        return string[len(prefix):]
+    return string

requirements.txt

@@ -8,3 +8,4 @@ netaddr==0.7.18
 testfixtures==4.1.2
 paramiko
 troposphere>=1.0.0
+pycrypto

tests/test_fab_tasks.py

@@ -1,12 +1,14 @@
+import re
 import unittest
 
 import boto
 
 import yaml
 
-from mock import patch, Mock  # noqa
 
 from bootstrap_cfn import cloudformation, config, errors, fab_tasks, iam, r53
+from mock import patch, Mock  # noqa
+
 
 fake_profile = {'lol': {'aws_access_key_id': 'secretz', 'aws_secret_access_key': 'verysecretz'}}
 
@@ -144,6 +146,27 @@ def test_get_all_elbs(self, get_config_function):
         all_elbs = fab_tasks.get_all_elbs()
         self.assertEqual(all_elbs, ["unittest"])
 
+    @patch('bootstrap_cfn.fab_tasks.get_config')
+    def test_get_all_elbs_with_filter(self, get_config_function):
+        '''
+        Check if get_all_elbs() returns correct Internet facing ELB,
+        given a filter that would should match only a specific name.
+        Args:
+            get_config_function: mock of get_config() function
+
+        '''
+        basic_config_mock = yaml.load(set_up_basic_config())
+        basic_config_mock['elb'].append({'hosted_zone': 'unittest.dsd.io.',
+                                         'name': 'unittest2',
+                                         'scheme': 'internet-facing'})
+
+        get_config_function.return_value = config.ConfigParser(
+            basic_config_mock, "unittest_stack_name", "dev", "test")
+
+        regex = re.compile('unittest2')
+        filtered_elbs = fab_tasks.get_all_elbs(regex.match)
+        self.assertEqual(filtered_elbs, ["unittest2"])
+
     @patch('bootstrap_cfn.fab_tasks.get_all_elbs', return_value=["unittest_elb"])
     def test_get_first_public_elb(self, get_all_elbs_function):
         '''
@@ -170,16 +193,16 @@ def test_no_public_elb(self, get_all_elbs_function):
     @patch('bootstrap_cfn.fab_tasks.get_zone_name', return_value="dsd.io")
     @patch('bootstrap_cfn.fab_tasks.get_legacy_name', return_value="unittest-dev")
     @patch('bootstrap_cfn.fab_tasks.get_zone_id', return_value="ASDAKSLDK")
-    @patch('bootstrap_cfn.fab_tasks.get_first_public_elb', return_value="unittest_elb")
-    def test_get_active_stack(self, get_first_public_elb_function,
+    @patch('bootstrap_cfn.fab_tasks.get_public_elbs', return_value=["unittest_elb"])
+    def test_get_active_stack(self, get_public_elbs_function,
                               get_zone_id_function,
                               get_legacy_name_function,
                               get_zone_name_function,
                               get_connection_function):
         '''
         Return stack_id of m2 record defined in def r53_mock()
         Args:
-            get_first_public_elb_function:
+            get_public_elbs_function:
             get_zone_id_function:
             get_legacy_name_function:
             get_zone_name_function:
@@ -200,8 +223,10 @@ def test_get_active_stack(self, get_first_public_elb_function,
     @patch('bootstrap_cfn.fab_tasks.get_zone_name', return_value="dsd.io")
     @patch('bootstrap_cfn.fab_tasks.get_legacy_name', return_value="unittest-dev")
     @patch('bootstrap_cfn.fab_tasks.get_zone_id', return_value="ASDAKSLDK")
+    @patch('bootstrap_cfn.fab_tasks.get_public_elbs', return_value=["unittest_elb"])
     @patch('bootstrap_cfn.fab_tasks.get_first_public_elb', return_value="unittest_elb")
-    def test_set_active_stack(self, get_first_public_elb_function,
+    def test_set_active_stack(self, get_public_elbs_function,
+                              get_first_public_elb_function,
                               get_zone_id_function,
                               get_legacy_name_function,
                               get_zone_name_function,
@@ -211,6 +236,7 @@ def test_set_active_stack(self, get_first_public_elb_function,
         set stack tagged with "test" as active stack,
         using m4 record defined in def r53_mock()
         Args:
+            get_public_elbs_function:
             get_first_public_elb_function:
             get_zone_id_function:
             get_legacy_name_function:

tox.ini

@@ -1,4 +1,5 @@
 [flake8]
-max-line-length=160
+max-line-length = 160
 application-import-names = bootstrap_cfn,tests
 import-order-style = cryptography
+exclude = tox.ini,requirements.txt