diff --git a/License2Deploy/AWSConn.py b/License2Deploy/AWSConn.py index 7e4d1b6..6a819ce 100644 --- a/License2Deploy/AWSConn.py +++ b/License2Deploy/AWSConn.py @@ -4,6 +4,7 @@ import boto.ec2.autoscale as a import boto.ec2.elb as elb import boto.ec2.cloudwatch as cloudwatch +from boto3.session import Session import logging import yaml @@ -41,6 +42,12 @@ def aws_conn_cloudwatch(region, profile='default'): except Exception as e: logging.error("Unable to connect to region, please investigate: {0}".format(e)) + @staticmethod + def get_boto3_client(client_type, region, profile='default', session=None): + if not session: + session = Session(region_name=region, profile_name=profile) + return session.client(client_type) + @staticmethod def load_config(config): with open(config, 'r') as stream: diff --git a/License2Deploy/rolling_deploy.py b/License2Deploy/rolling_deploy.py index 2f71b60..496066e 100644 --- a/License2Deploy/rolling_deploy.py +++ b/License2Deploy/rolling_deploy.py @@ -11,21 +11,34 @@ class RollingDeploy(object): MAX_RETRIES = 10 - def __init__(self, env=None, project=None, buildNum=None, ami_id=None, profile_name=None, regions_conf=None): + def __init__(self, + env=None, + project=None, + build_number=None, + ami_id=None, + profile_name=None, + regions_conf=None, + stack_name=None, + session=None): self.env = env + self.session = session self.project = project.replace('-','') - self.buildNum = buildNum + self.build_number = build_number self.ami_id = ami_id self.profile_name = profile_name self.regions_conf = regions_conf + self.stack_name = stack_name + self.stack_resources = False + self.autoscaling_groups = False self.environments = AWSConn.load_config(self.regions_conf).get(self.env) self.region = AWSConn.determine_region(self.environments) self.conn_ec2 = AWSConn.aws_conn_ec2(self.region, self.profile_name) self.conn_elb = AWSConn.aws_conn_elb(self.region, self.profile_name) self.conn_auto = AWSConn.aws_conn_auto(self.region, self.profile_name) self.conn_cloudwatch = AWSConn.aws_conn_cloudwatch(self.region, self.profile_name) + self.cloudformation_client = AWSConn.get_boto3_client('cloudformation', self.region, self.profile_name, session) self.exit_error_code = 2 - self.load_balancer = self.get_lb() + self.load_balancer = False def get_ami_id_state(self, ami_id): try: @@ -65,8 +78,19 @@ def get_group_info(self, group_name=None): def get_autoscale_group_name(self): ''' Search for project in autoscale groups and return autoscale group name ''' - proj_name = next((instance.name for instance in filter(lambda n: n.name, self.get_group_info()) if self.project in instance.name and self.env in instance.name), None) - return proj_name + if self.stack_name: + return self.get_autoscaling_group_name_from_cloudformation() + return next((instance.name for instance in filter(lambda n: n.name, self.get_group_info()) if self.project in instance.name and self.env in instance.name), None) + + def get_autoscaling_group_name_from_cloudformation(self): + if not self.autoscaling_groups: + self.autoscaling_groups = [asg for asg in self.get_stack_resources() if asg['ResourceType'] == 'AWS::AutoScaling::AutoScalingGroup'] + return [asg['PhysicalResourceId'] for asg in self.autoscaling_groups if self.project in asg['PhysicalResourceId']][0] + + def get_stack_resources(self): + if not self.stack_resources: + self.stack_resources = self.cloudformation_client.list_stack_resources(StackName=self.stack_name)['StackResourceSummaries'] + return self.stack_resources def get_lb(self): try: @@ -189,7 +213,7 @@ def confirm_lb_has_only_new_instances(self, wait_time=60): instance_ids = self.conn_elb.describe_instance_health(self.load_balancer) for instance in instance_ids: build = self.conn_ec2.get_all_reservations(instance.instance_id)[0].instances[0].tags['BUILD'] - if build != self.buildNum: + if build != self.build_number: logging.error("There is still an old instance in the ELB: {0}. Please investigate".format(instance)) exit(self.exit_error_code) logging.info("Deployed instances {0} to ELB: {1}".format(instance_ids, self.load_balancer)) @@ -214,7 +238,7 @@ def tag_ami(self, ami_id, env): def gather_instance_info(self, group): #pragma: no cover instance_ids = self.get_all_instance_ids(group) - new_instance_ids = self.get_instance_ids_by_requested_build_tag(instance_ids, self.buildNum) + new_instance_ids = self.get_instance_ids_by_requested_build_tag(instance_ids, self.build_number) return new_instance_ids def healthcheck_new_instances(self, group_name): # pragma: no cover @@ -259,10 +283,11 @@ def enable_project_cloudwatch_alarms(self): exit(self.exit_error_code) def deploy(self): # pragma: no cover + self.load_balancer = self.get_lb() ''' Rollin Rollin Rollin, Rawhide! ''' group_name = self.get_autoscale_group_name() self.wait_ami_availability(self.ami_id) - logging.info("Build #: {0} ::: Autoscale Group: {1}".format(self.buildNum, group_name)) + logging.info("Build #: {0} ::: Autoscale Group: {1}".format(self.build_number, group_name)) self.disable_project_cloudwatch_alarms() self.set_autoscale_instance_desired_count(self.calculate_autoscale_desired_instance_count(group_name, 'increase'), group_name) logging.info("Sleeping for 240 seconds to allow for instances to spin up") @@ -292,16 +317,17 @@ def get_args(): # pragma: no cover parser = argparse.ArgumentParser() parser.add_argument('-e', '--environment', action='store', dest='env', help='Environment e.g. qa, stg, prd', type=str, required=True) parser.add_argument('-p', '--project', action='store', dest='project', help='Project name', type=str, required=True) - parser.add_argument('-b', '--build', action='store', dest='buildNum', help='Build Number', type=str, required=True) - parser.add_argument('-a', '--ami', action='store', dest='amiID', help='AMI ID to be deployed', type=str, required=True) + parser.add_argument('-b', '--build', action='store', dest='build_number', help='Build Number', type=str, required=True) + parser.add_argument('-a', '--ami', action='store', dest='ami_id', help='AMI ID to be deployed', type=str, required=True) parser.add_argument('-P', '--profile', default='default', action='store', dest='profile', help='Profile name as designated in aws credentials/config files', type=str) parser.add_argument('-c', '--config', default='/opt/License2Deploy/regions.yml', action='store', dest='config', help='Config file Location, eg. /opt/License2Deploy/regions.yml', type=str) + parser.add_argument('-s', '--stack', action='store', dest='stack_name', help='Stack name if AutoScaling Group created via CloudFormation', type=str) return parser.parse_args() def main(): # pragma: no cover args = get_args() SetLogging.setup_logging() - deployObj = RollingDeploy(args.env, args.project, args.buildNum, args.amiID, args.profile, args.config) + deployObj = RollingDeploy(args.env, args.project, args.build_number, args.ami_id, args.profile, args.config, args.stack_name) deployObj.deploy() if __name__ == "__main__": # pragma: no cover diff --git a/README.md b/README.md index ca5b4dc..c708fcd 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ The rolling deployment will: Usage ================== ``` -usage: rolling_deploy.py [-h] -e ENV -p PROJECT -b BUILDNUM -a AMIID - [-P PROFILE] [-c CONFIG] +usage: rolling_deploy.py [-h] -e ENV -p PROJECT -b BUILD_NUM -a AMI_ID + [-P PROFILE] [-c CONFIG] [-s STACK_NAME] optional arguments: -h, --help show this help message and exit @@ -29,9 +29,9 @@ optional arguments: Environment -p PROJECT, --project PROJECT Project name - -b BUILDNUM, --build BUILDNUM + -b BUILD_NUM, --build BUILD_NUM Build Number - -a AMIID, --ami AMIID + -a AMI_ID, --ami AMI_ID AMI ID -P PROFILE, --profile PROFILE Profile name as designated in aws credentials/config @@ -39,6 +39,8 @@ optional arguments: -c CONFIG, --config CONFIG Config file Location, eg. /opt/License2Deploy/config.yml + -s STACK_NAME, --stack STACK_NAME + Stack name if AutoScaling Group created via CloudFormation ``` Requirements ================== diff --git a/setup.py b/setup.py index 5f825e9..f3cbc57 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,17 @@ install_requires = [ "boto", "PyYaml", - "argparse" + "argparse", + 'boto3' ] tests_require = [ "mock", "boto", "moto", - "PyYaml" + "PyYaml", + 'placebo', + 'boto3' ] def read(fname): diff --git a/tests/cloudformation_client_test.py b/tests/cloudformation_client_test.py new file mode 100644 index 0000000..83d1b3c --- /dev/null +++ b/tests/cloudformation_client_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +import os +import placebo +from boto3.session import Session +import unittest + +from License2Deploy.rolling_deploy import RollingDeploy + + +class CloudformationClientTest(unittest.TestCase): + + def setUp(self): + session = Session(region_name='us-west-1') + current_dir = os.path.dirname(os.path.realpath(__file__)) + pill = placebo.attach(session, '{0}/test_data'.format(current_dir)) + pill.playback() + self.rolling_deploy = RollingDeploy('stg', 'server-gms-extender', '0', 'ami-abcd1234', None, './regions.yml', stack_name='test-stack-name', session=session) + + def test_get_autoscaling_group_name_via_cloudformation(self): + self.assertEquals(self.rolling_deploy.stack_resources, False) + self.assertEquals(self.rolling_deploy.autoscaling_groups, False) + asg_name = self.rolling_deploy.get_autoscale_group_name() + self.assertTrue(self.rolling_deploy.stack_resources) + self.assertTrue(self.rolling_deploy.autoscaling_groups) + self.assertEquals(asg_name, 'dnbi-backend-qa-servergmsextenderASGqa-QO8UAEHUFJD') + diff --git a/tests/rolling_deploy_test.py b/tests/rolling_deploy_test.py index 4cb524d..9ef830e 100644 --- a/tests/rolling_deploy_test.py +++ b/tests/rolling_deploy_test.py @@ -2,11 +2,9 @@ import unittest import boto -import os from boto.ec2.autoscale.launchconfig import LaunchConfiguration from boto.ec2.autoscale.group import AutoScalingGroup from boto.ec2.cloudwatch.alarm import MetricAlarm -from boto.ec2.cloudwatch.dimension import Dimension from moto import mock_autoscaling from moto import mock_ec2 from moto import mock_elb @@ -206,6 +204,7 @@ def test_wait_ami_availability(self): @mock_elb def test_confirm_lb_has_only_new_instances(self): instance_ids = self.setUpEC2()[1] + self.rolling_deploy.load_balancer = self.rolling_deploy.get_lb() self.assertEqual(len(instance_ids), len(self.rolling_deploy.confirm_lb_has_only_new_instances(1))) #Return All LB's with the proper build number @mock_elb @@ -220,7 +219,8 @@ def test_get_lb_failure(self): if sys.version_info >= (2, 7): self.setUpELB() with self.assertRaises(SystemExit) as rolling_deploy: - RollingDeploy('stg', 'fake-server-gms-extender', '0', 'bad', 'server-deploy', './regions.yml') + bad_rolling_deploy = RollingDeploy('stg', 'fake-gms-extender', '0', 'bad', None, './regions.yml') + bad_rolling_deploy.load_balancer = bad_rolling_deploy.get_lb() self.assertEqual(2, rolling_deploy.exception.code) @mock_ec2 @@ -348,4 +348,4 @@ def main(): unittest.main() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tests/test_data/cloudformation.ListStackResources_1.json b/tests/test_data/cloudformation.ListStackResources_1.json new file mode 100644 index 0000000..790b46b --- /dev/null +++ b/tests/test_data/cloudformation.ListStackResources_1.json @@ -0,0 +1,139 @@ +{ + "status_code": 200, + "data": { + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "27063939-3" + }, + "StackResourceSummaries": [ + { + "ResourceType": "AWS::S3::Bucket", + "PhysicalResourceId": "buket-1", + "LastUpdatedTimestamp": { + "hour": 20, + "__class__": "datetime", + "month": 2, + "second": 21, + "microsecond": 285000, + "year": 2016, + "day": 23, + "minute": 21 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "bucketintropo" + }, + { + "ResourceType": "AWS::S3::Bucket", + "PhysicalResourceId": "bucket-2", + "LastUpdatedTimestamp": { + "hour": 20, + "__class__": "datetime", + "month": 2, + "second": 21, + "microsecond": 266000, + "year": 2016, + "day": 23, + "minute": 21 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "buckettropo2" + }, + { + "ResourceType": "AWS::AutoScaling::AutoScalingGroup", + "PhysicalResourceId": "dnbi-backend-qa-dnbicacheASGqa-89YEUHYGHFJ", + "LastUpdatedTimestamp": { + "hour": 22, + "__class__": "datetime", + "month": 6, + "second": 15, + "microsecond": 62000, + "year": 2016, + "day": 14, + "minute": 26 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "ASG1" + }, + { + "ResourceType": "AWS::ElasticLoadBalancing::LoadBalancer", + "PhysicalResourceId": "ELB-1", + "LastUpdatedTimestamp": { + "hour": 20, + "__class__": "datetime", + "month": 2, + "second": 22, + "microsecond": 596000, + "year": 2016, + "day": 23, + "minute": 21 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "ELB1" + }, + { + "ResourceType": "AWS::AutoScaling::LaunchConfiguration", + "PhysicalResourceId": "dnbi-backend-qa-dnbicacheLCqa-89SYRUIFHEUI", + "LastUpdatedTimestamp": { + "hour": 22, + "__class__": "datetime", + "month": 6, + "second": 8, + "microsecond": 201000, + "year": 2016, + "day": 14, + "minute": 26 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "LC1" + }, + { + "ResourceType": "AWS::AutoScaling::AutoScalingGroup", + "PhysicalResourceId": "dnbi-backend-qa-servergmsextenderASGqa-QO8UAEHUFJD", + "LastUpdatedTimestamp": { + "hour": 21, + "__class__": "datetime", + "month": 6, + "second": 18, + "microsecond": 223000, + "year": 2016, + "day": 23, + "minute": 7 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "ASG2" + }, + { + "ResourceType": "AWS::ElasticLoadBalancing::LoadBalancer", + "PhysicalResourceId": "dnbiservergmsextenderELB", + "LastUpdatedTimestamp": { + "hour": 20, + "__class__": "datetime", + "month": 2, + "second": 23, + "microsecond": 442000, + "year": 2016, + "day": 23, + "minute": 21 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "ELB2" + }, + { + "ResourceType": "AWS::AutoScaling::LaunchConfiguration", + "PhysicalResourceId": "dnbi-backend-qa-servergmsextenderLCqa-IUEAHJKDIUJ", + "LastUpdatedTimestamp": { + "hour": 21, + "__class__": "datetime", + "month": 6, + "second": 12, + "microsecond": 144000, + "year": 2016, + "day": 23, + "minute": 7 + }, + "ResourceStatus": "UPDATE_COMPLETE", + "LogicalResourceId": "LC2" + } + ] + } +}