From 78216eed2757e22ec5a69bc1838f5b0df411262e Mon Sep 17 00:00:00 2001 From: Matthew Aynalem Date: Tue, 1 May 2018 13:34:00 -0700 Subject: [PATCH 1/2] DEVOPS-11114: python 3 compatibility / add tox / pytest runner --- .editorconfig | 13 +++++++++++++ .gitignore | 3 +++ .travis.yml | 10 ++++++---- License2Deploy/rolling_deploy.py | 18 +++++++++--------- setup.cfg | 13 +++++++++++++ setup.py | 6 ++++-- tests/__init__.py | 1 - tests/cloudformation_client_test.py | 10 +++++----- tests/rolling_deploy_test.py | 20 +++++++------------- tox.ini | 18 ++++++++++++++++++ 10 files changed, 78 insertions(+), 34 deletions(-) create mode 100644 .editorconfig create mode 100644 setup.cfg create mode 100644 tox.ini diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..503acc3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +# top-most EditorConfig file +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +trim_trailing_whitespace = true + +[*.py] +max_line_length = 119 +indent_size = 4 diff --git a/.gitignore b/.gitignore index 22a1b42..fbc521d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ *htmlcov .DS_Store .localized +*.idea +.tox +.pytest_cache diff --git a/.travis.yml b/.travis.yml index a560a33..fc08604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,19 @@ language: python python: - - "2.7" - "2.6" + - "2.7" + - "3.6" install: - - python setup.py install - pip install coveralls + - pip install tox-travis + - pip install tox==2.6.0 before_script: - export PYTHONPATH=$PYTHONPATH:$PWD script: - - coverage run --source License2Deploy setup.py test + - tox after_success: - coveralls + - if [ "$TRAVIS_PYTHON_VERSION" == "2.7" ]; then coveralls; fi diff --git a/License2Deploy/rolling_deploy.py b/License2Deploy/rolling_deploy.py index c991001..057b474 100644 --- a/License2Deploy/rolling_deploy.py +++ b/License2Deploy/rolling_deploy.py @@ -4,8 +4,8 @@ import argparse from sys import exit, argv from time import sleep, time -from AWSConn import AWSConn -from set_logging import SetLogging +from .AWSConn import AWSConn +from .set_logging import SetLogging from retry.api import retry_call class RollingDeploy(object): @@ -94,7 +94,7 @@ def get_autoscale_group_name(self): ''' Search for project in autoscale groups and return autoscale group name ''' if self.stack_name: return self.get_autoscaling_group_name_from_cloudformation() - return next((instance.name for instance in filter(lambda n: n.name, self.get_group_info()) if self.project in instance.name and self.env in instance.name), None) + return next((instance.name for instance in [n for n in self.get_group_info() if n.name] if self.project in instance.name and self.env in instance.name), None) def get_autoscaling_group_name_from_cloudformation(self): if not self.autoscaling_group: @@ -133,7 +133,7 @@ def calculate_autoscale_desired_instance_count(self, group_name, desired_state): except Exception as e: logging.error("Please make sure the desired_state is set to either increase or decrease: {0}".format(e)) exit(self.exit_error_code) - + def double_autoscale_instance_count(self, count): ''' Multiply current count by 2 ''' return count * 2 @@ -223,16 +223,16 @@ def wait_for_new_instances(self, instance_ids, retry=10, wait_time=30): def lb_healthcheck(self, new_ids): ''' Confirm that the healthchecks report back OK in the LB. ''' instance_ids = self.conn_elb.describe_instance_health(self.load_balancer, new_ids) - status = filter(lambda instance: instance.state != "InService", instance_ids) + status = [instance for instance in instance_ids if instance.state != "InService"] if status: raise Exception('Must check load balancer again. Following instance(s) are not "InService": {0}'.format(status)) else: logging.info('ELB healthcheck OK') return True - + def calculate_max_minutes(self, tries, delay): return tries * delay / 60 - + def only_new_instances_check(self): instance_ids = self.conn_elb.describe_instance_health(self.load_balancer) for instance in instance_ids: @@ -254,7 +254,7 @@ def tag_ami(self, ami_id, env): ''' Tagging AMI with DEPLOYED tag ''' try: current_tag = self.conn_ec2.get_all_images(image_ids=ami_id)[0].tags.get('deployed') - if not current_tag: + if not current_tag: logging.info("No DEPLOY tags exist, tagging with {0}".format(env)) self.conn_ec2.create_tags([self.ami_id], {"deployed": env}) elif env not in current_tag: @@ -407,6 +407,6 @@ def main(): # pragma: no cover SetLogging.setup_logging() deployObj = RollingDeploy(args.env, args.project, args.build_number, args.ami_id, args.profile, args.config, args.stack_name, args.force_redeploy, None, args.creation_wait, args.ready_wait, args.health_wait, args.only_new_wait) deployObj.deploy() - + if __name__ == "__main__": # pragma: no cover main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b44f221 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,13 @@ +[coverage:run] +branch = true +source = License2Deploy + +[coverage:report] +omit = + tests + .tox + +[tool:pytest] +addopts = --verbose +testpaths = + tests diff --git a/setup.py b/setup.py index 2167f43..073311a 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,13 @@ tests_require = [ "mock", "boto", - "moto", + "moto==1.0.1", "PyYaml", 'placebo', 'boto3', - 'retry' + 'retry', + 'pytest', + 'coverage', ] def read(fname): diff --git a/tests/__init__.py b/tests/__init__.py index 792d600..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +0,0 @@ -# diff --git a/tests/cloudformation_client_test.py b/tests/cloudformation_client_test.py index 5721dcd..6558634 100644 --- a/tests/cloudformation_client_test.py +++ b/tests/cloudformation_client_test.py @@ -17,15 +17,15 @@ def setUp(self): self.rolling_deploy = RollingDeploy('stg', 'server-gms-extender', '0', 'ami-abcd1234', None, './regions.yml', stack_name='test-stack-name', session=session) def test_get_autoscaling_group_name_via_cloudformation(self): - self.assertEquals(self.rolling_deploy.autoscaling_group, False) + self.assertEqual(self.rolling_deploy.autoscaling_group, False) asg_name = self.rolling_deploy.get_autoscale_group_name() self.assertTrue(self.rolling_deploy.autoscaling_group) - self.assertEquals(asg_name, 'dnbi-backend-qa-dnbigmsextenderASGqa-1NP5ZBSVZRD0N') + self.assertEqual(asg_name, 'dnbi-backend-qa-dnbigmsextenderASGqa-1NP5ZBSVZRD0N') def test_retrieve_project_cloudwatch_alarms(self): - self.assertEquals(self.rolling_deploy.stack_resources, False) - self.assertEquals(self.rolling_deploy.cloudwatch_alarms, False) + self.assertEqual(self.rolling_deploy.stack_resources, False) + self.assertEqual(self.rolling_deploy.cloudwatch_alarms, False) cloudwatch_alarms = self.rolling_deploy.retrieve_project_cloudwatch_alarms() self.assertTrue(self.rolling_deploy.stack_resources) - self.assertEquals(cloudwatch_alarms, ['dnbi-servergmsextender-SCALEDOWNALARMqa-123123', 'dnbi-servergmsextender-SCALEUPALARMqa-4asdhjks']) + self.assertEqual(cloudwatch_alarms, ['dnbi-servergmsextender-SCALEDOWNALARMqa-123123', 'dnbi-servergmsextender-SCALEUPALARMqa-4asdhjks']) diff --git a/tests/rolling_deploy_test.py b/tests/rolling_deploy_test.py index e144195..76c4f22 100644 --- a/tests/rolling_deploy_test.py +++ b/tests/rolling_deploy_test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import sys +import pytest import unittest import boto from boto.ec2.autoscale.launchconfig import LaunchConfiguration @@ -10,7 +11,6 @@ from moto import mock_ec2_deprecated from moto import mock_elb_deprecated from moto.cloudwatch import mock_cloudwatch_deprecated -from nose.tools import raises from License2Deploy.rolling_deploy import RollingDeploy from License2Deploy.AWSConn import AWSConn @@ -137,7 +137,7 @@ def test_retrieve_project_cloudwatch_alarms(self): instance_ids = self.setUpEC2() self.setUpCloudWatch(instance_ids) cloud_watch_alarms = self.rolling_deploy.retrieve_project_cloudwatch_alarms() - print cloud_watch_alarms + print(cloud_watch_alarms) self.assertEqual(1, len(cloud_watch_alarms)) @mock_cloudwatch_deprecated @@ -214,7 +214,7 @@ def test_confirm_lb_has_only_new_instances(self): @mock_elb_deprecated def test_get_lb(self): self.setUpELB() - self.assertEqual(u'servergmsextenderELBstg', self.rolling_deploy.get_lb()) #Return All LB's with the proper build number + self.assertEqual('servergmsextenderELBstg', self.rolling_deploy.get_lb()) #Return All LB's with the proper build number # assertRaises is a context manager since Python 2.7. Only testing in Python 2.7 # https://docs.python.org/2.7/library/unittest.html @@ -295,15 +295,15 @@ def test_is_redeploy(self): self.setUpEC2() self.assertTrue(self.rolling_deploy.is_redeploy()) - @raises(SystemExit) @mock_ec2_deprecated def test_is_redeploy_fails(self): self.setUpEC2(tag=False) - self.assertRaises(self.rolling_deploy.is_redeploy(), Exception) + with pytest.raises(SystemExit): + self.rolling_deploy.is_redeploy() - @raises(SystemExit) def test_stop_deploy(self): - self.assertRaises(self.rolling_deploy.stop_deploy('error!'), Exception) + with pytest.raises(SystemExit): + self.rolling_deploy.stop_deploy('error!') @mock_ec2_deprecated @mock_autoscaling_deprecated @@ -389,9 +389,3 @@ def test_double_autoscale_instance_count(self): def test_decrease_autoscale_instance_count(self): self.assertEqual(self.rolling_deploy.decrease_autoscale_instance_count(4), 2) - -def main(): - unittest.main() - -if __name__ == "__main__": - main() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f527402 --- /dev/null +++ b/tox.ini @@ -0,0 +1,18 @@ +[tox] +minversion=2.6.0 +envlist = py26,py27,py36 +skip_missing_interpreters = True + +[testenv] +usedevelop = true +extras = + test +whitelist_externals = + find +commands= + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -delete + {envbindir}/coverage erase + {envbindir}/coverage run \ + {envbindir}/pytest --basetemp={envtmpdir} {posargs} + {envbindir}/coverage html -d {toxinidir}/htmlcov/{envname} From ba4410b72eaeffbbf7169bc6cff11eda6799fc87 Mon Sep 17 00:00:00 2001 From: Matthew Aynalem Date: Wed, 16 May 2018 15:50:53 -0700 Subject: [PATCH 2/2] add cli entry --- .gitignore | 2 ++ CHANGES | 1 + CustomInstall.py | 9 ------- License2Deploy/AWSConn.py | 9 ++++--- License2Deploy/__init__.py | 1 - License2Deploy/rolling_deploy.py | 3 +-- License2Deploy/set_logging.py | 5 +--- README.md | 12 ++++++---- setup.py | 37 ++++++++++++++++------------- tests/cloudformation_client_test.py | 1 - tests/rolling_deploy_test.py | 2 -- 11 files changed, 36 insertions(+), 46 deletions(-) delete mode 100644 CustomInstall.py diff --git a/.gitignore b/.gitignore index fbc521d..f535a59 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ *.idea .tox .pytest_cache +dist +.cache diff --git a/CHANGES b/CHANGES index 6749871..32815a2 100644 --- a/CHANGES +++ b/CHANGES @@ -1 +1,2 @@ v0.0.1 First working version +v0.2.0 python 3 compatibility, add cli entrypoint diff --git a/CustomInstall.py b/CustomInstall.py deleted file mode 100644 index 9d58cc1..0000000 --- a/CustomInstall.py +++ /dev/null @@ -1,9 +0,0 @@ -from setuptools.command.install import install -from subprocess import call - -class CustomInstall(install): - - def run(self): - install.run(self) - #print("running custom install steps...") - #implement custom install steps if needed diff --git a/License2Deploy/AWSConn.py b/License2Deploy/AWSConn.py index 6a819ce..4573614 100644 --- a/License2Deploy/AWSConn.py +++ b/License2Deploy/AWSConn.py @@ -1,5 +1,3 @@ -#!/usr/bin/python - import boto.ec2 as ec2 import boto.ec2.autoscale as a import boto.ec2.elb as elb @@ -8,6 +6,7 @@ import logging import yaml + class AWSConn(object): @staticmethod @@ -20,7 +19,7 @@ def aws_conn_auto(region, profile='default'): @staticmethod def aws_conn_ec2(region, profile='default'): - try: + try: conn = ec2.connect_to_region(region, profile_name=profile) return conn except Exception as e: @@ -36,7 +35,7 @@ def aws_conn_elb(region, profile='default'): @staticmethod def aws_conn_cloudwatch(region, profile='default'): - try: + try: conn = cloudwatch.connect_to_region(region, profile_name=profile) return conn except Exception as e: @@ -51,7 +50,7 @@ def get_boto3_client(client_type, region, profile='default', session=None): @staticmethod def load_config(config): with open(config, 'r') as stream: - return yaml.load(stream) + return yaml.safe_load(stream) @staticmethod def determine_region(reg): diff --git a/License2Deploy/__init__.py b/License2Deploy/__init__.py index 1e0ca6a..e69de29 100644 --- a/License2Deploy/__init__.py +++ b/License2Deploy/__init__.py @@ -1 +0,0 @@ -# diff --git a/License2Deploy/rolling_deploy.py b/License2Deploy/rolling_deploy.py index 057b474..1ad97e0 100644 --- a/License2Deploy/rolling_deploy.py +++ b/License2Deploy/rolling_deploy.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import logging import argparse from sys import exit, argv @@ -8,6 +6,7 @@ from .set_logging import SetLogging from retry.api import retry_call + class RollingDeploy(object): MAX_RETRIES = 10 diff --git a/License2Deploy/set_logging.py b/License2Deploy/set_logging.py index 19b6cf2..1fd09ea 100644 --- a/License2Deploy/set_logging.py +++ b/License2Deploy/set_logging.py @@ -1,7 +1,6 @@ -#!/usr/bin/python - import logging + class SetLogging(object): @staticmethod @@ -9,5 +8,3 @@ def setup_logging(): # pragma: no cover logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s',level=logging.INFO) logging.info("Begin Logging...") -if __name__ == '__main__': - SetLogging.setup_logging() # pragma: no cover diff --git a/README.md b/README.md index 9859fa9..9cc4d5d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Rolling deploys for AWS AutoScale Groups What is License2Deploy? ================== -License2Deploy is an automated solution for rolling deployments in AWS written in python. +License2Deploy is an automated solution for rolling deployments in AWS written in python. The rolling deployment will: - Double your instances in your autoscale group @@ -20,7 +20,7 @@ The rolling deployment will: Usage ================== ``` -usage: rolling_deploy.py [-h] -e ENV -p PROJECT -b BUILD_NUM -a AMI_ID +usage: rolling_deploy [-h] -e ENV -p PROJECT -b BUILD_NUM -a AMI_ID [-P PROFILE] [-c CONFIG] [-s STACK_NAME] [-f FORCE_REDEPLOY] [-C CREATION_WAIT] [-r READY_WAIT] [-H HEALTH_WAIT] [-o ONLY_NEW_WAIT] @@ -72,7 +72,7 @@ There are a few requirements in order for the automated rolling deployments to w 4. All instances in the autoscale group need to be tagged with a build number * This is an important step as when the script runs, it will differentiate the old builds from the new builds based off of the build number that is passed in as a command line parameter -5. The credentials for the user need to be in the ~/.aws/credentials file and if not passed in as a +5. The credentials for the user need to be in the ~/.aws/credentials file and if not passed in as a command line argument, the script will look at the 'default' profile. 6. The script needs the AMI ID of the instances that will be built. * The reason for the AMI ID is to ensure that if it was just created, it is not in a pending state @@ -84,8 +84,10 @@ There are a few requirements in order for the automated rolling deployments to w Development ============ -python setup.py install +To run unit tests: -python setup.py test +```sh +$ tox +``` python License2Deploy/rolling_deploy.py diff --git a/setup.py b/setup.py index 073311a..377a05d 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,5 @@ -#!/usr/bin/python - import os -import sys -from CustomInstall import CustomInstall -from setuptools import setup, Command +from setuptools import setup install_requires = [ "boto", @@ -25,23 +21,30 @@ 'coverage', ] + def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() + setup( - name = "License2Deploy", - version = "0.0.1", - author = "Dun and Bradstreet", - author_email = "license2deploy@dandb.com", - description = ("Rolling deploys by changing desired amount of instances AWS EC2 Autoscale Group"), - license = "GPLv3", - keywords = "AWS EC2 AutoScale Group AMI desired capacity", - url = "https://github.com/dandb/License2Deploy", + name="License2Deploy", + version="0.2.0", + author="Dun and Bradstreet", + author_email="license2deploy@dandb.com", + description="Rolling deploys by changing desired amount of instances AWS EC2 Autoscale Group", + license="GPLv3", + keywords="AWS EC2 AutoScale Group AMI desired capacity", + url="https://github.com/dandb/License2Deploy", packages=['License2Deploy'], + entry_points={ + 'console_scripts': [ + 'rolling_deploy = License2Deploy.rolling_deploy:main' + ] + }, include_package_data=True, - install_requires = install_requires, - tests_require = tests_require, + install_requires=install_requires, + tests_require=tests_require, extras_require={'test': tests_require}, - long_description=read('README.md') + '\n\n' + read('CHANGES'), - test_suite = 'tests' + long_description=read('README.md'), + test_suite='tests' ) diff --git a/tests/cloudformation_client_test.py b/tests/cloudformation_client_test.py index 6558634..ee0115c 100644 --- a/tests/cloudformation_client_test.py +++ b/tests/cloudformation_client_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python import os import placebo from boto3.session import Session diff --git a/tests/rolling_deploy_test.py b/tests/rolling_deploy_test.py index 76c4f22..2d397dc 100644 --- a/tests/rolling_deploy_test.py +++ b/tests/rolling_deploy_test.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - import sys import pytest import unittest