From 90954fe91028f324c143e2713e7b7aad313e97aa Mon Sep 17 00:00:00 2001 From: Ash Berlin Date: Tue, 24 Mar 2015 10:25:10 +0000 Subject: [PATCH 1/5] Make getting connection objects easier in fab_tasks We were returning more and more things from get_config, not all of which was used everywhere (and we wanted to add 2 more) and that was just getting silly. Most of what we were returning was only used to pass in to EC2 or Cloudformation constructors, so lets just make it easier to create those instances. --- bootstrap_cfn/fab_tasks.py | 62 ++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index b19ea3c..8bbb577 100644 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -69,7 +69,7 @@ def get_stack_name(): return env.stack_name return "%s-%s" % (env.application, env.environment) -def get_config(): +def _validate_fabric_env(): if env.aws is None: print "\n[ERROR] Please specify an AWS account, e.g 'aws:dev'" sys.exit(1) @@ -85,15 +85,21 @@ def get_config(): if not hasattr(env, 'stack_passwords'): env.stack_passwords={} - aws_config = AWSConfig(env.aws) + +def get_config(): + _validate_fabric_env() project_config = ProjectConfig( env.config, env.environment, passwords=env.stack_passwords) cfn_config = ConfigParser(project_config.config, get_stack_name()) - cfn = Cloudformation(aws_config) - return aws_config, cfn, cfn_config + return cfn_config + +def get_connection(klass): + _validate_fabric_env() + aws_config = AWSConfig(env.aws) + return klass(aws_config) @task def cfn_delete(force=False): @@ -102,7 +108,8 @@ def cfn_delete(force=False): if not x in ['y','Y','Yes','yes']: sys.exit(1) stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() + cfn_config = get_config() + cfn = get_connection(Cloudformation) cfn.delete(stack_name) print "\n\nSTACK {0} DELETING...".format(stack_name) @@ -116,16 +123,18 @@ def cfn_delete(force=False): cfn.wait_for_stack_missing(stack_name) print "Stack successfully deleted" if 'ssl' in cfn_config.data: - iam = IAM(aws_config) + iam = get_connection(IAM) iam.delete_ssl_certificate(cfn_config.ssl(), stack_name) @task def cfn_create(): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() + cfn_config = get_config() + + cfn = get_connection(Cloudformation) #Upload any SSL certs that we may need for the stack. if 'ssl' in cfn_config.data: - iam = IAM(aws_config) + iam = get_connection(IAM) iam.upload_ssl_certificate(cfn_config.ssl(), stack_name) #Useful for debug #print cfn_config.process() @@ -150,29 +159,17 @@ def cfn_create(): print 'Failed to create stack: {0}'.format(stack) #So delete the SSL cert that we uploaded if 'ssl' in cfn_config.data: - iam = IAM(aws_config) iam.delete_ssl_certificate(cfn_config.ssl(), stack_name) def get_stack_instances_ips(stack_name): - if env.aws is None: - print "\n[ERROR] Please specify an AWS account, e.g 'aws:dev'" - sys.exit(1) - - aws_config = AWSConfig(env.aws) - cfn = Cloudformation(aws_config) - ec2 = EC2(aws_config) + cfn = get_connection(Cloudformation) + ec2 = get_connection(EC2) instance_id_list = cfn.get_stack_instance_ids(stack_name) return ec2.get_instance_public_ips(instance_id_list) @task def get_stack_addresses(): - if env.environment is None: - print "\n[ERROR] Please specify an environment, e.g 'environment:dev'" - sys.exit(1) - if env.application is None: - print "\n[ERROR] Please specify an application, e.g 'application:peoplefinder'" - sys.exit(1) stack_name = get_stack_name() res = get_stack_instances_ips(stack_name) print res @@ -182,9 +179,7 @@ def get_stack_addresses(): @task def find_master(): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() - ec2 = EC2(aws_config) - stack_name = get_stack_name() + ec2 = get_connection(EC2) master = ec2.get_master_instance(stack_name).ip_address print 'Salt master public address: {0}'.format(master) return master @@ -192,15 +187,14 @@ def find_master(): def get_stack_instances_ips(stack_name): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() - ec2 = EC2(aws_config) + ec2 = get_connection(EC2) instance_id_list = cfn.get_stack_instance_ids(stack_name) def get_candidate_minions(): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() - ec2 = EC2(aws_config) + cfn = get_connection(Cloudformation) + ec2 = get_connection(EC2) instance_ids = cfn.get_stack_instance_ids(stack_name) stack_name = get_stack_name() master_instance_id = ec2.get_master_instance(stack_name).id @@ -211,8 +205,7 @@ def get_candidate_minions(): @task def install_minions(): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() - ec2 = EC2(aws_config) + ec2 = get_connection(EC2) print "Waiting for SSH on all instances..." ec2.wait_for_ssh(stack_name) candidates = get_candidate_minions() @@ -244,8 +237,8 @@ def install_minions(): @task def install_master(): stack_name = get_stack_name() - aws_config, cfn, cfn_config = get_config() - ec2 = EC2(aws_config) + ec2 = get_connection(EC2) + cfn = get_connection(Cloudformation) print "Waiting for SSH on all instances..." ec2.wait_for_ssh(stack_name) instance_ids = cfn.get_stack_instance_ids(stack_name) @@ -311,6 +304,9 @@ def rsync(): sys.exit(1) stack_name = get_stack_name() + + ec2 = get_connection(EC2) + work_dir = os.path.join('..', '{0}-deploy'.format(env.application)) # LOAD AWS CONFIG FROM ~/.config.yaml aws_config = AWSConfig(env.aws) From 6a29998e7cd6a236156e2257e591bc0d0c3054f8 Mon Sep 17 00:00:00 2001 From: Ash Berlin Date: Tue, 24 Mar 2015 10:34:39 +0000 Subject: [PATCH 2/5] Remove unused task and functions. I know it was unused because the implementation would end up returning IDs, not IPs. --- bootstrap_cfn/fab_tasks.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index 8bbb577..6dc269e 100644 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -161,20 +161,6 @@ def cfn_create(): if 'ssl' in cfn_config.data: iam.delete_ssl_certificate(cfn_config.ssl(), stack_name) -def get_stack_instances_ips(stack_name): - cfn = get_connection(Cloudformation) - ec2 = get_connection(EC2) - instance_id_list = cfn.get_stack_instance_ids(stack_name) - return ec2.get_instance_public_ips(instance_id_list) - - -@task -def get_stack_addresses(): - stack_name = get_stack_name() - res = get_stack_instances_ips(stack_name) - print res - return res - @task def find_master(): @@ -185,12 +171,6 @@ def find_master(): return master -def get_stack_instances_ips(stack_name): - stack_name = get_stack_name() - ec2 = get_connection(EC2) - instance_id_list = cfn.get_stack_instance_ids(stack_name) - - def get_candidate_minions(): stack_name = get_stack_name() cfn = get_connection(Cloudformation) From 3038c09589cdc17894d1e9014e8c849a424ad8a2 Mon Sep 17 00:00:00 2001 From: Ash Berlin Date: Tue, 24 Mar 2015 10:40:50 +0000 Subject: [PATCH 3/5] PEP8 fixes Including showing us that we have some dead code paths :/ --- bootstrap_cfn/fab_tasks.py | 48 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index 6dc269e..15c59be 100644 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -1,11 +1,7 @@ #!/usr/bin/env python import sys -import urllib2 -import json import random -import yaml -import time from bootstrap_cfn.config import AWSConfig, ProjectConfig, ConfigParser from bootstrap_cfn.cloudformation import Cloudformation from bootstrap_cfn.ec2 import EC2 @@ -13,9 +9,8 @@ import os -from fabric.api import env, task, sudo, execute, run, parallel, settings -from fabric.contrib.project import rsync_project, upload_project -from fabric.operations import put +from fabric.api import env, task, sudo +from fabric.contrib.project import upload_project # GLOBAL VARIABLES env.application = None @@ -30,6 +25,8 @@ # imported in a fabfile. path = env.real_fabfile or os.getcwd() sys.path.append(os.path.dirname(path)) + + @task def aws(x): env.aws = str(x).lower() @@ -64,11 +61,13 @@ def blocking(x): def user(x): env.user = x + def get_stack_name(): if hasattr(env, 'stack_name'): return env.stack_name return "%s-%s" % (env.application, env.environment) + def _validate_fabric_env(): if env.aws is None: print "\n[ERROR] Please specify an AWS account, e.g 'aws:dev'" @@ -84,7 +83,8 @@ def _validate_fabric_env(): sys.exit(1) if not hasattr(env, 'stack_passwords'): - env.stack_passwords={} + env.stack_passwords = {} + def get_config(): _validate_fabric_env() @@ -96,16 +96,18 @@ def get_config(): cfn_config = ConfigParser(project_config.config, get_stack_name()) return cfn_config + def get_connection(klass): _validate_fabric_env() aws_config = AWSConfig(env.aws) return klass(aws_config) + @task def cfn_delete(force=False): if not force: x = raw_input("Are you really sure you want to blow away the whole stack!? (y/n)\n") - if not x in ['y','Y','Yes','yes']: + if x not in ['y', 'Y', 'Yes', 'yes']: sys.exit(1) stack_name = get_stack_name() cfn_config = get_config() @@ -114,7 +116,6 @@ def cfn_delete(force=False): print "\n\nSTACK {0} DELETING...".format(stack_name) if hasattr(env, 'blocking') and env.blocking.lower() == 'false': - print stacks print 'Running in non blocking mode. Exiting.' sys.exit(0) @@ -126,24 +127,24 @@ def cfn_delete(force=False): iam = get_connection(IAM) iam.delete_ssl_certificate(cfn_config.ssl(), stack_name) + @task def cfn_create(): stack_name = get_stack_name() cfn_config = get_config() cfn = get_connection(Cloudformation) - #Upload any SSL certs that we may need for the stack. + # Upload any SSL certs that we may need for the stack. if 'ssl' in cfn_config.data: iam = get_connection(IAM) iam.upload_ssl_certificate(cfn_config.ssl(), stack_name) - #Useful for debug - #print cfn_config.process() + # Useful for debug + # print cfn_config.process() # Inject security groups in stack template and create stacks. stack = cfn.create(stack_name, cfn_config.process()) print "\n\nSTACK {0} CREATING...".format(stack_name) if hasattr(env, 'blocking') and env.blocking.lower() == 'false': - print stacks print 'Running in non blocking mode. Exiting.' sys.exit(0) @@ -157,7 +158,7 @@ def cfn_create(): print 'Successfully built stack {0}.'.format(stack) else: print 'Failed to create stack: {0}'.format(stack) - #So delete the SSL cert that we uploaded + # So delete the SSL cert that we uploaded if 'ssl' in cfn_config.data: iam.delete_ssl_certificate(cfn_config.ssl(), stack_name) @@ -214,6 +215,7 @@ def install_minions(): env.host_string = 'ubuntu@%s' % master_public_ip sudo('salt-key -y -A') + @task def install_master(): stack_name = get_stack_name() @@ -283,38 +285,32 @@ def rsync(): print "\n[ERROR] Please specify a config file, e.g 'config:/tmp/sample-application.yaml'" sys.exit(1) - stack_name = get_stack_name() - - ec2 = get_connection(EC2) - work_dir = os.path.join('..', '{0}-deploy'.format(env.application)) - # LOAD AWS CONFIG FROM ~/.config.yaml - aws_config = AWSConfig(env.aws) project_config = ProjectConfig(env.config, env.environment) cfg = project_config.config local_salt_dir = os.path.join( work_dir, - cfg.get('salt',{}).get( + cfg.get('salt', {}).get( 'local_salt_dir', 'salt'), '.') local_pillar_dir = os.path.join( work_dir, - cfg.get('salt',{}).get( + cfg.get('salt', {}).get( 'local_pillar_dir', 'pillar'), '.') local_vendor_dir = os.path.join( work_dir, - cfg.get('salt',{}).get( + cfg.get('salt', {}).get( 'local_vendor_dir', 'vendor'), '.') - remote_state_dir = cfg.get('salt',{}).get('remote_state_dir', '/srv/salt') - remote_pillar_dir = cfg.get('salt',{}).get('remote_pillar_dir', '/srv/pillar') + remote_state_dir = cfg.get('salt', {}).get('remote_state_dir', '/srv/salt') + remote_pillar_dir = cfg.get('salt', {}).get('remote_pillar_dir', '/srv/pillar') # if not os.path.exists(local_state_dir): # shake(work_dir) From e051375712c34997a8ddd4220e1c495edab26cf1 Mon Sep 17 00:00:00 2001 From: Ash Berlin Date: Tue, 24 Mar 2015 10:57:35 +0000 Subject: [PATCH 4/5] Tidy up config handling in rsync task --- bootstrap_cfn/fab_tasks.py | 48 +++++++++----------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/bootstrap_cfn/fab_tasks.py b/bootstrap_cfn/fab_tasks.py index 15c59be..670e708 100644 --- a/bootstrap_cfn/fab_tasks.py +++ b/bootstrap_cfn/fab_tasks.py @@ -267,53 +267,35 @@ def install_master(): @task def rsync(): - if env.aws is None: - print "\n[ERROR] Please specify an AWS account, e.g 'aws:dev'" - sys.exit(1) - if env.environment is None: - print "\n[ERROR] Please specify an environment, e.g 'environment:dev'" - sys.exit(1) - if env.application is None: - print "\n[ERROR] Please specify an application, e.g 'application:peoplefinder'" - sys.exit(1) if env.config is None: # check if there is a deploy repo in a predefined location app_yaml = '../{0}-deploy/{0}.yaml'.format(env.application) if os.path.exists(app_yaml): env.config = app_yaml - else: - print "\n[ERROR] Please specify a config file, e.g 'config:/tmp/sample-application.yaml'" - sys.exit(1) + + _validate_fabric_env() work_dir = os.path.join('..', '{0}-deploy'.format(env.application)) project_config = ProjectConfig(env.config, env.environment) cfg = project_config.config + salt_cfg = cfg.get('salt', {}) local_salt_dir = os.path.join( work_dir, - cfg.get('salt', {}).get( - 'local_salt_dir', - 'salt'), + salt_cfg.get('local_salt_dir', 'salt'), '.') local_pillar_dir = os.path.join( work_dir, - cfg.get('salt', {}).get( - 'local_pillar_dir', - 'pillar'), + salt_cfg.get('local_pillar_dir', 'pillar'), '.') local_vendor_dir = os.path.join( work_dir, - cfg.get('salt', {}).get( - 'local_vendor_dir', - 'vendor'), + salt_cfg.get('local_vendor_dir', 'vendor'), '.') - remote_state_dir = cfg.get('salt', {}).get('remote_state_dir', '/srv/salt') - remote_pillar_dir = cfg.get('salt', {}).get('remote_pillar_dir', '/srv/pillar') - - # if not os.path.exists(local_state_dir): - # shake(work_dir) + remote_state_dir = salt_cfg.get('remote_state_dir', '/srv/salt') + remote_pillar_dir = salt_cfg.get('remote_pillar_dir', '/srv/pillar') master_ip = find_master() env.host_string = '{0}@{1}'.format(env.user, master_ip) @@ -321,16 +303,11 @@ def rsync(): sudo('mkdir -p {0}'.format(remote_pillar_dir)) upload_project( remote_dir=remote_state_dir, - local_dir=os.path.join( - local_vendor_dir, - '_root', - '.'), + local_dir=os.path.join(local_vendor_dir, '_root', '.'), use_sudo=True) upload_project( remote_dir='/srv/', - local_dir=os.path.join( - local_vendor_dir, - 'formula-repos'), + local_dir=os.path.join(local_vendor_dir, 'formula-repos'), use_sudo=True) upload_project( remote_dir=remote_state_dir, @@ -338,8 +315,5 @@ def rsync(): use_sudo=True) upload_project( remote_dir=remote_pillar_dir, - local_dir=os.path.join( - local_pillar_dir, - env.environment, - '.'), + local_dir=os.path.join(local_pillar_dir, env.environment, '.'), use_sudo=True) From 81da6acd4cbd6e7b17b6e420504ef78abd63eba9 Mon Sep 17 00:00:00 2001 From: mattmb Date: Tue, 24 Mar 2015 11:18:57 +0000 Subject: [PATCH 5/5] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b47392..27b548e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * Move bootstrap script execution to fabric tasks. * Fix bug in wait_for_ssh when no instances are running. * Add conditional statement in fabfile to check for ssl cert on roll back before trying to delete it. + * Refactor fab_tasks get_config method to not return *every* config item. Also PEP8 fixes and removing unused functions. ## Version 0.1