Merge pull request #713 from OpenSourcePolicyCenter/bhard/fabfile_dep…

…loy_simplification

Improvements to Create and Update scripts

deploy/fab/ec2-fabfile.py → deploy/fab/ec2_fabfile.py

@@ -36,8 +36,10 @@
 DEPLOYMENT_VERSIONS_ARGS = deploy_versions_cli(ip_address='skip')
 if not 'OSPC_ANACONDA_TOKEN' in os.environ:
     raise ValueError('OSPC_ANACONDA_TOKEN must be defined in env vars')
-check_unmodified()
-#based on https://github.com/ContinuumIO/wakari-deploy/blob/master/ami_creation/fabfile.py
+
+if not os.environ.get("ALLOW_UNCOMMITTED", None) == 'True':
+    check_unmodified()
+
 ssh_transport = logging.getLogger("ssh.transport")
 ssh_transport.disabled = True
 log = logging.getLogger(__name__)
@@ -52,15 +54,6 @@
 # log.addHandler(handler)
 log.addHandler(console)
 
-# ami-65ce000e
-# ami-849e0c84
-# ami-1998e77c
-# ami-af5591eb
-# ami-9f92edfa
-# ami-1d4b1f78
-#AMI_ID = 'ami-1d4b1f78' #SSD in us-east-1
-#AMI_ID = 'ami-dfcab0b5' #SSD in us-east-1
-#AMI_ID = 'ami-dfcab0b5' #SSD in us-east-1
 AMI_ID = 'ami-6c276206' #SSD in us-east-1
 
 use_latest_key = 'True' == os.environ.get('USE_LATEST_KEY', False)
@@ -90,16 +83,14 @@
 env.timeout = 50
 
 KEYNAME = "{}-aei-dropq-{}".format(AMI_ID,os.environ.get('USER', 'ospc'))
-#KEYNAME = "keypair-ospc"
-#SECURITY_GROUP = 'launch-wizard-26'
-#SECURITY_GROUP = 'dropq-security-group'
+KEYFILE = "./" + KEYNAME + ".pem"
 SECURITY_GROUP = 'dropq-iam-group'
 NODE_COUNT = int(os.environ.get('WORKER_NODE_COUNT', 1))
 
 def create_box():
     old_ids = set(i.id for i in ec2.get_only_instances())
     machine = ec2.run_instances(AMI_ID, key_name=KEYNAME, min_count=NODE_COUNT, max_count=NODE_COUNT,
-        security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'm3.medium'))
+        security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'r3.large'))
     new_instances = [i for i in ec2.get_only_instances() if i.id not in old_ids]
     for new_instance in new_instances:
         print("new instance:", new_instance.id)
@@ -186,6 +177,16 @@ def test_ssh2():
         '''
     run('pwd')
 
+def add_public_keys():
+    # Add the necessary public key information to the instance.
+    from boto.s3.connection import S3Connection
+    conn = S3Connection(os.environ.get('AWS_ID'), os.environ.get('AWS_SECRET'))
+    bucket = conn.get_bucket("taxbrain-deploy-assets")
+
+    k = bucket.get_key("keys.txt")
+    keyfile = k.get_contents_as_string()
+    run("echo $'%s' >> ~/.ssh/authorized_keys" % keyfile)
+
 def test_ssh(instance, key_file):
     # needed to convert from unicode to ascii?
     key_file = str(key_file)
@@ -309,36 +310,26 @@ def wait_for_login_prompt(instance, retry_count=20):
     public_dns_name = 'ec2-{}.compute-1.amazonaws.com'.format(ip_address.replace('.','-'))
     connect_to_existing_machine(ip_address, key_filename)
 else:
-    if not use_latest_key:
-        raise ValueError()
-        key_filename = create_keypair()
-
-    else:
-        key_filename = "./latest.pem"
-
     instances = create_box()
     public_dns_names = []
     ip_addresses = []
     for instance in instances:
         print("trying this instance :", instance)
-        subprocess.check_output(['cp', key_filename,
-                                os.path.join(os.path.dirname(key_filename), 'ec2-{}.pem'.format(instance.ip_address))])
+        subprocess.check_output(['cp', KEYFILE,
+                                os.path.join(os.path.dirname(KEYFILE), 'ec2-{}.pem'.format(instance.ip_address))])
         public_dns_names.append(instance.public_dns_name)
         ip_addresses.append(instance.ip_address)
 
 if 'quitafterec2spinup' in sys.argv:
     quit()
 
-#import pdb;pdb.set_trace()
-#test_all_ssh(instances, key_filename)
 
-key_file = str(key_filename)
 fqdns = [str(instance.public_dns_name) for instance in instances]
 env.hosts = [fqdn for fqdn in fqdns]
 env.user = 'ubuntu'
 env.password = ''
-env.key_file = key_file
-env.key_filename = key_file
+env.key_file = KEYFILE
+env.key_filename = KEYFILE
 # forward ssh agent -- equivalent of ssh -A
 env.forward_agent = True
 
@@ -359,6 +350,7 @@ def wait_for_login_prompt(instance, retry_count=20):
 
 
 execute(test_ssh2)
+execute(add_public_keys)
 execute(apt_installs)
 execute(install_deploy_repo)
 execute(install_ogusa_repo)
@@ -368,7 +360,8 @@ def wait_for_login_prompt(instance, retry_count=20):
 execute(reset_server)
 
 for instance in instances:
-    ssh_command = 'ssh -i {key} ubuntu@{ip} "'.format(ip=instance.ip_address, key=key_filename)
+    ssh_command = 'ssh -i {key} ubuntu@{ip} "'.format(ip=instance.ip_address,
+            key=KEYNAME)
     with open("log_{}.log".format(instance.ip_address), 'w') as f:
         f.write(ssh_command)
     print(ssh_command)