From 05e49c280efa9f26fbb427f694b1d6f36ed84441 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Mon, 10 Oct 2016 16:00:51 -0700 Subject: [PATCH 01/34] Fix start to set initial_size and min_size correctly --- cli/cfncluster/cfncluster.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index 7755a91d53..cee69a5bcc 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -173,11 +173,13 @@ def start(args): # Set asg limits max_queue_size = [param[1] for param in config.parameters if param[0] == 'MaxQueueSize'] max_queue_size = max_queue_size[0] if len(max_queue_size) > 0 else 10 - initial_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] if args.reset_desired else [0] - initial_queue_size = initial_queue_size[0] if len(initial_queue_size) > 0 else 0 + desired_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] + desired_queue_size = desired_queue_size[0] if len(desired_queue_size) > 0 else 2 + min_queue_size = [desired_queue_size for param in config.parameters if param[0] == 'MaintainInitialSize' and param[1] == "true"] + min_queue_size = min_queue_size[0] if len(min_queue_size) > 0 else 0 asg = get_asg(stack_name=stack_name, config=config) - set_asg_limits(asg=asg, min=initial_queue_size, max=max_queue_size, desired=initial_queue_size) + set_asg_limits(asg=asg, min=min_queue_size, max=max_queue_size, desired=desired_queue_size) # Poll for status poll_master_server_state(stack_name, config) From d29a0b267a4026ac0f731ea703b79fde8b08fb4d Mon Sep 17 00:00:00 2001 From: duff Date: Tue, 11 Oct 2016 16:03:15 -0700 Subject: [PATCH 02/34] adding m4.16xlarge instance --- cloudformation/cfncluster.cfn.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index b2860ea489..58f13ad45d 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -756,6 +756,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -810,6 +811,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -1724,6 +1726,10 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, + "m4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "m4.10xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" From 125c23a79ea99876c185d235887482dcaa552192 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Mon, 17 Oct 2016 08:18:41 -0700 Subject: [PATCH 03/34] * Fixing p2 instance types. Fixes #143 --- cloudformation/cfncluster.cfn.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 58f13ad45d..4dac9b5808 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -740,7 +740,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", @@ -795,7 +795,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", From 06aece4a25d45409762dbe54a1e76cfe452db475 Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Mon, 10 Oct 2016 16:00:51 -0700 Subject: [PATCH 04/34] Fix start to set initial_size and min_size correctly --- cli/cfncluster/cfncluster.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index f2af9414f9..c1e7c869c0 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -173,11 +173,13 @@ def start(args): # Set asg limits max_queue_size = [param[1] for param in config.parameters if param[0] == 'MaxQueueSize'] max_queue_size = max_queue_size[0] if len(max_queue_size) > 0 else 10 - initial_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] if args.reset_desired else [0] - initial_queue_size = initial_queue_size[0] if len(initial_queue_size) > 0 else 0 + desired_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] + desired_queue_size = desired_queue_size[0] if len(desired_queue_size) > 0 else 2 + min_queue_size = [desired_queue_size for param in config.parameters if param[0] == 'MaintainInitialSize' and param[1] == "true"] + min_queue_size = min_queue_size[0] if len(min_queue_size) > 0 else 0 asg = get_asg(stack_name=stack_name, config=config) - set_asg_limits(asg=asg, min=initial_queue_size, max=max_queue_size, desired=initial_queue_size) + set_asg_limits(asg=asg, min=min_queue_size, max=max_queue_size, desired=desired_queue_size) # Poll for status poll_master_server_state(stack_name, config) From b81a164c932fef3d5a9ea312ed8d391cfefe414b Mon Sep 17 00:00:00 2001 From: duff Date: Tue, 11 Oct 2016 16:03:15 -0700 Subject: [PATCH 05/34] adding m4.16xlarge instance --- cloudformation/cfncluster.cfn.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index b2860ea489..58f13ad45d 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -756,6 +756,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -810,6 +811,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -1724,6 +1726,10 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, + "m4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "m4.10xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" From 68a8b3e0db4f549f813df0eb6a2a4979162d7cf5 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Mon, 17 Oct 2016 08:18:41 -0700 Subject: [PATCH 06/34] * Fixing p2 instance types. Fixes #143 --- cloudformation/cfncluster.cfn.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 58f13ad45d..4dac9b5808 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -740,7 +740,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", @@ -795,7 +795,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", From 9d0b024c0d3f11c090ad1e446e7ac02e01d31058 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 11:34:25 -0800 Subject: [PATCH 07/34] stage1 changes from futurize --- cli/cfncluster/cfncluster.py | 42 +++++++++++++++++---------------- cli/cfncluster/cfnconfig.py | 8 ++++--- cli/cfncluster/cli.py | 5 ++-- cli/cfncluster/config_sanity.py | 5 ++-- cli/cfncluster/easyconfig.py | 20 +++++++++------- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index c1e7c869c0..8be8873d84 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -19,13 +21,13 @@ import socket import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') def version(args): config = cfnconfig.CfnClusterConfig(args) - print config.version + print(config.version) def create(args): print('Starting: %s' % (args.cluster_name)) @@ -50,7 +52,7 @@ def create(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -75,12 +77,12 @@ def create(args): time.sleep(5) outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) else: status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -118,7 +120,7 @@ def update(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -142,7 +144,7 @@ def update(args): status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -161,7 +163,7 @@ def start(args): response = ec2conn.start_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -203,7 +205,7 @@ def stop(args): response = ec2conn.stop_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -226,7 +228,7 @@ def list(args): print('%s' % (stack.stack_name[11:])) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) else: raise e except KeyboardInterrupt: @@ -245,7 +247,7 @@ def get_master_server_id(stack_name, config): resources = cfnconn.describe_stack_resources(stack_name) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -282,7 +284,7 @@ def poll_master_server_state(stack_name, config): sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -305,7 +307,7 @@ def get_ec2_instances(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -336,7 +338,7 @@ def get_asg(stack_name, config): return asgconn.get_all_groups(names=[asg_id])[0] except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -365,7 +367,7 @@ def get_asg_ids(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -435,22 +437,22 @@ def status(args): if state == 'running': outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) elif ((status == 'ROLLBACK_COMPLETE') or (status == 'CREATE_FAILED') or (status == 'DELETE_FAILED') or (status == 'UPDATE_ROLLBACK_COMPLETE')): events = cfnconn.describe_stack_events(stack) for event in events: if ((event.resource_status == 'CREATE_FAILED') or (event.resource_status == 'DELETE_FAILED') or (event.resource_status == 'UPDATE_FAILED')): - print event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ - event.resource_status_reason + print(event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ + event.resource_status_reason) else: sys.stdout.write('\n') sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): sys.stdout.write('\r') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -491,7 +493,7 @@ def delete(args): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index c77177d09e..9380fb1bc1 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -16,7 +18,7 @@ import pkg_resources import json import urllib2 -import config_sanity +from . import config_sanity import boto.cloudformation def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): @@ -36,7 +38,7 @@ def __init__(self, args): self.parameters = [] self.version = pkg_resources.get_distribution("cfncluster").version self.__DEFAULT_CONFIG = False - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ # Determine config file name based on args or default if args.config_file is not None: @@ -119,7 +121,7 @@ def __init__(self, args): except ConfigParser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True: pass else: diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index ddb6743d5e..f317af989e 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -15,8 +16,8 @@ import platform import json -import cfncluster -import easyconfig +from . import cfncluster +from . import easyconfig def create(args): cfncluster.create(args) diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index e8f11ed9df..643c135df6 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -93,10 +94,10 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty else: try: urllib2.urlopen(resource_value) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: print(e.code) sys.exit(1) - except urllib2.URLError, e: + except urllib2.URLError as e: print(e.args) sys.exit(1) # EC2 EBS Snapshot Id diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index 7273599a14..a735618163 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance with the @@ -16,7 +18,7 @@ import os import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') @@ -31,9 +33,9 @@ def prompt(prompt, default_value=None, hidden=False, options=None): user_prompt = user_prompt + ']: ' if isinstance(options, list): - print 'Acceptable Values for %s: ' % prompt + print('Acceptable Values for %s: ' % prompt) for o in options: - print ' %s' % o + print(' %s' % o) var = raw_input(user_prompt) @@ -79,8 +81,8 @@ def list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name): keynames.append(key.name) if not keynames: - print 'ERROR: No keys found in region ' + aws_region_name - print 'Please create an EC2 keypair before continuing' + print('ERROR: No keys found in region ' + aws_region_name) + print('Please create an EC2 keypair before continuing') sys.exit(1) return keynames @@ -93,8 +95,8 @@ def list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name): vpcids.append(vpc.id) if not vpcids: - print 'ERROR: No vpcs found in region ' + aws_region_name - print 'Please create an EC2 vpcpair before continuing' + print('ERROR: No vpcs found in region ' + aws_region_name) + print('Please create an EC2 vpcpair before continuing') sys.exit(1) return vpcids @@ -107,8 +109,8 @@ def list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_ subnetids.append(subnet.id) if not subnetids: - print 'ERROR: No subnets found in region ' + aws_region_name - print 'Please create an EC2 subnetpair before continuing' + print('ERROR: No subnets found in region ' + aws_region_name) + print('Please create an EC2 subnetpair before continuing') sys.exit(1) return subnetids From f8d5ebe938268e2fb5485c363b7e07e24886632c Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 14:10:59 -0800 Subject: [PATCH 08/34] Initial support for Python 3 --- cli/cfncluster/cfncluster.py | 1 + cli/cfncluster/cfnconfig.py | 46 ++++++++++++++++++--------------- cli/cfncluster/cli.py | 2 ++ cli/cfncluster/config_sanity.py | 12 +++++---- cli/cfncluster/easyconfig.py | 13 ++++++---- cli/setup.py | 5 +++- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index 8be8873d84..9d019ab806 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -11,6 +11,7 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from builtins import str import sys import boto.cloudformation import boto.ec2.autoscale diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index 9380fb1bc1..f92f328875 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -11,13 +11,17 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import object +import configparser import os import sys import inspect import pkg_resources import json -import urllib2 +import urllib.request, urllib.error, urllib.parse from . import config_sanity import boto.cloudformation @@ -31,7 +35,7 @@ def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): return __cli_template -class CfnClusterConfig: +class CfnClusterConfig(object): def __init__(self, args): self.args = args @@ -60,7 +64,7 @@ def __init__(self, args): sys.exit(1) - __config = ConfigParser.ConfigParser() + __config = configparser.ConfigParser() __config.read(self.__config_file) # Determine the EC2 region to used used or default to us-east-1 @@ -73,17 +77,17 @@ def __init__(self, args): else: try: self.region = __config.get('aws', 'aws_region_name') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.region = 'us-east-1' # Check if credentials have been provided in config try: self.aws_access_key_id = __config.get('aws', 'aws_access_key_id') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_access_key_id=None try: self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_secret_access_key=None # Determine which cluster template will be used @@ -104,12 +108,12 @@ def __init__(self, args): # Check if package updates should be checked try: self.__update_check = __config.getboolean('global', 'update_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__update_check = True if self.__update_check == True: try: - __latest = json.loads(urllib2.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] + __latest = json.loads(urllib.request.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] if self.version < __latest: print('warning: There is a newer version %s of cfncluster available.' % __latest) except Exception: @@ -118,7 +122,7 @@ def __init__(self, args): # Check if config sanity should be run try: self.__sanity_check = __config.getboolean('global', 'sanity_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it __args_func = self.args.func.__name__ @@ -136,7 +140,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'EC2KeyPair', self.key_name) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: print("ERROR: Missing key_name option in [%s] section." % self.__cluster_section) sys.exit(1) self.parameters.append(('KeyName', self.key_name)) @@ -155,7 +159,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'URL', self.template_url) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: if self.region == 'us-gov-west-1': self.template_url = ('https://s3-%s.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.region, self.version)) @@ -191,7 +195,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__vpc_options.get(key)[1],__temp__) self.parameters.append((self.__vpc_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all cluster section options @@ -225,7 +229,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__cluster_options.get(key)[1],__temp__) self.parameters.append((self.__cluster_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Merge tags from config with tags from command line args @@ -234,7 +238,7 @@ def __init__(self, args): try: tags = __config.get(self.__cluster_section, 'tags') self.tags = json.loads(tags); - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: if args.tags is not None: @@ -251,7 +255,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__ebs_section = ('ebs %s' % self.__ebs_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all EBS options @@ -274,7 +278,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__ebs_options.get(key)[1],__temp__) self.parameters.append((self.__ebs_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -287,7 +291,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__scaling_section = ('scaling %s' % self.__scaling_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all scaling options @@ -309,7 +313,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__scaling_options.get(key)[1],__temp__) self.parameters.append((self.__scaling_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -318,9 +322,9 @@ def __init__(self, args): try: if self.args.extra_parameters is not None: self.parameters = dict(self.parameters) - self.__temp_dict = dict(self.parameters.items() + self.args.extra_parameters.items()) + self.__temp_dict = dict(list(self.parameters.items()) + list(self.args.extra_parameters.items())) self.__dictlist = [] - for key, value in self.__temp_dict.iteritems(): + for key, value in self.__temp_dict.items(): temp = [str(key),str(value)] self.__dictlist.append(temp) self.parameters = self.__dictlist diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index f317af989e..1b14dbbd10 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -83,6 +83,8 @@ def main(): help='do not wait for stack events, after executing stack command') subparsers = parser.add_subparsers() + subparsers.required = True + subparsers.dest = 'command' pcreate = subparsers.add_parser('create', help='creates a cluster') pcreate.add_argument("cluster_name", type=str, default=None, diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index 643c135df6..d7503f9081 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -10,12 +10,14 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from future import standard_library +standard_library.install_aliases() __author__ = 'dougalb' import boto.ec2 import boto.vpc -import urllib2 -from urlparse import urlparse +import urllib.request, urllib.error, urllib.parse +from urllib.parse import urlparse import boto.exception import sys @@ -93,11 +95,11 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty pass else: try: - urllib2.urlopen(resource_value) - except urllib2.HTTPError as e: + urllib.request.urlopen(resource_value) + except urllib.error.HTTPError as e: print(e.code) sys.exit(1) - except urllib2.URLError as e: + except urllib.error.URLError as e: print(e.args) sys.exit(1) # EC2 EBS Snapshot Id diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index a735618163..b8575c7522 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -11,7 +11,10 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import input +import configparser import sys import boto.ec2 import boto.vpc @@ -37,7 +40,7 @@ def prompt(prompt, default_value=None, hidden=False, options=None): for o in options: print(' %s' % o) - var = raw_input(user_prompt) + var = input(user_prompt) if var == '': return default_value @@ -123,7 +126,7 @@ def configure(args): else: config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config')) - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() # Check if configuration file exists if os.path.isfile(config_file): @@ -156,9 +159,9 @@ def configure(args): for section in sections: try: config.add_section(section['__name__']) - except ConfigParser.DuplicateSectionError: + except configparser.DuplicateSectionError: pass - for key, value in section.iteritems(): + for key, value in section.items(): # Only update configuration if not set if value is not None and key is not '__name__': config.set(section['__name__'], key, value) diff --git a/cli/setup.py b/cli/setup.py index 6b3671aa52..9560c1e9c7 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -21,13 +21,16 @@ def read(fname): console_scripts = ['cfncluster = cfncluster.cli:main'] version = "1.3.1" -requires = ['boto>=2.42.0', 'awscli>=1.10.56'] +requires = ['boto>=2.42.0', 'awscli>=1.10.56', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): # For python2.6 we have to require argparse since it # was not in stdlib until 2.7. requires.append('argparse>=1.4.0') +if sys.version_info.major == 2: + requires.append('configparser>=3.5.0') + setup( name = "cfncluster", version = version, From f871f6ab69394f17da430c03216b3954cad271ac Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:03 -0800 Subject: [PATCH 09/34] Correcting version check for Python 2.6 --- cli/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/setup.py b/cli/setup.py index 9560c1e9c7..0924e78229 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -28,7 +28,7 @@ def read(fname): # was not in stdlib until 2.7. requires.append('argparse>=1.4.0') -if sys.version_info.major == 2: +if sys.version_info[0] == 2: requires.append('configparser>=3.5.0') setup( From c1c7972a4f82667380f706b7d2e4a78d14e741db Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:22 -0800 Subject: [PATCH 10/34] Improving tests --- tests/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index d9bc48948d..ed783e0541 100644 --- a/tests/test.sh +++ b/tests/test.sh @@ -3,4 +3,6 @@ set -x echo $PATH -which cfncluster \ No newline at end of file +which cfncluster +cfncluster version +cfncluster --help From 27df3ca839f98c979517dcadf1052854576b8abc Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:42 -0800 Subject: [PATCH 11/34] Updated CHANGELOG.rst --- CHANGELOG.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ce00b7698d..a7d45b2b03 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ CHANGELOG ========= +develop +======= +* updates:``cfncluster``: Moved to Apache 2.0 license +* updates:``cfncluster``: Support for Python 3 + 1.3.1 ===== * feature:``ami``: Added support for Ubuntu 16.04 LTS @@ -198,4 +203,4 @@ changes should be non-instrusive. 0.0.5 ===== -* Release on GitHub and PyPi \ No newline at end of file +* Release on GitHub and PyPi From 33bd656fbf2b8fab96ddce127d3bb9837cbd81e0 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:31:27 -0800 Subject: [PATCH 12/34] Adding Python 3.6 build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3c0ddaf190..9440395f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" install: - cd cli From 7a64eef5d48196b9f4d810c997ee4cb269b13daf Mon Sep 17 00:00:00 2001 From: t859531 Date: Wed, 22 Mar 2017 19:05:52 -0400 Subject: [PATCH 13/34] Documentation fixes configuration.rst 1. Corrected syntax error in Tag usage example 2. Corrected minimum size of root vol for master and compute node to 15 instead of 10 iam.rst 1. Added additional IAM permissions needed for clean start and clean delete/rollback --- docs/source/configuration.rst | 10 +++++----- docs/source/iam.rst | 20 ++++++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index eb9e4ad09f..9af4945bae 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -273,17 +273,17 @@ master_root_volume_size """"""""""""""""""""""" MasterServer root volume size in GB. (AMI must support growroot) -Defaults to 10 in default template. :: +Defaults to 15 in default template. :: - master_root_volume_size = 10 + master_root_volume_size = 15 compute_root_volume_size """""""""""""""""""""""" ComputeFleet root volume size in GB. (AMI must support growroot) -Defaults to 10 in default template. :: +Defaults to 15 in default template. :: - compute_root_volume_size = 10 + compute_root_volume_size = 15 base_os """"""" @@ -378,7 +378,7 @@ Tags are JSON formatted and should not have quotes outside the curly braces. See `AWS CloudFormation Resource Tags Type `_. :: - tags = {"key": "value", "key2", "value2"} + tags = {"key" : "value", "key2" : "value2"} .. _vpc_section: diff --git a/docs/source/iam.rst b/docs/source/iam.rst index 21e05cc2b2..c990181799 100644 --- a/docs/source/iam.rst +++ b/docs/source/iam.rst @@ -144,7 +144,8 @@ CfnClusterUserPolicy "ec2:DescribeVpcAttribute", "ec2:DescribeAddresses", "ec2:CreateTags", - "ec2:DescribeNetworkInterfaces" + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeAvailabilityZones" ], "Effect": "Allow", "Resource": "*" @@ -162,7 +163,10 @@ CfnClusterUserPolicy "ec2:CreateNetworkInterface", "ec2:CreateSecurityGroup", "ec2:ModifyVolumeAttribute", - "ec2:ModifyNetworkInterfaceAttribute" + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:DeleteNetworkInterface", + "ec2:DeleteVolume", + "ec2:TerminateInstances" ], "Effect": "Allow", "Resource": "*" @@ -200,7 +204,8 @@ CfnClusterUserPolicy { "Sid": "DynamoDBModify", "Action": [ - "dynamodb:CreateTable" + "dynamodb:CreateTable", + "dynamodb:DeleteTable" ], "Effect": "Allow", "Resource": "*" @@ -225,7 +230,8 @@ CfnClusterUserPolicy "Sid": "SQSModify", "Action": [ "sqs:CreateQueue", - "sqs:SetQueueAttributes" + "sqs:SetQueueAttributes", + "sqs:DeleteQueue" ], "Effect": "Allow", "Resource": "*" @@ -233,7 +239,8 @@ CfnClusterUserPolicy { "Sid": "SNSDescribe", "Action": [ - "sns:ListTopics" + "sns:ListTopics", + “sns:GetTopicAttributes" ], "Effect": "Allow", "Resource": "*" @@ -242,7 +249,8 @@ CfnClusterUserPolicy "Sid": "SNSModify", "Action": [ "sns:CreateTopic", - "sns:Subscribe" + "sns:Subscribe", + “sns:DeleteTopic” ], "Effect": "Allow", "Resource": "*" From a0af8254626399e18aa4e7c06615f588f09e387e Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 27 Apr 2017 16:24:17 -0700 Subject: [PATCH 14/34] Adding YAML version of the template. :-) --- cloudformation/cfncluster.cfn.yaml | 3093 ++++++++++++++++++++++++++++ 1 file changed, 3093 insertions(+) create mode 100644 cloudformation/cfncluster.cfn.yaml diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml new file mode 100644 index 0000000000..31b8a99229 --- /dev/null +++ b/cloudformation/cfncluster.cfn.yaml @@ -0,0 +1,3093 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' + +Description: 'AWS CloudFormation Sample Template cfncluster.cfn.yaml: Sample template + showing an framework for deploying master + compute type clusters on AWS. **WARNING** + This template creates AWS resources. You will be billed for the AWS resources used + if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1' + +Metadata: + AWS::CloudFormation::Designer: + 0824f592-a72b-4bb5-8c67-4ce6df509cdd: + embeds: [] + position: + x: -430 + y: 30 + size: + height: 240 + width: 240 + z: 1 + 0c5a1e92-ffdd-4344-8ba0-65270e5fd472: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 390 + y: 30 + size: + height: 60 + width: 60 + z: 1 + 127f78c3-d47a-4770-8aad-3fd733c3e1b0: + embeds: [] + isrelatedto: + - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + position: + x: 60 + y: 680 + size: + height: 60 + width: 60 + z: 1 + 210b5501-b505-4ee1-917c-a20cae3ac837: + embeds: [] + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + position: + x: 360 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 23ed10e0-7726-4099-8370-4116c0bc78a2: + embeds: [] + isrelatedto: + - 8e63b592-a50f-4b89-a40b-e991adaef77b + position: + x: -50 + y: 680 + size: + height: 60 + width: 60 + z: 1 + 2522b048-2b7c-44ed-9d35-6cc66069ca5b: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 390 + y: 130 + size: + height: 60 + width: 60 + z: 1 + 2904615e-c0ad-4325-b0c5-b9ea7c482b4e: + embeds: [] + position: + x: 60 + y: 390 + size: + height: 60 + width: 60 + z: 1 + 2f599618-92e8-49cf-92bd-551ba9775f39: + embeds: [] + position: + x: 450 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 3fa053e5-8ab7-4cdf-a737-4d20386e4717: + dependson: + - 88da1502-8179-4be3-9123-fdc5eac0b6a3 + embeds: [] + isassociatedwith: + - d3204db6-c821-400d-bf31-6afd5739923c + isrelatedto: + - bb5a532f-bb43-472a-9965-c4054cd61cc9 + - 210b5501-b505-4ee1-917c-a20cae3ac837 + - ee5f3006-419e-4786-b527-e9503a662e5e + position: + x: -50 + y: 480 + size: + height: 60 + width: 60 + z: 1 + 45360a4f-64aa-49f8-b016-78fe5ed86819: + embeds: [] + ismemberof: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + position: + x: 60 + y: 310 + size: + height: 60 + width: 60 + z: 1 + 466554d3-0e10-4ae4-bfee-51b0aadbf923: + source: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + target: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + 596ddc76-44b9-4747-921c-c000c5b636c0: + embeds: [] + position: + x: 60 + y: 150 + references: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + size: + height: 60 + width: 60 + z: 0 + 619070e4-82e5-445a-93a0-4795ff69d61c: + embeds: [] + isrelatedto: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + position: + x: 180 + y: 480 + size: + height: 60 + width: 60 + z: 1 + 6a874346-80de-4412-9677-00590371a8a2: + embeds: [] + isassociatedwith: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + isrelatedto: + - 210b5501-b505-4ee1-917c-a20cae3ac837 + position: + x: 450 + y: 470 + size: + height: 60 + width: 60 + z: 1 + 789cb442-3b1a-4673-a07e-8acb1c59c271: + embeds: [] + position: + x: 270 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 7b0e0a3f-3a02-4e70-9e79-7647b98fd579: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + - 789cb442-3b1a-4673-a07e-8acb1c59c271 + position: + x: 270 + y: 130 + size: + height: 60 + width: 60 + z: 1 + 88da1502-8179-4be3-9123-fdc5eac0b6a3: + embeds: [] + isconnectedto: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + - ee5f3006-419e-4786-b527-e9503a662e5e + position: + x: -50 + y: 310 + size: + height: 60 + width: 60 + z: 1 + 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c: + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + position: + x: 60 + y: 600 + size: + height: 60 + width: 60 + z: 1 + 8e63b592-a50f-4b89-a40b-e991adaef77b: + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + position: + x: -50 + y: 600 + size: + height: 60 + width: 60 + z: 1 + 9dd9772e-03a3-4983-b28e-a52ac42f0ffe: + source: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + target: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + a8fbc80f-2514-4cc9-aee8-ba7936add0a7: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 170 + y: 30 + size: + height: 60 + width: 60 + z: 1 + bb5a532f-bb43-472a-9965-c4054cd61cc9: + embeds: [] + position: + x: -150 + y: 70 + size: + height: 150 + width: 150 + z: 1 + c8d3da13-fd27-4377-b606-0f41728c17e1: + embeds: [] + position: + x: -170 + y: 310 + size: + height: 60 + width: 60 + z: 1 + ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb: + embeds: [] + position: + x: 270 + y: 30 + size: + height: 60 + width: 60 + z: 1 + d3204db6-c821-400d-bf31-6afd5739923c: + embeds: [] + ismemberof: + - 619070e4-82e5-445a-93a0-4795ff69d61c + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + position: + x: 60 + y: 480 + size: + height: 60 + width: 60 + z: 1 + d5bad569-ad14-487d-bcc6-5569373bc8cd: + embeds: [] + position: + x: 180 + y: 310 + size: + height: 60 + width: 60 + z: 1 + e9ac6c41-8110-49b9-a92c-de57be8a292a: + source: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + target: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + ee5f3006-419e-4786-b527-e9503a662e5e: + embeds: [] + position: + x: -170 + y: 390 + size: + height: 60 + width: 60 + z: 1 + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Network - Basic Settings + Parameters: + - VPCId + - MasterSubnetId + - Label: + default: Cluster - Basic Setting + Parameters: + - KeyName + - AvailabilityZone + - MasterInstanceType + - ComputeInstanceType + - BaseOS + - Scheduler + - Label: + default: EBS - Basic Settings + Parameters: + - VolumeSize + - Label: + default: Network - Advanced Settings + Parameters: + - AccessFrom + - ComputeSubnetId + - ComputeSubnetCidr + - UsePublicIps + - AdditionalSG + - VPCSecurityGroupId + - Label: + default: Cluster - Advanced Setting + Parameters: + - InitialQueueSize + - ComputeWaitConditionCount + - MaxQueueSize + - SpotPrice + - ClusterType + - ProxyServer + - CustomAMI + - MaintainInitialSize + - PreInstallScript + - PreInstallArgs + - PostInstallArgs + - PostInstallScript + - S3ReadResource + - S3ReadWriteResource + - Placement + - PlacementGroup + - EncryptedEphemeral + - EphemeralDir + - SharedDir + - CWLRegion + - CWLLogGroup + - CustomChefRunList + - CustomChefCookbook + - ExtraJson + - Tenancy + - EphemeralKMSKeyId + - ClusterReadyScript + - MasterRootVolumeSize + - ComputeRootVolumeSize + - EC2IAMRoleName + - Label: + default: EBS - Advanced Settings + Parameters: + - VolumeType + - EBSSnapshotId + - VolumeIOPS + - EBSEncryption + - EBSKMSKeyId + - EBSVolumeId + - Label: + default: Scaling Settings + Parameters: + - ScalingPeriod + - ScalingEvaluationPeriods + - ScalingThreshold + - ScalingAdjustment + - ScalingThreshold2 + - ScalingAdjustment2 + - ScalingCooldown + - Label: + default: Additonal Settings + Parameters: + - CLITemplate + ParameterLabels: + AccessFrom: + AccessFrom: + default: access_from + default: access_from + AdditionalSG: + default: additional_sg + AvailabilityZone: + default: AVAILABILITY ZONE + BaseOS: + default: base_os + CLITemplate: + default: CLI TEMPLATE + CWLLogGroup: + default: cwl_log_group + CWLRegion: + default: cwl_region + ClusterReadyScript: + default: cluster_ready + ClusterType: + default: cluster_type + ComputeInstanceType: + default: compute_instance_type + ComputeRootVolumeSize: + default: compute_volume_size + ComputeSubnetCidr: + default: compute_subnet_cidr + ComputeSubnetId: + default: compute_subnet_id + ComputeWaitConditionCount: + default: WAIT COUNT + CustomAMI: + default: custom_ami + CustomChefCookbook: + default: custom_chef_cookbook + CustomChefRunList: + default: custom_chef_runlist + EBSEncryption: + default: ebs_encryption + EBSKMSKeyId: + default: ebs_kms_key_id + EBSSnapshotId: + default: ebs_snapshot_id + EBSVolumeId: + default: ebs_volume_id + EC2IAMRoleName: + default: ec2_iam_role + EncryptedEphemeral: + default: encrypted_ephemeral + EphemeralDir: + default: ephemeral_dir + EphemeralKMSKeyId: + default: ephemeral_kms_key_id + ExtraJson: + default: extra_json + InitialQueueSize: + default: initial_queue_size + KeyName: + default: key_name + MaintainInitialSize: + default: maintain_initial_size + MasterInstanceType: + default: master_instance_type + MasterRootVolumeSize: + default: master_root_volume_size + MasterSubnetId: + default: master_subnet_id + MaxQueueSize: + default: max_queue_size + Placement: + default: placement + PlacementGroup: + default: placement_group + PostInstallArgs: + default: post_install_args + PostInstallScript: + default: post_install + PreInstallArgs: + default: pre_install_args + PreInstallScript: + default: pre_install + ProxyServer: + default: proxy_server + S3ReadResource: + default: s3_read_resource + S3ReadWriteResource: + default: s3_read_write_resource + ScalingAdjustment: + default: scaling_adjustment + ScalingAdjustment2: + default: scaling_adjustment2 + ScalingCooldown: + default: scaling_cooldown + ScalingEvaluationPeriods: + default: scaling_evaluation_periods + ScalingPeriod: + default: scaling_period + ScalingThreshold: + default: scaling_threshold + ScalingThreshold2: + default: scaling_threshold2 + Scheduler: + default: scheduler + SharedDir: + default: shared_dir + SpotPrice: + default: spot_price + Tenancy: + default: tenancy + UsePublicIps: + default: use_public_ips + VPCId: + default: vpc_id + VPCSecurityGroupId: + default: vpc_security_group_id + VolumeIOPS: + default: volume_iops + VolumeSize: + default: volume_size + VolumeType: + default: volume_type + +Parameters: + AccessFrom: + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + Default: 0.0.0.0/0 + Description: Lockdown SSH/HTTP access (default can be accessed from anywhere) + MaxLength: '18' + MinLength: '9' + Type: String + AdditionalCfnTemplate: + Default: NONE + Description: A second CloudFormation template to launch with the cluster + Type: String + AdditionalSG: + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + Default: NONE + Description: Additional VPC secuirty group to be added to instances. Defaults + to NONE + Type: String + AvailabilityZone: + Description: Availability Zone the cluster will launch into. THIS IS REQUIRED + Type: AWS::EC2::AvailabilityZone::Name + BaseOS: + AllowedValues: + - centos6 + - centos7 + - alinux + - ubuntu1404 + - ubuntu1604 + ConstraintDescription: must be a supported base OS + Default: alinux + Description: Base OS type for cluster AMI + Type: String + CLITemplate: + Default: default + Type: String + CWLLogGroup: + Default: NONE + Description: CloudWatch Logs LogGroup + Type: String + CWLRegion: + Default: NONE + Description: CloudWatch Logs region + Type: String + ClusterReadyScript: + Default: NONE + Description: Cluster ready script URL. This is only on the MasterServer, when + the cluster reaches CREATE_COMPLETE. + Type: String + ClusterType: + AllowedValues: + - ondemand + - spot + ConstraintDescription: 'Must be a supported cluster type: ondemand, spot' + Default: ondemand + Description: Type of cluster to launch. Can either be "ondemand" or "spot". Choosing + "spot" will cause the ComputeFleet AutoScaling group to launch EC2 Spot instances. + Default value is "ondemand". + Type: String + ComputeInstanceType: + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.32xlarge + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + Default: t2.micro + Description: ComputeFleet EC2 instance type + Type: String + ComputeRootVolumeSize: + Default: '15' + Description: Size of ComputeFleet EBS root volume in GB + MinValue: '15' + Type: Number + ComputeSubnetCidr: + AllowedPattern: (NONE|(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + Default: NONE + Description: CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited + list and can support multiple CIDR ranges for a multi-AZ cluster. The order + and length of this list MUST match the AvailabilityZones parameter. + Type: String + ComputeSubnetId: + AllowedPattern: (NONE|^subnet-[0-9a-z]{8}$) + Default: NONE + Description: ID of the Subnet you want to provision the Compute Servers into + Type: String + ComputeWaitConditionCount: + Default: '2' + Description: Specific number of instances to wait for while creating the cluster + Type: Number + CustomAMI: + AllowedPattern: (NONE|^ami-[0-9a-z]{8}$|^ami-[0-9a-z]{17}$) + Default: NONE + Description: ID of a Custom AMI, to use instead of published AMI's + Type: String + CustomChefCookbook: + Default: NONE + Description: URL of custom cookbook that will override the default. This will + be unpacked and then dependencies resolved with Berkshelf. + Type: String + CustomChefRunList: + Default: NONE + Description: Custom run list, which will override the default + Type: String + EBSEncryption: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to use EBS encryption for /shared volume. (Not to be + used for snapshots) + Type: String + EBSKMSKeyId: + Default: NONE + Description: KMS ARN for customer created master key, will be used for EBS encryption + Type: String + EBSSnapshotId: + AllowedPattern: (NONE|^snap-[0-9a-z]{8}$|^snap-[0-9a-z]{17}$) + Default: NONE + Description: Id of EBS snapshot if using snapshot as source for volume + Type: String + EBSVolumeId: + AllowedPattern: (NONE|^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$) + Default: NONE + Description: Existing EBS volume Id + Type: String + EC2IAMRoleName: + Default: NONE + Description: Existing EC2 IAM role name + Type: String + EncryptedEphemeral: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to encrypt local ephemeral drives. The keys are in-memory + and non-recoverable. + Type: String + EphemeralDir: + Default: /scratch + Description: The path/mountpoint for the ephemeral drive + Type: String + EphemeralKMSKeyId: + Default: NONE + Description: KMS ARN for customer created master key, will be used for ephemeral + encryption + Type: String + ExtraJson: + Default: '{}' + Description: Extra json to be added to Chef dna.json + Type: String + InitialQueueSize: + Default: '2' + Description: Initial number of EC2 instances to launch as compute nodes within + the cluster. This value maps to the DesiredSize parameter for the ComputeFleet + AutoScaling Group. + Type: Number + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instances + using the default cluster user. + Type: AWS::EC2::KeyPair::KeyName + MaintainInitialSize: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to set autoscaling group to maintain initial size and + scale back + Type: String + MasterInstanceType: + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.32xlarge + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + Default: t2.micro + Description: MasterServer EC2 instance type + Type: String + MasterRootVolumeSize: + Default: '15' + Description: Size of MasterServer EBS root volume in GB + MinValue: '15' + Type: Number + MasterSubnetId: + Description: ID of the Subnet you want to provision the Master server into + Type: AWS::EC2::Subnet::Id + MaxQueueSize: + Default: '10' + Description: Maximum number of EC2 instances that can be launched in the cluster. + This value maps to the MaxSize parameter for the ComputeFleet AutoScaling Group. + Type: Number + Placement: + AllowedValues: + - cluster + - compute + Default: cluster + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + PlacementGroup: + Default: NONE + Description: The name of an existing placement group + Type: String + PostInstallArgs: + Default: NONE + Description: Postinstall script args passed to the postinstall script. + Type: String + PostInstallScript: + Default: NONE + Description: Postinstall script URL. This is run before any host configuration. + Type: String + PreInstallArgs: + Default: NONE + Description: Preinstall script args passed to the preinstall script. + Type: String + PreInstallScript: + Default: NONE + Description: Preinstall script URL. This is run before any host configuration. + Type: String + ProxyServer: + Default: NONE + Description: hostname and port of HTTP proxy server for cfn-init, boto and yum + i.e. proxy.example.com:8080 + Type: String + S3ReadResource: + Default: NONE + Description: S3 resource with read access from cfncluster nodes + Type: String + S3ReadWriteResource: + Default: NONE + Description: Addtional policy document to be added to EC2 IAM role created and + assigned to all nodes. + Type: String + ScalingAdjustment: + Default: '1' + Description: Number of instances to add to cluster when the CloudWatch ScaleUp + action is called. + Type: String + ScalingAdjustment2: + Default: '10' + Description: Number of instances to add to cluster when the CloudWatch ScaleUp2 + action is called. + Type: String + ScalingCooldown: + Default: '300' + Description: Period in seconds to wait before allowing further scaling actions + Type: String + ScalingEvaluationPeriods: + Default: '2' + Description: Number of periods consective periods required to trigger the scaling + adjustment + Type: String + ScalingPeriod: + Default: '60' + Description: Period in seconds to measure ScalingThreshold + Type: String + ScalingThreshold: + Default: '1' + Description: Threshold for triggering CloudWatch ScaleUp action + Type: String + ScalingThreshold2: + Default: '200' + Description: Threshold for triggering CloudWatch ScaleUp2 action + Type: String + Scheduler: + AllowedValues: + - sge + - openlava + - torque + - slurm + - custom + - test + - pbspro + ConstraintDescription: must be a supported scheduler + Default: sge + Description: Cluster scheduler + Type: String + SharedDir: + Default: /shared + Description: The path/mountpoint for the shared drive + Type: String + SpotPrice: + Default: '0.00' + Description: Spot bid price for the ComputeFleet AutoScaling Group when the ClusterType + = "spot". + Type: Number + Tenancy: + AllowedValues: + - default + - dedicated + Default: default + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + UsePublicIps: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'true' + Description: Boolean flag to use public IP's for instances. If false, the VPC + must be correctly setup to use NAT for all traffic. + Type: String + VPCId: + Description: ID of the VPC you want to provision cluster into. Only used with + UseVPCBase=false + Type: AWS::EC2::VPC::Id + VPCSecurityGroupId: + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + Default: NONE + Description: Existing VPC security group Id + Type: String + VolumeIOPS: + Default: '100' + Description: Number of IOPS for volume type io1. Not used for other volume types. + Type: Number + VolumeSize: + Default: '20' + Description: Size of EBS volume in GB, if creating a new one + Type: Number + VolumeType: + AllowedValues: + - standard + - gp2 + - io1 + - st1 + - sc1 + ConstraintDescription: 'must be a supported volume type: standard, io1, gp2, st1, + sc1' + Default: gp2 + Description: Type of volume to create either new or from snapshot + Type: String + +Mappings: + AWSInstanceType2Capabilites: + c3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + c3.large: + Arch: 64HVM + EBSOpt: 'False' + c3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.large: + Arch: 64HVM + EBSOpt: 'True' + c4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + cc2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + cg1.4xlarge: + Arch: 64HVM + EBSOpt: 'False' + cr1.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + d2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + g2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + g2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + i2.large: + Arch: 64HVM + EBSOpt: 'False' + i2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.large: + Arch: 64HVM + EBSOpt: 'False' + m3.medium: + Arch: 64HVM + EBSOpt: 'False' + m3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.10xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.large: + Arch: 64HVM + EBSOpt: 'True' + m4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.large: + Arch: 64HVM + EBSOpt: 'True' + r3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + r3.large: + Arch: 64HVM + EBSOpt: 'False' + r3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + t2.large: + Arch: 64HVM + EBSOpt: 'False' + t2.medium: + Arch: 64HVM + EBSOpt: 'False' + t2.micro: + Arch: 64HVM + EBSOpt: 'False' + t2.nano: + Arch: 64HVM + EBSOpt: 'False' + t2.small: + Arch: 64HVM + EBSOpt: 'False' + x1.32xlarge: + Arch: 64HVM + EBSOpt: 'True' + AWSRegion2Capabilites: + ap-northeast-1: + arn: aws + ap-northeast-2: + arn: aws + ap-south-1: + arn: aws + ap-southeast-1: + arn: aws + ap-southeast-2: + arn: aws + eu-central-1: + arn: aws + eu-west-1: + arn: aws + sa-east-1: + arn: aws + us-east-1: + arn: aws + us-gov-west-1: + arn: aws-us-gov + us-west-1: + arn: aws + us-west-2: + arn: aws + AWSRegionOS2AMI: + ap-northeast-1: + alinux: ami-50bd7731 + centos6: ami-46b17b27 + centos7: ami-43c90322 + ubuntu1404: ami-89bd77e8 + ubuntu1604: ami-eabe748b + ap-northeast-2: + alinux: ami-afe431c1 + centos6: ami-75e1341b + centos7: ami-5cec3932 + ubuntu1404: ami-bae732d4 + ubuntu1604: ami-7eef3a10 + ap-south-1: + alinux: ami-0495e06b + centos6: ami-a392e7cc + centos7: ami-bf90e5d0 + ubuntu1404: ami-489de827 + ubuntu1604: ami-5d90e532 + ap-southeast-1: + alinux: ami-9af229f9 + centos6: ami-23f22940 + centos7: ami-d5f52eb6 + ubuntu1404: ami-95f229f6 + ubuntu1604: ami-23f32840 + ap-southeast-2: + alinux: ami-51b98f32 + centos6: ami-0ea6906d + centos7: ami-79a5931a + ubuntu1404: ami-d3b98fb0 + ubuntu1604: ami-c1b98fa2 + eu-central-1: + alinux: ami-96db28f9 + centos6: ami-5fd92a30 + centos7: ami-fcda2993 + ubuntu1404: ami-89db28e6 + ubuntu1604: ami-9dd427f2 + eu-west-1: + alinux: ami-d8f785ab + centos6: ami-27f78554 + centos7: ami-5af48629 + ubuntu1404: ami-3ff0824c + ubuntu1604: ami-99f082ea + sa-east-1: + alinux: ami-8bd848e7 + centos6: ami-8ad848e6 + centos7: ami-f8d84894 + ubuntu1404: ami-3f34a553 + ubuntu1604: ami-8d0e9fe1 + us-east-1: + alinux: ami-d5751ec2 + centos6: ami-504b2047 + centos7: ami-e94328fe + ubuntu1404: ami-ba4a21ad + ubuntu1604: ami-d6751ec1 + us-gov-west-1: + alinux: NOT_SUPPORTED + centos6: NOT_SUPPORTED + centos7: NOT_SUPPORTED + ubuntu1404: NOT_SUPPORTED + ubuntu1604: NOT_SUPPORTED + us-west-1: + alinux: ami-5f19553f + centos6: ami-051f5365 + centos7: ami-a51955c5 + ubuntu1404: ami-40185420 + ubuntu1604: ami-751f5315 + us-west-2: + alinux: ami-1e31e07e + centos6: ami-6e30e10e + centos7: ami-2e1dcc4e + ubuntu1404: ami-4633e226 + ubuntu1604: ami-870edfe7 + CfnClusterVersions: + default: + ami: '201609030709' + berkshelf: 4.6.1 + cfncluster: cfncluster-1.3.1 + chef: 12.13.30 + cookbook: cfncluster-cookbook-1.3.0 + ridley: 4.5.1 + OSFeatures: + alinux: + RootDevice: /dev/xvda + User: ec2-user + centos6: + RootDevice: /dev/sda1 + User: centos + centos7: + RootDevice: /dev/sda1 + User: centos + ubuntu1404: + RootDevice: /dev/sda1 + User: ubuntu + ubuntu1604: + RootDevice: /dev/sda1 + User: ubuntu + +Conditions: + AddAdditionalSG: + Fn::Not: + - Fn::Equals: + - Ref: AdditionalSG + - NONE + CloudWatchLogs: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: CWLRegion + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: CWLLogGroup + - NONE + ComputePublicIps: + Fn::And: + - Fn::Equals: + - Ref: UsePublicIps + - 'true' + - Condition: UseMasterSubnetForCompute + CreateComputeSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + CreateEBSVolume: + Fn::Equals: + - Ref: EBSVolumeId + - NONE + CreateEC2IAMRole: + Fn::Equals: + - Ref: EC2IAMRoleName + - NONE + CreatePlacementGroup: + Fn::And: + - Fn::Equals: + - Ref: PlacementGroup + - DYNAMIC + - Condition: UsePlacementGroup + CreateSecurityGroups: + Fn::Equals: + - Ref: VPCSecurityGroupId + - NONE + CreateSubStack: + Fn::Not: + - Fn::Equals: + - Ref: AdditionalCfnTemplate + - NONE + MaintainInitialASGSize: + Fn::Equals: + - Ref: MaintainInitialSize + - 'true' + MasterPublicIp: + Fn::Equals: + - Ref: UsePublicIps + - 'true' + UseClusterPlacement: + Fn::And: + - Fn::Equals: + - Ref: Placement + - cluster + - Condition: UsePlacementGroup + UseComputeSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + UseCustomAMI: + Fn::Not: + - Fn::Equals: + - Ref: CustomAMI + - NONE + UseCustomRunList: + Fn::Not: + - Fn::Equals: + - Ref: CustomChefRunList + - NONE + UseDedicatedTenancy: + Fn::Equals: + - Ref: Tenancy + - dedicated + UseEBSEncryption: + Fn::Equals: + - Ref: EBSEncryption + - 'true' + UseEBSKMSKey: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: EBSKMSKeyId + - NONE + - Condition: UseEBSEncryption + UseEBSPIOPS: + Fn::Equals: + - Ref: VolumeType + - io1 + UseEBSSnapshot: + Fn::Not: + - Fn::Equals: + - Ref: EBSSnapshotId + - NONE + UseEC2IAMRole: + Fn::Not: + - Fn::Equals: + - Ref: EC2IAMRoleName + - NONE + UseEphemeralKMSKey: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: EphemeralKMSKeyId + - NONE + - Fn::Equals: + - Ref: EncryptedEphemeral + - 'true' + UseExistingEBSVolume: + Fn::Not: + - Fn::Equals: + - Ref: EBSVolumeId + - NONE + UseExistingSecurityGroup: + Fn::Not: + - Fn::Equals: + - Ref: VPCSecurityGroupId + - NONE + UseMasterSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + UsePlacementGroup: + Fn::Not: + - Fn::Equals: + - Ref: PlacementGroup + - NONE + UseS3ReadPolicy: + Fn::Not: + - Fn::Equals: + - Ref: S3ReadResource + - NONE + UseS3ReadWritePolicy: + Fn::Not: + - Fn::Equals: + - Ref: S3ReadWriteResource + - NONE + UseSpotInstances: + Fn::Equals: + - Ref: ClusterType + - spot + +Resources: + AddCapacityAlarm: + Metadata: + AWS::CloudFormation::Designer: + id: 23ed10e0-7726-4099-8370-4116c0bc78a2 + Properties: + AlarmActions: + - Ref: ScaleUpPolicy + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: + Ref: AWS::StackName + EvaluationPeriods: + Ref: ScalingEvaluationPeriods + MetricName: pending + Namespace: cfncluster + Period: + Ref: ScalingPeriod + Statistic: Average + Threshold: + Ref: ScalingThreshold + Type: AWS::CloudWatch::Alarm + AddCapacityAlarm2: + Metadata: + AWS::CloudFormation::Designer: + id: 127f78c3-d47a-4770-8aad-3fd733c3e1b0 + Properties: + AlarmActions: + - Ref: ScaleUpPolicy2 + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: + Ref: AWS::StackName + EvaluationPeriods: + Ref: ScalingEvaluationPeriods + MetricName: pending + Namespace: cfncluster + Period: + Ref: ScalingPeriod + Statistic: Sum + Threshold: + Ref: ScalingThreshold2 + Type: AWS::CloudWatch::Alarm + AdditionalCfnStack: + Condition: CreateSubStack + Properties: + TemplateURL: + Ref: AdditionalCfnTemplate + Type: AWS::CloudFormation::Stack + AssociateEIP: + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: e9ac6c41-8110-49b9-a92c-de57be8a292a + Properties: + AllocationId: + Fn::GetAtt: + - MasterEIP + - AllocationId + NetworkInterfaceId: + Ref: MasterENI + Type: AWS::EC2::EIPAssociation + CfnClusterPolicies: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 7b0e0a3f-3a02-4e70-9e79-7647b98fd579 + Properties: + PolicyDocument: + Statement: + - Action: + - ec2:AttachVolume + - ec2:DescribeInstanceAttribute + - ec2:DescribeInstanceStatus + - ec2:DescribeInstances + Effect: Allow + Resource: + - '*' + Sid: EC2 + - Action: + - dynamodb:ListTables + Effect: Allow + Resource: + - '*' + Sid: DynamoDBList + - Action: + - sqs:SendMessage + - sqs:ReceiveMessage + - sqs:ChangeMessageVisibility + - sqs:DeleteMessage + - sqs:GetQueueUrl + Effect: Allow + Resource: + - Fn::GetAtt: + - SQS + - Arn + Sid: SQSQueue + - Action: + - autoscaling:DescribeAutoScalingGroups + - autoscaling:TerminateInstanceInAutoScalingGroup + - autoscaling:SetDesiredCapacity + Effect: Allow + Resource: + - '*' + Sid: Autoscaling + - Action: + - cloudwatch:PutMetricData + Effect: Allow + Resource: + - '*' + Sid: CloudWatch + - Action: + - dynamodb:PutItem + - dynamodb:Query + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:DescribeTable + Effect: Allow + Resource: + - Fn::Join: + - '' + - - 'arn:' + - Fn::FindInMap: + - AWSRegion2Capabilites + - Ref: AWS::Region + - arn + - ':dynamodb:' + - Ref: AWS::Region + - ':' + - Ref: AWS::AccountId + - :table/ + - Ref: DynamoDBTable + Sid: DynamoDBTable + - Action: + - sqs:ListQueues + Effect: Allow + Resource: + - '*' + Sid: SQSList + - Action: + - logs:* + Effect: Allow + Resource: + - Fn::Join: + - '' + - - 'arn:' + - Fn::FindInMap: + - AWSRegion2Capabilites + - Ref: AWS::Region + - arn + - :logs:*:*:* + Sid: CloudWatchLogs + PolicyName: cfncluster + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + ComputeFleet: + CreationPolicy: + ResourceSignal: + Count: + Ref: ComputeWaitConditionCount + Timeout: PT30M + DependsOn: MasterServer + Metadata: + AWS::CloudFormation::Designer: + id: 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + Properties: + DesiredCapacity: + Ref: InitialQueueSize + LaunchConfigurationName: + Ref: ComputeServerLaunchConfig + MaxSize: + Ref: MaxQueueSize + MinSize: + Fn::If: + - MaintainInitialASGSize + - Ref: InitialQueueSize + - '0' + NotificationConfiguration: + NotificationTypes: + - autoscaling:EC2_INSTANCE_TERMINATE + TopicARN: + Ref: SNS + PlacementGroup: + Fn::If: + - UsePlacementGroup + - Fn::If: + - CreatePlacementGroup + - Ref: DynamicPlacementGroup + - Ref: PlacementGroup + - Ref: AWS::NoValue + Tags: + - Key: Name + PropagateAtLaunch: 'true' + Value: Compute + VPCZoneIdentifier: + - Fn::If: + - UseMasterSubnetForCompute + - Ref: MasterSubnetId + - Fn::If: + - CreateComputeSubnetForCompute + - Ref: ComputeSubnet + - Ref: ComputeSubnetId + Type: AWS::AutoScaling::AutoScalingGroup + ComputeRoute: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 596ddc76-44b9-4747-921c-c000c5b636c0 + Properties: + DestinationCidrBlock: 0.0.0.0/0 + NetworkInterfaceId: + Ref: MasterENI + RouteTableId: + Ref: ComputeRouteTable + Type: AWS::EC2::Route + ComputeRouteTable: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + Properties: + Tags: + - Key: Application + Value: + Ref: AWS::StackName + - Key: Network + Value: ComputeSubnet + VpcId: + Ref: VPCId + Type: AWS::EC2::RouteTable + ComputeSecurityGroup: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + Properties: + GroupDescription: Allow access to resources in subnets behind front + SecurityGroupIngress: + - FromPort: '0' + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: MasterSecurityGroup + ToPort: '65535' + VpcId: + Ref: VPCId + Type: AWS::EC2::SecurityGroup + ComputeSecurityGroupIngress: + Condition: CreateSecurityGroups + Properties: + FromPort: '0' + GroupId: + Ref: ComputeSecurityGroup + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: ComputeSecurityGroup + ToPort: '65535' + Type: AWS::EC2::SecurityGroupIngress + ComputeServerLaunchConfig: + Metadata: + AWS::CloudFormation::Designer: + id: d3204db6-c821-400d-bf31-6afd5739923c + AWS::CloudFormation::Init: + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + - signalComputeReady + deployConfigFiles: + commands: + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + files: + /etc/chef/client.rb: + content: + Fn::Join: + - '' + - - cookbook_path ['/etc/chef/cookbooks'] + group: root + mode: '000644' + owner: root + /tmp/dna.json: + content: + cfncluster: + cfn_cluster_user: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - User + cfn_encrypted_ephemeral: + Ref: EncryptedEphemeral + cfn_ephemeral_dir: + Ref: EphemeralDir + cfn_master: + Fn::GetAtt: + - MasterServer + - PrivateDnsName + cfn_node_type: ComputeFleet + cfn_postinstall: + Ref: PostInstallScript + cfn_postinstall_args: + Ref: PostInstallArgs + cfn_preinstall: + Ref: PreInstallScript + cfn_preinstall_args: + Ref: PreInstallArgs + cfn_proxy: + Ref: ProxyServer + cfn_region: + Ref: AWS::Region + cfn_scheduler: + Ref: Scheduler + cfn_shared_dir: + Ref: SharedDir + cfn_sqs_queue: + Ref: SQS + stack_name: + Ref: AWS::StackName + run_list: + Fn::If: + - UseCustomRunList + - Ref: CustomChefRunList + - Fn::Join: + - '' + - - 'recipe[cfncluster::' + - Ref: Scheduler + - _config] + group: root + mode: '000644' + owner: root + /tmp/extra.json: + content: + Ref: ExtraJson + group: root + mode: '000644' + owner: root + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + signalComputeReady: + commands: + compute_ready: + command: /opt/cfncluster/scripts/compute_ready + Comment: cfncluster Compute server + Properties: + AssociatePublicIpAddress: + Fn::If: + - ComputePublicIps + - 'true' + - 'false' + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - RootDevice + Ebs: + VolumeSize: + Ref: ComputeRootVolumeSize + VolumeType: gp2 + IamInstanceProfile: + Fn::If: + - UseEC2IAMRole + - Ref: EC2IAMRoleName + - Ref: RootInstanceProfile + ImageId: + Fn::If: + - UseCustomAMI + - Ref: CustomAMI + - Fn::FindInMap: + - AWSRegionOS2AMI + - Ref: AWS::Region + - Ref: BaseOS + InstanceMonitoring: 'false' + InstanceType: + Ref: ComputeInstanceType + KeyName: + Ref: KeyName + PlacementTenancy: + Fn::If: + - UseDedicatedTenancy + - Ref: Tenancy + - Ref: AWS::NoValue + SecurityGroups: + - Fn::If: + - CreateSecurityGroups + - Ref: ComputeSecurityGroup + - Ref: AWS::NoValue + - Fn::If: + - AddAdditionalSG + - Ref: AdditionalSG + - Ref: AWS::NoValue + - Fn::If: + - UseExistingSecurityGroup + - Ref: VPCSecurityGroupId + - Ref: AWS::NoValue + SpotPrice: + Fn::If: + - UseSpotInstances + - Ref: SpotPrice + - Ref: AWS::NoValue + UserData: + Fn::Base64: + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - Ref: ProxyServer + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - Ref: ProxyServer + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - Ref: AWS::StackName + - ' --resource=ComputeFleet --region=' + - Ref: AWS::Region + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - Ref: ProxyServer + - ' + + ' + - custom_cookbook= + - Ref: CustomChefCookbook + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - Ref: AWS::Region + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - Ref: AWS::Region + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - Ref: AWS::Region + - /cookbooks/ + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cfncluster + - ' + + ' + - export cookbook_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - ' + + ' + - export chef_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - chef + - ' + + ' + - export ridley_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - ridley + - ' + + ' + - export berkshelf_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - berkshelf + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - Ref: AWS::StackName + - ' -v -c default -r ComputeServerLaunchConfig --region ' + - Ref: AWS::Region + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup + complete" --stack= + - Ref: AWS::StackName + - ' --resource=ComputeFleet --region=' + - Ref: AWS::Region + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Type: AWS::AutoScaling::LaunchConfiguration + ComputeSubnet: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + Properties: + AvailabilityZone: + Ref: AvailabilityZone + CidrBlock: + Ref: ComputeSubnetCidr + Tags: + - Key: Network + Value: ComputeSubnet + VpcId: + Ref: VPCId + Type: AWS::EC2::Subnet + ComputeSubnetRouteTableAssociation: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 9dd9772e-03a3-4983-b28e-a52ac42f0ffe + Properties: + RouteTableId: + Ref: ComputeRouteTable + SubnetId: + Ref: ComputeSubnet + Type: AWS::EC2::SubnetRouteTableAssociation + DynamicPlacementGroup: + Condition: CreatePlacementGroup + Metadata: + AWS::CloudFormation::Designer: + id: ee5f3006-419e-4786-b527-e9503a662e5e + Properties: + Strategy: cluster + Type: AWS::EC2::PlacementGroup + DynamoDBTable: + Metadata: + AWS::CloudFormation::Designer: + id: 789cb442-3b1a-4673-a07e-8acb1c59c271 + Properties: + AttributeDefinitions: + - AttributeName: instanceId + AttributeType: S + KeySchema: + - AttributeName: instanceId + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' + Type: AWS::DynamoDB::Table + MasterEIP: + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + Properties: + Domain: vpc + Type: AWS::EC2::EIP + MasterENI: + Metadata: + AWS::CloudFormation::Designer: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + Properties: + Description: cfncluster Master Server + GroupSet: + - Fn::If: + - CreateSecurityGroups + - Ref: MasterSecurityGroup + - Ref: AWS::NoValue + - Fn::If: + - AddAdditionalSG + - Ref: AdditionalSG + - Ref: AWS::NoValue + - Fn::If: + - UseExistingSecurityGroup + - Ref: VPCSecurityGroupId + - Ref: AWS::NoValue + SourceDestCheck: 'false' + SubnetId: + Ref: MasterSubnetId + Type: AWS::EC2::NetworkInterface + MasterSecurityGroup: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + Properties: + GroupDescription: Enable access to the Master host + SecurityGroupIngress: + - CidrIp: + Ref: AccessFrom + FromPort: '22' + IpProtocol: tcp + ToPort: '22' + - CidrIp: + Ref: AccessFrom + FromPort: '80' + IpProtocol: tcp + ToPort: '80' + VpcId: + Ref: VPCId + Type: AWS::EC2::SecurityGroup + MasterSecurityGroupIngress: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 466554d3-0e10-4ae4-bfee-51b0aadbf923 + Properties: + FromPort: '0' + GroupId: + Ref: MasterSecurityGroup + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: ComputeSecurityGroup + ToPort: '65535' + Type: AWS::EC2::SecurityGroupIngress + MasterServer: + CreationPolicy: + ResourceSignal: + Count: '1' + Timeout: PT30M + Metadata: + AWS::CloudFormation::Designer: + id: 88da1502-8179-4be3-9123-fdc5eac0b6a3 + AWS::CloudFormation::Init: + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + deployConfigFiles: + commands: + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + files: + /etc/chef/client.rb: + content: + Fn::Join: + - '' + - - cookbook_path ['/etc/chef/cookbooks'] + group: root + mode: '000644' + owner: root + /tmp/dna.json: + content: + cfncluster: + cfn_cluster_user: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - User + cfn_ddb_table: + Ref: DynamoDBTable + cfn_encrypted_ephemeral: + Ref: EncryptedEphemeral + cfn_ephemeral_dir: + Ref: EphemeralDir + cfn_node_type: MasterServer + cfn_postinstall: + Ref: PostInstallScript + cfn_postinstall_args: + Ref: PostInstallArgs + cfn_preinstall: + Ref: PreInstallScript + cfn_preinstall_args: + Ref: PreInstallArgs + cfn_proxy: + Ref: ProxyServer + cfn_region: + Ref: AWS::Region + cfn_scheduler: + Ref: Scheduler + cfn_shared_dir: + Ref: SharedDir + cfn_sqs_queue: + Fn::GetAtt: + - SQS + - QueueName + cfn_volume: + Fn::If: + - UseExistingEBSVolume + - Ref: EBSVolumeId + - Ref: SharedVolume + stack_name: + Ref: AWS::StackName + run_list: + Fn::If: + - UseCustomRunList + - Ref: CustomChefRunList + - Fn::Join: + - '' + - - 'recipe[cfncluster::' + - Ref: Scheduler + - _config] + group: root + mode: '000644' + owner: root + /tmp/extra.json: + content: + Ref: ExtraJson + group: root + mode: '000644' + owner: root + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + Comment: cfncluster Master server + Properties: + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - RootDevice + Ebs: + VolumeSize: + Ref: MasterRootVolumeSize + VolumeType: gp2 + EbsOptimized: + Fn::FindInMap: + - AWSInstanceType2Capabilites + - Ref: MasterInstanceType + - EBSOpt + IamInstanceProfile: + Fn::If: + - UseEC2IAMRole + - Ref: EC2IAMRoleName + - Ref: RootInstanceProfile + ImageId: + Fn::If: + - UseCustomAMI + - Ref: CustomAMI + - Fn::FindInMap: + - AWSRegionOS2AMI + - Ref: AWS::Region + - Ref: BaseOS + InstanceType: + Ref: MasterInstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - DeviceIndex: '0' + NetworkInterfaceId: + Ref: MasterENI + PlacementGroupName: + Fn::If: + - UseClusterPlacement + - Fn::If: + - CreatePlacementGroup + - Ref: DynamicPlacementGroup + - Ref: PlacementGroup + - Ref: AWS::NoValue + Tags: + - Key: Application + Value: + Ref: AWS::StackName + - Key: Name + Value: Master + Tenancy: + Ref: Tenancy + UserData: + Fn::Base64: + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - Ref: ProxyServer + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - Ref: ProxyServer + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - Ref: AWS::StackName + - ' --resource=MasterServer --region=' + - Ref: AWS::Region + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - Ref: ProxyServer + - ' + + ' + - custom_cookbook= + - Ref: CustomChefCookbook + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - Ref: AWS::Region + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - Ref: AWS::Region + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - Ref: AWS::Region + - /cookbooks/ + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cfncluster + - ' + + ' + - export cookbook_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - ' + + ' + - export chef_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - chef + - ' + + ' + - export ridley_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - ridley + - ' + + ' + - export berkshelf_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - berkshelf + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - Ref: AWS::StackName + - ' -v -c default -r MasterServer --region ' + - Ref: AWS::Region + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup + complete" --stack= + - Ref: AWS::StackName + - ' --resource=MasterServer --region=' + - Ref: AWS::Region + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Type: AWS::EC2::Instance + RootInstanceProfile: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 2522b048-2b7c-44ed-9d35-6cc66069ca5b + Properties: + Path: / + Roles: + - Ref: RootRole + Type: AWS::IAM::InstanceProfile + RootRole: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: + - sts:AssumeRole + Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Version: '2012-10-17' + Path: / + Type: AWS::IAM::Role + S3ReadRolePolicies: + Condition: UseS3ReadPolicy + Metadata: + AWS::CloudFormation::Designer: + id: 0c5a1e92-ffdd-4344-8ba0-65270e5fd472 + Properties: + PolicyDocument: + Statement: + - Action: + - s3:Get* + - s3:List* + Effect: Allow + Resource: + - Ref: S3ReadResource + Sid: S3Read + Version: '2012-10-17' + PolicyName: S3Read + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + S3ReadWriteRolePolicies: + Condition: UseS3ReadWritePolicy + Metadata: + AWS::CloudFormation::Designer: + id: a8fbc80f-2514-4cc9-aee8-ba7936add0a7 + Properties: + PolicyDocument: + Statement: + - Action: + - s3:* + Effect: Allow + Resource: + - Ref: S3ReadWriteResource + Sid: S3ReadWrite + Version: '2012-10-17' + PolicyName: S3ReadWrite + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + SNS: + Metadata: + AWS::CloudFormation::Designer: + id: 210b5501-b505-4ee1-917c-a20cae3ac837 + Properties: + Subscription: + - Endpoint: + Fn::GetAtt: + - SQS + - Arn + Protocol: sqs + Type: AWS::SNS::Topic + SQS: + Metadata: + AWS::CloudFormation::Designer: + id: 2f599618-92e8-49cf-92bd-551ba9775f39 + Properties: {} + Type: AWS::SQS::Queue + SQSPolicy: + Metadata: + AWS::CloudFormation::Designer: + id: 6a874346-80de-4412-9677-00590371a8a2 + Properties: + PolicyDocument: + Id: MyQueuePolicy + Statement: + - Action: + - sqs:SendMessage + Condition: + ArnEquals: + aws:SourceArn: + Ref: SNS + Effect: Allow + Principal: + AWS: '*' + Resource: '*' + Sid: Allow-SendMessage-From-AS-SNS-Topic + Queues: + - Ref: SQS + Type: AWS::SQS::QueuePolicy + ScaleUpPolicy: + Metadata: + AWS::CloudFormation::Designer: + id: 8e63b592-a50f-4b89-a40b-e991adaef77b + Properties: + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: + Ref: ComputeFleet + Cooldown: + Ref: ScalingCooldown + ScalingAdjustment: + Ref: ScalingAdjustment + Type: AWS::AutoScaling::ScalingPolicy + ScaleUpPolicy2: + Metadata: + AWS::CloudFormation::Designer: + id: 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + Properties: + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: + Ref: ComputeFleet + Cooldown: + Ref: ScalingCooldown + ScalingAdjustment: + Ref: ScalingAdjustment2 + Type: AWS::AutoScaling::ScalingPolicy + SharedVolume: + Condition: CreateEBSVolume + Metadata: + AWS::CloudFormation::Designer: + id: c8d3da13-fd27-4377-b606-0f41728c17e1 + Properties: + AvailabilityZone: + Ref: AvailabilityZone + Encrypted: + Fn::If: + - UseEBSEncryption + - Ref: EBSEncryption + - Ref: AWS::NoValue + Iops: + Fn::If: + - UseEBSPIOPS + - Ref: VolumeIOPS + - Ref: AWS::NoValue + KmsKeyId: + Fn::If: + - UseEBSKMSKey + - Ref: EBSKMSKeyId + - Ref: AWS::NoValue + Size: + Fn::If: + - UseEBSSnapshot + - Ref: AWS::NoValue + - Ref: VolumeSize + SnapshotId: + Fn::If: + - UseEBSSnapshot + - Ref: EBSSnapshotId + - Ref: AWS::NoValue + VolumeType: + Ref: VolumeType + Type: AWS::EC2::Volume + +Outputs: + GangliaPrivateURL: + Description: Private URL to access Ganglia + Value: + Fn::Join: + - '' + - - http:// + - Fn::GetAtt: + - MasterServer + - PrivateIp + - /ganglia/ + GangliaPublicURL: + Condition: MasterPublicIp + Description: Public URL to access Ganglia + Value: + Fn::Join: + - '' + - - http:// + - Fn::GetAtt: + - MasterServer + - PublicIp + - /ganglia/ + MasterPrivateIP: + Description: Private IP Address of the Master host + Value: + Fn::GetAtt: + - MasterServer + - PrivateIp + MasterPublicIP: + Condition: MasterPublicIp + Description: Public IP Address of the Master host + Value: + Fn::GetAtt: + - MasterServer + - PublicIp \ No newline at end of file From d79e5012330145e1800f2ff0761e4bb3f2068a59 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Thu, 25 May 2017 12:22:15 -0700 Subject: [PATCH 15/34] Update ami_development.rst --- docs/source/ami_development.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/ami_development.rst b/docs/source/ami_development.rst index 19e291156b..aebbbb5a80 100644 --- a/docs/source/ami_development.rst +++ b/docs/source/ami_development.rst @@ -16,7 +16,7 @@ Steps This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubuntu machine you can easily get an `EC2 instance `_ running Ubuntu. -#. :code:`sudo apt-get install build-essential` +#. :code:`sudo apt-get -y install build-essential git` #. Go to https://downloads.chef.io/chef-dk, grab the latest version for your OS and install. For example: @@ -36,6 +36,7 @@ This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubu sudo tar xf ~/go1.7.linux-amd64.tar.gz echo 'export GOPATH=~/work' >> ~/.bashrc echo 'export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH' >> ~/.bashrc + . ~/.bashrc #. Install packer from source From 59a0fe2059afdbc05b0175944e79beef1c2ca62c Mon Sep 17 00:00:00 2001 From: Adam Boeglin Date: Mon, 10 Oct 2016 16:00:51 -0700 Subject: [PATCH 16/34] Fix start to set initial_size and min_size correctly --- cli/cfncluster/cfncluster.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index f2af9414f9..c1e7c869c0 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -173,11 +173,13 @@ def start(args): # Set asg limits max_queue_size = [param[1] for param in config.parameters if param[0] == 'MaxQueueSize'] max_queue_size = max_queue_size[0] if len(max_queue_size) > 0 else 10 - initial_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] if args.reset_desired else [0] - initial_queue_size = initial_queue_size[0] if len(initial_queue_size) > 0 else 0 + desired_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] + desired_queue_size = desired_queue_size[0] if len(desired_queue_size) > 0 else 2 + min_queue_size = [desired_queue_size for param in config.parameters if param[0] == 'MaintainInitialSize' and param[1] == "true"] + min_queue_size = min_queue_size[0] if len(min_queue_size) > 0 else 0 asg = get_asg(stack_name=stack_name, config=config) - set_asg_limits(asg=asg, min=initial_queue_size, max=max_queue_size, desired=initial_queue_size) + set_asg_limits(asg=asg, min=min_queue_size, max=max_queue_size, desired=desired_queue_size) # Poll for status poll_master_server_state(stack_name, config) From 17220145942038579f0a1e252c05b53a5aa00582 Mon Sep 17 00:00:00 2001 From: duff Date: Tue, 11 Oct 2016 16:03:15 -0700 Subject: [PATCH 17/34] adding m4.16xlarge instance --- cloudformation/cfncluster.cfn.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index b2860ea489..58f13ad45d 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -756,6 +756,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -810,6 +811,7 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", + "m4.16xlarge", "x1.32xlarge" ] }, @@ -1724,6 +1726,10 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, + "m4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "m4.10xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" From a1c94d5fd55ff5fad2301a252e4eba04cd08680b Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Mon, 17 Oct 2016 08:18:41 -0700 Subject: [PATCH 18/34] * Fixing p2 instance types. Fixes #143 --- cloudformation/cfncluster.cfn.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 58f13ad45d..4dac9b5808 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -740,7 +740,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", @@ -795,7 +795,7 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", From b64372f24aeb68caacdb10636c4472c30bda886a Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 11:34:25 -0800 Subject: [PATCH 19/34] stage1 changes from futurize --- cli/cfncluster/cfncluster.py | 42 +++++++++++++++++---------------- cli/cfncluster/cfnconfig.py | 8 ++++--- cli/cfncluster/cli.py | 5 ++-- cli/cfncluster/config_sanity.py | 5 ++-- cli/cfncluster/easyconfig.py | 20 +++++++++------- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index c1e7c869c0..8be8873d84 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -19,13 +21,13 @@ import socket import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') def version(args): config = cfnconfig.CfnClusterConfig(args) - print config.version + print(config.version) def create(args): print('Starting: %s' % (args.cluster_name)) @@ -50,7 +52,7 @@ def create(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -75,12 +77,12 @@ def create(args): time.sleep(5) outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) else: status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -118,7 +120,7 @@ def update(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -142,7 +144,7 @@ def update(args): status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -161,7 +163,7 @@ def start(args): response = ec2conn.start_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -203,7 +205,7 @@ def stop(args): response = ec2conn.stop_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -226,7 +228,7 @@ def list(args): print('%s' % (stack.stack_name[11:])) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) else: raise e except KeyboardInterrupt: @@ -245,7 +247,7 @@ def get_master_server_id(stack_name, config): resources = cfnconn.describe_stack_resources(stack_name) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -282,7 +284,7 @@ def poll_master_server_state(stack_name, config): sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -305,7 +307,7 @@ def get_ec2_instances(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -336,7 +338,7 @@ def get_asg(stack_name, config): return asgconn.get_all_groups(names=[asg_id])[0] except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -365,7 +367,7 @@ def get_asg_ids(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -435,22 +437,22 @@ def status(args): if state == 'running': outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) elif ((status == 'ROLLBACK_COMPLETE') or (status == 'CREATE_FAILED') or (status == 'DELETE_FAILED') or (status == 'UPDATE_ROLLBACK_COMPLETE')): events = cfnconn.describe_stack_events(stack) for event in events: if ((event.resource_status == 'CREATE_FAILED') or (event.resource_status == 'DELETE_FAILED') or (event.resource_status == 'UPDATE_FAILED')): - print event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ - event.resource_status_reason + print(event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ + event.resource_status_reason) else: sys.stdout.write('\n') sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): sys.stdout.write('\r') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -491,7 +493,7 @@ def delete(args): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index c77177d09e..9380fb1bc1 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -16,7 +18,7 @@ import pkg_resources import json import urllib2 -import config_sanity +from . import config_sanity import boto.cloudformation def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): @@ -36,7 +38,7 @@ def __init__(self, args): self.parameters = [] self.version = pkg_resources.get_distribution("cfncluster").version self.__DEFAULT_CONFIG = False - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ # Determine config file name based on args or default if args.config_file is not None: @@ -119,7 +121,7 @@ def __init__(self, args): except ConfigParser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True: pass else: diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index ddb6743d5e..f317af989e 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -15,8 +16,8 @@ import platform import json -import cfncluster -import easyconfig +from . import cfncluster +from . import easyconfig def create(args): cfncluster.create(args) diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index e8f11ed9df..643c135df6 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -93,10 +94,10 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty else: try: urllib2.urlopen(resource_value) - except urllib2.HTTPError, e: + except urllib2.HTTPError as e: print(e.code) sys.exit(1) - except urllib2.URLError, e: + except urllib2.URLError as e: print(e.args) sys.exit(1) # EC2 EBS Snapshot Id diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index 7273599a14..a735618163 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance with the @@ -16,7 +18,7 @@ import os import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') @@ -31,9 +33,9 @@ def prompt(prompt, default_value=None, hidden=False, options=None): user_prompt = user_prompt + ']: ' if isinstance(options, list): - print 'Acceptable Values for %s: ' % prompt + print('Acceptable Values for %s: ' % prompt) for o in options: - print ' %s' % o + print(' %s' % o) var = raw_input(user_prompt) @@ -79,8 +81,8 @@ def list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name): keynames.append(key.name) if not keynames: - print 'ERROR: No keys found in region ' + aws_region_name - print 'Please create an EC2 keypair before continuing' + print('ERROR: No keys found in region ' + aws_region_name) + print('Please create an EC2 keypair before continuing') sys.exit(1) return keynames @@ -93,8 +95,8 @@ def list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name): vpcids.append(vpc.id) if not vpcids: - print 'ERROR: No vpcs found in region ' + aws_region_name - print 'Please create an EC2 vpcpair before continuing' + print('ERROR: No vpcs found in region ' + aws_region_name) + print('Please create an EC2 vpcpair before continuing') sys.exit(1) return vpcids @@ -107,8 +109,8 @@ def list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_ subnetids.append(subnet.id) if not subnetids: - print 'ERROR: No subnets found in region ' + aws_region_name - print 'Please create an EC2 subnetpair before continuing' + print('ERROR: No subnets found in region ' + aws_region_name) + print('Please create an EC2 subnetpair before continuing') sys.exit(1) return subnetids From 91f58d6e3c905f3b541df8577d5582d91860631c Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 14:10:59 -0800 Subject: [PATCH 20/34] Initial support for Python 3 --- cli/cfncluster/cfncluster.py | 1 + cli/cfncluster/cfnconfig.py | 46 ++++++++++++++++++--------------- cli/cfncluster/cli.py | 2 ++ cli/cfncluster/config_sanity.py | 12 +++++---- cli/cfncluster/easyconfig.py | 13 ++++++---- cli/setup.py | 5 +++- 6 files changed, 47 insertions(+), 32 deletions(-) diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index 8be8873d84..9d019ab806 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -11,6 +11,7 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from builtins import str import sys import boto.cloudformation import boto.ec2.autoscale diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index 9380fb1bc1..f92f328875 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -11,13 +11,17 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import object +import configparser import os import sys import inspect import pkg_resources import json -import urllib2 +import urllib.request, urllib.error, urllib.parse from . import config_sanity import boto.cloudformation @@ -31,7 +35,7 @@ def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): return __cli_template -class CfnClusterConfig: +class CfnClusterConfig(object): def __init__(self, args): self.args = args @@ -60,7 +64,7 @@ def __init__(self, args): sys.exit(1) - __config = ConfigParser.ConfigParser() + __config = configparser.ConfigParser() __config.read(self.__config_file) # Determine the EC2 region to used used or default to us-east-1 @@ -73,17 +77,17 @@ def __init__(self, args): else: try: self.region = __config.get('aws', 'aws_region_name') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.region = 'us-east-1' # Check if credentials have been provided in config try: self.aws_access_key_id = __config.get('aws', 'aws_access_key_id') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_access_key_id=None try: self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_secret_access_key=None # Determine which cluster template will be used @@ -104,12 +108,12 @@ def __init__(self, args): # Check if package updates should be checked try: self.__update_check = __config.getboolean('global', 'update_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__update_check = True if self.__update_check == True: try: - __latest = json.loads(urllib2.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] + __latest = json.loads(urllib.request.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] if self.version < __latest: print('warning: There is a newer version %s of cfncluster available.' % __latest) except Exception: @@ -118,7 +122,7 @@ def __init__(self, args): # Check if config sanity should be run try: self.__sanity_check = __config.getboolean('global', 'sanity_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it __args_func = self.args.func.__name__ @@ -136,7 +140,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'EC2KeyPair', self.key_name) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: print("ERROR: Missing key_name option in [%s] section." % self.__cluster_section) sys.exit(1) self.parameters.append(('KeyName', self.key_name)) @@ -155,7 +159,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'URL', self.template_url) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: if self.region == 'us-gov-west-1': self.template_url = ('https://s3-%s.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.region, self.version)) @@ -191,7 +195,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__vpc_options.get(key)[1],__temp__) self.parameters.append((self.__vpc_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all cluster section options @@ -225,7 +229,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__cluster_options.get(key)[1],__temp__) self.parameters.append((self.__cluster_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Merge tags from config with tags from command line args @@ -234,7 +238,7 @@ def __init__(self, args): try: tags = __config.get(self.__cluster_section, 'tags') self.tags = json.loads(tags); - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: if args.tags is not None: @@ -251,7 +255,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__ebs_section = ('ebs %s' % self.__ebs_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all EBS options @@ -274,7 +278,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__ebs_options.get(key)[1],__temp__) self.parameters.append((self.__ebs_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -287,7 +291,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__scaling_section = ('scaling %s' % self.__scaling_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all scaling options @@ -309,7 +313,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__scaling_options.get(key)[1],__temp__) self.parameters.append((self.__scaling_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -318,9 +322,9 @@ def __init__(self, args): try: if self.args.extra_parameters is not None: self.parameters = dict(self.parameters) - self.__temp_dict = dict(self.parameters.items() + self.args.extra_parameters.items()) + self.__temp_dict = dict(list(self.parameters.items()) + list(self.args.extra_parameters.items())) self.__dictlist = [] - for key, value in self.__temp_dict.iteritems(): + for key, value in self.__temp_dict.items(): temp = [str(key),str(value)] self.__dictlist.append(temp) self.parameters = self.__dictlist diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index f317af989e..1b14dbbd10 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -83,6 +83,8 @@ def main(): help='do not wait for stack events, after executing stack command') subparsers = parser.add_subparsers() + subparsers.required = True + subparsers.dest = 'command' pcreate = subparsers.add_parser('create', help='creates a cluster') pcreate.add_argument("cluster_name", type=str, default=None, diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index 643c135df6..d7503f9081 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -10,12 +10,14 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from future import standard_library +standard_library.install_aliases() __author__ = 'dougalb' import boto.ec2 import boto.vpc -import urllib2 -from urlparse import urlparse +import urllib.request, urllib.error, urllib.parse +from urllib.parse import urlparse import boto.exception import sys @@ -93,11 +95,11 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty pass else: try: - urllib2.urlopen(resource_value) - except urllib2.HTTPError as e: + urllib.request.urlopen(resource_value) + except urllib.error.HTTPError as e: print(e.code) sys.exit(1) - except urllib2.URLError as e: + except urllib.error.URLError as e: print(e.args) sys.exit(1) # EC2 EBS Snapshot Id diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index a735618163..b8575c7522 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -11,7 +11,10 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import input +import configparser import sys import boto.ec2 import boto.vpc @@ -37,7 +40,7 @@ def prompt(prompt, default_value=None, hidden=False, options=None): for o in options: print(' %s' % o) - var = raw_input(user_prompt) + var = input(user_prompt) if var == '': return default_value @@ -123,7 +126,7 @@ def configure(args): else: config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config')) - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() # Check if configuration file exists if os.path.isfile(config_file): @@ -156,9 +159,9 @@ def configure(args): for section in sections: try: config.add_section(section['__name__']) - except ConfigParser.DuplicateSectionError: + except configparser.DuplicateSectionError: pass - for key, value in section.iteritems(): + for key, value in section.items(): # Only update configuration if not set if value is not None and key is not '__name__': config.set(section['__name__'], key, value) diff --git a/cli/setup.py b/cli/setup.py index 6b3671aa52..9560c1e9c7 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -21,13 +21,16 @@ def read(fname): console_scripts = ['cfncluster = cfncluster.cli:main'] version = "1.3.1" -requires = ['boto>=2.42.0', 'awscli>=1.10.56'] +requires = ['boto>=2.42.0', 'awscli>=1.10.56', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): # For python2.6 we have to require argparse since it # was not in stdlib until 2.7. requires.append('argparse>=1.4.0') +if sys.version_info.major == 2: + requires.append('configparser>=3.5.0') + setup( name = "cfncluster", version = version, From 061d4fd6d776cfaf92db4c76d7e4292dd1f571a1 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:03 -0800 Subject: [PATCH 21/34] Correcting version check for Python 2.6 --- cli/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/setup.py b/cli/setup.py index 9560c1e9c7..0924e78229 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -28,7 +28,7 @@ def read(fname): # was not in stdlib until 2.7. requires.append('argparse>=1.4.0') -if sys.version_info.major == 2: +if sys.version_info[0] == 2: requires.append('configparser>=3.5.0') setup( From 918dc0dc4efc9d75b9ccce20a6714a66e72f61c0 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:22 -0800 Subject: [PATCH 22/34] Improving tests --- tests/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test.sh b/tests/test.sh index d9bc48948d..ed783e0541 100644 --- a/tests/test.sh +++ b/tests/test.sh @@ -3,4 +3,6 @@ set -x echo $PATH -which cfncluster \ No newline at end of file +which cfncluster +cfncluster version +cfncluster --help From ff794fbbdfd2cd7e4039854407e598983cfd5e16 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:00:42 -0800 Subject: [PATCH 23/34] Updated CHANGELOG.rst --- CHANGELOG.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ce00b7698d..a7d45b2b03 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ CHANGELOG ========= +develop +======= +* updates:``cfncluster``: Moved to Apache 2.0 license +* updates:``cfncluster``: Support for Python 3 + 1.3.1 ===== * feature:``ami``: Added support for Ubuntu 16.04 LTS @@ -198,4 +203,4 @@ changes should be non-instrusive. 0.0.5 ===== -* Release on GitHub and PyPi \ No newline at end of file +* Release on GitHub and PyPi From 1271e8175b749daa5dfdea5bc4f2782e1515a782 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 9 Feb 2017 15:31:27 -0800 Subject: [PATCH 24/34] Adding Python 3.6 build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3c0ddaf190..9440395f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" install: - cd cli From 649bf95a906c1eac20ac0d2a373d7c52b6074ba7 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 27 Apr 2017 16:24:17 -0700 Subject: [PATCH 25/34] Adding YAML version of the template. :-) --- cloudformation/cfncluster.cfn.yaml | 3093 ++++++++++++++++++++++++++++ 1 file changed, 3093 insertions(+) create mode 100644 cloudformation/cfncluster.cfn.yaml diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml new file mode 100644 index 0000000000..31b8a99229 --- /dev/null +++ b/cloudformation/cfncluster.cfn.yaml @@ -0,0 +1,3093 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' + +Description: 'AWS CloudFormation Sample Template cfncluster.cfn.yaml: Sample template + showing an framework for deploying master + compute type clusters on AWS. **WARNING** + This template creates AWS resources. You will be billed for the AWS resources used + if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1' + +Metadata: + AWS::CloudFormation::Designer: + 0824f592-a72b-4bb5-8c67-4ce6df509cdd: + embeds: [] + position: + x: -430 + y: 30 + size: + height: 240 + width: 240 + z: 1 + 0c5a1e92-ffdd-4344-8ba0-65270e5fd472: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 390 + y: 30 + size: + height: 60 + width: 60 + z: 1 + 127f78c3-d47a-4770-8aad-3fd733c3e1b0: + embeds: [] + isrelatedto: + - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + position: + x: 60 + y: 680 + size: + height: 60 + width: 60 + z: 1 + 210b5501-b505-4ee1-917c-a20cae3ac837: + embeds: [] + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + position: + x: 360 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 23ed10e0-7726-4099-8370-4116c0bc78a2: + embeds: [] + isrelatedto: + - 8e63b592-a50f-4b89-a40b-e991adaef77b + position: + x: -50 + y: 680 + size: + height: 60 + width: 60 + z: 1 + 2522b048-2b7c-44ed-9d35-6cc66069ca5b: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 390 + y: 130 + size: + height: 60 + width: 60 + z: 1 + 2904615e-c0ad-4325-b0c5-b9ea7c482b4e: + embeds: [] + position: + x: 60 + y: 390 + size: + height: 60 + width: 60 + z: 1 + 2f599618-92e8-49cf-92bd-551ba9775f39: + embeds: [] + position: + x: 450 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 3fa053e5-8ab7-4cdf-a737-4d20386e4717: + dependson: + - 88da1502-8179-4be3-9123-fdc5eac0b6a3 + embeds: [] + isassociatedwith: + - d3204db6-c821-400d-bf31-6afd5739923c + isrelatedto: + - bb5a532f-bb43-472a-9965-c4054cd61cc9 + - 210b5501-b505-4ee1-917c-a20cae3ac837 + - ee5f3006-419e-4786-b527-e9503a662e5e + position: + x: -50 + y: 480 + size: + height: 60 + width: 60 + z: 1 + 45360a4f-64aa-49f8-b016-78fe5ed86819: + embeds: [] + ismemberof: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + position: + x: 60 + y: 310 + size: + height: 60 + width: 60 + z: 1 + 466554d3-0e10-4ae4-bfee-51b0aadbf923: + source: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + target: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + 596ddc76-44b9-4747-921c-c000c5b636c0: + embeds: [] + position: + x: 60 + y: 150 + references: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + size: + height: 60 + width: 60 + z: 0 + 619070e4-82e5-445a-93a0-4795ff69d61c: + embeds: [] + isrelatedto: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + position: + x: 180 + y: 480 + size: + height: 60 + width: 60 + z: 1 + 6a874346-80de-4412-9677-00590371a8a2: + embeds: [] + isassociatedwith: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + isrelatedto: + - 210b5501-b505-4ee1-917c-a20cae3ac837 + position: + x: 450 + y: 470 + size: + height: 60 + width: 60 + z: 1 + 789cb442-3b1a-4673-a07e-8acb1c59c271: + embeds: [] + position: + x: 270 + y: 380 + size: + height: 60 + width: 60 + z: 1 + 7b0e0a3f-3a02-4e70-9e79-7647b98fd579: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + - 789cb442-3b1a-4673-a07e-8acb1c59c271 + position: + x: 270 + y: 130 + size: + height: 60 + width: 60 + z: 1 + 88da1502-8179-4be3-9123-fdc5eac0b6a3: + embeds: [] + isconnectedto: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + - ee5f3006-419e-4786-b527-e9503a662e5e + position: + x: -50 + y: 310 + size: + height: 60 + width: 60 + z: 1 + 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c: + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + position: + x: 60 + y: 600 + size: + height: 60 + width: 60 + z: 1 + 8e63b592-a50f-4b89-a40b-e991adaef77b: + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + position: + x: -50 + y: 600 + size: + height: 60 + width: 60 + z: 1 + 9dd9772e-03a3-4983-b28e-a52ac42f0ffe: + source: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + target: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + a8fbc80f-2514-4cc9-aee8-ba7936add0a7: + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + position: + x: 170 + y: 30 + size: + height: 60 + width: 60 + z: 1 + bb5a532f-bb43-472a-9965-c4054cd61cc9: + embeds: [] + position: + x: -150 + y: 70 + size: + height: 150 + width: 150 + z: 1 + c8d3da13-fd27-4377-b606-0f41728c17e1: + embeds: [] + position: + x: -170 + y: 310 + size: + height: 60 + width: 60 + z: 1 + ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb: + embeds: [] + position: + x: 270 + y: 30 + size: + height: 60 + width: 60 + z: 1 + d3204db6-c821-400d-bf31-6afd5739923c: + embeds: [] + ismemberof: + - 619070e4-82e5-445a-93a0-4795ff69d61c + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + position: + x: 60 + y: 480 + size: + height: 60 + width: 60 + z: 1 + d5bad569-ad14-487d-bcc6-5569373bc8cd: + embeds: [] + position: + x: 180 + y: 310 + size: + height: 60 + width: 60 + z: 1 + e9ac6c41-8110-49b9-a92c-de57be8a292a: + source: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + target: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + ee5f3006-419e-4786-b527-e9503a662e5e: + embeds: [] + position: + x: -170 + y: 390 + size: + height: 60 + width: 60 + z: 1 + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Network - Basic Settings + Parameters: + - VPCId + - MasterSubnetId + - Label: + default: Cluster - Basic Setting + Parameters: + - KeyName + - AvailabilityZone + - MasterInstanceType + - ComputeInstanceType + - BaseOS + - Scheduler + - Label: + default: EBS - Basic Settings + Parameters: + - VolumeSize + - Label: + default: Network - Advanced Settings + Parameters: + - AccessFrom + - ComputeSubnetId + - ComputeSubnetCidr + - UsePublicIps + - AdditionalSG + - VPCSecurityGroupId + - Label: + default: Cluster - Advanced Setting + Parameters: + - InitialQueueSize + - ComputeWaitConditionCount + - MaxQueueSize + - SpotPrice + - ClusterType + - ProxyServer + - CustomAMI + - MaintainInitialSize + - PreInstallScript + - PreInstallArgs + - PostInstallArgs + - PostInstallScript + - S3ReadResource + - S3ReadWriteResource + - Placement + - PlacementGroup + - EncryptedEphemeral + - EphemeralDir + - SharedDir + - CWLRegion + - CWLLogGroup + - CustomChefRunList + - CustomChefCookbook + - ExtraJson + - Tenancy + - EphemeralKMSKeyId + - ClusterReadyScript + - MasterRootVolumeSize + - ComputeRootVolumeSize + - EC2IAMRoleName + - Label: + default: EBS - Advanced Settings + Parameters: + - VolumeType + - EBSSnapshotId + - VolumeIOPS + - EBSEncryption + - EBSKMSKeyId + - EBSVolumeId + - Label: + default: Scaling Settings + Parameters: + - ScalingPeriod + - ScalingEvaluationPeriods + - ScalingThreshold + - ScalingAdjustment + - ScalingThreshold2 + - ScalingAdjustment2 + - ScalingCooldown + - Label: + default: Additonal Settings + Parameters: + - CLITemplate + ParameterLabels: + AccessFrom: + AccessFrom: + default: access_from + default: access_from + AdditionalSG: + default: additional_sg + AvailabilityZone: + default: AVAILABILITY ZONE + BaseOS: + default: base_os + CLITemplate: + default: CLI TEMPLATE + CWLLogGroup: + default: cwl_log_group + CWLRegion: + default: cwl_region + ClusterReadyScript: + default: cluster_ready + ClusterType: + default: cluster_type + ComputeInstanceType: + default: compute_instance_type + ComputeRootVolumeSize: + default: compute_volume_size + ComputeSubnetCidr: + default: compute_subnet_cidr + ComputeSubnetId: + default: compute_subnet_id + ComputeWaitConditionCount: + default: WAIT COUNT + CustomAMI: + default: custom_ami + CustomChefCookbook: + default: custom_chef_cookbook + CustomChefRunList: + default: custom_chef_runlist + EBSEncryption: + default: ebs_encryption + EBSKMSKeyId: + default: ebs_kms_key_id + EBSSnapshotId: + default: ebs_snapshot_id + EBSVolumeId: + default: ebs_volume_id + EC2IAMRoleName: + default: ec2_iam_role + EncryptedEphemeral: + default: encrypted_ephemeral + EphemeralDir: + default: ephemeral_dir + EphemeralKMSKeyId: + default: ephemeral_kms_key_id + ExtraJson: + default: extra_json + InitialQueueSize: + default: initial_queue_size + KeyName: + default: key_name + MaintainInitialSize: + default: maintain_initial_size + MasterInstanceType: + default: master_instance_type + MasterRootVolumeSize: + default: master_root_volume_size + MasterSubnetId: + default: master_subnet_id + MaxQueueSize: + default: max_queue_size + Placement: + default: placement + PlacementGroup: + default: placement_group + PostInstallArgs: + default: post_install_args + PostInstallScript: + default: post_install + PreInstallArgs: + default: pre_install_args + PreInstallScript: + default: pre_install + ProxyServer: + default: proxy_server + S3ReadResource: + default: s3_read_resource + S3ReadWriteResource: + default: s3_read_write_resource + ScalingAdjustment: + default: scaling_adjustment + ScalingAdjustment2: + default: scaling_adjustment2 + ScalingCooldown: + default: scaling_cooldown + ScalingEvaluationPeriods: + default: scaling_evaluation_periods + ScalingPeriod: + default: scaling_period + ScalingThreshold: + default: scaling_threshold + ScalingThreshold2: + default: scaling_threshold2 + Scheduler: + default: scheduler + SharedDir: + default: shared_dir + SpotPrice: + default: spot_price + Tenancy: + default: tenancy + UsePublicIps: + default: use_public_ips + VPCId: + default: vpc_id + VPCSecurityGroupId: + default: vpc_security_group_id + VolumeIOPS: + default: volume_iops + VolumeSize: + default: volume_size + VolumeType: + default: volume_type + +Parameters: + AccessFrom: + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + Default: 0.0.0.0/0 + Description: Lockdown SSH/HTTP access (default can be accessed from anywhere) + MaxLength: '18' + MinLength: '9' + Type: String + AdditionalCfnTemplate: + Default: NONE + Description: A second CloudFormation template to launch with the cluster + Type: String + AdditionalSG: + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + Default: NONE + Description: Additional VPC secuirty group to be added to instances. Defaults + to NONE + Type: String + AvailabilityZone: + Description: Availability Zone the cluster will launch into. THIS IS REQUIRED + Type: AWS::EC2::AvailabilityZone::Name + BaseOS: + AllowedValues: + - centos6 + - centos7 + - alinux + - ubuntu1404 + - ubuntu1604 + ConstraintDescription: must be a supported base OS + Default: alinux + Description: Base OS type for cluster AMI + Type: String + CLITemplate: + Default: default + Type: String + CWLLogGroup: + Default: NONE + Description: CloudWatch Logs LogGroup + Type: String + CWLRegion: + Default: NONE + Description: CloudWatch Logs region + Type: String + ClusterReadyScript: + Default: NONE + Description: Cluster ready script URL. This is only on the MasterServer, when + the cluster reaches CREATE_COMPLETE. + Type: String + ClusterType: + AllowedValues: + - ondemand + - spot + ConstraintDescription: 'Must be a supported cluster type: ondemand, spot' + Default: ondemand + Description: Type of cluster to launch. Can either be "ondemand" or "spot". Choosing + "spot" will cause the ComputeFleet AutoScaling group to launch EC2 Spot instances. + Default value is "ondemand". + Type: String + ComputeInstanceType: + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.32xlarge + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + Default: t2.micro + Description: ComputeFleet EC2 instance type + Type: String + ComputeRootVolumeSize: + Default: '15' + Description: Size of ComputeFleet EBS root volume in GB + MinValue: '15' + Type: Number + ComputeSubnetCidr: + AllowedPattern: (NONE|(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + Default: NONE + Description: CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited + list and can support multiple CIDR ranges for a multi-AZ cluster. The order + and length of this list MUST match the AvailabilityZones parameter. + Type: String + ComputeSubnetId: + AllowedPattern: (NONE|^subnet-[0-9a-z]{8}$) + Default: NONE + Description: ID of the Subnet you want to provision the Compute Servers into + Type: String + ComputeWaitConditionCount: + Default: '2' + Description: Specific number of instances to wait for while creating the cluster + Type: Number + CustomAMI: + AllowedPattern: (NONE|^ami-[0-9a-z]{8}$|^ami-[0-9a-z]{17}$) + Default: NONE + Description: ID of a Custom AMI, to use instead of published AMI's + Type: String + CustomChefCookbook: + Default: NONE + Description: URL of custom cookbook that will override the default. This will + be unpacked and then dependencies resolved with Berkshelf. + Type: String + CustomChefRunList: + Default: NONE + Description: Custom run list, which will override the default + Type: String + EBSEncryption: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to use EBS encryption for /shared volume. (Not to be + used for snapshots) + Type: String + EBSKMSKeyId: + Default: NONE + Description: KMS ARN for customer created master key, will be used for EBS encryption + Type: String + EBSSnapshotId: + AllowedPattern: (NONE|^snap-[0-9a-z]{8}$|^snap-[0-9a-z]{17}$) + Default: NONE + Description: Id of EBS snapshot if using snapshot as source for volume + Type: String + EBSVolumeId: + AllowedPattern: (NONE|^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$) + Default: NONE + Description: Existing EBS volume Id + Type: String + EC2IAMRoleName: + Default: NONE + Description: Existing EC2 IAM role name + Type: String + EncryptedEphemeral: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to encrypt local ephemeral drives. The keys are in-memory + and non-recoverable. + Type: String + EphemeralDir: + Default: /scratch + Description: The path/mountpoint for the ephemeral drive + Type: String + EphemeralKMSKeyId: + Default: NONE + Description: KMS ARN for customer created master key, will be used for ephemeral + encryption + Type: String + ExtraJson: + Default: '{}' + Description: Extra json to be added to Chef dna.json + Type: String + InitialQueueSize: + Default: '2' + Description: Initial number of EC2 instances to launch as compute nodes within + the cluster. This value maps to the DesiredSize parameter for the ComputeFleet + AutoScaling Group. + Type: Number + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instances + using the default cluster user. + Type: AWS::EC2::KeyPair::KeyName + MaintainInitialSize: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'false' + Description: Boolean flag to set autoscaling group to maintain initial size and + scale back + Type: String + MasterInstanceType: + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.32xlarge + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + Default: t2.micro + Description: MasterServer EC2 instance type + Type: String + MasterRootVolumeSize: + Default: '15' + Description: Size of MasterServer EBS root volume in GB + MinValue: '15' + Type: Number + MasterSubnetId: + Description: ID of the Subnet you want to provision the Master server into + Type: AWS::EC2::Subnet::Id + MaxQueueSize: + Default: '10' + Description: Maximum number of EC2 instances that can be launched in the cluster. + This value maps to the MaxSize parameter for the ComputeFleet AutoScaling Group. + Type: Number + Placement: + AllowedValues: + - cluster + - compute + Default: cluster + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + PlacementGroup: + Default: NONE + Description: The name of an existing placement group + Type: String + PostInstallArgs: + Default: NONE + Description: Postinstall script args passed to the postinstall script. + Type: String + PostInstallScript: + Default: NONE + Description: Postinstall script URL. This is run before any host configuration. + Type: String + PreInstallArgs: + Default: NONE + Description: Preinstall script args passed to the preinstall script. + Type: String + PreInstallScript: + Default: NONE + Description: Preinstall script URL. This is run before any host configuration. + Type: String + ProxyServer: + Default: NONE + Description: hostname and port of HTTP proxy server for cfn-init, boto and yum + i.e. proxy.example.com:8080 + Type: String + S3ReadResource: + Default: NONE + Description: S3 resource with read access from cfncluster nodes + Type: String + S3ReadWriteResource: + Default: NONE + Description: Addtional policy document to be added to EC2 IAM role created and + assigned to all nodes. + Type: String + ScalingAdjustment: + Default: '1' + Description: Number of instances to add to cluster when the CloudWatch ScaleUp + action is called. + Type: String + ScalingAdjustment2: + Default: '10' + Description: Number of instances to add to cluster when the CloudWatch ScaleUp2 + action is called. + Type: String + ScalingCooldown: + Default: '300' + Description: Period in seconds to wait before allowing further scaling actions + Type: String + ScalingEvaluationPeriods: + Default: '2' + Description: Number of periods consective periods required to trigger the scaling + adjustment + Type: String + ScalingPeriod: + Default: '60' + Description: Period in seconds to measure ScalingThreshold + Type: String + ScalingThreshold: + Default: '1' + Description: Threshold for triggering CloudWatch ScaleUp action + Type: String + ScalingThreshold2: + Default: '200' + Description: Threshold for triggering CloudWatch ScaleUp2 action + Type: String + Scheduler: + AllowedValues: + - sge + - openlava + - torque + - slurm + - custom + - test + - pbspro + ConstraintDescription: must be a supported scheduler + Default: sge + Description: Cluster scheduler + Type: String + SharedDir: + Default: /shared + Description: The path/mountpoint for the shared drive + Type: String + SpotPrice: + Default: '0.00' + Description: Spot bid price for the ComputeFleet AutoScaling Group when the ClusterType + = "spot". + Type: Number + Tenancy: + AllowedValues: + - default + - dedicated + Default: default + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + UsePublicIps: + AllowedValues: + - 'true' + - 'false' + ConstraintDescription: true/false + Default: 'true' + Description: Boolean flag to use public IP's for instances. If false, the VPC + must be correctly setup to use NAT for all traffic. + Type: String + VPCId: + Description: ID of the VPC you want to provision cluster into. Only used with + UseVPCBase=false + Type: AWS::EC2::VPC::Id + VPCSecurityGroupId: + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + Default: NONE + Description: Existing VPC security group Id + Type: String + VolumeIOPS: + Default: '100' + Description: Number of IOPS for volume type io1. Not used for other volume types. + Type: Number + VolumeSize: + Default: '20' + Description: Size of EBS volume in GB, if creating a new one + Type: Number + VolumeType: + AllowedValues: + - standard + - gp2 + - io1 + - st1 + - sc1 + ConstraintDescription: 'must be a supported volume type: standard, io1, gp2, st1, + sc1' + Default: gp2 + Description: Type of volume to create either new or from snapshot + Type: String + +Mappings: + AWSInstanceType2Capabilites: + c3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + c3.large: + Arch: 64HVM + EBSOpt: 'False' + c3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.large: + Arch: 64HVM + EBSOpt: 'True' + c4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + cc2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + cg1.4xlarge: + Arch: 64HVM + EBSOpt: 'False' + cr1.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + d2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + g2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + g2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + i2.large: + Arch: 64HVM + EBSOpt: 'False' + i2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.large: + Arch: 64HVM + EBSOpt: 'False' + m3.medium: + Arch: 64HVM + EBSOpt: 'False' + m3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.10xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.large: + Arch: 64HVM + EBSOpt: 'True' + m4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.large: + Arch: 64HVM + EBSOpt: 'True' + r3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + r3.large: + Arch: 64HVM + EBSOpt: 'False' + r3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + t2.large: + Arch: 64HVM + EBSOpt: 'False' + t2.medium: + Arch: 64HVM + EBSOpt: 'False' + t2.micro: + Arch: 64HVM + EBSOpt: 'False' + t2.nano: + Arch: 64HVM + EBSOpt: 'False' + t2.small: + Arch: 64HVM + EBSOpt: 'False' + x1.32xlarge: + Arch: 64HVM + EBSOpt: 'True' + AWSRegion2Capabilites: + ap-northeast-1: + arn: aws + ap-northeast-2: + arn: aws + ap-south-1: + arn: aws + ap-southeast-1: + arn: aws + ap-southeast-2: + arn: aws + eu-central-1: + arn: aws + eu-west-1: + arn: aws + sa-east-1: + arn: aws + us-east-1: + arn: aws + us-gov-west-1: + arn: aws-us-gov + us-west-1: + arn: aws + us-west-2: + arn: aws + AWSRegionOS2AMI: + ap-northeast-1: + alinux: ami-50bd7731 + centos6: ami-46b17b27 + centos7: ami-43c90322 + ubuntu1404: ami-89bd77e8 + ubuntu1604: ami-eabe748b + ap-northeast-2: + alinux: ami-afe431c1 + centos6: ami-75e1341b + centos7: ami-5cec3932 + ubuntu1404: ami-bae732d4 + ubuntu1604: ami-7eef3a10 + ap-south-1: + alinux: ami-0495e06b + centos6: ami-a392e7cc + centos7: ami-bf90e5d0 + ubuntu1404: ami-489de827 + ubuntu1604: ami-5d90e532 + ap-southeast-1: + alinux: ami-9af229f9 + centos6: ami-23f22940 + centos7: ami-d5f52eb6 + ubuntu1404: ami-95f229f6 + ubuntu1604: ami-23f32840 + ap-southeast-2: + alinux: ami-51b98f32 + centos6: ami-0ea6906d + centos7: ami-79a5931a + ubuntu1404: ami-d3b98fb0 + ubuntu1604: ami-c1b98fa2 + eu-central-1: + alinux: ami-96db28f9 + centos6: ami-5fd92a30 + centos7: ami-fcda2993 + ubuntu1404: ami-89db28e6 + ubuntu1604: ami-9dd427f2 + eu-west-1: + alinux: ami-d8f785ab + centos6: ami-27f78554 + centos7: ami-5af48629 + ubuntu1404: ami-3ff0824c + ubuntu1604: ami-99f082ea + sa-east-1: + alinux: ami-8bd848e7 + centos6: ami-8ad848e6 + centos7: ami-f8d84894 + ubuntu1404: ami-3f34a553 + ubuntu1604: ami-8d0e9fe1 + us-east-1: + alinux: ami-d5751ec2 + centos6: ami-504b2047 + centos7: ami-e94328fe + ubuntu1404: ami-ba4a21ad + ubuntu1604: ami-d6751ec1 + us-gov-west-1: + alinux: NOT_SUPPORTED + centos6: NOT_SUPPORTED + centos7: NOT_SUPPORTED + ubuntu1404: NOT_SUPPORTED + ubuntu1604: NOT_SUPPORTED + us-west-1: + alinux: ami-5f19553f + centos6: ami-051f5365 + centos7: ami-a51955c5 + ubuntu1404: ami-40185420 + ubuntu1604: ami-751f5315 + us-west-2: + alinux: ami-1e31e07e + centos6: ami-6e30e10e + centos7: ami-2e1dcc4e + ubuntu1404: ami-4633e226 + ubuntu1604: ami-870edfe7 + CfnClusterVersions: + default: + ami: '201609030709' + berkshelf: 4.6.1 + cfncluster: cfncluster-1.3.1 + chef: 12.13.30 + cookbook: cfncluster-cookbook-1.3.0 + ridley: 4.5.1 + OSFeatures: + alinux: + RootDevice: /dev/xvda + User: ec2-user + centos6: + RootDevice: /dev/sda1 + User: centos + centos7: + RootDevice: /dev/sda1 + User: centos + ubuntu1404: + RootDevice: /dev/sda1 + User: ubuntu + ubuntu1604: + RootDevice: /dev/sda1 + User: ubuntu + +Conditions: + AddAdditionalSG: + Fn::Not: + - Fn::Equals: + - Ref: AdditionalSG + - NONE + CloudWatchLogs: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: CWLRegion + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: CWLLogGroup + - NONE + ComputePublicIps: + Fn::And: + - Fn::Equals: + - Ref: UsePublicIps + - 'true' + - Condition: UseMasterSubnetForCompute + CreateComputeSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + CreateEBSVolume: + Fn::Equals: + - Ref: EBSVolumeId + - NONE + CreateEC2IAMRole: + Fn::Equals: + - Ref: EC2IAMRoleName + - NONE + CreatePlacementGroup: + Fn::And: + - Fn::Equals: + - Ref: PlacementGroup + - DYNAMIC + - Condition: UsePlacementGroup + CreateSecurityGroups: + Fn::Equals: + - Ref: VPCSecurityGroupId + - NONE + CreateSubStack: + Fn::Not: + - Fn::Equals: + - Ref: AdditionalCfnTemplate + - NONE + MaintainInitialASGSize: + Fn::Equals: + - Ref: MaintainInitialSize + - 'true' + MasterPublicIp: + Fn::Equals: + - Ref: UsePublicIps + - 'true' + UseClusterPlacement: + Fn::And: + - Fn::Equals: + - Ref: Placement + - cluster + - Condition: UsePlacementGroup + UseComputeSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + - Fn::Not: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + UseCustomAMI: + Fn::Not: + - Fn::Equals: + - Ref: CustomAMI + - NONE + UseCustomRunList: + Fn::Not: + - Fn::Equals: + - Ref: CustomChefRunList + - NONE + UseDedicatedTenancy: + Fn::Equals: + - Ref: Tenancy + - dedicated + UseEBSEncryption: + Fn::Equals: + - Ref: EBSEncryption + - 'true' + UseEBSKMSKey: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: EBSKMSKeyId + - NONE + - Condition: UseEBSEncryption + UseEBSPIOPS: + Fn::Equals: + - Ref: VolumeType + - io1 + UseEBSSnapshot: + Fn::Not: + - Fn::Equals: + - Ref: EBSSnapshotId + - NONE + UseEC2IAMRole: + Fn::Not: + - Fn::Equals: + - Ref: EC2IAMRoleName + - NONE + UseEphemeralKMSKey: + Fn::And: + - Fn::Not: + - Fn::Equals: + - Ref: EphemeralKMSKeyId + - NONE + - Fn::Equals: + - Ref: EncryptedEphemeral + - 'true' + UseExistingEBSVolume: + Fn::Not: + - Fn::Equals: + - Ref: EBSVolumeId + - NONE + UseExistingSecurityGroup: + Fn::Not: + - Fn::Equals: + - Ref: VPCSecurityGroupId + - NONE + UseMasterSubnetForCompute: + Fn::And: + - Fn::Equals: + - Ref: ComputeSubnetId + - NONE + - Fn::Equals: + - Ref: ComputeSubnetCidr + - NONE + UsePlacementGroup: + Fn::Not: + - Fn::Equals: + - Ref: PlacementGroup + - NONE + UseS3ReadPolicy: + Fn::Not: + - Fn::Equals: + - Ref: S3ReadResource + - NONE + UseS3ReadWritePolicy: + Fn::Not: + - Fn::Equals: + - Ref: S3ReadWriteResource + - NONE + UseSpotInstances: + Fn::Equals: + - Ref: ClusterType + - spot + +Resources: + AddCapacityAlarm: + Metadata: + AWS::CloudFormation::Designer: + id: 23ed10e0-7726-4099-8370-4116c0bc78a2 + Properties: + AlarmActions: + - Ref: ScaleUpPolicy + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: + Ref: AWS::StackName + EvaluationPeriods: + Ref: ScalingEvaluationPeriods + MetricName: pending + Namespace: cfncluster + Period: + Ref: ScalingPeriod + Statistic: Average + Threshold: + Ref: ScalingThreshold + Type: AWS::CloudWatch::Alarm + AddCapacityAlarm2: + Metadata: + AWS::CloudFormation::Designer: + id: 127f78c3-d47a-4770-8aad-3fd733c3e1b0 + Properties: + AlarmActions: + - Ref: ScaleUpPolicy2 + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: + Ref: AWS::StackName + EvaluationPeriods: + Ref: ScalingEvaluationPeriods + MetricName: pending + Namespace: cfncluster + Period: + Ref: ScalingPeriod + Statistic: Sum + Threshold: + Ref: ScalingThreshold2 + Type: AWS::CloudWatch::Alarm + AdditionalCfnStack: + Condition: CreateSubStack + Properties: + TemplateURL: + Ref: AdditionalCfnTemplate + Type: AWS::CloudFormation::Stack + AssociateEIP: + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: e9ac6c41-8110-49b9-a92c-de57be8a292a + Properties: + AllocationId: + Fn::GetAtt: + - MasterEIP + - AllocationId + NetworkInterfaceId: + Ref: MasterENI + Type: AWS::EC2::EIPAssociation + CfnClusterPolicies: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 7b0e0a3f-3a02-4e70-9e79-7647b98fd579 + Properties: + PolicyDocument: + Statement: + - Action: + - ec2:AttachVolume + - ec2:DescribeInstanceAttribute + - ec2:DescribeInstanceStatus + - ec2:DescribeInstances + Effect: Allow + Resource: + - '*' + Sid: EC2 + - Action: + - dynamodb:ListTables + Effect: Allow + Resource: + - '*' + Sid: DynamoDBList + - Action: + - sqs:SendMessage + - sqs:ReceiveMessage + - sqs:ChangeMessageVisibility + - sqs:DeleteMessage + - sqs:GetQueueUrl + Effect: Allow + Resource: + - Fn::GetAtt: + - SQS + - Arn + Sid: SQSQueue + - Action: + - autoscaling:DescribeAutoScalingGroups + - autoscaling:TerminateInstanceInAutoScalingGroup + - autoscaling:SetDesiredCapacity + Effect: Allow + Resource: + - '*' + Sid: Autoscaling + - Action: + - cloudwatch:PutMetricData + Effect: Allow + Resource: + - '*' + Sid: CloudWatch + - Action: + - dynamodb:PutItem + - dynamodb:Query + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:DescribeTable + Effect: Allow + Resource: + - Fn::Join: + - '' + - - 'arn:' + - Fn::FindInMap: + - AWSRegion2Capabilites + - Ref: AWS::Region + - arn + - ':dynamodb:' + - Ref: AWS::Region + - ':' + - Ref: AWS::AccountId + - :table/ + - Ref: DynamoDBTable + Sid: DynamoDBTable + - Action: + - sqs:ListQueues + Effect: Allow + Resource: + - '*' + Sid: SQSList + - Action: + - logs:* + Effect: Allow + Resource: + - Fn::Join: + - '' + - - 'arn:' + - Fn::FindInMap: + - AWSRegion2Capabilites + - Ref: AWS::Region + - arn + - :logs:*:*:* + Sid: CloudWatchLogs + PolicyName: cfncluster + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + ComputeFleet: + CreationPolicy: + ResourceSignal: + Count: + Ref: ComputeWaitConditionCount + Timeout: PT30M + DependsOn: MasterServer + Metadata: + AWS::CloudFormation::Designer: + id: 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + Properties: + DesiredCapacity: + Ref: InitialQueueSize + LaunchConfigurationName: + Ref: ComputeServerLaunchConfig + MaxSize: + Ref: MaxQueueSize + MinSize: + Fn::If: + - MaintainInitialASGSize + - Ref: InitialQueueSize + - '0' + NotificationConfiguration: + NotificationTypes: + - autoscaling:EC2_INSTANCE_TERMINATE + TopicARN: + Ref: SNS + PlacementGroup: + Fn::If: + - UsePlacementGroup + - Fn::If: + - CreatePlacementGroup + - Ref: DynamicPlacementGroup + - Ref: PlacementGroup + - Ref: AWS::NoValue + Tags: + - Key: Name + PropagateAtLaunch: 'true' + Value: Compute + VPCZoneIdentifier: + - Fn::If: + - UseMasterSubnetForCompute + - Ref: MasterSubnetId + - Fn::If: + - CreateComputeSubnetForCompute + - Ref: ComputeSubnet + - Ref: ComputeSubnetId + Type: AWS::AutoScaling::AutoScalingGroup + ComputeRoute: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 596ddc76-44b9-4747-921c-c000c5b636c0 + Properties: + DestinationCidrBlock: 0.0.0.0/0 + NetworkInterfaceId: + Ref: MasterENI + RouteTableId: + Ref: ComputeRouteTable + Type: AWS::EC2::Route + ComputeRouteTable: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + Properties: + Tags: + - Key: Application + Value: + Ref: AWS::StackName + - Key: Network + Value: ComputeSubnet + VpcId: + Ref: VPCId + Type: AWS::EC2::RouteTable + ComputeSecurityGroup: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + Properties: + GroupDescription: Allow access to resources in subnets behind front + SecurityGroupIngress: + - FromPort: '0' + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: MasterSecurityGroup + ToPort: '65535' + VpcId: + Ref: VPCId + Type: AWS::EC2::SecurityGroup + ComputeSecurityGroupIngress: + Condition: CreateSecurityGroups + Properties: + FromPort: '0' + GroupId: + Ref: ComputeSecurityGroup + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: ComputeSecurityGroup + ToPort: '65535' + Type: AWS::EC2::SecurityGroupIngress + ComputeServerLaunchConfig: + Metadata: + AWS::CloudFormation::Designer: + id: d3204db6-c821-400d-bf31-6afd5739923c + AWS::CloudFormation::Init: + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + - signalComputeReady + deployConfigFiles: + commands: + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + files: + /etc/chef/client.rb: + content: + Fn::Join: + - '' + - - cookbook_path ['/etc/chef/cookbooks'] + group: root + mode: '000644' + owner: root + /tmp/dna.json: + content: + cfncluster: + cfn_cluster_user: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - User + cfn_encrypted_ephemeral: + Ref: EncryptedEphemeral + cfn_ephemeral_dir: + Ref: EphemeralDir + cfn_master: + Fn::GetAtt: + - MasterServer + - PrivateDnsName + cfn_node_type: ComputeFleet + cfn_postinstall: + Ref: PostInstallScript + cfn_postinstall_args: + Ref: PostInstallArgs + cfn_preinstall: + Ref: PreInstallScript + cfn_preinstall_args: + Ref: PreInstallArgs + cfn_proxy: + Ref: ProxyServer + cfn_region: + Ref: AWS::Region + cfn_scheduler: + Ref: Scheduler + cfn_shared_dir: + Ref: SharedDir + cfn_sqs_queue: + Ref: SQS + stack_name: + Ref: AWS::StackName + run_list: + Fn::If: + - UseCustomRunList + - Ref: CustomChefRunList + - Fn::Join: + - '' + - - 'recipe[cfncluster::' + - Ref: Scheduler + - _config] + group: root + mode: '000644' + owner: root + /tmp/extra.json: + content: + Ref: ExtraJson + group: root + mode: '000644' + owner: root + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + signalComputeReady: + commands: + compute_ready: + command: /opt/cfncluster/scripts/compute_ready + Comment: cfncluster Compute server + Properties: + AssociatePublicIpAddress: + Fn::If: + - ComputePublicIps + - 'true' + - 'false' + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - RootDevice + Ebs: + VolumeSize: + Ref: ComputeRootVolumeSize + VolumeType: gp2 + IamInstanceProfile: + Fn::If: + - UseEC2IAMRole + - Ref: EC2IAMRoleName + - Ref: RootInstanceProfile + ImageId: + Fn::If: + - UseCustomAMI + - Ref: CustomAMI + - Fn::FindInMap: + - AWSRegionOS2AMI + - Ref: AWS::Region + - Ref: BaseOS + InstanceMonitoring: 'false' + InstanceType: + Ref: ComputeInstanceType + KeyName: + Ref: KeyName + PlacementTenancy: + Fn::If: + - UseDedicatedTenancy + - Ref: Tenancy + - Ref: AWS::NoValue + SecurityGroups: + - Fn::If: + - CreateSecurityGroups + - Ref: ComputeSecurityGroup + - Ref: AWS::NoValue + - Fn::If: + - AddAdditionalSG + - Ref: AdditionalSG + - Ref: AWS::NoValue + - Fn::If: + - UseExistingSecurityGroup + - Ref: VPCSecurityGroupId + - Ref: AWS::NoValue + SpotPrice: + Fn::If: + - UseSpotInstances + - Ref: SpotPrice + - Ref: AWS::NoValue + UserData: + Fn::Base64: + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - Ref: ProxyServer + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - Ref: ProxyServer + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - Ref: AWS::StackName + - ' --resource=ComputeFleet --region=' + - Ref: AWS::Region + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - Ref: ProxyServer + - ' + + ' + - custom_cookbook= + - Ref: CustomChefCookbook + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - Ref: AWS::Region + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - Ref: AWS::Region + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - Ref: AWS::Region + - /cookbooks/ + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cfncluster + - ' + + ' + - export cookbook_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - ' + + ' + - export chef_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - chef + - ' + + ' + - export ridley_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - ridley + - ' + + ' + - export berkshelf_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - berkshelf + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - Ref: AWS::StackName + - ' -v -c default -r ComputeServerLaunchConfig --region ' + - Ref: AWS::Region + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup + complete" --stack= + - Ref: AWS::StackName + - ' --resource=ComputeFleet --region=' + - Ref: AWS::Region + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Type: AWS::AutoScaling::LaunchConfiguration + ComputeSubnet: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + Properties: + AvailabilityZone: + Ref: AvailabilityZone + CidrBlock: + Ref: ComputeSubnetCidr + Tags: + - Key: Network + Value: ComputeSubnet + VpcId: + Ref: VPCId + Type: AWS::EC2::Subnet + ComputeSubnetRouteTableAssociation: + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 9dd9772e-03a3-4983-b28e-a52ac42f0ffe + Properties: + RouteTableId: + Ref: ComputeRouteTable + SubnetId: + Ref: ComputeSubnet + Type: AWS::EC2::SubnetRouteTableAssociation + DynamicPlacementGroup: + Condition: CreatePlacementGroup + Metadata: + AWS::CloudFormation::Designer: + id: ee5f3006-419e-4786-b527-e9503a662e5e + Properties: + Strategy: cluster + Type: AWS::EC2::PlacementGroup + DynamoDBTable: + Metadata: + AWS::CloudFormation::Designer: + id: 789cb442-3b1a-4673-a07e-8acb1c59c271 + Properties: + AttributeDefinitions: + - AttributeName: instanceId + AttributeType: S + KeySchema: + - AttributeName: instanceId + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' + Type: AWS::DynamoDB::Table + MasterEIP: + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + Properties: + Domain: vpc + Type: AWS::EC2::EIP + MasterENI: + Metadata: + AWS::CloudFormation::Designer: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + Properties: + Description: cfncluster Master Server + GroupSet: + - Fn::If: + - CreateSecurityGroups + - Ref: MasterSecurityGroup + - Ref: AWS::NoValue + - Fn::If: + - AddAdditionalSG + - Ref: AdditionalSG + - Ref: AWS::NoValue + - Fn::If: + - UseExistingSecurityGroup + - Ref: VPCSecurityGroupId + - Ref: AWS::NoValue + SourceDestCheck: 'false' + SubnetId: + Ref: MasterSubnetId + Type: AWS::EC2::NetworkInterface + MasterSecurityGroup: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + Properties: + GroupDescription: Enable access to the Master host + SecurityGroupIngress: + - CidrIp: + Ref: AccessFrom + FromPort: '22' + IpProtocol: tcp + ToPort: '22' + - CidrIp: + Ref: AccessFrom + FromPort: '80' + IpProtocol: tcp + ToPort: '80' + VpcId: + Ref: VPCId + Type: AWS::EC2::SecurityGroup + MasterSecurityGroupIngress: + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 466554d3-0e10-4ae4-bfee-51b0aadbf923 + Properties: + FromPort: '0' + GroupId: + Ref: MasterSecurityGroup + IpProtocol: '-1' + SourceSecurityGroupId: + Ref: ComputeSecurityGroup + ToPort: '65535' + Type: AWS::EC2::SecurityGroupIngress + MasterServer: + CreationPolicy: + ResourceSignal: + Count: '1' + Timeout: PT30M + Metadata: + AWS::CloudFormation::Designer: + id: 88da1502-8179-4be3-9123-fdc5eac0b6a3 + AWS::CloudFormation::Init: + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + deployConfigFiles: + commands: + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + files: + /etc/chef/client.rb: + content: + Fn::Join: + - '' + - - cookbook_path ['/etc/chef/cookbooks'] + group: root + mode: '000644' + owner: root + /tmp/dna.json: + content: + cfncluster: + cfn_cluster_user: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - User + cfn_ddb_table: + Ref: DynamoDBTable + cfn_encrypted_ephemeral: + Ref: EncryptedEphemeral + cfn_ephemeral_dir: + Ref: EphemeralDir + cfn_node_type: MasterServer + cfn_postinstall: + Ref: PostInstallScript + cfn_postinstall_args: + Ref: PostInstallArgs + cfn_preinstall: + Ref: PreInstallScript + cfn_preinstall_args: + Ref: PreInstallArgs + cfn_proxy: + Ref: ProxyServer + cfn_region: + Ref: AWS::Region + cfn_scheduler: + Ref: Scheduler + cfn_shared_dir: + Ref: SharedDir + cfn_sqs_queue: + Fn::GetAtt: + - SQS + - QueueName + cfn_volume: + Fn::If: + - UseExistingEBSVolume + - Ref: EBSVolumeId + - Ref: SharedVolume + stack_name: + Ref: AWS::StackName + run_list: + Fn::If: + - UseCustomRunList + - Ref: CustomChefRunList + - Fn::Join: + - '' + - - 'recipe[cfncluster::' + - Ref: Scheduler + - _config] + group: root + mode: '000644' + owner: root + /tmp/extra.json: + content: + Ref: ExtraJson + group: root + mode: '000644' + owner: root + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + Comment: cfncluster Master server + Properties: + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: + Fn::FindInMap: + - OSFeatures + - Ref: BaseOS + - RootDevice + Ebs: + VolumeSize: + Ref: MasterRootVolumeSize + VolumeType: gp2 + EbsOptimized: + Fn::FindInMap: + - AWSInstanceType2Capabilites + - Ref: MasterInstanceType + - EBSOpt + IamInstanceProfile: + Fn::If: + - UseEC2IAMRole + - Ref: EC2IAMRoleName + - Ref: RootInstanceProfile + ImageId: + Fn::If: + - UseCustomAMI + - Ref: CustomAMI + - Fn::FindInMap: + - AWSRegionOS2AMI + - Ref: AWS::Region + - Ref: BaseOS + InstanceType: + Ref: MasterInstanceType + KeyName: + Ref: KeyName + NetworkInterfaces: + - DeviceIndex: '0' + NetworkInterfaceId: + Ref: MasterENI + PlacementGroupName: + Fn::If: + - UseClusterPlacement + - Fn::If: + - CreatePlacementGroup + - Ref: DynamicPlacementGroup + - Ref: PlacementGroup + - Ref: AWS::NoValue + Tags: + - Key: Application + Value: + Ref: AWS::StackName + - Key: Name + Value: Master + Tenancy: + Ref: Tenancy + UserData: + Fn::Base64: + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - Ref: ProxyServer + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - Ref: ProxyServer + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - Ref: AWS::StackName + - ' --resource=MasterServer --region=' + - Ref: AWS::Region + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - Ref: ProxyServer + - ' + + ' + - custom_cookbook= + - Ref: CustomChefCookbook + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - Ref: AWS::Region + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - Ref: AWS::Region + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - Ref: AWS::Region + - /cookbooks/ + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cfncluster + - ' + + ' + - export cookbook_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - cookbook + - ' + + ' + - export chef_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - chef + - ' + + ' + - export ridley_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - ridley + - ' + + ' + - export berkshelf_version= + - Fn::FindInMap: + - CfnClusterVersions + - default + - berkshelf + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - Ref: AWS::StackName + - ' -v -c default -r MasterServer --region ' + - Ref: AWS::Region + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup + complete" --stack= + - Ref: AWS::StackName + - ' --resource=MasterServer --region=' + - Ref: AWS::Region + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Type: AWS::EC2::Instance + RootInstanceProfile: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 2522b048-2b7c-44ed-9d35-6cc66069ca5b + Properties: + Path: / + Roles: + - Ref: RootRole + Type: AWS::IAM::InstanceProfile + RootRole: + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: + - sts:AssumeRole + Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Version: '2012-10-17' + Path: / + Type: AWS::IAM::Role + S3ReadRolePolicies: + Condition: UseS3ReadPolicy + Metadata: + AWS::CloudFormation::Designer: + id: 0c5a1e92-ffdd-4344-8ba0-65270e5fd472 + Properties: + PolicyDocument: + Statement: + - Action: + - s3:Get* + - s3:List* + Effect: Allow + Resource: + - Ref: S3ReadResource + Sid: S3Read + Version: '2012-10-17' + PolicyName: S3Read + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + S3ReadWriteRolePolicies: + Condition: UseS3ReadWritePolicy + Metadata: + AWS::CloudFormation::Designer: + id: a8fbc80f-2514-4cc9-aee8-ba7936add0a7 + Properties: + PolicyDocument: + Statement: + - Action: + - s3:* + Effect: Allow + Resource: + - Ref: S3ReadWriteResource + Sid: S3ReadWrite + Version: '2012-10-17' + PolicyName: S3ReadWrite + Roles: + - Ref: RootRole + Type: AWS::IAM::Policy + SNS: + Metadata: + AWS::CloudFormation::Designer: + id: 210b5501-b505-4ee1-917c-a20cae3ac837 + Properties: + Subscription: + - Endpoint: + Fn::GetAtt: + - SQS + - Arn + Protocol: sqs + Type: AWS::SNS::Topic + SQS: + Metadata: + AWS::CloudFormation::Designer: + id: 2f599618-92e8-49cf-92bd-551ba9775f39 + Properties: {} + Type: AWS::SQS::Queue + SQSPolicy: + Metadata: + AWS::CloudFormation::Designer: + id: 6a874346-80de-4412-9677-00590371a8a2 + Properties: + PolicyDocument: + Id: MyQueuePolicy + Statement: + - Action: + - sqs:SendMessage + Condition: + ArnEquals: + aws:SourceArn: + Ref: SNS + Effect: Allow + Principal: + AWS: '*' + Resource: '*' + Sid: Allow-SendMessage-From-AS-SNS-Topic + Queues: + - Ref: SQS + Type: AWS::SQS::QueuePolicy + ScaleUpPolicy: + Metadata: + AWS::CloudFormation::Designer: + id: 8e63b592-a50f-4b89-a40b-e991adaef77b + Properties: + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: + Ref: ComputeFleet + Cooldown: + Ref: ScalingCooldown + ScalingAdjustment: + Ref: ScalingAdjustment + Type: AWS::AutoScaling::ScalingPolicy + ScaleUpPolicy2: + Metadata: + AWS::CloudFormation::Designer: + id: 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + Properties: + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: + Ref: ComputeFleet + Cooldown: + Ref: ScalingCooldown + ScalingAdjustment: + Ref: ScalingAdjustment2 + Type: AWS::AutoScaling::ScalingPolicy + SharedVolume: + Condition: CreateEBSVolume + Metadata: + AWS::CloudFormation::Designer: + id: c8d3da13-fd27-4377-b606-0f41728c17e1 + Properties: + AvailabilityZone: + Ref: AvailabilityZone + Encrypted: + Fn::If: + - UseEBSEncryption + - Ref: EBSEncryption + - Ref: AWS::NoValue + Iops: + Fn::If: + - UseEBSPIOPS + - Ref: VolumeIOPS + - Ref: AWS::NoValue + KmsKeyId: + Fn::If: + - UseEBSKMSKey + - Ref: EBSKMSKeyId + - Ref: AWS::NoValue + Size: + Fn::If: + - UseEBSSnapshot + - Ref: AWS::NoValue + - Ref: VolumeSize + SnapshotId: + Fn::If: + - UseEBSSnapshot + - Ref: EBSSnapshotId + - Ref: AWS::NoValue + VolumeType: + Ref: VolumeType + Type: AWS::EC2::Volume + +Outputs: + GangliaPrivateURL: + Description: Private URL to access Ganglia + Value: + Fn::Join: + - '' + - - http:// + - Fn::GetAtt: + - MasterServer + - PrivateIp + - /ganglia/ + GangliaPublicURL: + Condition: MasterPublicIp + Description: Public URL to access Ganglia + Value: + Fn::Join: + - '' + - - http:// + - Fn::GetAtt: + - MasterServer + - PublicIp + - /ganglia/ + MasterPrivateIP: + Description: Private IP Address of the Master host + Value: + Fn::GetAtt: + - MasterServer + - PrivateIp + MasterPublicIP: + Condition: MasterPublicIp + Description: Public IP Address of the Master host + Value: + Fn::GetAtt: + - MasterServer + - PublicIp \ No newline at end of file From c92755e41f60cb815feaf18b426c7fe9e4418483 Mon Sep 17 00:00:00 2001 From: Dougal Ballantyne Date: Thu, 25 May 2017 12:22:15 -0700 Subject: [PATCH 26/34] Update ami_development.rst --- docs/source/ami_development.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/ami_development.rst b/docs/source/ami_development.rst index 19e291156b..aebbbb5a80 100644 --- a/docs/source/ami_development.rst +++ b/docs/source/ami_development.rst @@ -16,7 +16,7 @@ Steps This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubuntu machine you can easily get an `EC2 instance `_ running Ubuntu. -#. :code:`sudo apt-get install build-essential` +#. :code:`sudo apt-get -y install build-essential git` #. Go to https://downloads.chef.io/chef-dk, grab the latest version for your OS and install. For example: @@ -36,6 +36,7 @@ This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubu sudo tar xf ~/go1.7.linux-amd64.tar.gz echo 'export GOPATH=~/work' >> ~/.bashrc echo 'export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH' >> ~/.bashrc + . ~/.bashrc #. Install packer from source From 2e8b996436e80a630912fcc8b168ff2c5c8e8a84 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Fri, 26 May 2017 08:48:22 -0700 Subject: [PATCH 27/34] Updates to templates to match https://github.com/awslabs/cfncluster-cookbook/pull/22 --- cloudformation/cfncluster.cfn.json | 12 ++++++------ cloudformation/cfncluster.cfn.yaml | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 4dac9b5808..7d3e4bf737 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1865,12 +1865,12 @@ }, "CfnClusterVersions" : { "default" : { - "cfncluster" : "cfncluster-1.3.1", - "cookbook" : "cfncluster-cookbook-1.3.0", - "chef" : "12.13.30", - "ridley" : "4.5.1", - "berkshelf" : "4.6.1", - "ami" : "201609030709" + "cfncluster" : "cfncluster-1.3.2", + "cookbook" : "cfncluster-cookbook-1.3.2", + "chef" : "12.19.36", + "ridley" : "5.1.0", + "berkshelf" : "5.6.4", + "ami" : "dev" } }, "AWSRegion2Capabilites" : { diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml index 31b8a99229..18070dd679 100644 --- a/cloudformation/cfncluster.cfn.yaml +++ b/cloudformation/cfncluster.cfn.yaml @@ -4,7 +4,7 @@ AWSTemplateFormatVersion: '2010-09-09' Description: 'AWS CloudFormation Sample Template cfncluster.cfn.yaml: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used - if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1' + if you create a stack from this template. Version: ami-dev cfncluster-1.3.2' Metadata: AWS::CloudFormation::Designer: @@ -1173,12 +1173,12 @@ Mappings: ubuntu1604: ami-870edfe7 CfnClusterVersions: default: - ami: '201609030709' - berkshelf: 4.6.1 - cfncluster: cfncluster-1.3.1 - chef: 12.13.30 - cookbook: cfncluster-cookbook-1.3.0 - ridley: 4.5.1 + ami: 'dev' + berkshelf: 5.6.4 + cfncluster: cfncluster-1.3.2 + chef: 12.19.36 + cookbook: cfncluster-cookbook-1.3.2 + ridley: 5.1.0 OSFeatures: alinux: RootDevice: /dev/xvda @@ -3090,4 +3090,4 @@ Outputs: Value: Fn::GetAtt: - MasterServer - - PublicIp \ No newline at end of file + - PublicIp From 3fc182a4d6e55d0601f03a9f0fc669d0fd80d8e7 Mon Sep 17 00:00:00 2001 From: Ballantyne Date: Thu, 1 Jun 2017 07:40:07 -0700 Subject: [PATCH 28/34] Small fixes to improve HTTP config sanity check error messages. --- cli/cfncluster/config_sanity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index d7503f9081..6ba896c9eb 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -97,10 +97,10 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty try: urllib.request.urlopen(resource_value) except urllib.error.HTTPError as e: - print(e.code) + print('Config sanity error:', resource_value, e.code, e.reason) sys.exit(1) except urllib.error.URLError as e: - print(e.args) + print('Config sanity error:', resource_value, e.reason) sys.exit(1) # EC2 EBS Snapshot Id elif resource_type == 'EC2Snapshot': From e63bf8357893bba9a89b5d51426129029f755a85 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Tue, 6 Jun 2017 18:54:25 +0000 Subject: [PATCH 29/34] Update instance type list Add t2.xlarge/t2.2xlarge, x1.16xlarge, r4.*, f1.*, and i3.* instance types to the supported list. Signed-off-by: Brian Barrett --- cloudformation/cfncluster.cfn.json | 126 +- cloudformation/cfncluster.cfn.yaml | 4346 +++++++++++++--------------- 2 files changed, 2114 insertions(+), 2358 deletions(-) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 7d3e4bf737..44bfc75021 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -727,10 +727,22 @@ "r3.2xlarge", "r3.xlarge", "r3.large", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", "i2.8xlarge", "i2.4xlarge", "i2.2xlarge", "i2.xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", "cr1.8xlarge", "cg1.4xlarge", "m3.medium", @@ -747,6 +759,8 @@ "t2.small", "t2.medium", "t2.large", + "t2.xlarge", + "t2.2xlarge", "d2.8xlarge", "d2.4xlarge", "d2.2xlarge", @@ -757,7 +771,10 @@ "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", - "x1.32xlarge" + "x1.16xlarge", + "x1.32xlarge", + "f1.2xlarge", + "f1.16xlarge" ] }, "ComputeInstanceType" : { @@ -782,10 +799,22 @@ "r3.2xlarge", "r3.xlarge", "r3.large", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", "i2.8xlarge", "i2.4xlarge", "i2.2xlarge", "i2.xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", "cr1.8xlarge", "cg1.4xlarge", "m3.medium", @@ -802,6 +831,8 @@ "t2.small", "t2.medium", "t2.large", + "t2.xlarge", + "t2.2xlarge", "d2.8xlarge", "d2.4xlarge", "d2.2xlarge", @@ -812,7 +843,10 @@ "m4.4xlarge", "m4.10xlarge", "m4.16xlarge", - "x1.32xlarge" + "x1.16xlarge", + "x1.32xlarge", + "f1.2xlarge", + "f1.16xlarge" ] }, "InitialQueueSize" : { @@ -1578,7 +1612,7 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, - "p2.large" : { + "p2.xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" }, @@ -1666,6 +1700,34 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "r3.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.large" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.4xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.8xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "i2.8xlarge" : { "Arch" : "64HVM", "EBSOpt" : "False" @@ -1686,10 +1748,38 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "i3.large" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.4xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.8xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "cg1.4xlarge" : { "Arch" : "64HVM", "EBSOpt" : "False" }, + "t2.nano" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, "t2.micro" : { "Arch" : "64HVM", "EBSOpt" : "False" @@ -1702,6 +1792,18 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "t2.large" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, + "t2.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, + "t2.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, "d2.8xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" @@ -1718,14 +1820,22 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, - "t2.large" : { + "x1.16xlarge" : { "Arch" : "64HVM", - "EBSOpt" : "False" + "EBSOpt" : "True" }, "x1.32xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" }, + "f1.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "f1.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "m4.16xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" @@ -1749,10 +1859,6 @@ "m4.large" : { "Arch" : "64HVM", "EBSOpt" : "True" - }, - "t2.nano" : { - "Arch" : "64HVM", - "EBSOpt" : "False" } }, "AWSRegionOS2AMI" : { @@ -4114,4 +4220,4 @@ "Condition" : "MasterPublicIp" } } -} \ No newline at end of file +} diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml index 18070dd679..fcc71595ca 100644 --- a/cloudformation/cfncluster.cfn.yaml +++ b/cloudformation/cfncluster.cfn.yaml @@ -1,725 +1,514 @@ ---- AWSTemplateFormatVersion: '2010-09-09' - -Description: 'AWS CloudFormation Sample Template cfncluster.cfn.yaml: Sample template +Description: 'AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used - if you create a stack from this template. Version: ami-dev cfncluster-1.3.2' - + if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1' Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Network - Basic Settings + Parameters: + - VPCId + - MasterSubnetId + - Label: + default: Cluster - Basic Setting + Parameters: + - KeyName + - AvailabilityZone + - MasterInstanceType + - ComputeInstanceType + - BaseOS + - Scheduler + - Label: + default: EBS - Basic Settings + Parameters: + - VolumeSize + - Label: + default: Network - Advanced Settings + Parameters: + - AccessFrom + - ComputeSubnetId + - ComputeSubnetCidr + - UsePublicIps + - AdditionalSG + - VPCSecurityGroupId + - Label: + default: Cluster - Advanced Setting + Parameters: + - InitialQueueSize + - ComputeWaitConditionCount + - MaxQueueSize + - SpotPrice + - ClusterType + - ProxyServer + - CustomAMI + - MaintainInitialSize + - PreInstallScript + - PreInstallArgs + - PostInstallArgs + - PostInstallScript + - S3ReadResource + - S3ReadWriteResource + - Placement + - PlacementGroup + - EncryptedEphemeral + - EphemeralDir + - SharedDir + - CWLRegion + - CWLLogGroup + - CustomChefRunList + - CustomChefCookbook + - ExtraJson + - Tenancy + - EphemeralKMSKeyId + - ClusterReadyScript + - MasterRootVolumeSize + - ComputeRootVolumeSize + - EC2IAMRoleName + - Label: + default: EBS - Advanced Settings + Parameters: + - VolumeType + - EBSSnapshotId + - VolumeIOPS + - EBSEncryption + - EBSKMSKeyId + - EBSVolumeId + - Label: + default: Scaling Settings + Parameters: + - ScalingPeriod + - ScalingEvaluationPeriods + - ScalingThreshold + - ScalingAdjustment + - ScalingThreshold2 + - ScalingAdjustment2 + - ScalingCooldown + - Label: + default: Additonal Settings + Parameters: + - CLITemplate + ParameterLabels: + KeyName: + default: key_name + AccessFrom: + default: access_from + AccessFrom: + default: access_from + VPCId: + default: vpc_id + MasterSubnetId: + default: master_subnet_id + ComputeSubnetId: + default: compute_subnet_id + ComputeSubnetCidr: + default: compute_subnet_cidr + UsePublicIps: + default: use_public_ips + AdditionalSG: + default: additional_sg + VPCSecurityGroupId: + default: vpc_security_group_id + ComputeInstanceType: + default: compute_instance_type + MasterInstanceType: + default: master_instance_type + InitialQueueSize: + default: initial_queue_size + MaxQueueSize: + default: max_queue_size + MaintainInitialSize: + default: maintain_initial_size + Scheduler: + default: scheduler + ClusterType: + default: cluster_type + EphemeralDir: + default: ephemeral_dir + SpotPrice: + default: spot_price + CustomAMI: + default: custom_ami + PreInstallScript: + default: pre_install + PostInstallScript: + default: post_install + PreInstallArgs: + default: pre_install_args + PostInstallArgs: + default: post_install_args + S3ReadResource: + default: s3_read_resource + S3ReadWriteResource: + default: s3_read_write_resource + CWLRegion: + default: cwl_region + CWLLogGroup: + default: cwl_log_group + ProxyServer: + default: proxy_server + Placement: + default: placement + PlacementGroup: + default: placement_group + EncryptedEphemeral: + default: encrypted_ephemeral + SharedDir: + default: shared_dir + Tenancy: + default: tenancy + EphemeralKMSKeyId: + default: ephemeral_kms_key_id + ClusterReadyScript: + default: cluster_ready + MasterRootVolumeSize: + default: master_root_volume_size + ComputeRootVolumeSize: + default: compute_volume_size + BaseOS: + default: base_os + EC2IAMRoleName: + default: ec2_iam_role + ExtraJson: + default: extra_json + CustomChefCookbook: + default: custom_chef_cookbook + CustomChefRunList: + default: custom_chef_runlist + EBSSnapshotId: + default: ebs_snapshot_id + VolumeType: + default: volume_type + VolumeSize: + default: volume_size + EBSKMSKeyId: + default: ebs_kms_key_id + VolumeIOPS: + default: volume_iops + EBSEncryption: + default: ebs_encryption + EBSVolumeId: + default: ebs_volume_id + ScalingThreshold: + default: scaling_threshold + ScalingPeriod: + default: scaling_period + ScalingEvaluationPeriods: + default: scaling_evaluation_periods + ScalingAdjustment: + default: scaling_adjustment + ScalingAdjustment2: + default: scaling_adjustment2 + ScalingCooldown: + default: scaling_cooldown + ScalingThreshold2: + default: scaling_threshold2 + ComputeWaitConditionCount: + default: WAIT COUNT + AvailabilityZone: + default: AVAILABILITY ZONE + CLITemplate: + default: CLI TEMPLATE AWS::CloudFormation::Designer: - 0824f592-a72b-4bb5-8c67-4ce6df509cdd: - embeds: [] - position: - x: -430 - y: 30 + ee5f3006-419e-4786-b527-e9503a662e5e: size: - height: 240 - width: 240 + width: 60 + height: 60 + position: + x: -170 + y: 390 z: 1 - 0c5a1e92-ffdd-4344-8ba0-65270e5fd472: embeds: [] - isassociatedwith: - - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb - position: - x: 390 - y: 30 + c8d3da13-fd27-4377-b606-0f41728c17e1: size: - height: 60 width: 60 + height: 60 + position: + x: -170 + y: 310 z: 1 - 127f78c3-d47a-4770-8aad-3fd733c3e1b0: embeds: [] - isrelatedto: - - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c - position: - x: 60 - y: 680 + 2904615e-c0ad-4325-b0c5-b9ea7c482b4e: size: - height: 60 width: 60 + height: 60 + position: + x: 60 + y: 390 z: 1 - 210b5501-b505-4ee1-917c-a20cae3ac837: embeds: [] - isrelatedto: - - 2f599618-92e8-49cf-92bd-551ba9775f39 - position: - x: 360 - y: 380 + 45360a4f-64aa-49f8-b016-78fe5ed86819: size: - height: 60 width: 60 - z: 1 - 23ed10e0-7726-4099-8370-4116c0bc78a2: - embeds: [] - isrelatedto: - - 8e63b592-a50f-4b89-a40b-e991adaef77b - position: - x: -50 - y: 680 - size: - height: 60 - width: 60 - z: 1 - 2522b048-2b7c-44ed-9d35-6cc66069ca5b: - embeds: [] - isassociatedwith: - - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb - position: - x: 390 - y: 130 - size: height: 60 - width: 60 - z: 1 - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e: - embeds: [] position: x: 60 - y: 390 - size: - height: 60 - width: 60 + y: 310 z: 1 - 2f599618-92e8-49cf-92bd-551ba9775f39: embeds: [] - position: - x: 450 - y: 380 + ismemberof: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + 619070e4-82e5-445a-93a0-4795ff69d61c: size: - height: 60 width: 60 - z: 1 - 3fa053e5-8ab7-4cdf-a737-4d20386e4717: - dependson: - - 88da1502-8179-4be3-9123-fdc5eac0b6a3 - embeds: [] - isassociatedwith: - - d3204db6-c821-400d-bf31-6afd5739923c - isrelatedto: - - bb5a532f-bb43-472a-9965-c4054cd61cc9 - - 210b5501-b505-4ee1-917c-a20cae3ac837 - - ee5f3006-419e-4786-b527-e9503a662e5e + height: 60 position: - x: -50 + x: 180 y: 480 - size: - height: 60 - width: 60 z: 1 - 45360a4f-64aa-49f8-b016-78fe5ed86819: embeds: [] - ismemberof: + isrelatedto: - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e - position: - x: 60 - y: 310 - size: - height: 60 - width: 60 - z: 1 466554d3-0e10-4ae4-bfee-51b0aadbf923: source: id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e target: id: 619070e4-82e5-445a-93a0-4795ff69d61c - 596ddc76-44b9-4747-921c-c000c5b636c0: + 0824f592-a72b-4bb5-8c67-4ce6df509cdd: + size: + width: 240 + height: 240 + position: + x: -430 + y: 30 + z: 1 embeds: [] + 596ddc76-44b9-4747-921c-c000c5b636c0: + size: + width: 60 + height: 60 position: x: 60 y: 150 + z: 0 + embeds: [] references: - 45360a4f-64aa-49f8-b016-78fe5ed86819 + bb5a532f-bb43-472a-9965-c4054cd61cc9: size: - height: 60 - width: 60 - z: 0 - 619070e4-82e5-445a-93a0-4795ff69d61c: + width: 150 + height: 150 + position: + x: -150 + y: 70 + z: 1 embeds: [] - isrelatedto: - - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + 9dd9772e-03a3-4983-b28e-a52ac42f0ffe: + source: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + target: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + d5bad569-ad14-487d-bcc6-5569373bc8cd: + size: + width: 60 + height: 60 position: x: 180 - y: 480 + y: 310 + z: 1 + embeds: [] + e9ac6c41-8110-49b9-a92c-de57be8a292a: + source: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + target: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb: size: - height: 60 width: 60 + height: 60 + position: + x: 270 + y: 30 z: 1 - 6a874346-80de-4412-9677-00590371a8a2: embeds: [] - isassociatedwith: - - 2f599618-92e8-49cf-92bd-551ba9775f39 - isrelatedto: - - 210b5501-b505-4ee1-917c-a20cae3ac837 - position: - x: 450 - y: 470 + a8fbc80f-2514-4cc9-aee8-ba7936add0a7: size: - height: 60 width: 60 + height: 60 + position: + x: 170 + y: 30 z: 1 - 789cb442-3b1a-4673-a07e-8acb1c59c271: embeds: [] - position: - x: 270 - y: 380 + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + 0c5a1e92-ffdd-4344-8ba0-65270e5fd472: size: - height: 60 width: 60 + height: 60 + position: + x: 390 + y: 30 z: 1 - 7b0e0a3f-3a02-4e70-9e79-7647b98fd579: embeds: [] isassociatedwith: - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb - isrelatedto: - - 2f599618-92e8-49cf-92bd-551ba9775f39 - - 789cb442-3b1a-4673-a07e-8acb1c59c271 + 2522b048-2b7c-44ed-9d35-6cc66069ca5b: + size: + width: 60 + height: 60 position: - x: 270 + x: 390 y: 130 + z: 1 + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + d3204db6-c821-400d-bf31-6afd5739923c: size: - height: 60 width: 60 + height: 60 + position: + x: 60 + y: 480 z: 1 + embeds: [] + ismemberof: + - 619070e4-82e5-445a-93a0-4795ff69d61c + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b 88da1502-8179-4be3-9123-fdc5eac0b6a3: + size: + width: 60 + height: 60 + position: + x: -50 + y: 310 + z: 1 embeds: [] isconnectedto: - 45360a4f-64aa-49f8-b016-78fe5ed86819 isrelatedto: - 2522b048-2b7c-44ed-9d35-6cc66069ca5b - ee5f3006-419e-4786-b527-e9503a662e5e - position: - x: -50 - y: 310 + 789cb442-3b1a-4673-a07e-8acb1c59c271: size: - height: 60 width: 60 + height: 60 + position: + x: 270 + y: 380 z: 1 - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c: embeds: [] - isassociatedwith: - - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 - position: - x: 60 - y: 600 + 2f599618-92e8-49cf-92bd-551ba9775f39: size: - height: 60 width: 60 + height: 60 + position: + x: 450 + y: 380 z: 1 - 8e63b592-a50f-4b89-a40b-e991adaef77b: embeds: [] - isassociatedwith: - - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 - position: - x: -50 - y: 600 + 7b0e0a3f-3a02-4e70-9e79-7647b98fd579: size: - height: 60 width: 60 + height: 60 + position: + x: 270 + y: 130 z: 1 - 9dd9772e-03a3-4983-b28e-a52ac42f0ffe: - source: - id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd - target: - id: bb5a532f-bb43-472a-9965-c4054cd61cc9 - a8fbc80f-2514-4cc9-aee8-ba7936add0a7: embeds: [] isassociatedwith: - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb - position: - x: 170 - y: 30 + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + - 789cb442-3b1a-4673-a07e-8acb1c59c271 + 210b5501-b505-4ee1-917c-a20cae3ac837: size: - height: 60 width: 60 - z: 1 - bb5a532f-bb43-472a-9965-c4054cd61cc9: - embeds: [] + height: 60 position: - x: -150 - y: 70 - size: - height: 150 - width: 150 + x: 360 + y: 380 z: 1 - c8d3da13-fd27-4377-b606-0f41728c17e1: embeds: [] - position: - x: -170 - y: 310 + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + 3fa053e5-8ab7-4cdf-a737-4d20386e4717: size: - height: 60 width: 60 - z: 1 - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb: - embeds: [] - position: - x: 270 - y: 30 - size: height: 60 - width: 60 + position: + x: -50 + y: 480 z: 1 - d3204db6-c821-400d-bf31-6afd5739923c: embeds: [] - ismemberof: - - 619070e4-82e5-445a-93a0-4795ff69d61c + isassociatedwith: + - d3204db6-c821-400d-bf31-6afd5739923c + dependson: + - 88da1502-8179-4be3-9123-fdc5eac0b6a3 isrelatedto: - - 2522b048-2b7c-44ed-9d35-6cc66069ca5b - position: - x: 60 - y: 480 + - bb5a532f-bb43-472a-9965-c4054cd61cc9 + - 210b5501-b505-4ee1-917c-a20cae3ac837 + - ee5f3006-419e-4786-b527-e9503a662e5e + 8e63b592-a50f-4b89-a40b-e991adaef77b: size: - height: 60 width: 60 + height: 60 + position: + x: -50 + y: 600 z: 1 - d5bad569-ad14-487d-bcc6-5569373bc8cd: embeds: [] - position: - x: 180 - y: 310 + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + 23ed10e0-7726-4099-8370-4116c0bc78a2: size: - height: 60 width: 60 + height: 60 + position: + x: -50 + y: 680 z: 1 - e9ac6c41-8110-49b9-a92c-de57be8a292a: - source: - id: d5bad569-ad14-487d-bcc6-5569373bc8cd - target: - id: 45360a4f-64aa-49f8-b016-78fe5ed86819 - ee5f3006-419e-4786-b527-e9503a662e5e: embeds: [] + isrelatedto: + - 8e63b592-a50f-4b89-a40b-e991adaef77b + 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c: + size: + width: 60 + height: 60 position: - x: -170 - y: 390 + x: 60 + y: 600 + z: 1 + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + 127f78c3-d47a-4770-8aad-3fd733c3e1b0: size: + width: 60 height: 60 + position: + x: 60 + y: 680 + z: 1 + embeds: [] + isrelatedto: + - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + 6a874346-80de-4412-9677-00590371a8a2: + size: width: 60 + height: 60 + position: + x: 450 + y: 470 z: 1 - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Network - Basic Settings - Parameters: - - VPCId - - MasterSubnetId - - Label: - default: Cluster - Basic Setting - Parameters: - - KeyName - - AvailabilityZone - - MasterInstanceType - - ComputeInstanceType - - BaseOS - - Scheduler - - Label: - default: EBS - Basic Settings - Parameters: - - VolumeSize - - Label: - default: Network - Advanced Settings - Parameters: - - AccessFrom - - ComputeSubnetId - - ComputeSubnetCidr - - UsePublicIps - - AdditionalSG - - VPCSecurityGroupId - - Label: - default: Cluster - Advanced Setting - Parameters: - - InitialQueueSize - - ComputeWaitConditionCount - - MaxQueueSize - - SpotPrice - - ClusterType - - ProxyServer - - CustomAMI - - MaintainInitialSize - - PreInstallScript - - PreInstallArgs - - PostInstallArgs - - PostInstallScript - - S3ReadResource - - S3ReadWriteResource - - Placement - - PlacementGroup - - EncryptedEphemeral - - EphemeralDir - - SharedDir - - CWLRegion - - CWLLogGroup - - CustomChefRunList - - CustomChefCookbook - - ExtraJson - - Tenancy - - EphemeralKMSKeyId - - ClusterReadyScript - - MasterRootVolumeSize - - ComputeRootVolumeSize - - EC2IAMRoleName - - Label: - default: EBS - Advanced Settings - Parameters: - - VolumeType - - EBSSnapshotId - - VolumeIOPS - - EBSEncryption - - EBSKMSKeyId - - EBSVolumeId - - Label: - default: Scaling Settings - Parameters: - - ScalingPeriod - - ScalingEvaluationPeriods - - ScalingThreshold - - ScalingAdjustment - - ScalingThreshold2 - - ScalingAdjustment2 - - ScalingCooldown - - Label: - default: Additonal Settings - Parameters: - - CLITemplate - ParameterLabels: - AccessFrom: - AccessFrom: - default: access_from - default: access_from - AdditionalSG: - default: additional_sg - AvailabilityZone: - default: AVAILABILITY ZONE - BaseOS: - default: base_os - CLITemplate: - default: CLI TEMPLATE - CWLLogGroup: - default: cwl_log_group - CWLRegion: - default: cwl_region - ClusterReadyScript: - default: cluster_ready - ClusterType: - default: cluster_type - ComputeInstanceType: - default: compute_instance_type - ComputeRootVolumeSize: - default: compute_volume_size - ComputeSubnetCidr: - default: compute_subnet_cidr - ComputeSubnetId: - default: compute_subnet_id - ComputeWaitConditionCount: - default: WAIT COUNT - CustomAMI: - default: custom_ami - CustomChefCookbook: - default: custom_chef_cookbook - CustomChefRunList: - default: custom_chef_runlist - EBSEncryption: - default: ebs_encryption - EBSKMSKeyId: - default: ebs_kms_key_id - EBSSnapshotId: - default: ebs_snapshot_id - EBSVolumeId: - default: ebs_volume_id - EC2IAMRoleName: - default: ec2_iam_role - EncryptedEphemeral: - default: encrypted_ephemeral - EphemeralDir: - default: ephemeral_dir - EphemeralKMSKeyId: - default: ephemeral_kms_key_id - ExtraJson: - default: extra_json - InitialQueueSize: - default: initial_queue_size - KeyName: - default: key_name - MaintainInitialSize: - default: maintain_initial_size - MasterInstanceType: - default: master_instance_type - MasterRootVolumeSize: - default: master_root_volume_size - MasterSubnetId: - default: master_subnet_id - MaxQueueSize: - default: max_queue_size - Placement: - default: placement - PlacementGroup: - default: placement_group - PostInstallArgs: - default: post_install_args - PostInstallScript: - default: post_install - PreInstallArgs: - default: pre_install_args - PreInstallScript: - default: pre_install - ProxyServer: - default: proxy_server - S3ReadResource: - default: s3_read_resource - S3ReadWriteResource: - default: s3_read_write_resource - ScalingAdjustment: - default: scaling_adjustment - ScalingAdjustment2: - default: scaling_adjustment2 - ScalingCooldown: - default: scaling_cooldown - ScalingEvaluationPeriods: - default: scaling_evaluation_periods - ScalingPeriod: - default: scaling_period - ScalingThreshold: - default: scaling_threshold - ScalingThreshold2: - default: scaling_threshold2 - Scheduler: - default: scheduler - SharedDir: - default: shared_dir - SpotPrice: - default: spot_price - Tenancy: - default: tenancy - UsePublicIps: - default: use_public_ips - VPCId: - default: vpc_id - VPCSecurityGroupId: - default: vpc_security_group_id - VolumeIOPS: - default: volume_iops - VolumeSize: - default: volume_size - VolumeType: - default: volume_type - + embeds: [] + isassociatedwith: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + isrelatedto: + - 210b5501-b505-4ee1-917c-a20cae3ac837 Parameters: - AccessFrom: - AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) - ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. - Default: 0.0.0.0/0 - Description: Lockdown SSH/HTTP access (default can be accessed from anywhere) - MaxLength: '18' - MinLength: '9' - Type: String - AdditionalCfnTemplate: - Default: NONE - Description: A second CloudFormation template to launch with the cluster - Type: String - AdditionalSG: - AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) - Default: NONE - Description: Additional VPC secuirty group to be added to instances. Defaults - to NONE - Type: String - AvailabilityZone: - Description: Availability Zone the cluster will launch into. THIS IS REQUIRED - Type: AWS::EC2::AvailabilityZone::Name - BaseOS: - AllowedValues: - - centos6 - - centos7 - - alinux - - ubuntu1404 - - ubuntu1604 - ConstraintDescription: must be a supported base OS - Default: alinux - Description: Base OS type for cluster AMI - Type: String - CLITemplate: - Default: default - Type: String - CWLLogGroup: - Default: NONE - Description: CloudWatch Logs LogGroup - Type: String - CWLRegion: - Default: NONE - Description: CloudWatch Logs region - Type: String - ClusterReadyScript: - Default: NONE - Description: Cluster ready script URL. This is only on the MasterServer, when - the cluster reaches CREATE_COMPLETE. - Type: String - ClusterType: - AllowedValues: - - ondemand - - spot - ConstraintDescription: 'Must be a supported cluster type: ondemand, spot' - Default: ondemand - Description: Type of cluster to launch. Can either be "ondemand" or "spot". Choosing - "spot" will cause the ComputeFleet AutoScaling group to launch EC2 Spot instances. - Default value is "ondemand". - Type: String - ComputeInstanceType: - AllowedValues: - - cc2.8xlarge - - c3.8xlarge - - c3.4xlarge - - c3.2xlarge - - c3.xlarge - - c3.large - - c4.8xlarge - - c4.4xlarge - - c4.2xlarge - - c4.xlarge - - c4.large - - r3.8xlarge - - r3.4xlarge - - r3.2xlarge - - r3.xlarge - - r3.large - - i2.8xlarge - - i2.4xlarge - - i2.2xlarge - - i2.xlarge - - cr1.8xlarge - - cg1.4xlarge - - m3.medium - - m3.large - - m3.xlarge - - m3.2xlarge - - hi1.4xlarge - - g2.2xlarge - - g2.8xlarge - - p2.xlarge - - p2.8xlarge - - p2.16xlarge - - t2.micro - - t2.small - - t2.medium - - t2.large - - d2.8xlarge - - d2.4xlarge - - d2.2xlarge - - d2.xlarge - - m4.large - - m4.xlarge - - m4.2xlarge - - m4.4xlarge - - m4.10xlarge - - m4.16xlarge - - x1.32xlarge - ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. - Default: t2.micro - Description: ComputeFleet EC2 instance type - Type: String - ComputeRootVolumeSize: - Default: '15' - Description: Size of ComputeFleet EBS root volume in GB - MinValue: '15' - Type: Number - ComputeSubnetCidr: - AllowedPattern: (NONE|(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})) - ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. - Default: NONE - Description: CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited - list and can support multiple CIDR ranges for a multi-AZ cluster. The order - and length of this list MUST match the AvailabilityZones parameter. - Type: String - ComputeSubnetId: - AllowedPattern: (NONE|^subnet-[0-9a-z]{8}$) - Default: NONE - Description: ID of the Subnet you want to provision the Compute Servers into - Type: String - ComputeWaitConditionCount: - Default: '2' - Description: Specific number of instances to wait for while creating the cluster - Type: Number - CustomAMI: - AllowedPattern: (NONE|^ami-[0-9a-z]{8}$|^ami-[0-9a-z]{17}$) - Default: NONE - Description: ID of a Custom AMI, to use instead of published AMI's - Type: String - CustomChefCookbook: - Default: NONE - Description: URL of custom cookbook that will override the default. This will - be unpacked and then dependencies resolved with Berkshelf. - Type: String - CustomChefRunList: - Default: NONE - Description: Custom run list, which will override the default - Type: String - EBSEncryption: - AllowedValues: - - 'true' - - 'false' - ConstraintDescription: true/false - Default: 'false' - Description: Boolean flag to use EBS encryption for /shared volume. (Not to be - used for snapshots) - Type: String - EBSKMSKeyId: - Default: NONE - Description: KMS ARN for customer created master key, will be used for EBS encryption - Type: String - EBSSnapshotId: - AllowedPattern: (NONE|^snap-[0-9a-z]{8}$|^snap-[0-9a-z]{17}$) - Default: NONE - Description: Id of EBS snapshot if using snapshot as source for volume - Type: String - EBSVolumeId: - AllowedPattern: (NONE|^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$) - Default: NONE - Description: Existing EBS volume Id - Type: String - EC2IAMRoleName: - Default: NONE - Description: Existing EC2 IAM role name - Type: String - EncryptedEphemeral: - AllowedValues: - - 'true' - - 'false' - ConstraintDescription: true/false - Default: 'false' - Description: Boolean flag to encrypt local ephemeral drives. The keys are in-memory - and non-recoverable. - Type: String - EphemeralDir: - Default: /scratch - Description: The path/mountpoint for the ephemeral drive - Type: String - EphemeralKMSKeyId: - Default: NONE - Description: KMS ARN for customer created master key, will be used for ephemeral - encryption - Type: String - ExtraJson: - Default: '{}' - Description: Extra json to be added to Chef dna.json - Type: String - InitialQueueSize: - Default: '2' - Description: Initial number of EC2 instances to launch as compute nodes within - the cluster. This value maps to the DesiredSize parameter for the ComputeFleet - AutoScaling Group. - Type: Number KeyName: Description: Name of an existing EC2 KeyPair to enable SSH access to the instances using the default cluster user. Type: AWS::EC2::KeyPair::KeyName - MaintainInitialSize: - AllowedValues: - - 'true' - - 'false' - ConstraintDescription: true/false - Default: 'false' - Description: Boolean flag to set autoscaling group to maintain initial size and - scale back - Type: String MasterInstanceType: + Description: MasterServer EC2 instance type + Type: String + Default: t2.micro + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. AllowedValues: - cc2.8xlarge - c3.8xlarge @@ -737,10 +526,22 @@ Parameters: - r3.2xlarge - r3.xlarge - r3.large + - r4.large + - r4.xlarge + - r4.2xlarge + - r4.4xlarge + - r4.8xlarge + - r4.16xlarge - i2.8xlarge - i2.4xlarge - i2.2xlarge - i2.xlarge + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge - cr1.8xlarge - cg1.4xlarge - m3.medium @@ -757,6 +558,8 @@ Parameters: - t2.small - t2.medium - t2.large + - t2.xlarge + - t2.2xlarge - d2.8xlarge - d2.4xlarge - d2.2xlarge @@ -767,98 +570,300 @@ Parameters: - m4.4xlarge - m4.10xlarge - m4.16xlarge + - x1.16xlarge - x1.32xlarge - ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. - Default: t2.micro - Description: MasterServer EC2 instance type + - f1.2xlarge + - f1.16xlarge + ComputeInstanceType: + Description: ComputeFleet EC2 instance type Type: String - MasterRootVolumeSize: - Default: '15' - Description: Size of MasterServer EBS root volume in GB - MinValue: '15' + Default: t2.micro + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - r4.large + - r4.xlarge + - r4.2xlarge + - r4.4xlarge + - r4.8xlarge + - r4.16xlarge + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - t2.xlarge + - t2.2xlarge + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.16xlarge + - x1.32xlarge + - f1.2xlarge + - f1.16xlarge + InitialQueueSize: + Description: Initial number of EC2 instances to launch as compute nodes within + the cluster. This value maps to the DesiredSize parameter for the ComputeFleet + AutoScaling Group. Type: Number - MasterSubnetId: - Description: ID of the Subnet you want to provision the Master server into - Type: AWS::EC2::Subnet::Id + Default: '2' MaxQueueSize: - Default: '10' Description: Maximum number of EC2 instances that can be launched in the cluster. This value maps to the MaxSize parameter for the ComputeFleet AutoScaling Group. Type: Number - Placement: + Default: '10' + ComputeSubnetId: + Description: ID of the Subnet you want to provision the Compute Servers into + Type: String + Default: NONE + AllowedPattern: (NONE|^subnet-[0-9a-z]{8}$) + ScalingThreshold: + Description: Threshold for triggering CloudWatch ScaleUp action + Type: String + Default: '1' + ScalingEvaluationPeriods: + Description: Number of periods consective periods required to trigger the scaling + adjustment + Type: String + Default: '2' + ScalingPeriod: + Description: Period in seconds to measure ScalingThreshold + Type: String + Default: '60' + SpotPrice: + Description: Spot bid price for the ComputeFleet AutoScaling Group when the ClusterType + = "spot". + Type: Number + Default: '0.00' + ClusterType: + Description: Type of cluster to launch. Can either be "ondemand" or "spot". Choosing + "spot" will cause the ComputeFleet AutoScaling group to launch EC2 Spot instances. + Default value is "ondemand". + Type: String + Default: ondemand + ConstraintDescription: 'Must be a supported cluster type: ondemand, spot' AllowedValues: - - cluster - - compute - Default: cluster - Description: Type of placement requird in cfncluster, it can either be cluster - or compute. + - ondemand + - spot + ProxyServer: + Description: hostname and port of HTTP proxy server for cfn-init, boto and yum + i.e. proxy.example.com:8080 Type: String - PlacementGroup: Default: NONE - Description: The name of an existing placement group + VolumeSize: + Description: Size of EBS volume in GB, if creating a new one + Type: Number + Default: '20' + VolumeType: + Description: Type of volume to create either new or from snapshot + Type: String + Default: gp2 + ConstraintDescription: 'must be a supported volume type: standard, io1, gp2, st1, + sc1' + AllowedValues: + - standard + - gp2 + - io1 + - st1 + - sc1 + MasterSubnetId: + Description: ID of the Subnet you want to provision the Master server into + Type: AWS::EC2::Subnet::Id + AvailabilityZone: + Description: Availability Zone the cluster will launch into. THIS IS REQUIRED + Type: AWS::EC2::AvailabilityZone::Name + EBSSnapshotId: + Description: Id of EBS snapshot if using snapshot as source for volume Type: String - PostInstallArgs: Default: NONE - Description: Postinstall script args passed to the postinstall script. + AllowedPattern: (NONE|^snap-[0-9a-z]{8}$|^snap-[0-9a-z]{17}$) + CustomAMI: + Description: ID of a Custom AMI, to use instead of published AMI's Type: String - PostInstallScript: Default: NONE - Description: Postinstall script URL. This is run before any host configuration. + AllowedPattern: (NONE|^ami-[0-9a-z]{8}$|^ami-[0-9a-z]{17}$) + VPCId: + Description: ID of the VPC you want to provision cluster into. Only used with + UseVPCBase=false + Type: AWS::EC2::VPC::Id + AccessFrom: + Description: Lockdown SSH/HTTP access (default can be accessed from anywhere) Type: String - PreInstallArgs: + MinLength: '9' + MaxLength: '18' + Default: 0.0.0.0/0 + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + ComputeSubnetCidr: + Description: CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited + list and can support multiple CIDR ranges for a multi-AZ cluster. The order + and length of this list MUST match the AvailabilityZones parameter. + Type: String + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + AllowedPattern: (NONE|(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})) Default: NONE - Description: Preinstall script args passed to the preinstall script. + MaintainInitialSize: + Description: Boolean flag to set autoscaling group to maintain initial size and + scale back + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + UsePublicIps: + Description: Boolean flag to use public IP's for instances. If false, the VPC + must be correctly setup to use NAT for all traffic. Type: String + Default: 'true' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + VolumeIOPS: + Description: Number of IOPS for volume type io1. Not used for other volume types. + Type: Number + Default: '100' PreInstallScript: - Default: NONE Description: Preinstall script URL. This is run before any host configuration. Type: String - ProxyServer: Default: NONE - Description: hostname and port of HTTP proxy server for cfn-init, boto and yum - i.e. proxy.example.com:8080 + PostInstallScript: + Description: Postinstall script URL. This is run before any host configuration. Type: String - S3ReadResource: Default: NONE + ComputeWaitConditionCount: + Description: Specific number of instances to wait for while creating the cluster + Type: Number + Default: '2' + S3ReadResource: Description: S3 resource with read access from cfncluster nodes Type: String - S3ReadWriteResource: Default: NONE + S3ReadWriteResource: Description: Addtional policy document to be added to EC2 IAM role created and assigned to all nodes. Type: String + Default: NONE + Placement: + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + Default: cluster + AllowedValues: + - cluster + - compute + PlacementGroup: + Description: The name of an existing placement group + Type: String + Default: NONE + EncryptedEphemeral: + Description: Boolean flag to encrypt local ephemeral drives. The keys are in-memory + and non-recoverable. + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + PreInstallArgs: + Description: Preinstall script args passed to the preinstall script. + Type: String + Default: NONE + PostInstallArgs: + Description: Postinstall script args passed to the postinstall script. + Type: String + Default: NONE + EBSEncryption: + Description: Boolean flag to use EBS encryption for /shared volume. (Not to be + used for snapshots) + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + EphemeralDir: + Description: The path/mountpoint for the ephemeral drive + Type: String + Default: /scratch + BaseOS: + Description: Base OS type for cluster AMI + Type: String + Default: alinux + ConstraintDescription: must be a supported base OS + AllowedValues: + - centos6 + - centos7 + - alinux + - ubuntu1404 + - ubuntu1604 + ScalingThreshold2: + Description: Threshold for triggering CloudWatch ScaleUp2 action + Type: String + Default: '200' + ScalingCooldown: + Description: Period in seconds to wait before allowing further scaling actions + Type: String + Default: '300' ScalingAdjustment: - Default: '1' Description: Number of instances to add to cluster when the CloudWatch ScaleUp action is called. Type: String + Default: '1' ScalingAdjustment2: - Default: '10' Description: Number of instances to add to cluster when the CloudWatch ScaleUp2 action is called. Type: String - ScalingCooldown: - Default: '300' - Description: Period in seconds to wait before allowing further scaling actions - Type: String - ScalingEvaluationPeriods: - Default: '2' - Description: Number of periods consective periods required to trigger the scaling - adjustment - Type: String - ScalingPeriod: - Default: '60' - Description: Period in seconds to measure ScalingThreshold - Type: String - ScalingThreshold: - Default: '1' - Description: Threshold for triggering CloudWatch ScaleUp action - Type: String - ScalingThreshold2: - Default: '200' - Description: Threshold for triggering CloudWatch ScaleUp2 action - Type: String + Default: '10' Scheduler: + Description: Cluster scheduler + Type: String + Default: sge + ConstraintDescription: must be a supported scheduler AllowedValues: - sge - openlava @@ -867,571 +872,531 @@ Parameters: - custom - test - pbspro - ConstraintDescription: must be a supported scheduler - Default: sge - Description: Cluster scheduler - Type: String SharedDir: - Default: /shared Description: The path/mountpoint for the shared drive Type: String - SpotPrice: - Default: '0.00' - Description: Spot bid price for the ComputeFleet AutoScaling Group when the ClusterType - = "spot". - Type: Number - Tenancy: - AllowedValues: - - default - - dedicated + Default: /shared + CLITemplate: + Type: String Default: default + AdditionalSG: + Description: Additional VPC secuirty group to be added to instances. Defaults + to NONE + Type: String + Default: NONE + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + CWLRegion: + Description: CloudWatch Logs region + Type: String + Default: NONE + CWLLogGroup: + Description: CloudWatch Logs LogGroup + Type: String + Default: NONE + CustomChefRunList: + Description: Custom run list, which will override the default + Type: String + Default: NONE + CustomChefCookbook: + Description: URL of custom cookbook that will override the default. This will + be unpacked and then dependencies resolved with Berkshelf. + Type: String + Default: NONE + ExtraJson: + Description: Extra json to be added to Chef dna.json + Type: String + Default: '{}' + Tenancy: Description: Type of placement requird in cfncluster, it can either be cluster or compute. Type: String - UsePublicIps: + Default: default AllowedValues: - - 'true' - - 'false' - ConstraintDescription: true/false - Default: 'true' - Description: Boolean flag to use public IP's for instances. If false, the VPC - must be correctly setup to use NAT for all traffic. + - default + - dedicated + EBSKMSKeyId: + Description: KMS ARN for customer created master key, will be used for EBS encryption Type: String - VPCId: - Description: ID of the VPC you want to provision cluster into. Only used with - UseVPCBase=false - Type: AWS::EC2::VPC::Id - VPCSecurityGroupId: - AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) Default: NONE - Description: Existing VPC security group Id + EphemeralKMSKeyId: + Description: KMS ARN for customer created master key, will be used for ephemeral + encryption Type: String - VolumeIOPS: - Default: '100' - Description: Number of IOPS for volume type io1. Not used for other volume types. + Default: NONE + ClusterReadyScript: + Description: Cluster ready script URL. This is only on the MasterServer, when + the cluster reaches CREATE_COMPLETE. + Type: String + Default: NONE + MasterRootVolumeSize: + Description: Size of MasterServer EBS root volume in GB Type: Number - VolumeSize: - Default: '20' - Description: Size of EBS volume in GB, if creating a new one + Default: '15' + MinValue: '15' + ComputeRootVolumeSize: + Description: Size of ComputeFleet EBS root volume in GB Type: Number - VolumeType: - AllowedValues: - - standard - - gp2 - - io1 - - st1 - - sc1 - ConstraintDescription: 'must be a supported volume type: standard, io1, gp2, st1, - sc1' - Default: gp2 - Description: Type of volume to create either new or from snapshot + Default: '15' + MinValue: '15' + EC2IAMRoleName: + Description: Existing EC2 IAM role name Type: String - + Default: NONE + VPCSecurityGroupId: + Description: Existing VPC security group Id + Type: String + Default: NONE + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + EBSVolumeId: + Description: Existing EBS volume Id + Type: String + Default: NONE + AllowedPattern: (NONE|^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$) + AdditionalCfnTemplate: + Description: A second CloudFormation template to launch with the cluster + Type: String + Default: NONE +Conditions: + UseSpotInstances: !Equals [!Ref 'ClusterType', spot] + CreateComputeSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetId', NONE], !Not [ + !Equals [!Ref 'ComputeSubnetCidr', NONE]]] + UseComputeSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetCidr', NONE], !Not [ + !Equals [!Ref 'ComputeSubnetId', NONE]]] + UseMasterSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetId', NONE], !Equals [ + !Ref 'ComputeSubnetCidr', NONE]] + UseEBSSnapshot: !Not [!Equals [!Ref 'EBSSnapshotId', NONE]] + UseCustomRunList: !Not [!Equals [!Ref 'CustomChefRunList', NONE]] + UseCustomAMI: !Not [!Equals [!Ref 'CustomAMI', NONE]] + MaintainInitialASGSize: !Equals [!Ref 'MaintainInitialSize', 'true'] + MasterPublicIp: !Equals [!Ref 'UsePublicIps', 'true'] + ComputePublicIps: !And [!Equals [!Ref 'UsePublicIps', 'true'], {Condition: UseMasterSubnetForCompute}] + UseEBSPIOPS: !Equals [!Ref 'VolumeType', io1] + UseS3ReadPolicy: !Not [!Equals [!Ref 'S3ReadResource', NONE]] + UsePlacementGroup: !Not [!Equals [!Ref 'PlacementGroup', NONE]] + UseClusterPlacement: !And [!Equals [!Ref 'Placement', cluster], {Condition: UsePlacementGroup}] + UseEBSEncryption: !Equals [!Ref 'EBSEncryption', 'true'] + UseS3ReadWritePolicy: !Not [!Equals [!Ref 'S3ReadWriteResource', NONE]] + CloudWatchLogs: !And [!Not [!Equals [!Ref 'CWLRegion', NONE]], !Not [!Equals [!Ref 'CWLLogGroup', + NONE]]] + AddAdditionalSG: !Not [!Equals [!Ref 'AdditionalSG', NONE]] + UseEBSKMSKey: !And [!Not [!Equals [!Ref 'EBSKMSKeyId', NONE]], {Condition: UseEBSEncryption}] + UseEphemeralKMSKey: !And [!Not [!Equals [!Ref 'EphemeralKMSKeyId', NONE]], !Equals [ + !Ref 'EncryptedEphemeral', 'true']] + UseDedicatedTenancy: !Equals [!Ref 'Tenancy', dedicated] + UseEC2IAMRole: !Not [!Equals [!Ref 'EC2IAMRoleName', NONE]] + CreateEC2IAMRole: !Equals [!Ref 'EC2IAMRoleName', NONE] + UseExistingSecurityGroup: !Not [!Equals [!Ref 'VPCSecurityGroupId', NONE]] + UseExistingEBSVolume: !Not [!Equals [!Ref 'EBSVolumeId', NONE]] + CreateEBSVolume: !Equals [!Ref 'EBSVolumeId', NONE] + CreateSecurityGroups: !Equals [!Ref 'VPCSecurityGroupId', NONE] + CreateSubStack: !Not [!Equals [!Ref 'AdditionalCfnTemplate', NONE]] + CreatePlacementGroup: !And [!Equals [!Ref 'PlacementGroup', DYNAMIC], {Condition: UsePlacementGroup}] Mappings: AWSInstanceType2Capabilites: - c3.2xlarge: + cc2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + cr1.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + g2.2xlarge: Arch: 64HVM EBSOpt: 'True' - c3.4xlarge: + g2.8xlarge: Arch: 64HVM EBSOpt: 'True' - c3.8xlarge: + p2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.medium: Arch: 64HVM EBSOpt: 'False' - c3.large: + m3.large: Arch: 64HVM EBSOpt: 'False' - c3.xlarge: + m3.xlarge: Arch: 64HVM EBSOpt: 'True' - c4.2xlarge: + m3.2xlarge: Arch: 64HVM EBSOpt: 'True' - c4.4xlarge: + c3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + c3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.xlarge: Arch: 64HVM EBSOpt: 'True' + c3.large: + Arch: 64HVM + EBSOpt: 'False' c4.8xlarge: Arch: 64HVM EBSOpt: 'True' - c4.large: + c4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.2xlarge: Arch: 64HVM EBSOpt: 'True' c4.xlarge: Arch: 64HVM EBSOpt: 'True' - cc2.8xlarge: + c4.large: Arch: 64HVM - EBSOpt: 'False' - cg1.4xlarge: + EBSOpt: 'True' + r3.8xlarge: Arch: 64HVM EBSOpt: 'False' - cr1.8xlarge: + r3.4xlarge: Arch: 64HVM - EBSOpt: 'False' - d2.2xlarge: + EBSOpt: 'True' + r3.2xlarge: Arch: 64HVM EBSOpt: 'True' - d2.4xlarge: + r3.xlarge: Arch: 64HVM EBSOpt: 'True' - d2.8xlarge: + r3.large: + Arch: 64HVM + EBSOpt: 'False' + r4.large: Arch: 64HVM EBSOpt: 'True' - d2.xlarge: + r4.xlarge: Arch: 64HVM EBSOpt: 'True' - g2.2xlarge: + r4.2xlarge: Arch: 64HVM EBSOpt: 'True' - g2.8xlarge: + r4.4xlarge: Arch: 64HVM EBSOpt: 'True' - i2.2xlarge: + r4.8xlarge: Arch: 64HVM EBSOpt: 'True' - i2.4xlarge: + r4.16xlarge: Arch: 64HVM EBSOpt: 'True' i2.8xlarge: Arch: 64HVM EBSOpt: 'False' + i2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.xlarge: + Arch: 64HVM + EBSOpt: 'True' i2.large: Arch: 64HVM EBSOpt: 'False' - i2.xlarge: + i3.large: Arch: 64HVM EBSOpt: 'True' - m3.2xlarge: + i3.xlarge: Arch: 64HVM EBSOpt: 'True' - m3.large: + i3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + cg1.4xlarge: + Arch: 64HVM + EBSOpt: 'False' + t2.nano: + Arch: 64HVM + EBSOpt: 'False' + t2.micro: + Arch: 64HVM + EBSOpt: 'False' + t2.small: + Arch: 64HVM + EBSOpt: 'False' + t2.medium: + Arch: 64HVM + EBSOpt: 'False' + t2.large: + Arch: 64HVM + EBSOpt: 'False' + t2.xlarge: Arch: 64HVM EBSOpt: 'False' - m3.medium: + t2.2xlarge: Arch: 64HVM EBSOpt: 'False' - m3.xlarge: + d2.8xlarge: Arch: 64HVM EBSOpt: 'True' - m4.10xlarge: + d2.4xlarge: Arch: 64HVM EBSOpt: 'True' - m4.16xlarge: + d2.2xlarge: Arch: 64HVM EBSOpt: 'True' - m4.2xlarge: + d2.xlarge: Arch: 64HVM EBSOpt: 'True' - m4.4xlarge: + x1.16xlarge: Arch: 64HVM EBSOpt: 'True' - m4.large: + x1.32xlarge: Arch: 64HVM EBSOpt: 'True' - m4.xlarge: + f1.2xlarge: Arch: 64HVM EBSOpt: 'True' - p2.16xlarge: + f1.16xlarge: Arch: 64HVM EBSOpt: 'True' - p2.8xlarge: + m4.16xlarge: Arch: 64HVM EBSOpt: 'True' - p2.large: + m4.10xlarge: Arch: 64HVM EBSOpt: 'True' - r3.2xlarge: + m4.4xlarge: Arch: 64HVM EBSOpt: 'True' - r3.4xlarge: + m4.2xlarge: Arch: 64HVM EBSOpt: 'True' - r3.8xlarge: - Arch: 64HVM - EBSOpt: 'False' - r3.large: - Arch: 64HVM - EBSOpt: 'False' - r3.xlarge: + m4.xlarge: Arch: 64HVM EBSOpt: 'True' - t2.large: - Arch: 64HVM - EBSOpt: 'False' - t2.medium: - Arch: 64HVM - EBSOpt: 'False' - t2.micro: - Arch: 64HVM - EBSOpt: 'False' - t2.nano: - Arch: 64HVM - EBSOpt: 'False' - t2.small: - Arch: 64HVM - EBSOpt: 'False' - x1.32xlarge: + m4.large: Arch: 64HVM EBSOpt: 'True' - AWSRegion2Capabilites: - ap-northeast-1: - arn: aws - ap-northeast-2: - arn: aws - ap-south-1: - arn: aws - ap-southeast-1: - arn: aws - ap-southeast-2: - arn: aws - eu-central-1: - arn: aws + AWSRegionOS2AMI: eu-west-1: - arn: aws - sa-east-1: - arn: aws - us-east-1: - arn: aws - us-gov-west-1: - arn: aws-us-gov - us-west-1: - arn: aws + centos6: ami-27f78554 + centos7: ami-5af48629 + alinux: ami-d8f785ab + ubuntu1404: ami-3ff0824c + ubuntu1604: ami-99f082ea us-west-2: - arn: aws - AWSRegionOS2AMI: - ap-northeast-1: - alinux: ami-50bd7731 - centos6: ami-46b17b27 - centos7: ami-43c90322 - ubuntu1404: ami-89bd77e8 - ubuntu1604: ami-eabe748b - ap-northeast-2: - alinux: ami-afe431c1 - centos6: ami-75e1341b - centos7: ami-5cec3932 - ubuntu1404: ami-bae732d4 - ubuntu1604: ami-7eef3a10 - ap-south-1: - alinux: ami-0495e06b - centos6: ami-a392e7cc - centos7: ami-bf90e5d0 - ubuntu1404: ami-489de827 - ubuntu1604: ami-5d90e532 - ap-southeast-1: - alinux: ami-9af229f9 - centos6: ami-23f22940 - centos7: ami-d5f52eb6 - ubuntu1404: ami-95f229f6 - ubuntu1604: ami-23f32840 - ap-southeast-2: - alinux: ami-51b98f32 - centos6: ami-0ea6906d - centos7: ami-79a5931a - ubuntu1404: ami-d3b98fb0 - ubuntu1604: ami-c1b98fa2 + centos6: ami-6e30e10e + centos7: ami-2e1dcc4e + alinux: ami-1e31e07e + ubuntu1404: ami-4633e226 + ubuntu1604: ami-870edfe7 eu-central-1: - alinux: ami-96db28f9 centos6: ami-5fd92a30 centos7: ami-fcda2993 + alinux: ami-96db28f9 ubuntu1404: ami-89db28e6 ubuntu1604: ami-9dd427f2 - eu-west-1: - alinux: ami-d8f785ab - centos6: ami-27f78554 - centos7: ami-5af48629 - ubuntu1404: ami-3ff0824c - ubuntu1604: ami-99f082ea sa-east-1: - alinux: ami-8bd848e7 centos6: ami-8ad848e6 centos7: ami-f8d84894 + alinux: ami-8bd848e7 ubuntu1404: ami-3f34a553 ubuntu1604: ami-8d0e9fe1 + ap-northeast-1: + centos6: ami-46b17b27 + centos7: ami-43c90322 + alinux: ami-50bd7731 + ubuntu1404: ami-89bd77e8 + ubuntu1604: ami-eabe748b us-east-1: - alinux: ami-d5751ec2 centos6: ami-504b2047 centos7: ami-e94328fe + alinux: ami-d5751ec2 ubuntu1404: ami-ba4a21ad ubuntu1604: ami-d6751ec1 - us-gov-west-1: - alinux: NOT_SUPPORTED - centos6: NOT_SUPPORTED - centos7: NOT_SUPPORTED - ubuntu1404: NOT_SUPPORTED - ubuntu1604: NOT_SUPPORTED us-west-1: - alinux: ami-5f19553f centos6: ami-051f5365 centos7: ami-a51955c5 + alinux: ami-5f19553f ubuntu1404: ami-40185420 ubuntu1604: ami-751f5315 - us-west-2: - alinux: ami-1e31e07e - centos6: ami-6e30e10e - centos7: ami-2e1dcc4e - ubuntu1404: ami-4633e226 - ubuntu1604: ami-870edfe7 - CfnClusterVersions: - default: - ami: 'dev' - berkshelf: 5.6.4 - cfncluster: cfncluster-1.3.2 - chef: 12.19.36 - cookbook: cfncluster-cookbook-1.3.2 - ridley: 5.1.0 + ap-southeast-2: + centos6: ami-0ea6906d + centos7: ami-79a5931a + alinux: ami-51b98f32 + ubuntu1404: ami-d3b98fb0 + ubuntu1604: ami-c1b98fa2 + ap-southeast-1: + centos6: ami-23f22940 + centos7: ami-d5f52eb6 + alinux: ami-9af229f9 + ubuntu1404: ami-95f229f6 + ubuntu1604: ami-23f32840 + us-gov-west-1: + centos6: NOT_SUPPORTED + centos7: NOT_SUPPORTED + alinux: NOT_SUPPORTED + ubuntu1404: NOT_SUPPORTED + ubuntu1604: NOT_SUPPORTED + ap-northeast-2: + centos6: ami-75e1341b + centos7: ami-5cec3932 + alinux: ami-afe431c1 + ubuntu1404: ami-bae732d4 + ubuntu1604: ami-7eef3a10 + ap-south-1: + centos6: ami-a392e7cc + centos7: ami-bf90e5d0 + alinux: ami-0495e06b + ubuntu1404: ami-489de827 + ubuntu1604: ami-5d90e532 OSFeatures: - alinux: - RootDevice: /dev/xvda - User: ec2-user centos6: - RootDevice: /dev/sda1 User: centos - centos7: RootDevice: /dev/sda1 + centos7: User: centos - ubuntu1404: RootDevice: /dev/sda1 + alinux: + User: ec2-user + RootDevice: /dev/xvda + ubuntu1404: User: ubuntu - ubuntu1604: RootDevice: /dev/sda1 + ubuntu1604: User: ubuntu - -Conditions: - AddAdditionalSG: - Fn::Not: - - Fn::Equals: - - Ref: AdditionalSG - - NONE - CloudWatchLogs: - Fn::And: - - Fn::Not: - - Fn::Equals: - - Ref: CWLRegion - - NONE - - Fn::Not: - - Fn::Equals: - - Ref: CWLLogGroup - - NONE - ComputePublicIps: - Fn::And: - - Fn::Equals: - - Ref: UsePublicIps - - 'true' - - Condition: UseMasterSubnetForCompute - CreateComputeSubnetForCompute: - Fn::And: - - Fn::Equals: - - Ref: ComputeSubnetId - - NONE - - Fn::Not: - - Fn::Equals: - - Ref: ComputeSubnetCidr - - NONE - CreateEBSVolume: - Fn::Equals: - - Ref: EBSVolumeId - - NONE - CreateEC2IAMRole: - Fn::Equals: - - Ref: EC2IAMRoleName - - NONE - CreatePlacementGroup: - Fn::And: - - Fn::Equals: - - Ref: PlacementGroup - - DYNAMIC - - Condition: UsePlacementGroup - CreateSecurityGroups: - Fn::Equals: - - Ref: VPCSecurityGroupId - - NONE - CreateSubStack: - Fn::Not: - - Fn::Equals: - - Ref: AdditionalCfnTemplate - - NONE - MaintainInitialASGSize: - Fn::Equals: - - Ref: MaintainInitialSize - - 'true' - MasterPublicIp: - Fn::Equals: - - Ref: UsePublicIps - - 'true' - UseClusterPlacement: - Fn::And: - - Fn::Equals: - - Ref: Placement - - cluster - - Condition: UsePlacementGroup - UseComputeSubnetForCompute: - Fn::And: - - Fn::Equals: - - Ref: ComputeSubnetCidr - - NONE - - Fn::Not: - - Fn::Equals: - - Ref: ComputeSubnetId - - NONE - UseCustomAMI: - Fn::Not: - - Fn::Equals: - - Ref: CustomAMI - - NONE - UseCustomRunList: - Fn::Not: - - Fn::Equals: - - Ref: CustomChefRunList - - NONE - UseDedicatedTenancy: - Fn::Equals: - - Ref: Tenancy - - dedicated - UseEBSEncryption: - Fn::Equals: - - Ref: EBSEncryption - - 'true' - UseEBSKMSKey: - Fn::And: - - Fn::Not: - - Fn::Equals: - - Ref: EBSKMSKeyId - - NONE - - Condition: UseEBSEncryption - UseEBSPIOPS: - Fn::Equals: - - Ref: VolumeType - - io1 - UseEBSSnapshot: - Fn::Not: - - Fn::Equals: - - Ref: EBSSnapshotId - - NONE - UseEC2IAMRole: - Fn::Not: - - Fn::Equals: - - Ref: EC2IAMRoleName - - NONE - UseEphemeralKMSKey: - Fn::And: - - Fn::Not: - - Fn::Equals: - - Ref: EphemeralKMSKeyId - - NONE - - Fn::Equals: - - Ref: EncryptedEphemeral - - 'true' - UseExistingEBSVolume: - Fn::Not: - - Fn::Equals: - - Ref: EBSVolumeId - - NONE - UseExistingSecurityGroup: - Fn::Not: - - Fn::Equals: - - Ref: VPCSecurityGroupId - - NONE - UseMasterSubnetForCompute: - Fn::And: - - Fn::Equals: - - Ref: ComputeSubnetId - - NONE - - Fn::Equals: - - Ref: ComputeSubnetCidr - - NONE - UsePlacementGroup: - Fn::Not: - - Fn::Equals: - - Ref: PlacementGroup - - NONE - UseS3ReadPolicy: - Fn::Not: - - Fn::Equals: - - Ref: S3ReadResource - - NONE - UseS3ReadWritePolicy: - Fn::Not: - - Fn::Equals: - - Ref: S3ReadWriteResource - - NONE - UseSpotInstances: - Fn::Equals: - - Ref: ClusterType - - spot - + RootDevice: /dev/sda1 + CfnClusterVersions: + default: + cfncluster: cfncluster-1.3.2 + cookbook: cfncluster-cookbook-1.3.2 + chef: 12.19.36 + ridley: 5.1.0 + berkshelf: 5.6.4 + ami: dev + AWSRegion2Capabilites: + eu-west-1: + arn: aws + us-east-1: + arn: aws + ap-northeast-1: + arn: aws + us-west-2: + arn: aws + sa-east-1: + arn: aws + us-west-1: + arn: aws + ap-southeast-1: + arn: aws + ap-southeast-2: + arn: aws + eu-central-1: + arn: aws + us-gov-west-1: + arn: aws-us-gov + ap-northeast-2: + arn: aws + ap-south-1: + arn: aws Resources: - AddCapacityAlarm: + SQS: + Type: AWS::SQS::Queue + Properties: {} + Metadata: + AWS::CloudFormation::Designer: + id: 2f599618-92e8-49cf-92bd-551ba9775f39 + SQSPolicy: + Type: AWS::SQS::QueuePolicy + Properties: + PolicyDocument: + Id: MyQueuePolicy + Statement: + - Sid: Allow-SendMessage-From-AS-SNS-Topic + Effect: Allow + Principal: + AWS: '*' + Action: + - sqs:SendMessage + Resource: '*' + Condition: + ArnEquals: + aws:SourceArn: !Ref 'SNS' + Queues: + - !Ref 'SQS' Metadata: AWS::CloudFormation::Designer: - id: 23ed10e0-7726-4099-8370-4116c0bc78a2 + id: 6a874346-80de-4412-9677-00590371a8a2 + SNS: + Type: AWS::SNS::Topic Properties: - AlarmActions: - - Ref: ScaleUpPolicy - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: Stack - Value: - Ref: AWS::StackName - EvaluationPeriods: - Ref: ScalingEvaluationPeriods - MetricName: pending - Namespace: cfncluster - Period: - Ref: ScalingPeriod - Statistic: Average - Threshold: - Ref: ScalingThreshold - Type: AWS::CloudWatch::Alarm - AddCapacityAlarm2: + Subscription: + - Endpoint: !GetAtt [SQS, Arn] + Protocol: sqs Metadata: AWS::CloudFormation::Designer: - id: 127f78c3-d47a-4770-8aad-3fd733c3e1b0 + id: 210b5501-b505-4ee1-917c-a20cae3ac837 + DynamoDBTable: + Type: AWS::DynamoDB::Table Properties: - AlarmActions: - - Ref: ScaleUpPolicy2 - ComparisonOperator: GreaterThanOrEqualToThreshold - Dimensions: - - Name: Stack - Value: - Ref: AWS::StackName - EvaluationPeriods: - Ref: ScalingEvaluationPeriods - MetricName: pending - Namespace: cfncluster - Period: - Ref: ScalingPeriod - Statistic: Sum - Threshold: - Ref: ScalingThreshold2 - Type: AWS::CloudWatch::Alarm - AdditionalCfnStack: - Condition: CreateSubStack + AttributeDefinitions: + - AttributeName: instanceId + AttributeType: S + KeySchema: + - AttributeName: instanceId + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' + Metadata: + AWS::CloudFormation::Designer: + id: 789cb442-3b1a-4673-a07e-8acb1c59c271 + RootRole: + Type: AWS::IAM::Role Properties: - TemplateURL: - Ref: AdditionalCfnTemplate - Type: AWS::CloudFormation::Stack - AssociateEIP: - Condition: MasterPublicIp + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Condition: CreateEC2IAMRole Metadata: AWS::CloudFormation::Designer: - id: e9ac6c41-8110-49b9-a92c-de57be8a292a + id: ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + RootInstanceProfile: + Type: AWS::IAM::InstanceProfile Properties: - AllocationId: - Fn::GetAtt: - - MasterEIP - - AllocationId - NetworkInterfaceId: - Ref: MasterENI - Type: AWS::EC2::EIPAssociation - CfnClusterPolicies: + Path: / + Roles: + - !Ref 'RootRole' Condition: CreateEC2IAMRole Metadata: AWS::CloudFormation::Designer: - id: 7b0e0a3f-3a02-4e70-9e79-7647b98fd579 + id: 2522b048-2b7c-44ed-9d35-6cc66069ca5b + CfnClusterPolicies: + Type: AWS::IAM::Policy Properties: + PolicyName: cfncluster PolicyDocument: Statement: - - Action: + - Sid: EC2 + Action: - ec2:AttachVolume - ec2:DescribeInstanceAttribute - ec2:DescribeInstanceStatus @@ -1439,14 +1404,14 @@ Resources: Effect: Allow Resource: - '*' - Sid: EC2 - - Action: + - Sid: DynamoDBList + Action: - dynamodb:ListTables Effect: Allow Resource: - '*' - Sid: DynamoDBList - - Action: + - Sid: SQSQueue + Action: - sqs:SendMessage - sqs:ReceiveMessage - sqs:ChangeMessageVisibility @@ -1454,25 +1419,23 @@ Resources: - sqs:GetQueueUrl Effect: Allow Resource: - - Fn::GetAtt: - - SQS - - Arn - Sid: SQSQueue - - Action: + - !GetAtt [SQS, Arn] + - Sid: Autoscaling + Action: - autoscaling:DescribeAutoScalingGroups - autoscaling:TerminateInstanceInAutoScalingGroup - autoscaling:SetDesiredCapacity Effect: Allow Resource: - '*' - Sid: Autoscaling - - Action: + - Sid: CloudWatch + Action: - cloudwatch:PutMetricData Effect: Allow Resource: - '*' - Sid: CloudWatch - - Action: + - Sid: DynamoDBTable + Action: - dynamodb:PutItem - dynamodb:Query - dynamodb:GetItem @@ -1480,280 +1443,79 @@ Resources: - dynamodb:DescribeTable Effect: Allow Resource: - - Fn::Join: - - '' - - - 'arn:' - - Fn::FindInMap: - - AWSRegion2Capabilites - - Ref: AWS::Region - - arn - - ':dynamodb:' - - Ref: AWS::Region - - ':' - - Ref: AWS::AccountId - - :table/ - - Ref: DynamoDBTable - Sid: DynamoDBTable - - Action: + - !Join ['', ['arn:', !FindInMap [AWSRegion2Capabilites, !Ref 'AWS::Region', + arn], ':dynamodb:', !Ref 'AWS::Region', ':', !Ref 'AWS::AccountId', + ':table/', !Ref 'DynamoDBTable']] + - Sid: SQSList + Action: - sqs:ListQueues Effect: Allow Resource: - '*' - Sid: SQSList - - Action: + - Sid: CloudWatchLogs + Action: - logs:* Effect: Allow Resource: - - Fn::Join: - - '' - - - 'arn:' - - Fn::FindInMap: - - AWSRegion2Capabilites - - Ref: AWS::Region - - arn - - :logs:*:*:* - Sid: CloudWatchLogs - PolicyName: cfncluster + - !Join ['', ['arn:', !FindInMap [AWSRegion2Capabilites, !Ref 'AWS::Region', + arn], ':logs:*:*:*']] Roles: - - Ref: RootRole - Type: AWS::IAM::Policy - ComputeFleet: - CreationPolicy: - ResourceSignal: - Count: - Ref: ComputeWaitConditionCount - Timeout: PT30M - DependsOn: MasterServer - Metadata: - AWS::CloudFormation::Designer: - id: 3fa053e5-8ab7-4cdf-a737-4d20386e4717 - Properties: - DesiredCapacity: - Ref: InitialQueueSize - LaunchConfigurationName: - Ref: ComputeServerLaunchConfig - MaxSize: - Ref: MaxQueueSize - MinSize: - Fn::If: - - MaintainInitialASGSize - - Ref: InitialQueueSize - - '0' - NotificationConfiguration: - NotificationTypes: - - autoscaling:EC2_INSTANCE_TERMINATE - TopicARN: - Ref: SNS - PlacementGroup: - Fn::If: - - UsePlacementGroup - - Fn::If: - - CreatePlacementGroup - - Ref: DynamicPlacementGroup - - Ref: PlacementGroup - - Ref: AWS::NoValue - Tags: - - Key: Name - PropagateAtLaunch: 'true' - Value: Compute - VPCZoneIdentifier: - - Fn::If: - - UseMasterSubnetForCompute - - Ref: MasterSubnetId - - Fn::If: - - CreateComputeSubnetForCompute - - Ref: ComputeSubnet - - Ref: ComputeSubnetId - Type: AWS::AutoScaling::AutoScalingGroup - ComputeRoute: - Condition: CreateComputeSubnetForCompute + - !Ref 'RootRole' + Condition: CreateEC2IAMRole Metadata: AWS::CloudFormation::Designer: - id: 596ddc76-44b9-4747-921c-c000c5b636c0 + id: 7b0e0a3f-3a02-4e70-9e79-7647b98fd579 + S3ReadRolePolicies: + Type: AWS::IAM::Policy Properties: - DestinationCidrBlock: 0.0.0.0/0 - NetworkInterfaceId: - Ref: MasterENI - RouteTableId: - Ref: ComputeRouteTable - Type: AWS::EC2::Route - ComputeRouteTable: - Condition: CreateComputeSubnetForCompute + PolicyName: S3Read + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: S3Read + Effect: Allow + Action: + - s3:Get* + - s3:List* + Resource: + - !Ref 'S3ReadResource' + Roles: + - !Ref 'RootRole' + Condition: UseS3ReadPolicy Metadata: AWS::CloudFormation::Designer: - id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + id: 0c5a1e92-ffdd-4344-8ba0-65270e5fd472 + S3ReadWriteRolePolicies: + Type: AWS::IAM::Policy Properties: - Tags: - - Key: Application - Value: - Ref: AWS::StackName - - Key: Network - Value: ComputeSubnet - VpcId: - Ref: VPCId - Type: AWS::EC2::RouteTable - ComputeSecurityGroup: - Condition: CreateSecurityGroups + PolicyName: S3ReadWrite + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: S3ReadWrite + Effect: Allow + Action: + - s3:* + Resource: + - !Ref 'S3ReadWriteResource' + Roles: + - !Ref 'RootRole' + Condition: UseS3ReadWritePolicy Metadata: AWS::CloudFormation::Designer: - id: 619070e4-82e5-445a-93a0-4795ff69d61c - Properties: - GroupDescription: Allow access to resources in subnets behind front - SecurityGroupIngress: - - FromPort: '0' - IpProtocol: '-1' - SourceSecurityGroupId: - Ref: MasterSecurityGroup - ToPort: '65535' - VpcId: - Ref: VPCId - Type: AWS::EC2::SecurityGroup - ComputeSecurityGroupIngress: - Condition: CreateSecurityGroups + id: a8fbc80f-2514-4cc9-aee8-ba7936add0a7 + MasterEIP: + Type: AWS::EC2::EIP Properties: - FromPort: '0' - GroupId: - Ref: ComputeSecurityGroup - IpProtocol: '-1' - SourceSecurityGroupId: - Ref: ComputeSecurityGroup - ToPort: '65535' - Type: AWS::EC2::SecurityGroupIngress - ComputeServerLaunchConfig: + Domain: vpc + Condition: MasterPublicIp Metadata: AWS::CloudFormation::Designer: - id: d3204db6-c821-400d-bf31-6afd5739923c - AWS::CloudFormation::Init: - chefConfig: - commands: - chef: - command: chef-client --local-mode --config /etc/chef/client.rb --log_level - auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes - /etc/chef/dna.json - cwd: /etc/chef - chefPrepEnv: - commands: - chef: - command: chef-client --local-mode --config /etc/chef/client.rb --log_level - auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes - /etc/chef/dna.json --override-runlist cfncluster::_prep_env - cwd: /etc/chef - configSets: - default: - - deployConfigFiles - - getCookbooks - - chefPrepEnv - - shellRunPreInstall - - chefConfig - - shellRunPostInstall - - shellForkClusterReadyInstall - - signalComputeReady - deployConfigFiles: - commands: - jq: - command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json - -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json - || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json - ) - mkdir: - command: mkdir -p /etc/chef/ohai/hints - touch: - command: touch /etc/chef/ohai/hints/ec2.json - files: - /etc/chef/client.rb: - content: - Fn::Join: - - '' - - - cookbook_path ['/etc/chef/cookbooks'] - group: root - mode: '000644' - owner: root - /tmp/dna.json: - content: - cfncluster: - cfn_cluster_user: - Fn::FindInMap: - - OSFeatures - - Ref: BaseOS - - User - cfn_encrypted_ephemeral: - Ref: EncryptedEphemeral - cfn_ephemeral_dir: - Ref: EphemeralDir - cfn_master: - Fn::GetAtt: - - MasterServer - - PrivateDnsName - cfn_node_type: ComputeFleet - cfn_postinstall: - Ref: PostInstallScript - cfn_postinstall_args: - Ref: PostInstallArgs - cfn_preinstall: - Ref: PreInstallScript - cfn_preinstall_args: - Ref: PreInstallArgs - cfn_proxy: - Ref: ProxyServer - cfn_region: - Ref: AWS::Region - cfn_scheduler: - Ref: Scheduler - cfn_shared_dir: - Ref: SharedDir - cfn_sqs_queue: - Ref: SQS - stack_name: - Ref: AWS::StackName - run_list: - Fn::If: - - UseCustomRunList - - Ref: CustomChefRunList - - Fn::Join: - - '' - - - 'recipe[cfncluster::' - - Ref: Scheduler - - _config] - group: root - mode: '000644' - owner: root - /tmp/extra.json: - content: - Ref: ExtraJson - group: root - mode: '000644' - owner: root - getCookbooks: - commands: - berk: - command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 - /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' - cwd: /tmp/cookbooks - env: - HOME: /tmp - shellForkClusterReadyInstall: - commands: - clusterreadyinstall: - command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall - shellRunPostInstall: - commands: - runpostinstall: - command: /opt/cfncluster/scripts/fetch_and_run -postinstall - shellRunPreInstall: - commands: - runpreinstall: - command: /opt/cfncluster/scripts/fetch_and_run -preinstall - signalComputeReady: - commands: - compute_ready: - command: /opt/cfncluster/scripts/compute_ready - Comment: cfncluster Compute server + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + MasterServer: + Type: AWS::EC2::Instance Properties: - AssociatePublicIpAddress: - Fn::If: - - ComputePublicIps - - 'true' - - 'false' + InstanceType: !Ref 'MasterInstanceType' BlockDeviceMappings: - DeviceName: /dev/xvdba VirtualName: ephemeral0 @@ -1803,537 +1565,352 @@ Resources: VirtualName: ephemeral22 - DeviceName: /dev/xvdbx VirtualName: ephemeral23 - - DeviceName: - Fn::FindInMap: - - OSFeatures - - Ref: BaseOS - - RootDevice + - DeviceName: !FindInMap [OSFeatures, !Ref 'BaseOS', RootDevice] Ebs: - VolumeSize: - Ref: ComputeRootVolumeSize + VolumeSize: !Ref 'MasterRootVolumeSize' VolumeType: gp2 - IamInstanceProfile: - Fn::If: - - UseEC2IAMRole - - Ref: EC2IAMRoleName - - Ref: RootInstanceProfile - ImageId: - Fn::If: - - UseCustomAMI - - Ref: CustomAMI - - Fn::FindInMap: - - AWSRegionOS2AMI - - Ref: AWS::Region - - Ref: BaseOS - InstanceMonitoring: 'false' - InstanceType: - Ref: ComputeInstanceType - KeyName: - Ref: KeyName - PlacementTenancy: - Fn::If: - - UseDedicatedTenancy - - Ref: Tenancy - - Ref: AWS::NoValue - SecurityGroups: - - Fn::If: - - CreateSecurityGroups - - Ref: ComputeSecurityGroup - - Ref: AWS::NoValue - - Fn::If: - - AddAdditionalSG - - Ref: AdditionalSG - - Ref: AWS::NoValue - - Fn::If: - - UseExistingSecurityGroup - - Ref: VPCSecurityGroupId - - Ref: AWS::NoValue - SpotPrice: - Fn::If: - - UseSpotInstances - - Ref: SpotPrice - - Ref: AWS::NoValue - UserData: - Fn::Base64: - Fn::Join: - - '' - - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + KeyName: !Ref 'KeyName' + Tags: + - Key: Application + Value: !Ref 'AWS::StackName' + - Key: Name + Value: Master + NetworkInterfaces: + - NetworkInterfaceId: !Ref 'MasterENI' + DeviceIndex: '0' + ImageId: !If [UseCustomAMI, !Ref 'CustomAMI', !FindInMap [AWSRegionOS2AMI, !Ref 'AWS::Region', + !Ref 'BaseOS']] + EbsOptimized: !FindInMap [AWSInstanceType2Capabilites, !Ref 'MasterInstanceType', + EBSOpt] + IamInstanceProfile: !If [UseEC2IAMRole, !Ref 'EC2IAMRoleName', !Ref 'RootInstanceProfile'] + PlacementGroupName: !If [UseClusterPlacement, !If [CreatePlacementGroup, !Ref 'DynamicPlacementGroup', + !Ref 'PlacementGroup'], !Ref 'AWS::NoValue'] + Tenancy: !Ref 'Tenancy' + UserData: !Base64 + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - !Ref 'ProxyServer' + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - !Ref 'ProxyServer' + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + - ' - - 'MIME-Version: 1.0 + ' + - '#!/bin/bash -x - ' - - '--==BOUNDARY== + ' + - 'function error_exit - ' - - 'Content-Type: text/cloud-config; charset="us-ascii" + ' + - '{ - ' - - 'MIME-Version: 1.0 + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - !Ref 'AWS::StackName' + - ' --resource=MasterServer --region=' + - !Ref 'AWS::Region' + - ' + ' + - ' exit 1 - ' - - '#cloud-config: + ' + - '} - ' - - 'runcmd: + ' + - 'function bootstrap_instance - ' - - ' - [ sh, "which yum && echo ' - - Ref: ProxyServer - - ' >> /etc/yum.conf" ] + ' + - '{ - ' - - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' - - Ref: ProxyServer - - '"; >> /etc/apt/aot.conf" ] + ' + - ' which yum 2>/dev/null; yum=$? - ' - - '--==BOUNDARY== + ' + - ' which apt-get 2>/dev/null; apt=$? - ' - - 'Content-Type: text/x-shellscript; charset="us-ascii" + ' + - ' if [ "$yum" == "0" ]; then - ' - - 'MIME-Version: 1.0 + ' + - ' yum -y groupinstall development && yum -y install curl wget + ' + - ' fi - ' - - '#!/bin/bash -x + ' + - ' if [ "$apt" == "0" ]; then + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget - ' - - 'function error_exit + ' + - ' fi - ' - - '{ + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) - ' - - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' - - Ref: AWS::StackName - - ' --resource=ComputeFleet --region=' - - Ref: AWS::Region - - ' + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef - ' - - ' exit 1 + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version - ' - - '} + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version - ' - - 'function bootstrap_instance + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url - ' - - '{ + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date - ' - - ' which yum 2>/dev/null; yum=$? + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 - ' - - ' which apt-get 2>/dev/null; apt=$? + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped - ' - - ' if [ "$yum" == "0" ]; then + ' + - '} - ' - - ' yum -y groupinstall development && yum -y install curl wget + ' + - proxy= + - !Ref 'ProxyServer' + - ' - ' - - ' fi + ' + - custom_cookbook= + - !Ref 'CustomChefCookbook' + - ' - ' - - ' if [ "$apt" == "0" ]; then + ' + - 'if [ "$proxy" != "NONE" ]; then - ' - - ' apt-cache search build-essential; apt-get clean; apt-get update; - apt-get -y install build-essential curl wget + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" - ' - - ' fi + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) - ' - - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; - easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) - ' - - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy - ' - - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy - ' - - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version - berkshelf:$berkshelf_version + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 - ' - - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + " >/tmp/proxy.sh - ' - - '} + ' + - ' echo -e "[Boto] - ' - - proxy= - - Ref: ProxyServer - - ' + proxy = ${proxy_host} - ' - - custom_cookbook= - - Ref: CustomChefCookbook - - ' + proxy_port = ${proxy_port} - ' - - 'if [ "$proxy" != "NONE" ]; then + " >/etc/boto.cfg - ' - - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + ' + - 'else - ' - - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + ' + - ' proxy_args="" - ' - - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + ' + - ' touch /tmp/proxy.sh - ' - - ' export http_proxy=$proxy; export https_proxy=$http_proxy + ' + - 'fi - ' - - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then - ' - - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + ' + - ' cookbook_url=$custom_cookbook - ' - - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + ' + - 'else - export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + ' + - ' if [ "' + - !Ref 'AWS::Region' + - '" == "us-east-1" ]; then - export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + ' + - ' s3_prefix=s3 - " >/tmp/proxy.sh + ' + - ' else - ' - - ' echo -e "[Boto] + ' + - ' s3_prefix=s3-' + - !Ref 'AWS::Region' + - ' - proxy = ${proxy_host} + ' + - ' fi - proxy_port = ${proxy_port} + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - !Ref 'AWS::Region' + - /cookbooks/ + - !FindInMap [CfnClusterVersions, default, cookbook] + - '.tgz - " >/etc/boto.cfg + ' + - 'fi - ' - - 'else + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin - ' - - ' proxy_args="" + ' + - export cfncluster_version= + - !FindInMap [CfnClusterVersions, default, cfncluster] + - ' - ' - - ' touch /tmp/proxy.sh + ' + - export cookbook_version= + - !FindInMap [CfnClusterVersions, default, cookbook] + - ' - ' - - 'fi + ' + - export chef_version= + - !FindInMap [CfnClusterVersions, default, chef] + - ' - ' - - 'if [ "$custom_cookbook" != "NONE" ]; then + ' + - export ridley_version= + - !FindInMap [CfnClusterVersions, default, ridley] + - ' - ' - - ' cookbook_url=$custom_cookbook + ' + - export berkshelf_version= + - !FindInMap [CfnClusterVersions, default, berkshelf] + - ' - ' - - 'else + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then - ' - - ' if [ "' - - Ref: AWS::Region - - '" == "us-east-1" ]; then + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) - ' - - ' s3_prefix=s3 + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then - ' - - ' else + ' + - ' bootstrap_instance - ' - - ' s3_prefix=s3-' - - Ref: AWS::Region - - ' + ' + - ' fi - ' - - ' fi + ' + - 'else - ' - - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' - - Ref: AWS::Region - - /cookbooks/ - - Fn::FindInMap: - - CfnClusterVersions - - default - - cookbook - - '.tgz + ' + - ' bootstrap_instance - ' - - 'fi + ' + - 'fi - ' - - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + ' + - 'mkdir /tmp/cookbooks - ' - - export cfncluster_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - cfncluster - - ' + ' + - 'cd /tmp/cookbooks - ' - - export cookbook_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - cookbook - - ' + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url - ' - - export chef_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - chef - - ' + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz - ' - - export ridley_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - ridley - - ' + ' + - 'cd /tmp - ' - - export berkshelf_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - berkshelf - - ' + ' + - '# Call CloudFormation - ' - - 'if [ -f /opt/cfncluster/.bootstrapped ]; then - - ' - - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) - - ' - - ' if [ "$cfncluster_version" != "$installed_version" ]; then - - ' - - ' bootstrap_instance - - ' - - ' fi - - ' - - 'else - - ' - - ' bootstrap_instance - - ' - - 'fi - - ' - - 'mkdir /tmp/cookbooks - - ' - - 'cd /tmp/cookbooks - - ' - - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" - $cookbook_url - - ' - - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz - - ' - - 'cd /tmp - - ' - - '# Call CloudFormation - - ' - - 'cfn-init ${proxy_args} -s ' - - Ref: AWS::StackName - - ' -v -c default -r ComputeServerLaunchConfig --region ' - - Ref: AWS::Region - - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, - check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' - - ' - - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup - complete" --stack= - - Ref: AWS::StackName - - ' --resource=ComputeFleet --region=' - - Ref: AWS::Region - - ' - - ' - - '# End of file - - ' - - '--==BOUNDARY== - - ' - Type: AWS::AutoScaling::LaunchConfiguration - ComputeSubnet: - Condition: CreateComputeSubnetForCompute - Metadata: - AWS::CloudFormation::Designer: - id: bb5a532f-bb43-472a-9965-c4054cd61cc9 - Properties: - AvailabilityZone: - Ref: AvailabilityZone - CidrBlock: - Ref: ComputeSubnetCidr - Tags: - - Key: Network - Value: ComputeSubnet - VpcId: - Ref: VPCId - Type: AWS::EC2::Subnet - ComputeSubnetRouteTableAssociation: - Condition: CreateComputeSubnetForCompute - Metadata: - AWS::CloudFormation::Designer: - id: 9dd9772e-03a3-4983-b28e-a52ac42f0ffe - Properties: - RouteTableId: - Ref: ComputeRouteTable - SubnetId: - Ref: ComputeSubnet - Type: AWS::EC2::SubnetRouteTableAssociation - DynamicPlacementGroup: - Condition: CreatePlacementGroup - Metadata: - AWS::CloudFormation::Designer: - id: ee5f3006-419e-4786-b527-e9503a662e5e - Properties: - Strategy: cluster - Type: AWS::EC2::PlacementGroup - DynamoDBTable: - Metadata: - AWS::CloudFormation::Designer: - id: 789cb442-3b1a-4673-a07e-8acb1c59c271 - Properties: - AttributeDefinitions: - - AttributeName: instanceId - AttributeType: S - KeySchema: - - AttributeName: instanceId - KeyType: HASH - ProvisionedThroughput: - ReadCapacityUnits: '5' - WriteCapacityUnits: '5' - Type: AWS::DynamoDB::Table - MasterEIP: - Condition: MasterPublicIp - Metadata: - AWS::CloudFormation::Designer: - id: d5bad569-ad14-487d-bcc6-5569373bc8cd - Properties: - Domain: vpc - Type: AWS::EC2::EIP - MasterENI: - Metadata: - AWS::CloudFormation::Designer: - id: 45360a4f-64aa-49f8-b016-78fe5ed86819 - Properties: - Description: cfncluster Master Server - GroupSet: - - Fn::If: - - CreateSecurityGroups - - Ref: MasterSecurityGroup - - Ref: AWS::NoValue - - Fn::If: - - AddAdditionalSG - - Ref: AdditionalSG - - Ref: AWS::NoValue - - Fn::If: - - UseExistingSecurityGroup - - Ref: VPCSecurityGroupId - - Ref: AWS::NoValue - SourceDestCheck: 'false' - SubnetId: - Ref: MasterSubnetId - Type: AWS::EC2::NetworkInterface - MasterSecurityGroup: - Condition: CreateSecurityGroups - Metadata: - AWS::CloudFormation::Designer: - id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e - Properties: - GroupDescription: Enable access to the Master host - SecurityGroupIngress: - - CidrIp: - Ref: AccessFrom - FromPort: '22' - IpProtocol: tcp - ToPort: '22' - - CidrIp: - Ref: AccessFrom - FromPort: '80' - IpProtocol: tcp - ToPort: '80' - VpcId: - Ref: VPCId - Type: AWS::EC2::SecurityGroup - MasterSecurityGroupIngress: - Condition: CreateSecurityGroups - Metadata: - AWS::CloudFormation::Designer: - id: 466554d3-0e10-4ae4-bfee-51b0aadbf923 - Properties: - FromPort: '0' - GroupId: - Ref: MasterSecurityGroup - IpProtocol: '-1' - SourceSecurityGroupId: - Ref: ComputeSecurityGroup - ToPort: '65535' - Type: AWS::EC2::SecurityGroupIngress - MasterServer: - CreationPolicy: - ResourceSignal: - Count: '1' - Timeout: PT30M + ' + - 'cfn-init ${proxy_args} -s ' + - !Ref 'AWS::StackName' + - ' -v -c default -r MasterServer --region ' + - !Ref 'AWS::Region' + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup complete" + --stack= + - !Ref 'AWS::StackName' + - ' --resource=MasterServer --region=' + - !Ref 'AWS::Region' + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' Metadata: - AWS::CloudFormation::Designer: - id: 88da1502-8179-4be3-9123-fdc5eac0b6a3 + Comment: cfncluster Master server AWS::CloudFormation::Init: - chefConfig: - commands: - chef: - command: chef-client --local-mode --config /etc/chef/client.rb --log_level - auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes - /etc/chef/dna.json - cwd: /etc/chef - chefPrepEnv: - commands: - chef: - command: chef-client --local-mode --config /etc/chef/client.rb --log_level - auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes - /etc/chef/dna.json --override-runlist cfncluster::_prep_env - cwd: /etc/chef configSets: default: - deployConfigFiles @@ -2344,85 +1921,51 @@ Resources: - shellRunPostInstall - shellForkClusterReadyInstall deployConfigFiles: - commands: - jq: - command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json - -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json - || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json - ) - mkdir: - command: mkdir -p /etc/chef/ohai/hints - touch: - command: touch /etc/chef/ohai/hints/ec2.json files: - /etc/chef/client.rb: - content: - Fn::Join: - - '' - - - cookbook_path ['/etc/chef/cookbooks'] - group: root + /tmp/dna.json: mode: '000644' owner: root - /tmp/dna.json: + group: root content: cfncluster: - cfn_cluster_user: - Fn::FindInMap: - - OSFeatures - - Ref: BaseOS - - User - cfn_ddb_table: - Ref: DynamoDBTable - cfn_encrypted_ephemeral: - Ref: EncryptedEphemeral - cfn_ephemeral_dir: - Ref: EphemeralDir + stack_name: !Ref 'AWS::StackName' + cfn_preinstall: !Ref 'PreInstallScript' + cfn_preinstall_args: !Ref 'PreInstallArgs' + cfn_postinstall: !Ref 'PostInstallScript' + cfn_postinstall_args: !Ref 'PostInstallArgs' + cfn_region: !Ref 'AWS::Region' + cfn_volume: !If [UseExistingEBSVolume, !Ref 'EBSVolumeId', !Ref 'SharedVolume'] + cfn_scheduler: !Ref 'Scheduler' + cfn_encrypted_ephemeral: !Ref 'EncryptedEphemeral' + cfn_ephemeral_dir: !Ref 'EphemeralDir' + cfn_shared_dir: !Ref 'SharedDir' + cfn_proxy: !Ref 'ProxyServer' cfn_node_type: MasterServer - cfn_postinstall: - Ref: PostInstallScript - cfn_postinstall_args: - Ref: PostInstallArgs - cfn_preinstall: - Ref: PreInstallScript - cfn_preinstall_args: - Ref: PreInstallArgs - cfn_proxy: - Ref: ProxyServer - cfn_region: - Ref: AWS::Region - cfn_scheduler: - Ref: Scheduler - cfn_shared_dir: - Ref: SharedDir - cfn_sqs_queue: - Fn::GetAtt: - - SQS - - QueueName - cfn_volume: - Fn::If: - - UseExistingEBSVolume - - Ref: EBSVolumeId - - Ref: SharedVolume - stack_name: - Ref: AWS::StackName - run_list: - Fn::If: - - UseCustomRunList - - Ref: CustomChefRunList - - Fn::Join: - - '' - - - 'recipe[cfncluster::' - - Ref: Scheduler - - _config] - group: root + cfn_cluster_user: !FindInMap [OSFeatures, !Ref 'BaseOS', User] + cfn_ddb_table: !Ref 'DynamoDBTable' + cfn_sqs_queue: !GetAtt [SQS, QueueName] + run_list: !If [UseCustomRunList, !Ref 'CustomChefRunList', !Join [ + '', ['recipe[cfncluster::', !Ref 'Scheduler', '_config]']]] + /etc/chef/client.rb: mode: '000644' owner: root - /tmp/extra.json: - content: - Ref: ExtraJson group: root + content: !Join ['', ['cookbook_path [''/etc/chef/cookbooks'']']] + /tmp/extra.json: mode: '000644' owner: root + group: root + content: !Ref 'ExtraJson' + commands: + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) getCookbooks: commands: berk: @@ -2431,20 +1974,81 @@ Resources: cwd: /tmp/cookbooks env: HOME: /tmp - shellForkClusterReadyInstall: + chefPrepEnv: commands: - clusterreadyinstall: - command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef shellRunPostInstall: commands: runpostinstall: command: /opt/cfncluster/scripts/fetch_and_run -postinstall - shellRunPreInstall: + shellForkClusterReadyInstall: commands: - runpreinstall: - command: /opt/cfncluster/scripts/fetch_and_run -preinstall - Comment: cfncluster Master server + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + AWS::CloudFormation::Designer: + id: 88da1502-8179-4be3-9123-fdc5eac0b6a3 + CreationPolicy: + ResourceSignal: + Count: '1' + Timeout: PT30M + ComputeFleet: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + MaxSize: !Ref 'MaxQueueSize' + VPCZoneIdentifier: + - !If [UseMasterSubnetForCompute, !Ref 'MasterSubnetId', !If [CreateComputeSubnetForCompute, + !Ref 'ComputeSubnet', !Ref 'ComputeSubnetId']] + LaunchConfigurationName: !Ref 'ComputeServerLaunchConfig' + MinSize: !If [MaintainInitialASGSize, !Ref 'InitialQueueSize', '0'] + DesiredCapacity: !Ref 'InitialQueueSize' + NotificationConfiguration: + TopicARN: !Ref 'SNS' + NotificationTypes: + - autoscaling:EC2_INSTANCE_TERMINATE + Tags: + - Key: Name + Value: Compute + PropagateAtLaunch: 'true' + PlacementGroup: !If [UsePlacementGroup, !If [CreatePlacementGroup, !Ref 'DynamicPlacementGroup', + !Ref 'PlacementGroup'], !Ref 'AWS::NoValue'] + DependsOn: MasterServer + CreationPolicy: + ResourceSignal: + Timeout: PT30M + Count: !Ref 'ComputeWaitConditionCount' + Metadata: + AWS::CloudFormation::Designer: + id: 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + ComputeServerLaunchConfig: + Type: AWS::AutoScaling::LaunchConfiguration Properties: + SecurityGroups: + - !If [CreateSecurityGroups, !Ref 'ComputeSecurityGroup', !Ref 'AWS::NoValue'] + - !If [AddAdditionalSG, !Ref 'AdditionalSG', !Ref 'AWS::NoValue'] + - !If [UseExistingSecurityGroup, !Ref 'VPCSecurityGroupId', !Ref 'AWS::NoValue'] + AssociatePublicIpAddress: !If [ComputePublicIps, 'true', 'false'] + InstanceType: !Ref 'ComputeInstanceType' + KeyName: !Ref 'KeyName' + IamInstanceProfile: !If [UseEC2IAMRole, !Ref 'EC2IAMRoleName', !Ref 'RootInstanceProfile'] + SpotPrice: !If [UseSpotInstances, !Ref 'SpotPrice', !Ref 'AWS::NoValue'] + ImageId: !If [UseCustomAMI, !Ref 'CustomAMI', !FindInMap [AWSRegionOS2AMI, !Ref 'AWS::Region', + !Ref 'BaseOS']] + InstanceMonitoring: 'false' BlockDeviceMappings: - DeviceName: /dev/xvdba VirtualName: ephemeral0 @@ -2494,600 +2098,646 @@ Resources: VirtualName: ephemeral22 - DeviceName: /dev/xvdbx VirtualName: ephemeral23 - - DeviceName: - Fn::FindInMap: - - OSFeatures - - Ref: BaseOS - - RootDevice + - DeviceName: !FindInMap [OSFeatures, !Ref 'BaseOS', RootDevice] Ebs: - VolumeSize: - Ref: MasterRootVolumeSize + VolumeSize: !Ref 'ComputeRootVolumeSize' VolumeType: gp2 - EbsOptimized: - Fn::FindInMap: - - AWSInstanceType2Capabilites - - Ref: MasterInstanceType - - EBSOpt - IamInstanceProfile: - Fn::If: - - UseEC2IAMRole - - Ref: EC2IAMRoleName - - Ref: RootInstanceProfile - ImageId: - Fn::If: - - UseCustomAMI - - Ref: CustomAMI - - Fn::FindInMap: - - AWSRegionOS2AMI - - Ref: AWS::Region - - Ref: BaseOS - InstanceType: - Ref: MasterInstanceType - KeyName: - Ref: KeyName - NetworkInterfaces: - - DeviceIndex: '0' - NetworkInterfaceId: - Ref: MasterENI - PlacementGroupName: - Fn::If: - - UseClusterPlacement - - Fn::If: - - CreatePlacementGroup - - Ref: DynamicPlacementGroup - - Ref: PlacementGroup - - Ref: AWS::NoValue - Tags: - - Key: Application - Value: - Ref: AWS::StackName - - Key: Name - Value: Master - Tenancy: - Ref: Tenancy - UserData: - Fn::Base64: - Fn::Join: - - '' - - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + PlacementTenancy: !If [UseDedicatedTenancy, !Ref 'Tenancy', !Ref 'AWS::NoValue'] + UserData: !Base64 + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" - ' - - 'MIME-Version: 1.0 + ' + - 'MIME-Version: 1.0 - ' - - '--==BOUNDARY== + ' + - '--==BOUNDARY== - ' - - 'Content-Type: text/cloud-config; charset="us-ascii" + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" - ' - - 'MIME-Version: 1.0 + ' + - 'MIME-Version: 1.0 - ' - - '#cloud-config: + ' + - '#cloud-config: - ' - - 'runcmd: + ' + - 'runcmd: - ' - - ' - [ sh, "which yum && echo ' - - Ref: ProxyServer - - ' >> /etc/yum.conf" ] + ' + - ' - [ sh, "which yum && echo ' + - !Ref 'ProxyServer' + - ' >> /etc/yum.conf" ] - ' - - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' - - Ref: ProxyServer - - '"; >> /etc/apt/aot.conf" ] + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - !Ref 'ProxyServer' + - '"; >> /etc/apt/aot.conf" ] - ' - - '--==BOUNDARY== + ' + - '--==BOUNDARY== - ' - - 'Content-Type: text/x-shellscript; charset="us-ascii" + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" - ' - - 'MIME-Version: 1.0 + ' + - 'MIME-Version: 1.0 - ' - - '#!/bin/bash -x + ' + - '#!/bin/bash -x - ' - - 'function error_exit + ' + - 'function error_exit - ' - - '{ + ' + - '{ - ' - - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' - - Ref: AWS::StackName - - ' --resource=MasterServer --region=' - - Ref: AWS::Region - - ' + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - !Ref 'AWS::StackName' + - ' --resource=ComputeFleet --region=' + - !Ref 'AWS::Region' + - ' - ' - - ' exit 1 + ' + - ' exit 1 - ' - - '} + ' + - '} - ' - - 'function bootstrap_instance + ' + - 'function bootstrap_instance - ' - - '{ + ' + - '{ - ' - - ' which yum 2>/dev/null; yum=$? + ' + - ' which yum 2>/dev/null; yum=$? - ' - - ' which apt-get 2>/dev/null; apt=$? + ' + - ' which apt-get 2>/dev/null; apt=$? - ' - - ' if [ "$yum" == "0" ]; then + ' + - ' if [ "$yum" == "0" ]; then - ' - - ' yum -y groupinstall development && yum -y install curl wget + ' + - ' yum -y groupinstall development && yum -y install curl wget - ' - - ' fi + ' + - ' fi - ' - - ' if [ "$apt" == "0" ]; then + ' + - ' if [ "$apt" == "0" ]; then - ' - - ' apt-cache search build-essential; apt-get clean; apt-get update; - apt-get -y install build-essential curl wget + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget - ' - - ' fi + ' + - ' fi - ' - - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; - easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) - ' - - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef - ' - - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version - ' - - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version - berkshelf:$berkshelf_version + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date - ' - - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 - ' - - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped - ' - - '} + ' + - '} - ' - - proxy= - - Ref: ProxyServer - - ' + ' + - proxy= + - !Ref 'ProxyServer' + - ' - ' - - custom_cookbook= - - Ref: CustomChefCookbook - - ' + ' + - custom_cookbook= + - !Ref 'CustomChefCookbook' + - ' - ' - - 'if [ "$proxy" != "NONE" ]; then + ' + - 'if [ "$proxy" != "NONE" ]; then - ' - - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" - ' - - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) - ' - - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) - ' - - ' export http_proxy=$proxy; export https_proxy=$http_proxy + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy - ' - - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy - ' - - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 - ' - - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy - export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy - export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 - " >/tmp/proxy.sh + " >/tmp/proxy.sh - ' - - ' echo -e "[Boto] + ' + - ' echo -e "[Boto] - proxy = ${proxy_host} + proxy = ${proxy_host} - proxy_port = ${proxy_port} + proxy_port = ${proxy_port} - " >/etc/boto.cfg + " >/etc/boto.cfg - ' - - 'else + ' + - 'else - ' - - ' proxy_args="" + ' + - ' proxy_args="" - ' - - ' touch /tmp/proxy.sh + ' + - ' touch /tmp/proxy.sh - ' - - 'fi + ' + - 'fi - ' - - 'if [ "$custom_cookbook" != "NONE" ]; then + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then - ' - - ' cookbook_url=$custom_cookbook + ' + - ' cookbook_url=$custom_cookbook - ' - - 'else + ' + - 'else - ' - - ' if [ "' - - Ref: AWS::Region - - '" == "us-east-1" ]; then + ' + - ' if [ "' + - !Ref 'AWS::Region' + - '" == "us-east-1" ]; then - ' - - ' s3_prefix=s3 + ' + - ' s3_prefix=s3 - ' - - ' else + ' + - ' else - ' - - ' s3_prefix=s3-' - - Ref: AWS::Region - - ' + ' + - ' s3_prefix=s3-' + - !Ref 'AWS::Region' + - ' - ' - - ' fi + ' + - ' fi - ' - - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' - - Ref: AWS::Region - - /cookbooks/ - - Fn::FindInMap: - - CfnClusterVersions - - default - - cookbook - - '.tgz + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - !Ref 'AWS::Region' + - /cookbooks/ + - !FindInMap [CfnClusterVersions, default, cookbook] + - '.tgz - ' - - 'fi + ' + - 'fi - ' - - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin - ' - - export cfncluster_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - cfncluster - - ' + ' + - export cfncluster_version= + - !FindInMap [CfnClusterVersions, default, cfncluster] + - ' - ' - - export cookbook_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - cookbook - - ' + ' + - export cookbook_version= + - !FindInMap [CfnClusterVersions, default, cookbook] + - ' - ' - - export chef_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - chef - - ' + ' + - export chef_version= + - !FindInMap [CfnClusterVersions, default, chef] + - ' - ' - - export ridley_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - ridley - - ' + ' + - export ridley_version= + - !FindInMap [CfnClusterVersions, default, ridley] + - ' - ' - - export berkshelf_version= - - Fn::FindInMap: - - CfnClusterVersions - - default - - berkshelf - - ' + ' + - export berkshelf_version= + - !FindInMap [CfnClusterVersions, default, berkshelf] + - ' - ' - - 'if [ -f /opt/cfncluster/.bootstrapped ]; then - - ' - - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) - - ' - - ' if [ "$cfncluster_version" != "$installed_version" ]; then - - ' - - ' bootstrap_instance - - ' - - ' fi - - ' - - 'else - - ' - - ' bootstrap_instance - - ' - - 'fi - - ' - - 'mkdir /tmp/cookbooks - - ' - - 'cd /tmp/cookbooks - - ' - - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" - $cookbook_url - - ' - - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz - - ' - - 'cd /tmp - - ' - - '# Call CloudFormation - - ' - - 'cfn-init ${proxy_args} -s ' - - Ref: AWS::StackName - - ' -v -c default -r MasterServer --region ' - - Ref: AWS::Region - - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, - check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' - - ' - - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup - complete" --stack= - - Ref: AWS::StackName - - ' --resource=MasterServer --region=' - - Ref: AWS::Region - - ' - - ' - - '# End of file - - ' - - '--==BOUNDARY== - - ' - Type: AWS::EC2::Instance - RootInstanceProfile: - Condition: CreateEC2IAMRole + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - !Ref 'AWS::StackName' + - ' -v -c default -r ComputeServerLaunchConfig --region ' + - !Ref 'AWS::Region' + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup complete" + --stack= + - !Ref 'AWS::StackName' + - ' --resource=ComputeFleet --region=' + - !Ref 'AWS::Region' + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Metadata: + Comment: cfncluster Compute server + AWS::CloudFormation::Init: + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + - signalComputeReady + deployConfigFiles: + files: + /tmp/dna.json: + mode: '000644' + owner: root + group: root + content: + cfncluster: + stack_name: !Ref 'AWS::StackName' + cfn_preinstall: !Ref 'PreInstallScript' + cfn_preinstall_args: !Ref 'PreInstallArgs' + cfn_postinstall: !Ref 'PostInstallScript' + cfn_postinstall_args: !Ref 'PostInstallArgs' + cfn_region: !Ref 'AWS::Region' + cfn_scheduler: !Ref 'Scheduler' + cfn_encrypted_ephemeral: !Ref 'EncryptedEphemeral' + cfn_ephemeral_dir: !Ref 'EphemeralDir' + cfn_shared_dir: !Ref 'SharedDir' + cfn_proxy: !Ref 'ProxyServer' + cfn_sqs_queue: !Ref 'SQS' + cfn_master: !GetAtt [MasterServer, PrivateDnsName] + cfn_node_type: ComputeFleet + cfn_cluster_user: !FindInMap [OSFeatures, !Ref 'BaseOS', User] + run_list: !If [UseCustomRunList, !Ref 'CustomChefRunList', !Join [ + '', ['recipe[cfncluster::', !Ref 'Scheduler', '_config]']]] + /etc/chef/client.rb: + mode: '000644' + owner: root + group: root + content: !Join ['', ['cookbook_path [''/etc/chef/cookbooks'']']] + /tmp/extra.json: + mode: '000644' + owner: root + group: root + content: !Ref 'ExtraJson' + commands: + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + signalComputeReady: + commands: + compute_ready: + command: /opt/cfncluster/scripts/compute_ready + AWS::CloudFormation::Designer: + id: d3204db6-c821-400d-bf31-6afd5739923c + ScaleUpPolicy2: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + Cooldown: !Ref 'ScalingCooldown' + ScalingAdjustment: !Ref 'ScalingAdjustment2' + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: !Ref 'ComputeFleet' Metadata: AWS::CloudFormation::Designer: - id: 2522b048-2b7c-44ed-9d35-6cc66069ca5b + id: 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + AddCapacityAlarm2: + Type: AWS::CloudWatch::Alarm Properties: - Path: / - Roles: - - Ref: RootRole - Type: AWS::IAM::InstanceProfile - RootRole: - Condition: CreateEC2IAMRole + Threshold: !Ref 'ScalingThreshold2' + Period: !Ref 'ScalingPeriod' + EvaluationPeriods: !Ref 'ScalingEvaluationPeriods' + Statistic: Sum + AlarmActions: + - !Ref 'ScaleUpPolicy2' + Namespace: cfncluster + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: !Ref 'AWS::StackName' + MetricName: pending Metadata: AWS::CloudFormation::Designer: - id: ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + id: 127f78c3-d47a-4770-8aad-3fd733c3e1b0 + ScaleUpPolicy: + Type: AWS::AutoScaling::ScalingPolicy Properties: - AssumeRolePolicyDocument: - Statement: - - Action: - - sts:AssumeRole - Effect: Allow - Principal: - Service: - - ec2.amazonaws.com - Version: '2012-10-17' - Path: / - Type: AWS::IAM::Role - S3ReadRolePolicies: - Condition: UseS3ReadPolicy + Cooldown: !Ref 'ScalingCooldown' + ScalingAdjustment: !Ref 'ScalingAdjustment' + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: !Ref 'ComputeFleet' Metadata: AWS::CloudFormation::Designer: - id: 0c5a1e92-ffdd-4344-8ba0-65270e5fd472 + id: 8e63b592-a50f-4b89-a40b-e991adaef77b + AddCapacityAlarm: + Type: AWS::CloudWatch::Alarm Properties: - PolicyDocument: - Statement: - - Action: - - s3:Get* - - s3:List* - Effect: Allow - Resource: - - Ref: S3ReadResource - Sid: S3Read - Version: '2012-10-17' - PolicyName: S3Read - Roles: - - Ref: RootRole - Type: AWS::IAM::Policy - S3ReadWriteRolePolicies: - Condition: UseS3ReadWritePolicy + Threshold: !Ref 'ScalingThreshold' + Period: !Ref 'ScalingPeriod' + EvaluationPeriods: !Ref 'ScalingEvaluationPeriods' + Statistic: Average + AlarmActions: + - !Ref 'ScaleUpPolicy' + Namespace: cfncluster + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: !Ref 'AWS::StackName' + MetricName: pending Metadata: AWS::CloudFormation::Designer: - id: a8fbc80f-2514-4cc9-aee8-ba7936add0a7 + id: 23ed10e0-7726-4099-8370-4116c0bc78a2 + ComputeSubnet: + Type: AWS::EC2::Subnet Properties: - PolicyDocument: - Statement: - - Action: - - s3:* - Effect: Allow - Resource: - - Ref: S3ReadWriteResource - Sid: S3ReadWrite - Version: '2012-10-17' - PolicyName: S3ReadWrite - Roles: - - Ref: RootRole - Type: AWS::IAM::Policy - SNS: + VpcId: !Ref 'VPCId' + CidrBlock: !Ref 'ComputeSubnetCidr' + Tags: + - Key: Network + Value: ComputeSubnet + AvailabilityZone: !Ref 'AvailabilityZone' + Condition: CreateComputeSubnetForCompute Metadata: AWS::CloudFormation::Designer: - id: 210b5501-b505-4ee1-917c-a20cae3ac837 + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + ComputeRouteTable: + Type: AWS::EC2::RouteTable Properties: - Subscription: - - Endpoint: - Fn::GetAtt: - - SQS - - Arn - Protocol: sqs - Type: AWS::SNS::Topic - SQS: + VpcId: !Ref 'VPCId' + Tags: + - Key: Application + Value: !Ref 'AWS::StackName' + - Key: Network + Value: ComputeSubnet + Condition: CreateComputeSubnetForCompute Metadata: AWS::CloudFormation::Designer: - id: 2f599618-92e8-49cf-92bd-551ba9775f39 - Properties: {} - Type: AWS::SQS::Queue - SQSPolicy: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + ComputeRoute: + Type: AWS::EC2::Route + Properties: + RouteTableId: !Ref 'ComputeRouteTable' + DestinationCidrBlock: 0.0.0.0/0 + NetworkInterfaceId: !Ref 'MasterENI' + Condition: CreateComputeSubnetForCompute Metadata: AWS::CloudFormation::Designer: - id: 6a874346-80de-4412-9677-00590371a8a2 + id: 596ddc76-44b9-4747-921c-c000c5b636c0 + ComputeSubnetRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation Properties: - PolicyDocument: - Id: MyQueuePolicy - Statement: - - Action: - - sqs:SendMessage - Condition: - ArnEquals: - aws:SourceArn: - Ref: SNS - Effect: Allow - Principal: - AWS: '*' - Resource: '*' - Sid: Allow-SendMessage-From-AS-SNS-Topic - Queues: - - Ref: SQS - Type: AWS::SQS::QueuePolicy - ScaleUpPolicy: + SubnetId: !Ref 'ComputeSubnet' + RouteTableId: !Ref 'ComputeRouteTable' + Condition: CreateComputeSubnetForCompute Metadata: AWS::CloudFormation::Designer: - id: 8e63b592-a50f-4b89-a40b-e991adaef77b + id: 9dd9772e-03a3-4983-b28e-a52ac42f0ffe + MasterSecurityGroup: + Type: AWS::EC2::SecurityGroup Properties: - AdjustmentType: ChangeInCapacity - AutoScalingGroupName: - Ref: ComputeFleet - Cooldown: - Ref: ScalingCooldown - ScalingAdjustment: - Ref: ScalingAdjustment - Type: AWS::AutoScaling::ScalingPolicy - ScaleUpPolicy2: + GroupDescription: Enable access to the Master host + VpcId: !Ref 'VPCId' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: '22' + ToPort: '22' + CidrIp: !Ref 'AccessFrom' + - IpProtocol: tcp + FromPort: '80' + ToPort: '80' + CidrIp: !Ref 'AccessFrom' + Condition: CreateSecurityGroups Metadata: AWS::CloudFormation::Designer: - id: 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + MasterSecurityGroupIngress: + Type: AWS::EC2::SecurityGroupIngress Properties: - AdjustmentType: ChangeInCapacity - AutoScalingGroupName: - Ref: ComputeFleet - Cooldown: - Ref: ScalingCooldown - ScalingAdjustment: - Ref: ScalingAdjustment2 - Type: AWS::AutoScaling::ScalingPolicy + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + SourceSecurityGroupId: !Ref 'ComputeSecurityGroup' + GroupId: !Ref 'MasterSecurityGroup' + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 466554d3-0e10-4ae4-bfee-51b0aadbf923 + ComputeSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allow access to resources in subnets behind front + VpcId: !Ref 'VPCId' + SecurityGroupIngress: + - SourceSecurityGroupId: !Ref 'MasterSecurityGroup' + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + ComputeSecurityGroupIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + SourceSecurityGroupId: !Ref 'ComputeSecurityGroup' + GroupId: !Ref 'ComputeSecurityGroup' + Condition: CreateSecurityGroups + MasterENI: + Type: AWS::EC2::NetworkInterface + Properties: + Description: cfncluster Master Server + SubnetId: !Ref 'MasterSubnetId' + SourceDestCheck: 'false' + GroupSet: + - !If [CreateSecurityGroups, !Ref 'MasterSecurityGroup', !Ref 'AWS::NoValue'] + - !If [AddAdditionalSG, !Ref 'AdditionalSG', !Ref 'AWS::NoValue'] + - !If [UseExistingSecurityGroup, !Ref 'VPCSecurityGroupId', !Ref 'AWS::NoValue'] + Metadata: + AWS::CloudFormation::Designer: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + AdditionalCfnStack: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: !Ref 'AdditionalCfnTemplate' + Condition: CreateSubStack SharedVolume: + Type: AWS::EC2::Volume + Properties: + AvailabilityZone: !Ref 'AvailabilityZone' + VolumeType: !Ref 'VolumeType' + Size: !If [UseEBSSnapshot, !Ref 'AWS::NoValue', !Ref 'VolumeSize'] + SnapshotId: !If [UseEBSSnapshot, !Ref 'EBSSnapshotId', !Ref 'AWS::NoValue'] + Iops: !If [UseEBSPIOPS, !Ref 'VolumeIOPS', !Ref 'AWS::NoValue'] + Encrypted: !If [UseEBSEncryption, !Ref 'EBSEncryption', !Ref 'AWS::NoValue'] + KmsKeyId: !If [UseEBSKMSKey, !Ref 'EBSKMSKeyId', !Ref 'AWS::NoValue'] Condition: CreateEBSVolume Metadata: AWS::CloudFormation::Designer: id: c8d3da13-fd27-4377-b606-0f41728c17e1 + AssociateEIP: + Type: AWS::EC2::EIPAssociation Properties: - AvailabilityZone: - Ref: AvailabilityZone - Encrypted: - Fn::If: - - UseEBSEncryption - - Ref: EBSEncryption - - Ref: AWS::NoValue - Iops: - Fn::If: - - UseEBSPIOPS - - Ref: VolumeIOPS - - Ref: AWS::NoValue - KmsKeyId: - Fn::If: - - UseEBSKMSKey - - Ref: EBSKMSKeyId - - Ref: AWS::NoValue - Size: - Fn::If: - - UseEBSSnapshot - - Ref: AWS::NoValue - - Ref: VolumeSize - SnapshotId: - Fn::If: - - UseEBSSnapshot - - Ref: EBSSnapshotId - - Ref: AWS::NoValue - VolumeType: - Ref: VolumeType - Type: AWS::EC2::Volume - + AllocationId: !GetAtt [MasterEIP, AllocationId] + NetworkInterfaceId: !Ref 'MasterENI' + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: e9ac6c41-8110-49b9-a92c-de57be8a292a + DynamicPlacementGroup: + Type: AWS::EC2::PlacementGroup + Properties: + Strategy: cluster + Condition: CreatePlacementGroup + Metadata: + AWS::CloudFormation::Designer: + id: ee5f3006-419e-4786-b527-e9503a662e5e Outputs: + MasterPrivateIP: + Description: Private IP Address of the Master host + Value: !GetAtt [MasterServer, PrivateIp] + MasterPublicIP: + Description: Public IP Address of the Master host + Value: !GetAtt [MasterServer, PublicIp] + Condition: MasterPublicIp GangliaPrivateURL: Description: Private URL to access Ganglia - Value: - Fn::Join: - - '' - - - http:// - - Fn::GetAtt: - - MasterServer - - PrivateIp - - /ganglia/ + Value: !Join ['', ['http://', !GetAtt [MasterServer, PrivateIp], /ganglia/]] GangliaPublicURL: - Condition: MasterPublicIp Description: Public URL to access Ganglia - Value: - Fn::Join: - - '' - - - http:// - - Fn::GetAtt: - - MasterServer - - PublicIp - - /ganglia/ - MasterPrivateIP: - Description: Private IP Address of the Master host - Value: - Fn::GetAtt: - - MasterServer - - PrivateIp - MasterPublicIP: + Value: !Join ['', ['http://', !GetAtt [MasterServer, PublicIp], /ganglia/]] Condition: MasterPublicIp - Description: Public IP Address of the Master host - Value: - Fn::GetAtt: - - MasterServer - - PublicIp From effcc3d965cffafbc82b1d9e7e9488650b2f911c Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Wed, 7 Jun 2017 02:45:41 +0000 Subject: [PATCH 30/34] Add support for new regions Add support for eu-west-2, us-east-2, and ca-central-1 regions. Signed-off-by: Brian Barrett --- cloudformation/cfncluster.cfn.json | 9 +++++++++ cloudformation/cfncluster.cfn.yaml | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 44bfc75021..fd498b3e1b 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1983,9 +1983,15 @@ "eu-west-1" : { "arn" : "aws" }, + "eu-west-2" : { + "arn" : "aws" + }, "us-east-1" : { "arn" : "aws" }, + "us-east-2" : { + "arn" : "aws" + }, "ap-northeast-1" : { "arn" : "aws" }, @@ -2015,6 +2021,9 @@ }, "ap-south-1" : { "arn" : "aws" + }, + "ca-central-1" : { + "arn" : "aws" } } }, diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml index fcc71595ca..46a9a8d09b 100644 --- a/cloudformation/cfncluster.cfn.yaml +++ b/cloudformation/cfncluster.cfn.yaml @@ -1288,8 +1288,12 @@ Mappings: AWSRegion2Capabilites: eu-west-1: arn: aws + eu-west-2: + arn: aws us-east-1: arn: aws + us-east-2: + arn: aws ap-northeast-1: arn: aws us-west-2: @@ -1310,6 +1314,8 @@ Mappings: arn: aws ap-south-1: arn: aws + ca-central-1: + arn: aws Resources: SQS: Type: AWS::SQS::Queue From ed9b58144218cd9c5d42a84da1f04fa780767b86 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Wed, 7 Jun 2017 17:37:48 +0000 Subject: [PATCH 31/34] Update version number to 1.3.2 Update release version to 1.3.2 as well as the required cfncluster-cookbook version to 1.3.2. Signed-off-by: Brian Barrett --- cli/setup.py | 2 +- cloudformation/cfncluster.cfn.json | 2 +- cloudformation/cfncluster.cfn.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/setup.py b/cli/setup.py index 0924e78229..69295d8842 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -20,7 +20,7 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() console_scripts = ['cfncluster = cfncluster.cli:main'] -version = "1.3.1" +version = "1.3.2" requires = ['boto>=2.42.0', 'awscli>=1.10.56', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index fd498b3e1b..9da73ae983 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1,6 +1,6 @@ { "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1", + "Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.2", "Metadata" : { "AWS::CloudFormation::Interface" : { "ParameterGroups" : [ diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml index 46a9a8d09b..7aa271dd37 100644 --- a/cloudformation/cfncluster.cfn.yaml +++ b/cloudformation/cfncluster.cfn.yaml @@ -2,7 +2,7 @@ AWSTemplateFormatVersion: '2010-09-09' Description: 'AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used - if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1' + if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.2' Metadata: AWS::CloudFormation::Interface: ParameterGroups: From 76ac6a482c39cf2ad8fdccf0a9e81526a22c0ea3 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Mon, 12 Jun 2017 16:07:04 +0000 Subject: [PATCH 32/34] Update AMI list for 1.3.2 release Add AMIs built on 6/11/2017 as part of the 1.3.2 release. Signed-off-by: Brian Barrett --- amis.txt | 125 ++++++++++++----------- cloudformation/cfncluster.cfn.json | 153 ++++++++++++++++------------- cloudformation/cfncluster.cfn.yaml | 142 ++++++++++++++------------ 3 files changed, 237 insertions(+), 183 deletions(-) diff --git a/amis.txt b/amis.txt index fe75348a84..4653a7ce46 100644 --- a/amis.txt +++ b/amis.txt @@ -1,60 +1,75 @@ # centos6 -ap-northeast-1: ami-46b17b27 -ap-northeast-2: ami-75e1341b -ap-south-1: ami-a392e7cc -ap-southeast-1: ami-23f22940 -ap-southeast-2: ami-0ea6906d -eu-central-1: ami-5fd92a30 -eu-west-1: ami-27f78554 -sa-east-1: ami-8ad848e6 -us-east-1: ami-504b2047 -us-west-1: ami-051f5365 -us-west-2: ami-6e30e10e +ap-northeast-1: ami-10686677 +ap-northeast-2: ami-bb77abd5 +ap-south-1: ami-0a710e65 +ap-southeast-1: ami-9b1596f8 +ap-southeast-2: ami-c7b9afa4 +ca-central-1: ami-a6f847c2 +eu-central-1: ami-56d67239 +eu-west-1: ami-abd5c9cd +eu-west-2: ami-cabcabae +sa-east-1: ami-b491f9d8 +us-east-1: ami-a293c9b4 +us-east-2: ami-2cd3f549 +us-west-1: ami-65c2e005 +us-west-2: ami-6bb1b812 # centos7 -ap-northeast-1: ami-43c90322 -ap-northeast-2: ami-5cec3932 -ap-south-1: ami-bf90e5d0 -ap-southeast-1: ami-d5f52eb6 -ap-southeast-2: ami-79a5931a -eu-central-1: ami-fcda2993 -eu-west-1: ami-5af48629 -sa-east-1: ami-f8d84894 -us-east-1: ami-e94328fe -us-west-1: ami-a51955c5 -us-west-2: ami-2e1dcc4e +ap-northeast-1: ami-0e6b6569 +ap-northeast-2: ami-9377abfd +ap-south-1: ami-546d123b +ap-southeast-1: ami-da1596b9 +ap-southeast-2: ami-afa6b0cc +ca-central-1: ami-62f94606 +eu-central-1: ami-9cd571f3 +eu-west-1: ami-6dd8c40b +eu-west-2: ami-9cb9aef8 +sa-east-1: ami-5092fa3c +us-east-1: ami-d29cc6c4 +us-east-2: ami-c3d2f4a6 +us-west-1: ami-17fcde77 +us-west-2: ami-22b2bb5b # alinux -ap-northeast-1: ami-50bd7731 -ap-northeast-2: ami-afe431c1 -ap-south-1: ami-0495e06b -ap-southeast-1: ami-9af229f9 -ap-southeast-2: ami-51b98f32 -eu-central-1: ami-96db28f9 -eu-west-1: ami-d8f785ab -sa-east-1: ami-8bd848e7 -us-east-1: ami-d5751ec2 -us-west-1: ami-5f19553f -us-west-2: ami-1e31e07e +ap-northeast-1: ami-586b653f +ap-northeast-2: ami-ea71ad84 +ap-south-1: ami-1f720d70 +ap-southeast-1: ami-1f1b987c +ap-southeast-2: ami-56b9af35 +ca-central-1: ami-65f94601 +eu-central-1: ami-7fd17510 +eu-west-1: ami-6ddac60b +eu-west-2: ami-f1bbac95 +sa-east-1: ami-a990f8c5 +us-east-1: ami-2492c832 +us-east-2: ami-a6d3f5c3 +us-west-1: ami-39c2e059 +us-west-2: ami-f4b0b98d # ubuntu1404 -ap-northeast-1: ami-89bd77e8 -ap-northeast-2: ami-bae732d4 -ap-south-1: ami-489de827 -ap-southeast-1: ami-95f229f6 -ap-southeast-2: ami-d3b98fb0 -eu-central-1: ami-89db28e6 -eu-west-1: ami-3ff0824c -sa-east-1: ami-3f34a553 -us-east-1: ami-ba4a21ad -us-west-1: ami-40185420 -us-west-2: ami-4633e226 +ap-northeast-1: ami-466c6221 +ap-northeast-2: ami-2877ab46 +ap-south-1: ami-11700f7e +ap-southeast-1: ami-be1497dd +ap-southeast-2: ami-37a4b254 +ca-central-1: ami-eefc438a +eu-central-1: ami-bcd773d3 +eu-west-1: ami-83d4c8e5 +eu-west-2: ami-5cb9ae38 +sa-east-1: ami-6d91f901 +us-east-1: ami-a192c8b7 +us-east-2: ami-e2d3f587 +us-west-1: ami-b5c2e0d5 +us-west-2: ami-f5b0b98c # ubuntu1604 -ap-northeast-1: ami-eabe748b -ap-northeast-2: ami-7eef3a10 -ap-south-1: ami-5d90e532 -ap-southeast-1: ami-23f32840 -ap-southeast-2: ami-c1b98fa2 -eu-central-1: ami-9dd427f2 -eu-west-1: ami-99f082ea -sa-east-1: ami-8d0e9fe1 -us-east-1: ami-d6751ec1 -us-west-1: ami-751f5315 -us-west-2: ami-870edfe7 +ap-northeast-1: ami-ce6c62a9 +ap-northeast-2: ami-d072aebe +ap-south-1: ami-446f102b +ap-southeast-1: ami-951596f6 +ap-southeast-2: ami-cca6b0af +ca-central-1: ami-a7fa45c3 +eu-central-1: ami-40d6722f +eu-west-1: ami-e6d5c980 +eu-west-2: ami-5eb9ae3a +sa-east-1: ami-df90f8b3 +us-east-1: ami-8c9fc59a +us-east-2: ami-54d5f331 +us-west-1: ami-0dfddf6d +us-west-2: ami-c4b1b8bd diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index 9da73ae983..3cb3cc3a4c 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1862,68 +1862,103 @@ } }, "AWSRegionOS2AMI" : { - "eu-west-1" : { - "centos6" : "ami-27f78554", - "centos7" : "ami-5af48629", - "alinux" : "ami-d8f785ab", - "ubuntu1404" : "ami-3ff0824c", - "ubuntu1604" : "ami-99f082ea" + "ap-northeast-1" : { + "alinux" : "ami-586b653f", + "centos6" : "ami-10686677", + "centos7" : "ami-0e6b6569", + "ubuntu1404" : "ami-466c6221", + "ubuntu1604" : "ami-ce6c62a9" }, - "us-west-2" : { - "centos6" : "ami-6e30e10e", - "centos7" : "ami-2e1dcc4e", - "alinux" : "ami-1e31e07e", - "ubuntu1404" : "ami-4633e226", - "ubuntu1604" : "ami-870edfe7" + "ap-northeast-2" : { + "alinux" : "ami-ea71ad84", + "centos6" : "ami-bb77abd5", + "centos7" : "ami-9377abfd", + "ubuntu1404" : "ami-2877ab46", + "ubuntu1604" : "ami-d072aebe" + }, + "ap-south-1" : { + "alinux" : "ami-1f720d70", + "centos6" : "ami-0a710e65", + "centos7" : "ami-546d123b", + "ubuntu1404" : "ami-11700f7e", + "ubuntu1604" : "ami-446f102b" + }, + "ap-southeast-1" : { + "alinux" : "ami-1f1b987c", + "centos6" : "ami-9b1596f8", + "centos7" : "ami-da1596b9", + "ubuntu1404" : "ami-be1497dd", + "ubuntu1604" : "ami-951596f6" + }, + "ap-southeast-2" : { + "alinux" : "ami-56b9af35", + "centos6" : "ami-c7b9afa4", + "centos7" : "ami-afa6b0cc", + "ubuntu1404" : "ami-37a4b254", + "ubuntu1604" : "ami-cca6b0af" + }, + "ca-central-1" : { + "alinux" : "ami-65f94601", + "centos6" : "ami-a6f847c2", + "centos7" : "ami-62f94606", + "ubuntu1404" : "ami-eefc438a", + "ubuntu1604" : "ami-a7fa45c3" }, "eu-central-1" : { - "centos6" : "ami-5fd92a30", - "centos7" : "ami-fcda2993", - "alinux" : "ami-96db28f9", - "ubuntu1404" : "ami-89db28e6", - "ubuntu1604" : "ami-9dd427f2" + "alinux" : "ami-7fd17510", + "centos6" : "ami-56d67239", + "centos7" : "ami-9cd571f3", + "ubuntu1404" : "ami-bcd773d3", + "ubuntu1604" : "ami-40d6722f" }, - "sa-east-1" : { - "centos6" : "ami-8ad848e6", - "centos7" : "ami-f8d84894", - "alinux" : "ami-8bd848e7", - "ubuntu1404" : "ami-3f34a553", - "ubuntu1604" : "ami-8d0e9fe1" + "eu-west-1" : { + "alinux" : "ami-6ddac60b", + "centos6" : "ami-abd5c9cd", + "centos7" : "ami-6dd8c40b", + "ubuntu1404" : "ami-83d4c8e5", + "ubuntu1604" : "ami-e6d5c980" }, - "ap-northeast-1" : { - "centos6" : "ami-46b17b27", - "centos7" : "ami-43c90322", - "alinux" : "ami-50bd7731", - "ubuntu1404" : "ami-89bd77e8", - "ubuntu1604" : "ami-eabe748b" + "eu-west-2" : { + "alinux" : "ami-f1bbac95", + "centos6" : "ami-cabcabae", + "centos7" : "ami-9cb9aef8", + "ubuntu1404" : "ami-5cb9ae38", + "ubuntu1604" : "ami-5eb9ae3a" + }, + "sa-east-1" : { + "alinux" : "ami-a990f8c5", + "centos6" : "ami-b491f9d8", + "centos7" : "ami-5092fa3c", + "ubuntu1404" : "ami-6d91f901", + "ubuntu1604" : "ami-df90f8b3" }, "us-east-1" : { - "centos6" : "ami-504b2047", - "centos7" : "ami-e94328fe", - "alinux" : "ami-d5751ec2", - "ubuntu1404" : "ami-ba4a21ad", - "ubuntu1604" : "ami-d6751ec1" + "alinux" : "ami-2492c832", + "centos6" : "ami-a293c9b4", + "centos7" : "ami-d29cc6c4", + "ubuntu1404" : "ami-a192c8b7", + "ubuntu1604" : "ami-8c9fc59a" }, - "us-west-1" : { - "centos6" : "ami-051f5365", - "centos7" : "ami-a51955c5", - "alinux" : "ami-5f19553f", - "ubuntu1404" : "ami-40185420", - "ubuntu1604" : "ami-751f5315" + "us-east-2" : { + "alinux" : "ami-a6d3f5c3", + "centos6" : "ami-2cd3f549", + "centos7" : "ami-c3d2f4a6", + "ubuntu1404" : "ami-e2d3f587", + "ubuntu1604" : "ami-54d5f331" }, - "ap-southeast-2" : { - "centos6" : "ami-0ea6906d", - "centos7" : "ami-79a5931a", - "alinux" : "ami-51b98f32", - "ubuntu1404" : "ami-d3b98fb0", - "ubuntu1604" : "ami-c1b98fa2" + "us-west-1" : { + "alinux" : "ami-39c2e059", + "centos6" : "ami-65c2e005", + "centos7" : "ami-17fcde77", + "ubuntu1404" : "ami-b5c2e0d5", + "ubuntu1604" : "ami-0dfddf6d" }, - "ap-southeast-1" : { - "centos6" : "ami-23f22940", - "centos7" : "ami-d5f52eb6", - "alinux" : "ami-9af229f9", - "ubuntu1404" : "ami-95f229f6", - "ubuntu1604" : "ami-23f32840" + "us-west-2" : { + "alinux" : "ami-f4b0b98d", + "centos6" : "ami-6bb1b812", + "centos7" : "ami-22b2bb5b", + "ubuntu1404" : "ami-f5b0b98c", + "ubuntu1604" : "ami-c4b1b8bd" }, "us-gov-west-1" : { "centos6" : "NOT_SUPPORTED", @@ -1931,20 +1966,6 @@ "alinux" : "NOT_SUPPORTED", "ubuntu1404" : "NOT_SUPPORTED", "ubuntu1604" : "NOT_SUPPORTED" - }, - "ap-northeast-2" : { - "centos6" : "ami-75e1341b", - "centos7" : "ami-5cec3932", - "alinux" : "ami-afe431c1", - "ubuntu1404" : "ami-bae732d4", - "ubuntu1604" : "ami-7eef3a10" - }, - "ap-south-1" : { - "centos6" : "ami-a392e7cc", - "centos7" : "ami-bf90e5d0", - "alinux" : "ami-0495e06b", - "ubuntu1404" : "ami-489de827", - "ubuntu1604" : "ami-5d90e532" } }, "OSFeatures" : { diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml index 7aa271dd37..4233197232 100644 --- a/cloudformation/cfncluster.cfn.yaml +++ b/cloudformation/cfncluster.cfn.yaml @@ -1189,78 +1189,96 @@ Mappings: Arch: 64HVM EBSOpt: 'True' AWSRegionOS2AMI: - eu-west-1: - centos6: ami-27f78554 - centos7: ami-5af48629 - alinux: ami-d8f785ab - ubuntu1404: ami-3ff0824c - ubuntu1604: ami-99f082ea - us-west-2: - centos6: ami-6e30e10e - centos7: ami-2e1dcc4e - alinux: ami-1e31e07e - ubuntu1404: ami-4633e226 - ubuntu1604: ami-870edfe7 + ap-northeast-1: + alinux: ami-586b653f + centos6: ami-10686677 + centos7: ami-0e6b6569 + ubuntu1404: ami-466c6221 + ubuntu1604: ami-ce6c62a9 + ap-northeast-2: + alinux: ami-ea71ad84 + centos6: ami-bb77abd5 + centos7: ami-9377abfd + ubuntu1404: ami-2877ab46 + ubuntu1604: ami-d072aebe + ap-south-1: + alinux: ami-1f720d70 + centos6: ami-0a710e65 + centos7: ami-546d123b + ubuntu1404: ami-11700f7e + ubuntu1604: ami-446f102b + ap-southeast-1: + alinux: ami-1f1b987c + centos6: ami-9b1596f8 + centos7: ami-da1596b9 + ubuntu1404: ami-be1497dd + ubuntu1604: ami-951596f6 + ap-southeast-2: + alinux: ami-56b9af35 + centos6: ami-c7b9afa4 + centos7: ami-afa6b0cc + ubuntu1404: ami-37a4b254 + ubuntu1604: ami-cca6b0af + ca-central-1: + alinux: ami-65f94601 + centos6: ami-a6f847c2 + centos7: ami-62f94606 + ubuntu1404: ami-eefc438a + ubuntu1604: ami-a7fa45c3 eu-central-1: - centos6: ami-5fd92a30 - centos7: ami-fcda2993 - alinux: ami-96db28f9 - ubuntu1404: ami-89db28e6 - ubuntu1604: ami-9dd427f2 + alinux: ami-7fd17510 + centos6: ami-56d67239 + centos7: ami-9cd571f3 + ubuntu1404: ami-bcd773d3 + ubuntu1604: ami-40d6722f + eu-west-1: + alinux: ami-6ddac60b + centos6: ami-abd5c9cd + centos7: ami-6dd8c40b + ubuntu1404: ami-83d4c8e5 + ubuntu1604: ami-e6d5c980 + eu-west-2: + alinux: ami-f1bbac95 + centos6: ami-cabcabae + centos7: ami-9cb9aef8 + ubuntu1404: ami-5cb9ae38 + ubuntu1604: ami-5eb9ae3a sa-east-1: - centos6: ami-8ad848e6 - centos7: ami-f8d84894 - alinux: ami-8bd848e7 - ubuntu1404: ami-3f34a553 - ubuntu1604: ami-8d0e9fe1 - ap-northeast-1: - centos6: ami-46b17b27 - centos7: ami-43c90322 - alinux: ami-50bd7731 - ubuntu1404: ami-89bd77e8 - ubuntu1604: ami-eabe748b + alinux: ami-a990f8c5 + centos6: ami-b491f9d8 + centos7: ami-5092fa3c + ubuntu1404: ami-6d91f901 + ubuntu1604: ami-df90f8b3 us-east-1: - centos6: ami-504b2047 - centos7: ami-e94328fe - alinux: ami-d5751ec2 - ubuntu1404: ami-ba4a21ad - ubuntu1604: ami-d6751ec1 + alinux: ami-2492c832 + centos6: ami-a293c9b4 + centos7: ami-d29cc6c4 + ubuntu1404: ami-a192c8b7 + ubuntu1604: ami-8c9fc59a + us-east-2: + alinux: ami-a6d3f5c3 + centos6: ami-2cd3f549 + centos7: ami-c3d2f4a6 + ubuntu1404: ami-e2d3f587 + ubuntu1604: ami-54d5f331 us-west-1: - centos6: ami-051f5365 - centos7: ami-a51955c5 - alinux: ami-5f19553f - ubuntu1404: ami-40185420 - ubuntu1604: ami-751f5315 - ap-southeast-2: - centos6: ami-0ea6906d - centos7: ami-79a5931a - alinux: ami-51b98f32 - ubuntu1404: ami-d3b98fb0 - ubuntu1604: ami-c1b98fa2 - ap-southeast-1: - centos6: ami-23f22940 - centos7: ami-d5f52eb6 - alinux: ami-9af229f9 - ubuntu1404: ami-95f229f6 - ubuntu1604: ami-23f32840 + alinux: ami-39c2e059 + centos6: ami-65c2e005 + centos7: ami-17fcde77 + ubuntu1404: ami-b5c2e0d5 + ubuntu1604: ami-0dfddf6d + us-west-2: + alinux: ami-f4b0b98d + centos6: ami-6bb1b812 + centos7: ami-22b2bb5b + ubuntu1404: ami-f5b0b98c + ubuntu1604: ami-c4b1b8bd us-gov-west-1: centos6: NOT_SUPPORTED centos7: NOT_SUPPORTED alinux: NOT_SUPPORTED ubuntu1404: NOT_SUPPORTED ubuntu1604: NOT_SUPPORTED - ap-northeast-2: - centos6: ami-75e1341b - centos7: ami-5cec3932 - alinux: ami-afe431c1 - ubuntu1404: ami-bae732d4 - ubuntu1604: ami-7eef3a10 - ap-south-1: - centos6: ami-a392e7cc - centos7: ami-bf90e5d0 - alinux: ami-0495e06b - ubuntu1404: ami-489de827 - ubuntu1604: ami-5d90e532 OSFeatures: centos6: User: centos From 2dae3551b371cce1d0bd5b870add4f40ee0571af Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Thu, 22 Jun 2017 03:16:11 +0000 Subject: [PATCH 33/34] text file updates for the 1.3.2 release Signed-off-by: Brian Barrett --- CHANGELOG.rst | 11 +++++++++-- CONTRIBUTORS.txt | 1 + README.rst | 13 ++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7d45b2b03..566ea4cb92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,15 @@ CHANGELOG ========= -develop -======= +1.3.2 +===== +* feature:``cfncluster``: Add support for r2.xlarge/t2.2xlarge, + x1.16xlarge, r4.*, f1.*, and i3.* instance types +* bugfix:``cfncluster``: Fix support for p2.2xlarge instance type +* feature:``cfncluster``: Add support for eu-west-2, us-east-2, and + ca-central-1 regions +* updates:``cfncluster-cookbook``: Updated to cfncluster-cookbook-1.3.2 +* updates:``ami``: Update all base AMIs to latest patch levels * updates:``cfncluster``: Moved to Apache 2.0 license * updates:``cfncluster``: Support for Python 3 diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 07e726ad2e..7468ae35f4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -3,6 +3,7 @@ List of contributors: - Dougal Ballantyne - dougalb at amazon dot com - Adam Boeglin - boeglina at amazon dot com - Sean Smith - sensmit at amazon dot com + - Brian Barrett - bbarrett at amazon dot com - Kenneth Daily - kmdaily at gmail dot com - John Lilley - johnbot at ltech dot edu diff --git a/README.rst b/README.rst index 3b8978dac4..2d18cf811c 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,15 @@ CfnCluster CfnCluster ("cloud formation cluster") is a framework that deploys and maintains high performance computing clusters on Amazon Web Services (AWS). Developed by AWS, CfnCluster facilitates both quick start proof of concepts (POCs) and production deployments. CfnCluster supports many different types of clustered applications and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS. +Known Issues +============ + +* CfnCluster 1.3.2 supports the I3 and F1 instance families. However, + it will not automatically setup a single, RAIDed volume across all + available instance store volumes (like it will for other instance + families which support instance storage). This will be fixed in a + future release. + Documentation ============= @@ -15,4 +24,6 @@ Documentation is part of the project and is published to - https://cfncluster.re Issues ====== -Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192 +Please open a GitHub issue for any feedback or issues: +https://github.com/awslabs/cfncluster. There is also an active AWS +HPC forum which may be helpful:https://forums.aws.amazon.com/forum.jspa?forumID=192. From 88c940f38c1ae7be1c45d374802be430bb13d3e7 Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Fri, 23 Jun 2017 16:41:46 +0000 Subject: [PATCH 34/34] Fix license string to match PyPI requirements --- cli/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/setup.py b/cli/setup.py index 69295d8842..6f9026cc7c 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -53,6 +53,6 @@ def read(fname): "Environment :: Console", "Programming Language :: Python", "Topic :: Scientific/Engineering", - "License :: Apache Software License", + "License :: OSI Approved :: Apache Software License", ], )