diff --git a/.travis.yml b/.travis.yml index 3c0ddaf190..9440395f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" install: - cd cli diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ce00b7698d..566ea4cb92 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,18 @@ CHANGELOG ========= +1.3.2 +===== +* feature:``cfncluster``: Add support for r2.xlarge/t2.2xlarge, + x1.16xlarge, r4.*, f1.*, and i3.* instance types +* bugfix:``cfncluster``: Fix support for p2.2xlarge instance type +* feature:``cfncluster``: Add support for eu-west-2, us-east-2, and + ca-central-1 regions +* updates:``cfncluster-cookbook``: Updated to cfncluster-cookbook-1.3.2 +* updates:``ami``: Update all base AMIs to latest patch levels +* updates:``cfncluster``: Moved to Apache 2.0 license +* updates:``cfncluster``: Support for Python 3 + 1.3.1 ===== * feature:``ami``: Added support for Ubuntu 16.04 LTS @@ -198,4 +210,4 @@ changes should be non-instrusive. 0.0.5 ===== -* Release on GitHub and PyPi \ No newline at end of file +* Release on GitHub and PyPi diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 07e726ad2e..7468ae35f4 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -3,6 +3,7 @@ List of contributors: - Dougal Ballantyne - dougalb at amazon dot com - Adam Boeglin - boeglina at amazon dot com - Sean Smith - sensmit at amazon dot com + - Brian Barrett - bbarrett at amazon dot com - Kenneth Daily - kmdaily at gmail dot com - John Lilley - johnbot at ltech dot edu diff --git a/README.rst b/README.rst index 3b8978dac4..2d18cf811c 100644 --- a/README.rst +++ b/README.rst @@ -7,6 +7,15 @@ CfnCluster CfnCluster ("cloud formation cluster") is a framework that deploys and maintains high performance computing clusters on Amazon Web Services (AWS). Developed by AWS, CfnCluster facilitates both quick start proof of concepts (POCs) and production deployments. CfnCluster supports many different types of clustered applications and can easily be extended to support different frameworks. The CLI is stateless, everything is done using CloudFormation or resources within AWS. +Known Issues +============ + +* CfnCluster 1.3.2 supports the I3 and F1 instance families. However, + it will not automatically setup a single, RAIDed volume across all + available instance store volumes (like it will for other instance + families which support instance storage). This will be fixed in a + future release. + Documentation ============= @@ -15,4 +24,6 @@ Documentation is part of the project and is published to - https://cfncluster.re Issues ====== -Any issues or feedback, should be posted in the AWS HPC forum - https://forums.aws.amazon.com/forum.jspa?forumID=192 +Please open a GitHub issue for any feedback or issues: +https://github.com/awslabs/cfncluster. There is also an active AWS +HPC forum which may be helpful:https://forums.aws.amazon.com/forum.jspa?forumID=192. diff --git a/amis.txt b/amis.txt index fe75348a84..4653a7ce46 100644 --- a/amis.txt +++ b/amis.txt @@ -1,60 +1,75 @@ # centos6 -ap-northeast-1: ami-46b17b27 -ap-northeast-2: ami-75e1341b -ap-south-1: ami-a392e7cc -ap-southeast-1: ami-23f22940 -ap-southeast-2: ami-0ea6906d -eu-central-1: ami-5fd92a30 -eu-west-1: ami-27f78554 -sa-east-1: ami-8ad848e6 -us-east-1: ami-504b2047 -us-west-1: ami-051f5365 -us-west-2: ami-6e30e10e +ap-northeast-1: ami-10686677 +ap-northeast-2: ami-bb77abd5 +ap-south-1: ami-0a710e65 +ap-southeast-1: ami-9b1596f8 +ap-southeast-2: ami-c7b9afa4 +ca-central-1: ami-a6f847c2 +eu-central-1: ami-56d67239 +eu-west-1: ami-abd5c9cd +eu-west-2: ami-cabcabae +sa-east-1: ami-b491f9d8 +us-east-1: ami-a293c9b4 +us-east-2: ami-2cd3f549 +us-west-1: ami-65c2e005 +us-west-2: ami-6bb1b812 # centos7 -ap-northeast-1: ami-43c90322 -ap-northeast-2: ami-5cec3932 -ap-south-1: ami-bf90e5d0 -ap-southeast-1: ami-d5f52eb6 -ap-southeast-2: ami-79a5931a -eu-central-1: ami-fcda2993 -eu-west-1: ami-5af48629 -sa-east-1: ami-f8d84894 -us-east-1: ami-e94328fe -us-west-1: ami-a51955c5 -us-west-2: ami-2e1dcc4e +ap-northeast-1: ami-0e6b6569 +ap-northeast-2: ami-9377abfd +ap-south-1: ami-546d123b +ap-southeast-1: ami-da1596b9 +ap-southeast-2: ami-afa6b0cc +ca-central-1: ami-62f94606 +eu-central-1: ami-9cd571f3 +eu-west-1: ami-6dd8c40b +eu-west-2: ami-9cb9aef8 +sa-east-1: ami-5092fa3c +us-east-1: ami-d29cc6c4 +us-east-2: ami-c3d2f4a6 +us-west-1: ami-17fcde77 +us-west-2: ami-22b2bb5b # alinux -ap-northeast-1: ami-50bd7731 -ap-northeast-2: ami-afe431c1 -ap-south-1: ami-0495e06b -ap-southeast-1: ami-9af229f9 -ap-southeast-2: ami-51b98f32 -eu-central-1: ami-96db28f9 -eu-west-1: ami-d8f785ab -sa-east-1: ami-8bd848e7 -us-east-1: ami-d5751ec2 -us-west-1: ami-5f19553f -us-west-2: ami-1e31e07e +ap-northeast-1: ami-586b653f +ap-northeast-2: ami-ea71ad84 +ap-south-1: ami-1f720d70 +ap-southeast-1: ami-1f1b987c +ap-southeast-2: ami-56b9af35 +ca-central-1: ami-65f94601 +eu-central-1: ami-7fd17510 +eu-west-1: ami-6ddac60b +eu-west-2: ami-f1bbac95 +sa-east-1: ami-a990f8c5 +us-east-1: ami-2492c832 +us-east-2: ami-a6d3f5c3 +us-west-1: ami-39c2e059 +us-west-2: ami-f4b0b98d # ubuntu1404 -ap-northeast-1: ami-89bd77e8 -ap-northeast-2: ami-bae732d4 -ap-south-1: ami-489de827 -ap-southeast-1: ami-95f229f6 -ap-southeast-2: ami-d3b98fb0 -eu-central-1: ami-89db28e6 -eu-west-1: ami-3ff0824c -sa-east-1: ami-3f34a553 -us-east-1: ami-ba4a21ad -us-west-1: ami-40185420 -us-west-2: ami-4633e226 +ap-northeast-1: ami-466c6221 +ap-northeast-2: ami-2877ab46 +ap-south-1: ami-11700f7e +ap-southeast-1: ami-be1497dd +ap-southeast-2: ami-37a4b254 +ca-central-1: ami-eefc438a +eu-central-1: ami-bcd773d3 +eu-west-1: ami-83d4c8e5 +eu-west-2: ami-5cb9ae38 +sa-east-1: ami-6d91f901 +us-east-1: ami-a192c8b7 +us-east-2: ami-e2d3f587 +us-west-1: ami-b5c2e0d5 +us-west-2: ami-f5b0b98c # ubuntu1604 -ap-northeast-1: ami-eabe748b -ap-northeast-2: ami-7eef3a10 -ap-south-1: ami-5d90e532 -ap-southeast-1: ami-23f32840 -ap-southeast-2: ami-c1b98fa2 -eu-central-1: ami-9dd427f2 -eu-west-1: ami-99f082ea -sa-east-1: ami-8d0e9fe1 -us-east-1: ami-d6751ec1 -us-west-1: ami-751f5315 -us-west-2: ami-870edfe7 +ap-northeast-1: ami-ce6c62a9 +ap-northeast-2: ami-d072aebe +ap-south-1: ami-446f102b +ap-southeast-1: ami-951596f6 +ap-southeast-2: ami-cca6b0af +ca-central-1: ami-a7fa45c3 +eu-central-1: ami-40d6722f +eu-west-1: ami-e6d5c980 +eu-west-2: ami-5eb9ae3a +sa-east-1: ami-df90f8b3 +us-east-1: ami-8c9fc59a +us-east-2: ami-54d5f331 +us-west-1: ami-0dfddf6d +us-west-2: ami-c4b1b8bd diff --git a/cli/cfncluster/cfncluster.py b/cli/cfncluster/cfncluster.py index f2af9414f9..9d019ab806 100644 --- a/cli/cfncluster/cfncluster.py +++ b/cli/cfncluster/cfncluster.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -9,6 +11,7 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from builtins import str import sys import boto.cloudformation import boto.ec2.autoscale @@ -19,13 +22,13 @@ import socket import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') def version(args): config = cfnconfig.CfnClusterConfig(args) - print config.version + print(config.version) def create(args): print('Starting: %s' % (args.cluster_name)) @@ -50,7 +53,7 @@ def create(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -75,12 +78,12 @@ def create(args): time.sleep(5) outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) else: status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -118,7 +121,7 @@ def update(args): aws_secret_access_key=config.aws_secret_access_key) availability_zone = str(vpcconn.get_all_subnets(subnet_ids=master_subnet_id)[0].availability_zone) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) config.parameters.append(('AvailabilityZone', availability_zone)) except ValueError: @@ -142,7 +145,7 @@ def update(args): status = cfnconn.describe_stacks(stack)[0].stack_status print('Status: %s' % status) except boto.exception.BotoServerError as e: - print e.message + print(e.message) sys.exit(1) except KeyboardInterrupt: print('\nExiting...') @@ -161,7 +164,7 @@ def start(args): response = ec2conn.start_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -173,11 +176,13 @@ def start(args): # Set asg limits max_queue_size = [param[1] for param in config.parameters if param[0] == 'MaxQueueSize'] max_queue_size = max_queue_size[0] if len(max_queue_size) > 0 else 10 - initial_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] if args.reset_desired else [0] - initial_queue_size = initial_queue_size[0] if len(initial_queue_size) > 0 else 0 + desired_queue_size = [param[1] for param in config.parameters if param[0] == 'InitialQueueSize'] + desired_queue_size = desired_queue_size[0] if len(desired_queue_size) > 0 else 2 + min_queue_size = [desired_queue_size for param in config.parameters if param[0] == 'MaintainInitialSize' and param[1] == "true"] + min_queue_size = min_queue_size[0] if len(min_queue_size) > 0 else 0 asg = get_asg(stack_name=stack_name, config=config) - set_asg_limits(asg=asg, min=initial_queue_size, max=max_queue_size, desired=initial_queue_size) + set_asg_limits(asg=asg, min=min_queue_size, max=max_queue_size, desired=desired_queue_size) # Poll for status poll_master_server_state(stack_name, config) @@ -201,7 +206,7 @@ def stop(args): response = ec2conn.stop_instances(master_server_id) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -224,7 +229,7 @@ def list(args): print('%s' % (stack.stack_name[11:])) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) else: raise e except KeyboardInterrupt: @@ -243,7 +248,7 @@ def get_master_server_id(stack_name, config): resources = cfnconn.describe_stack_resources(stack_name) except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -280,7 +285,7 @@ def poll_master_server_state(stack_name, config): sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -303,7 +308,7 @@ def get_ec2_instances(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -334,7 +339,7 @@ def get_asg(stack_name, config): return asgconn.get_all_groups(names=[asg_id])[0] except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -363,7 +368,7 @@ def get_asg_ids(stack, config): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -433,22 +438,22 @@ def status(args): if state == 'running': outputs = cfnconn.describe_stacks(stack)[0].outputs for output in outputs: - print output + print(output) elif ((status == 'ROLLBACK_COMPLETE') or (status == 'CREATE_FAILED') or (status == 'DELETE_FAILED') or (status == 'UPDATE_ROLLBACK_COMPLETE')): events = cfnconn.describe_stack_events(stack) for event in events: if ((event.resource_status == 'CREATE_FAILED') or (event.resource_status == 'DELETE_FAILED') or (event.resource_status == 'UPDATE_FAILED')): - print event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ - event.resource_status_reason + print(event.timestamp, event.resource_status, event.resource_type, event.logical_resource_id, \ + event.resource_status_reason) else: sys.stdout.write('\n') sys.stdout.flush() except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): sys.stdout.write('\r') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: @@ -489,7 +494,7 @@ def delete(args): except boto.exception.BotoServerError as e: if e.message.endswith("does not exist"): #sys.stdout.write('\r\n') - print e.message + print(e.message) sys.stdout.flush() sys.exit(0) else: diff --git a/cli/cfncluster/cfnconfig.py b/cli/cfncluster/cfnconfig.py index c77177d09e..f92f328875 100644 --- a/cli/cfncluster/cfnconfig.py +++ b/cli/cfncluster/cfnconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -9,14 +11,18 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import str +from builtins import object +import configparser import os import sys import inspect import pkg_resources import json -import urllib2 -import config_sanity +import urllib.request, urllib.error, urllib.parse +from . import config_sanity import boto.cloudformation def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): @@ -29,14 +35,14 @@ def getStackTemplate(region, aws_access_key_id, aws_secret_access_key, stack): return __cli_template -class CfnClusterConfig: +class CfnClusterConfig(object): def __init__(self, args): self.args = args self.parameters = [] self.version = pkg_resources.get_distribution("cfncluster").version self.__DEFAULT_CONFIG = False - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ # Determine config file name based on args or default if args.config_file is not None: @@ -58,7 +64,7 @@ def __init__(self, args): sys.exit(1) - __config = ConfigParser.ConfigParser() + __config = configparser.ConfigParser() __config.read(self.__config_file) # Determine the EC2 region to used used or default to us-east-1 @@ -71,17 +77,17 @@ def __init__(self, args): else: try: self.region = __config.get('aws', 'aws_region_name') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.region = 'us-east-1' # Check if credentials have been provided in config try: self.aws_access_key_id = __config.get('aws', 'aws_access_key_id') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_access_key_id=None try: self.aws_secret_access_key = __config.get('aws', 'aws_secret_access_key') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.aws_secret_access_key=None # Determine which cluster template will be used @@ -102,12 +108,12 @@ def __init__(self, args): # Check if package updates should be checked try: self.__update_check = __config.getboolean('global', 'update_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__update_check = True if self.__update_check == True: try: - __latest = json.loads(urllib2.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] + __latest = json.loads(urllib.request.urlopen("http://pypi.python.org/pypi/cfncluster/json").read())['info']['version'] if self.version < __latest: print('warning: There is a newer version %s of cfncluster available.' % __latest) except Exception: @@ -116,10 +122,10 @@ def __init__(self, args): # Check if config sanity should be run try: self.__sanity_check = __config.getboolean('global', 'sanity_check') - except ConfigParser.NoOptionError: + except configparser.NoOptionError: self.__sanity_check = False # Only check config on calls that mutate it - __args_func = self.args.func.func_name + __args_func = self.args.func.__name__ if (__args_func == 'create' or __args_func == 'update' or __args_func == 'configure') and self.__sanity_check is True: pass else: @@ -134,7 +140,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'EC2KeyPair', self.key_name) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: print("ERROR: Missing key_name option in [%s] section." % self.__cluster_section) sys.exit(1) self.parameters.append(('KeyName', self.key_name)) @@ -153,7 +159,7 @@ def __init__(self, args): if self.__sanity_check: config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, 'URL', self.template_url) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: if self.region == 'us-gov-west-1': self.template_url = ('https://s3-%s.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json' % (self.region, self.region, self.version)) @@ -189,7 +195,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__vpc_options.get(key)[1],__temp__) self.parameters.append((self.__vpc_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all cluster section options @@ -223,7 +229,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__cluster_options.get(key)[1],__temp__) self.parameters.append((self.__cluster_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Merge tags from config with tags from command line args @@ -232,7 +238,7 @@ def __init__(self, args): try: tags = __config.get(self.__cluster_section, 'tags') self.tags = json.loads(tags); - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass try: if args.tags is not None: @@ -249,7 +255,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__ebs_section = ('ebs %s' % self.__ebs_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all EBS options @@ -272,7 +278,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__ebs_options.get(key)[1],__temp__) self.parameters.append((self.__ebs_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -285,7 +291,7 @@ def __init__(self, args): % self.__cluster_section) sys.exit(1) self.__scaling_section = ('scaling %s' % self.__scaling_settings) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass # Dictionary list of all scaling options @@ -307,7 +313,7 @@ def __init__(self, args): config_sanity.check_resource(self.region,self.aws_access_key_id, self.aws_secret_access_key, self.__scaling_options.get(key)[1],__temp__) self.parameters.append((self.__scaling_options.get(key)[0],__temp__)) - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except AttributeError: pass @@ -316,9 +322,9 @@ def __init__(self, args): try: if self.args.extra_parameters is not None: self.parameters = dict(self.parameters) - self.__temp_dict = dict(self.parameters.items() + self.args.extra_parameters.items()) + self.__temp_dict = dict(list(self.parameters.items()) + list(self.args.extra_parameters.items())) self.__dictlist = [] - for key, value in self.__temp_dict.iteritems(): + for key, value in self.__temp_dict.items(): temp = [str(key),str(value)] self.__dictlist.append(temp) self.parameters = self.__dictlist diff --git a/cli/cfncluster/cli.py b/cli/cfncluster/cli.py index ddb6743d5e..1b14dbbd10 100644 --- a/cli/cfncluster/cli.py +++ b/cli/cfncluster/cli.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -15,8 +16,8 @@ import platform import json -import cfncluster -import easyconfig +from . import cfncluster +from . import easyconfig def create(args): cfncluster.create(args) @@ -82,6 +83,8 @@ def main(): help='do not wait for stack events, after executing stack command') subparsers = parser.add_subparsers() + subparsers.required = True + subparsers.dest = 'command' pcreate = subparsers.add_parser('create', help='creates a cluster') pcreate.add_argument("cluster_name", type=str, default=None, diff --git a/cli/cfncluster/config_sanity.py b/cli/cfncluster/config_sanity.py index e8f11ed9df..6ba896c9eb 100644 --- a/cli/cfncluster/config_sanity.py +++ b/cli/cfncluster/config_sanity.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the @@ -9,12 +10,14 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. +from future import standard_library +standard_library.install_aliases() __author__ = 'dougalb' import boto.ec2 import boto.vpc -import urllib2 -from urlparse import urlparse +import urllib.request, urllib.error, urllib.parse +from urllib.parse import urlparse import boto.exception import sys @@ -92,12 +95,12 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty pass else: try: - urllib2.urlopen(resource_value) - except urllib2.HTTPError, e: - print(e.code) + urllib.request.urlopen(resource_value) + except urllib.error.HTTPError as e: + print('Config sanity error:', resource_value, e.code, e.reason) sys.exit(1) - except urllib2.URLError, e: - print(e.args) + except urllib.error.URLError as e: + print('Config sanity error:', resource_value, e.reason) sys.exit(1) # EC2 EBS Snapshot Id elif resource_type == 'EC2Snapshot': diff --git a/cli/cfncluster/easyconfig.py b/cli/cfncluster/easyconfig.py index 7273599a14..b8575c7522 100644 --- a/cli/cfncluster/easyconfig.py +++ b/cli/cfncluster/easyconfig.py @@ -1,3 +1,5 @@ +from __future__ import print_function +from __future__ import absolute_import # Copyright 2013-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance with the @@ -9,14 +11,17 @@ # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and # limitations under the License. -import ConfigParser +from future import standard_library +standard_library.install_aliases() +from builtins import input +import configparser import sys import boto.ec2 import boto.vpc import os import logging -import cfnconfig +from . import cfnconfig logger = logging.getLogger('cfncluster.cfncluster') @@ -31,11 +36,11 @@ def prompt(prompt, default_value=None, hidden=False, options=None): user_prompt = user_prompt + ']: ' if isinstance(options, list): - print 'Acceptable Values for %s: ' % prompt + print('Acceptable Values for %s: ' % prompt) for o in options: - print ' %s' % o + print(' %s' % o) - var = raw_input(user_prompt) + var = input(user_prompt) if var == '': return default_value @@ -79,8 +84,8 @@ def list_keys(aws_access_key_id, aws_secret_access_key, aws_region_name): keynames.append(key.name) if not keynames: - print 'ERROR: No keys found in region ' + aws_region_name - print 'Please create an EC2 keypair before continuing' + print('ERROR: No keys found in region ' + aws_region_name) + print('Please create an EC2 keypair before continuing') sys.exit(1) return keynames @@ -93,8 +98,8 @@ def list_vpcs(aws_access_key_id, aws_secret_access_key, aws_region_name): vpcids.append(vpc.id) if not vpcids: - print 'ERROR: No vpcs found in region ' + aws_region_name - print 'Please create an EC2 vpcpair before continuing' + print('ERROR: No vpcs found in region ' + aws_region_name) + print('Please create an EC2 vpcpair before continuing') sys.exit(1) return vpcids @@ -107,8 +112,8 @@ def list_subnets(aws_access_key_id, aws_secret_access_key, aws_region_name, vpc_ subnetids.append(subnet.id) if not subnetids: - print 'ERROR: No subnets found in region ' + aws_region_name - print 'Please create an EC2 subnetpair before continuing' + print('ERROR: No subnets found in region ' + aws_region_name) + print('Please create an EC2 subnetpair before continuing') sys.exit(1) return subnetids @@ -121,7 +126,7 @@ def configure(args): else: config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config')) - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() # Check if configuration file exists if os.path.isfile(config_file): @@ -154,9 +159,9 @@ def configure(args): for section in sections: try: config.add_section(section['__name__']) - except ConfigParser.DuplicateSectionError: + except configparser.DuplicateSectionError: pass - for key, value in section.iteritems(): + for key, value in section.items(): # Only update configuration if not set if value is not None and key is not '__name__': config.set(section['__name__'], key, value) diff --git a/cli/setup.py b/cli/setup.py index 6b3671aa52..6f9026cc7c 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -20,14 +20,17 @@ def read(fname): return open(os.path.join(os.path.dirname(__file__), fname)).read() console_scripts = ['cfncluster = cfncluster.cli:main'] -version = "1.3.1" -requires = ['boto>=2.42.0', 'awscli>=1.10.56'] +version = "1.3.2" +requires = ['boto>=2.42.0', 'awscli>=1.10.56', 'future>=0.16.0'] if sys.version_info[:2] == (2, 6): # For python2.6 we have to require argparse since it # was not in stdlib until 2.7. requires.append('argparse>=1.4.0') +if sys.version_info[0] == 2: + requires.append('configparser>=3.5.0') + setup( name = "cfncluster", version = version, @@ -50,6 +53,6 @@ def read(fname): "Environment :: Console", "Programming Language :: Python", "Topic :: Scientific/Engineering", - "License :: Apache Software License", + "License :: OSI Approved :: Apache Software License", ], ) diff --git a/cloudformation/cfncluster.cfn.json b/cloudformation/cfncluster.cfn.json index b2860ea489..3cb3cc3a4c 100644 --- a/cloudformation/cfncluster.cfn.json +++ b/cloudformation/cfncluster.cfn.json @@ -1,6 +1,6 @@ { "AWSTemplateFormatVersion" : "2010-09-09", - "Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.1", + "Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.2", "Metadata" : { "AWS::CloudFormation::Interface" : { "ParameterGroups" : [ @@ -727,10 +727,22 @@ "r3.2xlarge", "r3.xlarge", "r3.large", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", "i2.8xlarge", "i2.4xlarge", "i2.2xlarge", "i2.xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", "cr1.8xlarge", "cg1.4xlarge", "m3.medium", @@ -740,13 +752,15 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", "t2.small", "t2.medium", "t2.large", + "t2.xlarge", + "t2.2xlarge", "d2.8xlarge", "d2.4xlarge", "d2.2xlarge", @@ -756,7 +770,11 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", - "x1.32xlarge" + "m4.16xlarge", + "x1.16xlarge", + "x1.32xlarge", + "f1.2xlarge", + "f1.16xlarge" ] }, "ComputeInstanceType" : { @@ -781,10 +799,22 @@ "r3.2xlarge", "r3.xlarge", "r3.large", + "r4.large", + "r4.xlarge", + "r4.2xlarge", + "r4.4xlarge", + "r4.8xlarge", + "r4.16xlarge", "i2.8xlarge", "i2.4xlarge", "i2.2xlarge", "i2.xlarge", + "i3.large", + "i3.xlarge", + "i3.2xlarge", + "i3.4xlarge", + "i3.8xlarge", + "i3.16xlarge", "cr1.8xlarge", "cg1.4xlarge", "m3.medium", @@ -794,13 +824,15 @@ "hi1.4xlarge", "g2.2xlarge", "g2.8xlarge", - "p2.large", + "p2.xlarge", "p2.8xlarge", "p2.16xlarge", "t2.micro", "t2.small", "t2.medium", "t2.large", + "t2.xlarge", + "t2.2xlarge", "d2.8xlarge", "d2.4xlarge", "d2.2xlarge", @@ -810,7 +842,11 @@ "m4.2xlarge", "m4.4xlarge", "m4.10xlarge", - "x1.32xlarge" + "m4.16xlarge", + "x1.16xlarge", + "x1.32xlarge", + "f1.2xlarge", + "f1.16xlarge" ] }, "InitialQueueSize" : { @@ -1576,7 +1612,7 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, - "p2.large" : { + "p2.xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" }, @@ -1664,6 +1700,34 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "r3.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.large" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.4xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.8xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "r4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "i2.8xlarge" : { "Arch" : "64HVM", "EBSOpt" : "False" @@ -1684,10 +1748,38 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "i3.large" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.4xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.8xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "i3.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "cg1.4xlarge" : { "Arch" : "64HVM", "EBSOpt" : "False" }, + "t2.nano" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, "t2.micro" : { "Arch" : "64HVM", "EBSOpt" : "False" @@ -1700,6 +1792,18 @@ "Arch" : "64HVM", "EBSOpt" : "False" }, + "t2.large" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, + "t2.xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, + "t2.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "False" + }, "d2.8xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" @@ -1716,14 +1820,26 @@ "Arch" : "64HVM", "EBSOpt" : "True" }, - "t2.large" : { + "x1.16xlarge" : { "Arch" : "64HVM", - "EBSOpt" : "False" + "EBSOpt" : "True" }, "x1.32xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" }, + "f1.2xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "f1.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, + "m4.16xlarge" : { + "Arch" : "64HVM", + "EBSOpt" : "True" + }, "m4.10xlarge" : { "Arch" : "64HVM", "EBSOpt" : "True" @@ -1743,75 +1859,106 @@ "m4.large" : { "Arch" : "64HVM", "EBSOpt" : "True" - }, - "t2.nano" : { - "Arch" : "64HVM", - "EBSOpt" : "False" } }, "AWSRegionOS2AMI" : { - "eu-west-1" : { - "centos6" : "ami-27f78554", - "centos7" : "ami-5af48629", - "alinux" : "ami-d8f785ab", - "ubuntu1404" : "ami-3ff0824c", - "ubuntu1604" : "ami-99f082ea" + "ap-northeast-1" : { + "alinux" : "ami-586b653f", + "centos6" : "ami-10686677", + "centos7" : "ami-0e6b6569", + "ubuntu1404" : "ami-466c6221", + "ubuntu1604" : "ami-ce6c62a9" }, - "us-west-2" : { - "centos6" : "ami-6e30e10e", - "centos7" : "ami-2e1dcc4e", - "alinux" : "ami-1e31e07e", - "ubuntu1404" : "ami-4633e226", - "ubuntu1604" : "ami-870edfe7" + "ap-northeast-2" : { + "alinux" : "ami-ea71ad84", + "centos6" : "ami-bb77abd5", + "centos7" : "ami-9377abfd", + "ubuntu1404" : "ami-2877ab46", + "ubuntu1604" : "ami-d072aebe" + }, + "ap-south-1" : { + "alinux" : "ami-1f720d70", + "centos6" : "ami-0a710e65", + "centos7" : "ami-546d123b", + "ubuntu1404" : "ami-11700f7e", + "ubuntu1604" : "ami-446f102b" + }, + "ap-southeast-1" : { + "alinux" : "ami-1f1b987c", + "centos6" : "ami-9b1596f8", + "centos7" : "ami-da1596b9", + "ubuntu1404" : "ami-be1497dd", + "ubuntu1604" : "ami-951596f6" + }, + "ap-southeast-2" : { + "alinux" : "ami-56b9af35", + "centos6" : "ami-c7b9afa4", + "centos7" : "ami-afa6b0cc", + "ubuntu1404" : "ami-37a4b254", + "ubuntu1604" : "ami-cca6b0af" + }, + "ca-central-1" : { + "alinux" : "ami-65f94601", + "centos6" : "ami-a6f847c2", + "centos7" : "ami-62f94606", + "ubuntu1404" : "ami-eefc438a", + "ubuntu1604" : "ami-a7fa45c3" }, "eu-central-1" : { - "centos6" : "ami-5fd92a30", - "centos7" : "ami-fcda2993", - "alinux" : "ami-96db28f9", - "ubuntu1404" : "ami-89db28e6", - "ubuntu1604" : "ami-9dd427f2" + "alinux" : "ami-7fd17510", + "centos6" : "ami-56d67239", + "centos7" : "ami-9cd571f3", + "ubuntu1404" : "ami-bcd773d3", + "ubuntu1604" : "ami-40d6722f" }, - "sa-east-1" : { - "centos6" : "ami-8ad848e6", - "centos7" : "ami-f8d84894", - "alinux" : "ami-8bd848e7", - "ubuntu1404" : "ami-3f34a553", - "ubuntu1604" : "ami-8d0e9fe1" + "eu-west-1" : { + "alinux" : "ami-6ddac60b", + "centos6" : "ami-abd5c9cd", + "centos7" : "ami-6dd8c40b", + "ubuntu1404" : "ami-83d4c8e5", + "ubuntu1604" : "ami-e6d5c980" + }, + "eu-west-2" : { + "alinux" : "ami-f1bbac95", + "centos6" : "ami-cabcabae", + "centos7" : "ami-9cb9aef8", + "ubuntu1404" : "ami-5cb9ae38", + "ubuntu1604" : "ami-5eb9ae3a" }, - "ap-northeast-1" : { - "centos6" : "ami-46b17b27", - "centos7" : "ami-43c90322", - "alinux" : "ami-50bd7731", - "ubuntu1404" : "ami-89bd77e8", - "ubuntu1604" : "ami-eabe748b" + "sa-east-1" : { + "alinux" : "ami-a990f8c5", + "centos6" : "ami-b491f9d8", + "centos7" : "ami-5092fa3c", + "ubuntu1404" : "ami-6d91f901", + "ubuntu1604" : "ami-df90f8b3" }, "us-east-1" : { - "centos6" : "ami-504b2047", - "centos7" : "ami-e94328fe", - "alinux" : "ami-d5751ec2", - "ubuntu1404" : "ami-ba4a21ad", - "ubuntu1604" : "ami-d6751ec1" + "alinux" : "ami-2492c832", + "centos6" : "ami-a293c9b4", + "centos7" : "ami-d29cc6c4", + "ubuntu1404" : "ami-a192c8b7", + "ubuntu1604" : "ami-8c9fc59a" + }, + "us-east-2" : { + "alinux" : "ami-a6d3f5c3", + "centos6" : "ami-2cd3f549", + "centos7" : "ami-c3d2f4a6", + "ubuntu1404" : "ami-e2d3f587", + "ubuntu1604" : "ami-54d5f331" }, "us-west-1" : { - "centos6" : "ami-051f5365", - "centos7" : "ami-a51955c5", - "alinux" : "ami-5f19553f", - "ubuntu1404" : "ami-40185420", - "ubuntu1604" : "ami-751f5315" - }, - "ap-southeast-2" : { - "centos6" : "ami-0ea6906d", - "centos7" : "ami-79a5931a", - "alinux" : "ami-51b98f32", - "ubuntu1404" : "ami-d3b98fb0", - "ubuntu1604" : "ami-c1b98fa2" + "alinux" : "ami-39c2e059", + "centos6" : "ami-65c2e005", + "centos7" : "ami-17fcde77", + "ubuntu1404" : "ami-b5c2e0d5", + "ubuntu1604" : "ami-0dfddf6d" }, - "ap-southeast-1" : { - "centos6" : "ami-23f22940", - "centos7" : "ami-d5f52eb6", - "alinux" : "ami-9af229f9", - "ubuntu1404" : "ami-95f229f6", - "ubuntu1604" : "ami-23f32840" + "us-west-2" : { + "alinux" : "ami-f4b0b98d", + "centos6" : "ami-6bb1b812", + "centos7" : "ami-22b2bb5b", + "ubuntu1404" : "ami-f5b0b98c", + "ubuntu1604" : "ami-c4b1b8bd" }, "us-gov-west-1" : { "centos6" : "NOT_SUPPORTED", @@ -1819,20 +1966,6 @@ "alinux" : "NOT_SUPPORTED", "ubuntu1404" : "NOT_SUPPORTED", "ubuntu1604" : "NOT_SUPPORTED" - }, - "ap-northeast-2" : { - "centos6" : "ami-75e1341b", - "centos7" : "ami-5cec3932", - "alinux" : "ami-afe431c1", - "ubuntu1404" : "ami-bae732d4", - "ubuntu1604" : "ami-7eef3a10" - }, - "ap-south-1" : { - "centos6" : "ami-a392e7cc", - "centos7" : "ami-bf90e5d0", - "alinux" : "ami-0495e06b", - "ubuntu1404" : "ami-489de827", - "ubuntu1604" : "ami-5d90e532" } }, "OSFeatures" : { @@ -1859,21 +1992,27 @@ }, "CfnClusterVersions" : { "default" : { - "cfncluster" : "cfncluster-1.3.1", - "cookbook" : "cfncluster-cookbook-1.3.0", - "chef" : "12.13.30", - "ridley" : "4.5.1", - "berkshelf" : "4.6.1", - "ami" : "201609030709" + "cfncluster" : "cfncluster-1.3.2", + "cookbook" : "cfncluster-cookbook-1.3.2", + "chef" : "12.19.36", + "ridley" : "5.1.0", + "berkshelf" : "5.6.4", + "ami" : "dev" } }, "AWSRegion2Capabilites" : { "eu-west-1" : { "arn" : "aws" }, + "eu-west-2" : { + "arn" : "aws" + }, "us-east-1" : { "arn" : "aws" }, + "us-east-2" : { + "arn" : "aws" + }, "ap-northeast-1" : { "arn" : "aws" }, @@ -1903,6 +2042,9 @@ }, "ap-south-1" : { "arn" : "aws" + }, + "ca-central-1" : { + "arn" : "aws" } } }, @@ -4108,4 +4250,4 @@ "Condition" : "MasterPublicIp" } } -} \ No newline at end of file +} diff --git a/cloudformation/cfncluster.cfn.yaml b/cloudformation/cfncluster.cfn.yaml new file mode 100644 index 0000000000..4233197232 --- /dev/null +++ b/cloudformation/cfncluster.cfn.yaml @@ -0,0 +1,2767 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: 'AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template + showing an framework for deploying master + compute type clusters on AWS. **WARNING** + This template creates AWS resources. You will be billed for the AWS resources used + if you create a stack from this template. Version: ami-201609030709 cfncluster-1.3.2' +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Network - Basic Settings + Parameters: + - VPCId + - MasterSubnetId + - Label: + default: Cluster - Basic Setting + Parameters: + - KeyName + - AvailabilityZone + - MasterInstanceType + - ComputeInstanceType + - BaseOS + - Scheduler + - Label: + default: EBS - Basic Settings + Parameters: + - VolumeSize + - Label: + default: Network - Advanced Settings + Parameters: + - AccessFrom + - ComputeSubnetId + - ComputeSubnetCidr + - UsePublicIps + - AdditionalSG + - VPCSecurityGroupId + - Label: + default: Cluster - Advanced Setting + Parameters: + - InitialQueueSize + - ComputeWaitConditionCount + - MaxQueueSize + - SpotPrice + - ClusterType + - ProxyServer + - CustomAMI + - MaintainInitialSize + - PreInstallScript + - PreInstallArgs + - PostInstallArgs + - PostInstallScript + - S3ReadResource + - S3ReadWriteResource + - Placement + - PlacementGroup + - EncryptedEphemeral + - EphemeralDir + - SharedDir + - CWLRegion + - CWLLogGroup + - CustomChefRunList + - CustomChefCookbook + - ExtraJson + - Tenancy + - EphemeralKMSKeyId + - ClusterReadyScript + - MasterRootVolumeSize + - ComputeRootVolumeSize + - EC2IAMRoleName + - Label: + default: EBS - Advanced Settings + Parameters: + - VolumeType + - EBSSnapshotId + - VolumeIOPS + - EBSEncryption + - EBSKMSKeyId + - EBSVolumeId + - Label: + default: Scaling Settings + Parameters: + - ScalingPeriod + - ScalingEvaluationPeriods + - ScalingThreshold + - ScalingAdjustment + - ScalingThreshold2 + - ScalingAdjustment2 + - ScalingCooldown + - Label: + default: Additonal Settings + Parameters: + - CLITemplate + ParameterLabels: + KeyName: + default: key_name + AccessFrom: + default: access_from + AccessFrom: + default: access_from + VPCId: + default: vpc_id + MasterSubnetId: + default: master_subnet_id + ComputeSubnetId: + default: compute_subnet_id + ComputeSubnetCidr: + default: compute_subnet_cidr + UsePublicIps: + default: use_public_ips + AdditionalSG: + default: additional_sg + VPCSecurityGroupId: + default: vpc_security_group_id + ComputeInstanceType: + default: compute_instance_type + MasterInstanceType: + default: master_instance_type + InitialQueueSize: + default: initial_queue_size + MaxQueueSize: + default: max_queue_size + MaintainInitialSize: + default: maintain_initial_size + Scheduler: + default: scheduler + ClusterType: + default: cluster_type + EphemeralDir: + default: ephemeral_dir + SpotPrice: + default: spot_price + CustomAMI: + default: custom_ami + PreInstallScript: + default: pre_install + PostInstallScript: + default: post_install + PreInstallArgs: + default: pre_install_args + PostInstallArgs: + default: post_install_args + S3ReadResource: + default: s3_read_resource + S3ReadWriteResource: + default: s3_read_write_resource + CWLRegion: + default: cwl_region + CWLLogGroup: + default: cwl_log_group + ProxyServer: + default: proxy_server + Placement: + default: placement + PlacementGroup: + default: placement_group + EncryptedEphemeral: + default: encrypted_ephemeral + SharedDir: + default: shared_dir + Tenancy: + default: tenancy + EphemeralKMSKeyId: + default: ephemeral_kms_key_id + ClusterReadyScript: + default: cluster_ready + MasterRootVolumeSize: + default: master_root_volume_size + ComputeRootVolumeSize: + default: compute_volume_size + BaseOS: + default: base_os + EC2IAMRoleName: + default: ec2_iam_role + ExtraJson: + default: extra_json + CustomChefCookbook: + default: custom_chef_cookbook + CustomChefRunList: + default: custom_chef_runlist + EBSSnapshotId: + default: ebs_snapshot_id + VolumeType: + default: volume_type + VolumeSize: + default: volume_size + EBSKMSKeyId: + default: ebs_kms_key_id + VolumeIOPS: + default: volume_iops + EBSEncryption: + default: ebs_encryption + EBSVolumeId: + default: ebs_volume_id + ScalingThreshold: + default: scaling_threshold + ScalingPeriod: + default: scaling_period + ScalingEvaluationPeriods: + default: scaling_evaluation_periods + ScalingAdjustment: + default: scaling_adjustment + ScalingAdjustment2: + default: scaling_adjustment2 + ScalingCooldown: + default: scaling_cooldown + ScalingThreshold2: + default: scaling_threshold2 + ComputeWaitConditionCount: + default: WAIT COUNT + AvailabilityZone: + default: AVAILABILITY ZONE + CLITemplate: + default: CLI TEMPLATE + AWS::CloudFormation::Designer: + ee5f3006-419e-4786-b527-e9503a662e5e: + size: + width: 60 + height: 60 + position: + x: -170 + y: 390 + z: 1 + embeds: [] + c8d3da13-fd27-4377-b606-0f41728c17e1: + size: + width: 60 + height: 60 + position: + x: -170 + y: 310 + z: 1 + embeds: [] + 2904615e-c0ad-4325-b0c5-b9ea7c482b4e: + size: + width: 60 + height: 60 + position: + x: 60 + y: 390 + z: 1 + embeds: [] + 45360a4f-64aa-49f8-b016-78fe5ed86819: + size: + width: 60 + height: 60 + position: + x: 60 + y: 310 + z: 1 + embeds: [] + ismemberof: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + 619070e4-82e5-445a-93a0-4795ff69d61c: + size: + width: 60 + height: 60 + position: + x: 180 + y: 480 + z: 1 + embeds: [] + isrelatedto: + - 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + 466554d3-0e10-4ae4-bfee-51b0aadbf923: + source: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + target: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + 0824f592-a72b-4bb5-8c67-4ce6df509cdd: + size: + width: 240 + height: 240 + position: + x: -430 + y: 30 + z: 1 + embeds: [] + 596ddc76-44b9-4747-921c-c000c5b636c0: + size: + width: 60 + height: 60 + position: + x: 60 + y: 150 + z: 0 + embeds: [] + references: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + bb5a532f-bb43-472a-9965-c4054cd61cc9: + size: + width: 150 + height: 150 + position: + x: -150 + y: 70 + z: 1 + embeds: [] + 9dd9772e-03a3-4983-b28e-a52ac42f0ffe: + source: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + target: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + d5bad569-ad14-487d-bcc6-5569373bc8cd: + size: + width: 60 + height: 60 + position: + x: 180 + y: 310 + z: 1 + embeds: [] + e9ac6c41-8110-49b9-a92c-de57be8a292a: + source: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + target: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb: + size: + width: 60 + height: 60 + position: + x: 270 + y: 30 + z: 1 + embeds: [] + a8fbc80f-2514-4cc9-aee8-ba7936add0a7: + size: + width: 60 + height: 60 + position: + x: 170 + y: 30 + z: 1 + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + 0c5a1e92-ffdd-4344-8ba0-65270e5fd472: + size: + width: 60 + height: 60 + position: + x: 390 + y: 30 + z: 1 + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + 2522b048-2b7c-44ed-9d35-6cc66069ca5b: + size: + width: 60 + height: 60 + position: + x: 390 + y: 130 + z: 1 + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + d3204db6-c821-400d-bf31-6afd5739923c: + size: + width: 60 + height: 60 + position: + x: 60 + y: 480 + z: 1 + embeds: [] + ismemberof: + - 619070e4-82e5-445a-93a0-4795ff69d61c + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + 88da1502-8179-4be3-9123-fdc5eac0b6a3: + size: + width: 60 + height: 60 + position: + x: -50 + y: 310 + z: 1 + embeds: [] + isconnectedto: + - 45360a4f-64aa-49f8-b016-78fe5ed86819 + isrelatedto: + - 2522b048-2b7c-44ed-9d35-6cc66069ca5b + - ee5f3006-419e-4786-b527-e9503a662e5e + 789cb442-3b1a-4673-a07e-8acb1c59c271: + size: + width: 60 + height: 60 + position: + x: 270 + y: 380 + z: 1 + embeds: [] + 2f599618-92e8-49cf-92bd-551ba9775f39: + size: + width: 60 + height: 60 + position: + x: 450 + y: 380 + z: 1 + embeds: [] + 7b0e0a3f-3a02-4e70-9e79-7647b98fd579: + size: + width: 60 + height: 60 + position: + x: 270 + y: 130 + z: 1 + embeds: [] + isassociatedwith: + - ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + - 789cb442-3b1a-4673-a07e-8acb1c59c271 + 210b5501-b505-4ee1-917c-a20cae3ac837: + size: + width: 60 + height: 60 + position: + x: 360 + y: 380 + z: 1 + embeds: [] + isrelatedto: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + 3fa053e5-8ab7-4cdf-a737-4d20386e4717: + size: + width: 60 + height: 60 + position: + x: -50 + y: 480 + z: 1 + embeds: [] + isassociatedwith: + - d3204db6-c821-400d-bf31-6afd5739923c + dependson: + - 88da1502-8179-4be3-9123-fdc5eac0b6a3 + isrelatedto: + - bb5a532f-bb43-472a-9965-c4054cd61cc9 + - 210b5501-b505-4ee1-917c-a20cae3ac837 + - ee5f3006-419e-4786-b527-e9503a662e5e + 8e63b592-a50f-4b89-a40b-e991adaef77b: + size: + width: 60 + height: 60 + position: + x: -50 + y: 600 + z: 1 + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + 23ed10e0-7726-4099-8370-4116c0bc78a2: + size: + width: 60 + height: 60 + position: + x: -50 + y: 680 + z: 1 + embeds: [] + isrelatedto: + - 8e63b592-a50f-4b89-a40b-e991adaef77b + 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c: + size: + width: 60 + height: 60 + position: + x: 60 + y: 600 + z: 1 + embeds: [] + isassociatedwith: + - 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + 127f78c3-d47a-4770-8aad-3fd733c3e1b0: + size: + width: 60 + height: 60 + position: + x: 60 + y: 680 + z: 1 + embeds: [] + isrelatedto: + - 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + 6a874346-80de-4412-9677-00590371a8a2: + size: + width: 60 + height: 60 + position: + x: 450 + y: 470 + z: 1 + embeds: [] + isassociatedwith: + - 2f599618-92e8-49cf-92bd-551ba9775f39 + isrelatedto: + - 210b5501-b505-4ee1-917c-a20cae3ac837 +Parameters: + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instances + using the default cluster user. + Type: AWS::EC2::KeyPair::KeyName + MasterInstanceType: + Description: MasterServer EC2 instance type + Type: String + Default: t2.micro + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - r4.large + - r4.xlarge + - r4.2xlarge + - r4.4xlarge + - r4.8xlarge + - r4.16xlarge + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - t2.xlarge + - t2.2xlarge + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.16xlarge + - x1.32xlarge + - f1.2xlarge + - f1.16xlarge + ComputeInstanceType: + Description: ComputeFleet EC2 instance type + Type: String + Default: t2.micro + ConstraintDescription: Must be a valid EC2 instance type, with support for HVM. + AllowedValues: + - cc2.8xlarge + - c3.8xlarge + - c3.4xlarge + - c3.2xlarge + - c3.xlarge + - c3.large + - c4.8xlarge + - c4.4xlarge + - c4.2xlarge + - c4.xlarge + - c4.large + - r3.8xlarge + - r3.4xlarge + - r3.2xlarge + - r3.xlarge + - r3.large + - r4.large + - r4.xlarge + - r4.2xlarge + - r4.4xlarge + - r4.8xlarge + - r4.16xlarge + - i2.8xlarge + - i2.4xlarge + - i2.2xlarge + - i2.xlarge + - i3.large + - i3.xlarge + - i3.2xlarge + - i3.4xlarge + - i3.8xlarge + - i3.16xlarge + - cr1.8xlarge + - cg1.4xlarge + - m3.medium + - m3.large + - m3.xlarge + - m3.2xlarge + - hi1.4xlarge + - g2.2xlarge + - g2.8xlarge + - p2.xlarge + - p2.8xlarge + - p2.16xlarge + - t2.micro + - t2.small + - t2.medium + - t2.large + - t2.xlarge + - t2.2xlarge + - d2.8xlarge + - d2.4xlarge + - d2.2xlarge + - d2.xlarge + - m4.large + - m4.xlarge + - m4.2xlarge + - m4.4xlarge + - m4.10xlarge + - m4.16xlarge + - x1.16xlarge + - x1.32xlarge + - f1.2xlarge + - f1.16xlarge + InitialQueueSize: + Description: Initial number of EC2 instances to launch as compute nodes within + the cluster. This value maps to the DesiredSize parameter for the ComputeFleet + AutoScaling Group. + Type: Number + Default: '2' + MaxQueueSize: + Description: Maximum number of EC2 instances that can be launched in the cluster. + This value maps to the MaxSize parameter for the ComputeFleet AutoScaling Group. + Type: Number + Default: '10' + ComputeSubnetId: + Description: ID of the Subnet you want to provision the Compute Servers into + Type: String + Default: NONE + AllowedPattern: (NONE|^subnet-[0-9a-z]{8}$) + ScalingThreshold: + Description: Threshold for triggering CloudWatch ScaleUp action + Type: String + Default: '1' + ScalingEvaluationPeriods: + Description: Number of periods consective periods required to trigger the scaling + adjustment + Type: String + Default: '2' + ScalingPeriod: + Description: Period in seconds to measure ScalingThreshold + Type: String + Default: '60' + SpotPrice: + Description: Spot bid price for the ComputeFleet AutoScaling Group when the ClusterType + = "spot". + Type: Number + Default: '0.00' + ClusterType: + Description: Type of cluster to launch. Can either be "ondemand" or "spot". Choosing + "spot" will cause the ComputeFleet AutoScaling group to launch EC2 Spot instances. + Default value is "ondemand". + Type: String + Default: ondemand + ConstraintDescription: 'Must be a supported cluster type: ondemand, spot' + AllowedValues: + - ondemand + - spot + ProxyServer: + Description: hostname and port of HTTP proxy server for cfn-init, boto and yum + i.e. proxy.example.com:8080 + Type: String + Default: NONE + VolumeSize: + Description: Size of EBS volume in GB, if creating a new one + Type: Number + Default: '20' + VolumeType: + Description: Type of volume to create either new or from snapshot + Type: String + Default: gp2 + ConstraintDescription: 'must be a supported volume type: standard, io1, gp2, st1, + sc1' + AllowedValues: + - standard + - gp2 + - io1 + - st1 + - sc1 + MasterSubnetId: + Description: ID of the Subnet you want to provision the Master server into + Type: AWS::EC2::Subnet::Id + AvailabilityZone: + Description: Availability Zone the cluster will launch into. THIS IS REQUIRED + Type: AWS::EC2::AvailabilityZone::Name + EBSSnapshotId: + Description: Id of EBS snapshot if using snapshot as source for volume + Type: String + Default: NONE + AllowedPattern: (NONE|^snap-[0-9a-z]{8}$|^snap-[0-9a-z]{17}$) + CustomAMI: + Description: ID of a Custom AMI, to use instead of published AMI's + Type: String + Default: NONE + AllowedPattern: (NONE|^ami-[0-9a-z]{8}$|^ami-[0-9a-z]{17}$) + VPCId: + Description: ID of the VPC you want to provision cluster into. Only used with + UseVPCBase=false + Type: AWS::EC2::VPC::Id + AccessFrom: + Description: Lockdown SSH/HTTP access (default can be accessed from anywhere) + Type: String + MinLength: '9' + MaxLength: '18' + Default: 0.0.0.0/0 + AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + ComputeSubnetCidr: + Description: CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited + list and can support multiple CIDR ranges for a multi-AZ cluster. The order + and length of this list MUST match the AvailabilityZones parameter. + Type: String + ConstraintDescription: must be a valid CIDR range of the form x.x.x.x/x. + AllowedPattern: (NONE|(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})) + Default: NONE + MaintainInitialSize: + Description: Boolean flag to set autoscaling group to maintain initial size and + scale back + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + UsePublicIps: + Description: Boolean flag to use public IP's for instances. If false, the VPC + must be correctly setup to use NAT for all traffic. + Type: String + Default: 'true' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + VolumeIOPS: + Description: Number of IOPS for volume type io1. Not used for other volume types. + Type: Number + Default: '100' + PreInstallScript: + Description: Preinstall script URL. This is run before any host configuration. + Type: String + Default: NONE + PostInstallScript: + Description: Postinstall script URL. This is run before any host configuration. + Type: String + Default: NONE + ComputeWaitConditionCount: + Description: Specific number of instances to wait for while creating the cluster + Type: Number + Default: '2' + S3ReadResource: + Description: S3 resource with read access from cfncluster nodes + Type: String + Default: NONE + S3ReadWriteResource: + Description: Addtional policy document to be added to EC2 IAM role created and + assigned to all nodes. + Type: String + Default: NONE + Placement: + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + Default: cluster + AllowedValues: + - cluster + - compute + PlacementGroup: + Description: The name of an existing placement group + Type: String + Default: NONE + EncryptedEphemeral: + Description: Boolean flag to encrypt local ephemeral drives. The keys are in-memory + and non-recoverable. + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + PreInstallArgs: + Description: Preinstall script args passed to the preinstall script. + Type: String + Default: NONE + PostInstallArgs: + Description: Postinstall script args passed to the postinstall script. + Type: String + Default: NONE + EBSEncryption: + Description: Boolean flag to use EBS encryption for /shared volume. (Not to be + used for snapshots) + Type: String + Default: 'false' + ConstraintDescription: true/false + AllowedValues: + - 'true' + - 'false' + EphemeralDir: + Description: The path/mountpoint for the ephemeral drive + Type: String + Default: /scratch + BaseOS: + Description: Base OS type for cluster AMI + Type: String + Default: alinux + ConstraintDescription: must be a supported base OS + AllowedValues: + - centos6 + - centos7 + - alinux + - ubuntu1404 + - ubuntu1604 + ScalingThreshold2: + Description: Threshold for triggering CloudWatch ScaleUp2 action + Type: String + Default: '200' + ScalingCooldown: + Description: Period in seconds to wait before allowing further scaling actions + Type: String + Default: '300' + ScalingAdjustment: + Description: Number of instances to add to cluster when the CloudWatch ScaleUp + action is called. + Type: String + Default: '1' + ScalingAdjustment2: + Description: Number of instances to add to cluster when the CloudWatch ScaleUp2 + action is called. + Type: String + Default: '10' + Scheduler: + Description: Cluster scheduler + Type: String + Default: sge + ConstraintDescription: must be a supported scheduler + AllowedValues: + - sge + - openlava + - torque + - slurm + - custom + - test + - pbspro + SharedDir: + Description: The path/mountpoint for the shared drive + Type: String + Default: /shared + CLITemplate: + Type: String + Default: default + AdditionalSG: + Description: Additional VPC secuirty group to be added to instances. Defaults + to NONE + Type: String + Default: NONE + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + CWLRegion: + Description: CloudWatch Logs region + Type: String + Default: NONE + CWLLogGroup: + Description: CloudWatch Logs LogGroup + Type: String + Default: NONE + CustomChefRunList: + Description: Custom run list, which will override the default + Type: String + Default: NONE + CustomChefCookbook: + Description: URL of custom cookbook that will override the default. This will + be unpacked and then dependencies resolved with Berkshelf. + Type: String + Default: NONE + ExtraJson: + Description: Extra json to be added to Chef dna.json + Type: String + Default: '{}' + Tenancy: + Description: Type of placement requird in cfncluster, it can either be cluster + or compute. + Type: String + Default: default + AllowedValues: + - default + - dedicated + EBSKMSKeyId: + Description: KMS ARN for customer created master key, will be used for EBS encryption + Type: String + Default: NONE + EphemeralKMSKeyId: + Description: KMS ARN for customer created master key, will be used for ephemeral + encryption + Type: String + Default: NONE + ClusterReadyScript: + Description: Cluster ready script URL. This is only on the MasterServer, when + the cluster reaches CREATE_COMPLETE. + Type: String + Default: NONE + MasterRootVolumeSize: + Description: Size of MasterServer EBS root volume in GB + Type: Number + Default: '15' + MinValue: '15' + ComputeRootVolumeSize: + Description: Size of ComputeFleet EBS root volume in GB + Type: Number + Default: '15' + MinValue: '15' + EC2IAMRoleName: + Description: Existing EC2 IAM role name + Type: String + Default: NONE + VPCSecurityGroupId: + Description: Existing VPC security group Id + Type: String + Default: NONE + AllowedPattern: (NONE|^sg-[0-9a-z]{8}$) + EBSVolumeId: + Description: Existing EBS volume Id + Type: String + Default: NONE + AllowedPattern: (NONE|^vol-[0-9a-z]{8}$|^vol-[0-9a-z]{17}$) + AdditionalCfnTemplate: + Description: A second CloudFormation template to launch with the cluster + Type: String + Default: NONE +Conditions: + UseSpotInstances: !Equals [!Ref 'ClusterType', spot] + CreateComputeSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetId', NONE], !Not [ + !Equals [!Ref 'ComputeSubnetCidr', NONE]]] + UseComputeSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetCidr', NONE], !Not [ + !Equals [!Ref 'ComputeSubnetId', NONE]]] + UseMasterSubnetForCompute: !And [!Equals [!Ref 'ComputeSubnetId', NONE], !Equals [ + !Ref 'ComputeSubnetCidr', NONE]] + UseEBSSnapshot: !Not [!Equals [!Ref 'EBSSnapshotId', NONE]] + UseCustomRunList: !Not [!Equals [!Ref 'CustomChefRunList', NONE]] + UseCustomAMI: !Not [!Equals [!Ref 'CustomAMI', NONE]] + MaintainInitialASGSize: !Equals [!Ref 'MaintainInitialSize', 'true'] + MasterPublicIp: !Equals [!Ref 'UsePublicIps', 'true'] + ComputePublicIps: !And [!Equals [!Ref 'UsePublicIps', 'true'], {Condition: UseMasterSubnetForCompute}] + UseEBSPIOPS: !Equals [!Ref 'VolumeType', io1] + UseS3ReadPolicy: !Not [!Equals [!Ref 'S3ReadResource', NONE]] + UsePlacementGroup: !Not [!Equals [!Ref 'PlacementGroup', NONE]] + UseClusterPlacement: !And [!Equals [!Ref 'Placement', cluster], {Condition: UsePlacementGroup}] + UseEBSEncryption: !Equals [!Ref 'EBSEncryption', 'true'] + UseS3ReadWritePolicy: !Not [!Equals [!Ref 'S3ReadWriteResource', NONE]] + CloudWatchLogs: !And [!Not [!Equals [!Ref 'CWLRegion', NONE]], !Not [!Equals [!Ref 'CWLLogGroup', + NONE]]] + AddAdditionalSG: !Not [!Equals [!Ref 'AdditionalSG', NONE]] + UseEBSKMSKey: !And [!Not [!Equals [!Ref 'EBSKMSKeyId', NONE]], {Condition: UseEBSEncryption}] + UseEphemeralKMSKey: !And [!Not [!Equals [!Ref 'EphemeralKMSKeyId', NONE]], !Equals [ + !Ref 'EncryptedEphemeral', 'true']] + UseDedicatedTenancy: !Equals [!Ref 'Tenancy', dedicated] + UseEC2IAMRole: !Not [!Equals [!Ref 'EC2IAMRoleName', NONE]] + CreateEC2IAMRole: !Equals [!Ref 'EC2IAMRoleName', NONE] + UseExistingSecurityGroup: !Not [!Equals [!Ref 'VPCSecurityGroupId', NONE]] + UseExistingEBSVolume: !Not [!Equals [!Ref 'EBSVolumeId', NONE]] + CreateEBSVolume: !Equals [!Ref 'EBSVolumeId', NONE] + CreateSecurityGroups: !Equals [!Ref 'VPCSecurityGroupId', NONE] + CreateSubStack: !Not [!Equals [!Ref 'AdditionalCfnTemplate', NONE]] + CreatePlacementGroup: !And [!Equals [!Ref 'PlacementGroup', DYNAMIC], {Condition: UsePlacementGroup}] +Mappings: + AWSInstanceType2Capabilites: + cc2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + cr1.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + g2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + g2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + p2.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.medium: + Arch: 64HVM + EBSOpt: 'False' + m3.large: + Arch: 64HVM + EBSOpt: 'False' + m3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + c3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + c3.large: + Arch: 64HVM + EBSOpt: 'False' + c4.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + c4.large: + Arch: 64HVM + EBSOpt: 'True' + r3.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + r3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + r3.large: + Arch: 64HVM + EBSOpt: 'False' + r4.large: + Arch: 64HVM + EBSOpt: 'True' + r4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + r4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + r4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + r4.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + r4.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.8xlarge: + Arch: 64HVM + EBSOpt: 'False' + i2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + i2.large: + Arch: 64HVM + EBSOpt: 'False' + i3.large: + Arch: 64HVM + EBSOpt: 'True' + i3.xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + i3.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + cg1.4xlarge: + Arch: 64HVM + EBSOpt: 'False' + t2.nano: + Arch: 64HVM + EBSOpt: 'False' + t2.micro: + Arch: 64HVM + EBSOpt: 'False' + t2.small: + Arch: 64HVM + EBSOpt: 'False' + t2.medium: + Arch: 64HVM + EBSOpt: 'False' + t2.large: + Arch: 64HVM + EBSOpt: 'False' + t2.xlarge: + Arch: 64HVM + EBSOpt: 'False' + t2.2xlarge: + Arch: 64HVM + EBSOpt: 'False' + d2.8xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + d2.xlarge: + Arch: 64HVM + EBSOpt: 'True' + x1.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + x1.32xlarge: + Arch: 64HVM + EBSOpt: 'True' + f1.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + f1.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.16xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.10xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.4xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.2xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.xlarge: + Arch: 64HVM + EBSOpt: 'True' + m4.large: + Arch: 64HVM + EBSOpt: 'True' + AWSRegionOS2AMI: + ap-northeast-1: + alinux: ami-586b653f + centos6: ami-10686677 + centos7: ami-0e6b6569 + ubuntu1404: ami-466c6221 + ubuntu1604: ami-ce6c62a9 + ap-northeast-2: + alinux: ami-ea71ad84 + centos6: ami-bb77abd5 + centos7: ami-9377abfd + ubuntu1404: ami-2877ab46 + ubuntu1604: ami-d072aebe + ap-south-1: + alinux: ami-1f720d70 + centos6: ami-0a710e65 + centos7: ami-546d123b + ubuntu1404: ami-11700f7e + ubuntu1604: ami-446f102b + ap-southeast-1: + alinux: ami-1f1b987c + centos6: ami-9b1596f8 + centos7: ami-da1596b9 + ubuntu1404: ami-be1497dd + ubuntu1604: ami-951596f6 + ap-southeast-2: + alinux: ami-56b9af35 + centos6: ami-c7b9afa4 + centos7: ami-afa6b0cc + ubuntu1404: ami-37a4b254 + ubuntu1604: ami-cca6b0af + ca-central-1: + alinux: ami-65f94601 + centos6: ami-a6f847c2 + centos7: ami-62f94606 + ubuntu1404: ami-eefc438a + ubuntu1604: ami-a7fa45c3 + eu-central-1: + alinux: ami-7fd17510 + centos6: ami-56d67239 + centos7: ami-9cd571f3 + ubuntu1404: ami-bcd773d3 + ubuntu1604: ami-40d6722f + eu-west-1: + alinux: ami-6ddac60b + centos6: ami-abd5c9cd + centos7: ami-6dd8c40b + ubuntu1404: ami-83d4c8e5 + ubuntu1604: ami-e6d5c980 + eu-west-2: + alinux: ami-f1bbac95 + centos6: ami-cabcabae + centos7: ami-9cb9aef8 + ubuntu1404: ami-5cb9ae38 + ubuntu1604: ami-5eb9ae3a + sa-east-1: + alinux: ami-a990f8c5 + centos6: ami-b491f9d8 + centos7: ami-5092fa3c + ubuntu1404: ami-6d91f901 + ubuntu1604: ami-df90f8b3 + us-east-1: + alinux: ami-2492c832 + centos6: ami-a293c9b4 + centos7: ami-d29cc6c4 + ubuntu1404: ami-a192c8b7 + ubuntu1604: ami-8c9fc59a + us-east-2: + alinux: ami-a6d3f5c3 + centos6: ami-2cd3f549 + centos7: ami-c3d2f4a6 + ubuntu1404: ami-e2d3f587 + ubuntu1604: ami-54d5f331 + us-west-1: + alinux: ami-39c2e059 + centos6: ami-65c2e005 + centos7: ami-17fcde77 + ubuntu1404: ami-b5c2e0d5 + ubuntu1604: ami-0dfddf6d + us-west-2: + alinux: ami-f4b0b98d + centos6: ami-6bb1b812 + centos7: ami-22b2bb5b + ubuntu1404: ami-f5b0b98c + ubuntu1604: ami-c4b1b8bd + us-gov-west-1: + centos6: NOT_SUPPORTED + centos7: NOT_SUPPORTED + alinux: NOT_SUPPORTED + ubuntu1404: NOT_SUPPORTED + ubuntu1604: NOT_SUPPORTED + OSFeatures: + centos6: + User: centos + RootDevice: /dev/sda1 + centos7: + User: centos + RootDevice: /dev/sda1 + alinux: + User: ec2-user + RootDevice: /dev/xvda + ubuntu1404: + User: ubuntu + RootDevice: /dev/sda1 + ubuntu1604: + User: ubuntu + RootDevice: /dev/sda1 + CfnClusterVersions: + default: + cfncluster: cfncluster-1.3.2 + cookbook: cfncluster-cookbook-1.3.2 + chef: 12.19.36 + ridley: 5.1.0 + berkshelf: 5.6.4 + ami: dev + AWSRegion2Capabilites: + eu-west-1: + arn: aws + eu-west-2: + arn: aws + us-east-1: + arn: aws + us-east-2: + arn: aws + ap-northeast-1: + arn: aws + us-west-2: + arn: aws + sa-east-1: + arn: aws + us-west-1: + arn: aws + ap-southeast-1: + arn: aws + ap-southeast-2: + arn: aws + eu-central-1: + arn: aws + us-gov-west-1: + arn: aws-us-gov + ap-northeast-2: + arn: aws + ap-south-1: + arn: aws + ca-central-1: + arn: aws +Resources: + SQS: + Type: AWS::SQS::Queue + Properties: {} + Metadata: + AWS::CloudFormation::Designer: + id: 2f599618-92e8-49cf-92bd-551ba9775f39 + SQSPolicy: + Type: AWS::SQS::QueuePolicy + Properties: + PolicyDocument: + Id: MyQueuePolicy + Statement: + - Sid: Allow-SendMessage-From-AS-SNS-Topic + Effect: Allow + Principal: + AWS: '*' + Action: + - sqs:SendMessage + Resource: '*' + Condition: + ArnEquals: + aws:SourceArn: !Ref 'SNS' + Queues: + - !Ref 'SQS' + Metadata: + AWS::CloudFormation::Designer: + id: 6a874346-80de-4412-9677-00590371a8a2 + SNS: + Type: AWS::SNS::Topic + Properties: + Subscription: + - Endpoint: !GetAtt [SQS, Arn] + Protocol: sqs + Metadata: + AWS::CloudFormation::Designer: + id: 210b5501-b505-4ee1-917c-a20cae3ac837 + DynamoDBTable: + Type: AWS::DynamoDB::Table + Properties: + AttributeDefinitions: + - AttributeName: instanceId + AttributeType: S + KeySchema: + - AttributeName: instanceId + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' + Metadata: + AWS::CloudFormation::Designer: + id: 789cb442-3b1a-4673-a07e-8acb1c59c271 + RootRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ec2.amazonaws.com + Action: + - sts:AssumeRole + Path: / + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: ce1cdcde-8551-4c33-b846-e1b0ab9ac8fb + RootInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref 'RootRole' + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 2522b048-2b7c-44ed-9d35-6cc66069ca5b + CfnClusterPolicies: + Type: AWS::IAM::Policy + Properties: + PolicyName: cfncluster + PolicyDocument: + Statement: + - Sid: EC2 + Action: + - ec2:AttachVolume + - ec2:DescribeInstanceAttribute + - ec2:DescribeInstanceStatus + - ec2:DescribeInstances + Effect: Allow + Resource: + - '*' + - Sid: DynamoDBList + Action: + - dynamodb:ListTables + Effect: Allow + Resource: + - '*' + - Sid: SQSQueue + Action: + - sqs:SendMessage + - sqs:ReceiveMessage + - sqs:ChangeMessageVisibility + - sqs:DeleteMessage + - sqs:GetQueueUrl + Effect: Allow + Resource: + - !GetAtt [SQS, Arn] + - Sid: Autoscaling + Action: + - autoscaling:DescribeAutoScalingGroups + - autoscaling:TerminateInstanceInAutoScalingGroup + - autoscaling:SetDesiredCapacity + Effect: Allow + Resource: + - '*' + - Sid: CloudWatch + Action: + - cloudwatch:PutMetricData + Effect: Allow + Resource: + - '*' + - Sid: DynamoDBTable + Action: + - dynamodb:PutItem + - dynamodb:Query + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:DescribeTable + Effect: Allow + Resource: + - !Join ['', ['arn:', !FindInMap [AWSRegion2Capabilites, !Ref 'AWS::Region', + arn], ':dynamodb:', !Ref 'AWS::Region', ':', !Ref 'AWS::AccountId', + ':table/', !Ref 'DynamoDBTable']] + - Sid: SQSList + Action: + - sqs:ListQueues + Effect: Allow + Resource: + - '*' + - Sid: CloudWatchLogs + Action: + - logs:* + Effect: Allow + Resource: + - !Join ['', ['arn:', !FindInMap [AWSRegion2Capabilites, !Ref 'AWS::Region', + arn], ':logs:*:*:*']] + Roles: + - !Ref 'RootRole' + Condition: CreateEC2IAMRole + Metadata: + AWS::CloudFormation::Designer: + id: 7b0e0a3f-3a02-4e70-9e79-7647b98fd579 + S3ReadRolePolicies: + Type: AWS::IAM::Policy + Properties: + PolicyName: S3Read + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: S3Read + Effect: Allow + Action: + - s3:Get* + - s3:List* + Resource: + - !Ref 'S3ReadResource' + Roles: + - !Ref 'RootRole' + Condition: UseS3ReadPolicy + Metadata: + AWS::CloudFormation::Designer: + id: 0c5a1e92-ffdd-4344-8ba0-65270e5fd472 + S3ReadWriteRolePolicies: + Type: AWS::IAM::Policy + Properties: + PolicyName: S3ReadWrite + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: S3ReadWrite + Effect: Allow + Action: + - s3:* + Resource: + - !Ref 'S3ReadWriteResource' + Roles: + - !Ref 'RootRole' + Condition: UseS3ReadWritePolicy + Metadata: + AWS::CloudFormation::Designer: + id: a8fbc80f-2514-4cc9-aee8-ba7936add0a7 + MasterEIP: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: d5bad569-ad14-487d-bcc6-5569373bc8cd + MasterServer: + Type: AWS::EC2::Instance + Properties: + InstanceType: !Ref 'MasterInstanceType' + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: !FindInMap [OSFeatures, !Ref 'BaseOS', RootDevice] + Ebs: + VolumeSize: !Ref 'MasterRootVolumeSize' + VolumeType: gp2 + KeyName: !Ref 'KeyName' + Tags: + - Key: Application + Value: !Ref 'AWS::StackName' + - Key: Name + Value: Master + NetworkInterfaces: + - NetworkInterfaceId: !Ref 'MasterENI' + DeviceIndex: '0' + ImageId: !If [UseCustomAMI, !Ref 'CustomAMI', !FindInMap [AWSRegionOS2AMI, !Ref 'AWS::Region', + !Ref 'BaseOS']] + EbsOptimized: !FindInMap [AWSInstanceType2Capabilites, !Ref 'MasterInstanceType', + EBSOpt] + IamInstanceProfile: !If [UseEC2IAMRole, !Ref 'EC2IAMRoleName', !Ref 'RootInstanceProfile'] + PlacementGroupName: !If [UseClusterPlacement, !If [CreatePlacementGroup, !Ref 'DynamicPlacementGroup', + !Ref 'PlacementGroup'], !Ref 'AWS::NoValue'] + Tenancy: !Ref 'Tenancy' + UserData: !Base64 + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - !Ref 'ProxyServer' + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - !Ref 'ProxyServer' + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - !Ref 'AWS::StackName' + - ' --resource=MasterServer --region=' + - !Ref 'AWS::Region' + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - !Ref 'ProxyServer' + - ' + + ' + - custom_cookbook= + - !Ref 'CustomChefCookbook' + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - !Ref 'AWS::Region' + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - !Ref 'AWS::Region' + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - !Ref 'AWS::Region' + - /cookbooks/ + - !FindInMap [CfnClusterVersions, default, cookbook] + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - !FindInMap [CfnClusterVersions, default, cfncluster] + - ' + + ' + - export cookbook_version= + - !FindInMap [CfnClusterVersions, default, cookbook] + - ' + + ' + - export chef_version= + - !FindInMap [CfnClusterVersions, default, chef] + - ' + + ' + - export ridley_version= + - !FindInMap [CfnClusterVersions, default, ridley] + - ' + + ' + - export berkshelf_version= + - !FindInMap [CfnClusterVersions, default, berkshelf] + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - !Ref 'AWS::StackName' + - ' -v -c default -r MasterServer --region ' + - !Ref 'AWS::Region' + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup complete" + --stack= + - !Ref 'AWS::StackName' + - ' --resource=MasterServer --region=' + - !Ref 'AWS::Region' + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Metadata: + Comment: cfncluster Master server + AWS::CloudFormation::Init: + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + deployConfigFiles: + files: + /tmp/dna.json: + mode: '000644' + owner: root + group: root + content: + cfncluster: + stack_name: !Ref 'AWS::StackName' + cfn_preinstall: !Ref 'PreInstallScript' + cfn_preinstall_args: !Ref 'PreInstallArgs' + cfn_postinstall: !Ref 'PostInstallScript' + cfn_postinstall_args: !Ref 'PostInstallArgs' + cfn_region: !Ref 'AWS::Region' + cfn_volume: !If [UseExistingEBSVolume, !Ref 'EBSVolumeId', !Ref 'SharedVolume'] + cfn_scheduler: !Ref 'Scheduler' + cfn_encrypted_ephemeral: !Ref 'EncryptedEphemeral' + cfn_ephemeral_dir: !Ref 'EphemeralDir' + cfn_shared_dir: !Ref 'SharedDir' + cfn_proxy: !Ref 'ProxyServer' + cfn_node_type: MasterServer + cfn_cluster_user: !FindInMap [OSFeatures, !Ref 'BaseOS', User] + cfn_ddb_table: !Ref 'DynamoDBTable' + cfn_sqs_queue: !GetAtt [SQS, QueueName] + run_list: !If [UseCustomRunList, !Ref 'CustomChefRunList', !Join [ + '', ['recipe[cfncluster::', !Ref 'Scheduler', '_config]']]] + /etc/chef/client.rb: + mode: '000644' + owner: root + group: root + content: !Join ['', ['cookbook_path [''/etc/chef/cookbooks'']']] + /tmp/extra.json: + mode: '000644' + owner: root + group: root + content: !Ref 'ExtraJson' + commands: + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + AWS::CloudFormation::Designer: + id: 88da1502-8179-4be3-9123-fdc5eac0b6a3 + CreationPolicy: + ResourceSignal: + Count: '1' + Timeout: PT30M + ComputeFleet: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + MaxSize: !Ref 'MaxQueueSize' + VPCZoneIdentifier: + - !If [UseMasterSubnetForCompute, !Ref 'MasterSubnetId', !If [CreateComputeSubnetForCompute, + !Ref 'ComputeSubnet', !Ref 'ComputeSubnetId']] + LaunchConfigurationName: !Ref 'ComputeServerLaunchConfig' + MinSize: !If [MaintainInitialASGSize, !Ref 'InitialQueueSize', '0'] + DesiredCapacity: !Ref 'InitialQueueSize' + NotificationConfiguration: + TopicARN: !Ref 'SNS' + NotificationTypes: + - autoscaling:EC2_INSTANCE_TERMINATE + Tags: + - Key: Name + Value: Compute + PropagateAtLaunch: 'true' + PlacementGroup: !If [UsePlacementGroup, !If [CreatePlacementGroup, !Ref 'DynamicPlacementGroup', + !Ref 'PlacementGroup'], !Ref 'AWS::NoValue'] + DependsOn: MasterServer + CreationPolicy: + ResourceSignal: + Timeout: PT30M + Count: !Ref 'ComputeWaitConditionCount' + Metadata: + AWS::CloudFormation::Designer: + id: 3fa053e5-8ab7-4cdf-a737-4d20386e4717 + ComputeServerLaunchConfig: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + SecurityGroups: + - !If [CreateSecurityGroups, !Ref 'ComputeSecurityGroup', !Ref 'AWS::NoValue'] + - !If [AddAdditionalSG, !Ref 'AdditionalSG', !Ref 'AWS::NoValue'] + - !If [UseExistingSecurityGroup, !Ref 'VPCSecurityGroupId', !Ref 'AWS::NoValue'] + AssociatePublicIpAddress: !If [ComputePublicIps, 'true', 'false'] + InstanceType: !Ref 'ComputeInstanceType' + KeyName: !Ref 'KeyName' + IamInstanceProfile: !If [UseEC2IAMRole, !Ref 'EC2IAMRoleName', !Ref 'RootInstanceProfile'] + SpotPrice: !If [UseSpotInstances, !Ref 'SpotPrice', !Ref 'AWS::NoValue'] + ImageId: !If [UseCustomAMI, !Ref 'CustomAMI', !FindInMap [AWSRegionOS2AMI, !Ref 'AWS::Region', + !Ref 'BaseOS']] + InstanceMonitoring: 'false' + BlockDeviceMappings: + - DeviceName: /dev/xvdba + VirtualName: ephemeral0 + - DeviceName: /dev/xvdbb + VirtualName: ephemeral1 + - DeviceName: /dev/xvdbc + VirtualName: ephemeral2 + - DeviceName: /dev/xvdbd + VirtualName: ephemeral3 + - DeviceName: /dev/xvdbe + VirtualName: ephemeral4 + - DeviceName: /dev/xvdbf + VirtualName: ephemeral5 + - DeviceName: /dev/xvdbg + VirtualName: ephemeral6 + - DeviceName: /dev/xvdbh + VirtualName: ephemeral7 + - DeviceName: /dev/xvdbi + VirtualName: ephemeral8 + - DeviceName: /dev/xvdbj + VirtualName: ephemeral9 + - DeviceName: /dev/xvdbk + VirtualName: ephemeral10 + - DeviceName: /dev/xvdbl + VirtualName: ephemeral11 + - DeviceName: /dev/xvdbm + VirtualName: ephemeral12 + - DeviceName: /dev/xvdbn + VirtualName: ephemeral13 + - DeviceName: /dev/xvdbo + VirtualName: ephemeral14 + - DeviceName: /dev/xvdbp + VirtualName: ephemeral15 + - DeviceName: /dev/xvdbq + VirtualName: ephemeral16 + - DeviceName: /dev/xvdbr + VirtualName: ephemeral17 + - DeviceName: /dev/xvdbs + VirtualName: ephemeral18 + - DeviceName: /dev/xvdbt + VirtualName: ephemeral19 + - DeviceName: /dev/xvdbu + VirtualName: ephemeral20 + - DeviceName: /dev/xvdbv + VirtualName: ephemeral21 + - DeviceName: /dev/xvdbw + VirtualName: ephemeral22 + - DeviceName: /dev/xvdbx + VirtualName: ephemeral23 + - DeviceName: !FindInMap [OSFeatures, !Ref 'BaseOS', RootDevice] + Ebs: + VolumeSize: !Ref 'ComputeRootVolumeSize' + VolumeType: gp2 + PlacementTenancy: !If [UseDedicatedTenancy, !Ref 'Tenancy', !Ref 'AWS::NoValue'] + UserData: !Base64 + Fn::Join: + - '' + - - 'Content-Type: multipart/mixed; boundary="==BOUNDARY==" + + ' + - 'MIME-Version: 1.0 + + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/cloud-config; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#cloud-config: + + ' + - 'runcmd: + + ' + - ' - [ sh, "which yum && echo ' + - !Ref 'ProxyServer' + - ' >> /etc/yum.conf" ] + + ' + - ' - [ sh, "which apt-get && echo Acquire::http::Proxy "' + - !Ref 'ProxyServer' + - '"; >> /etc/apt/aot.conf" ] + + ' + - '--==BOUNDARY== + + ' + - 'Content-Type: text/x-shellscript; charset="us-ascii" + + ' + - 'MIME-Version: 1.0 + + + ' + - '#!/bin/bash -x + + + ' + - 'function error_exit + + ' + - '{ + + ' + - ' cfn-signal ${proxy_args} --exit-code=1 --reason="$1" --stack=' + - !Ref 'AWS::StackName' + - ' --resource=ComputeFleet --region=' + - !Ref 'AWS::Region' + - ' + + ' + - ' exit 1 + + ' + - '} + + ' + - 'function bootstrap_instance + + ' + - '{ + + ' + - ' which yum 2>/dev/null; yum=$? + + ' + - ' which apt-get 2>/dev/null; apt=$? + + ' + - ' if [ "$yum" == "0" ]; then + + ' + - ' yum -y groupinstall development && yum -y install curl wget + + ' + - ' fi + + ' + - ' if [ "$apt" == "0" ]; then + + ' + - ' apt-cache search build-essential; apt-get clean; apt-get update; + apt-get -y install build-essential curl wget + + ' + - ' fi + + ' + - ' which cfn-init 2>/dev/null || ( curl -s -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz + https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz; + easy_install -U /tmp/aws-cfn-bootstrap-latest.tar.gz) + + ' + - ' mkdir -p /etc/chef && chown -R root:root /etc/chef + + ' + - ' curl -L https://www.chef.io/chef/install.sh | bash -s -- -v $chef_version + + ' + - ' /opt/chef/embedded/bin/gem install --no-rdoc --no-ri ridley:$ridley_version + berkshelf:$berkshelf_version + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz $cookbook_url + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.date $cookbook_url.date + + ' + - ' curl -s -L -o /etc/chef/cfncluster-cookbook.tgz.md5 $cookbook_url.md5 + + ' + - ' mkdir /opt/cfncluster && echo $cfncluster_version | tee /opt/cfncluster/.bootstrapped + + ' + - '} + + ' + - proxy= + - !Ref 'ProxyServer' + - ' + + ' + - custom_cookbook= + - !Ref 'CustomChefCookbook' + - ' + + ' + - 'if [ "$proxy" != "NONE" ]; then + + ' + - ' proxy_args="--http-proxy=${proxy} --https-proxy=${proxy}" + + ' + - ' proxy_host=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f1) + + ' + - ' proxy_port=$(echo "$proxy" | awk -F/ ''{print $3}'' | cut -d: -f2) + + ' + - ' export http_proxy=$proxy; export https_proxy=$http_proxy + + ' + - ' export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + ' + - ' export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + ' + - ' echo -e "export http_proxy=$proxy; export https_proxy=$http_proxy + + export HTTP_PROXY=$proxy; export HTTPS_PROXY=$http_proxy + + export no_proxy=169.254.169.254; export NO_PROXY=169.254.169.254 + + " >/tmp/proxy.sh + + ' + - ' echo -e "[Boto] + + proxy = ${proxy_host} + + proxy_port = ${proxy_port} + + " >/etc/boto.cfg + + ' + - 'else + + ' + - ' proxy_args="" + + ' + - ' touch /tmp/proxy.sh + + ' + - 'fi + + ' + - 'if [ "$custom_cookbook" != "NONE" ]; then + + ' + - ' cookbook_url=$custom_cookbook + + ' + - 'else + + ' + - ' if [ "' + - !Ref 'AWS::Region' + - '" == "us-east-1" ]; then + + ' + - ' s3_prefix=s3 + + ' + - ' else + + ' + - ' s3_prefix=s3-' + - !Ref 'AWS::Region' + - ' + + ' + - ' fi + + ' + - ' cookbook_url=https://${s3_prefix}.amazonaws.com/cfncluster-resources-' + - !Ref 'AWS::Region' + - /cookbooks/ + - !FindInMap [CfnClusterVersions, default, cookbook] + - '.tgz + + ' + - 'fi + + ' + - 'export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin + + ' + - export cfncluster_version= + - !FindInMap [CfnClusterVersions, default, cfncluster] + - ' + + ' + - export cookbook_version= + - !FindInMap [CfnClusterVersions, default, cookbook] + - ' + + ' + - export chef_version= + - !FindInMap [CfnClusterVersions, default, chef] + - ' + + ' + - export ridley_version= + - !FindInMap [CfnClusterVersions, default, ridley] + - ' + + ' + - export berkshelf_version= + - !FindInMap [CfnClusterVersions, default, berkshelf] + - ' + + ' + - 'if [ -f /opt/cfncluster/.bootstrapped ]; then + + ' + - ' installed_version=$(cat /opt/cfncluster/.bootstrapped) + + ' + - ' if [ "$cfncluster_version" != "$installed_version" ]; then + + ' + - ' bootstrap_instance + + ' + - ' fi + + ' + - 'else + + ' + - ' bootstrap_instance + + ' + - 'fi + + ' + - 'mkdir /tmp/cookbooks + + ' + - 'cd /tmp/cookbooks + + ' + - 'curl -v -L -o /etc/chef/cfncluster-cookbook.tgz -z "$(cat /etc/chef/cfncluster-cookbook.tgz.date)" + $cookbook_url + + ' + - 'tar -xzf /etc/chef/cfncluster-cookbook.tgz + + ' + - 'cd /tmp + + ' + - '# Call CloudFormation + + ' + - 'cfn-init ${proxy_args} -s ' + - !Ref 'AWS::StackName' + - ' -v -c default -r ComputeServerLaunchConfig --region ' + - !Ref 'AWS::Region' + - ' || error_exit ''Failed to run cfn-init. If --norollback was specified, + check /var/log/cfn-init.log and /var/log/cloud-init-output.log.'' + + ' + - cfn-signal ${proxy_args} --exit-code=0 --reason="MasterServer setup complete" + --stack= + - !Ref 'AWS::StackName' + - ' --resource=ComputeFleet --region=' + - !Ref 'AWS::Region' + - ' + + ' + - '# End of file + + ' + - '--==BOUNDARY== + + ' + Metadata: + Comment: cfncluster Compute server + AWS::CloudFormation::Init: + configSets: + default: + - deployConfigFiles + - getCookbooks + - chefPrepEnv + - shellRunPreInstall + - chefConfig + - shellRunPostInstall + - shellForkClusterReadyInstall + - signalComputeReady + deployConfigFiles: + files: + /tmp/dna.json: + mode: '000644' + owner: root + group: root + content: + cfncluster: + stack_name: !Ref 'AWS::StackName' + cfn_preinstall: !Ref 'PreInstallScript' + cfn_preinstall_args: !Ref 'PreInstallArgs' + cfn_postinstall: !Ref 'PostInstallScript' + cfn_postinstall_args: !Ref 'PostInstallArgs' + cfn_region: !Ref 'AWS::Region' + cfn_scheduler: !Ref 'Scheduler' + cfn_encrypted_ephemeral: !Ref 'EncryptedEphemeral' + cfn_ephemeral_dir: !Ref 'EphemeralDir' + cfn_shared_dir: !Ref 'SharedDir' + cfn_proxy: !Ref 'ProxyServer' + cfn_sqs_queue: !Ref 'SQS' + cfn_master: !GetAtt [MasterServer, PrivateDnsName] + cfn_node_type: ComputeFleet + cfn_cluster_user: !FindInMap [OSFeatures, !Ref 'BaseOS', User] + run_list: !If [UseCustomRunList, !Ref 'CustomChefRunList', !Join [ + '', ['recipe[cfncluster::', !Ref 'Scheduler', '_config]']]] + /etc/chef/client.rb: + mode: '000644' + owner: root + group: root + content: !Join ['', ['cookbook_path [''/etc/chef/cookbooks'']']] + /tmp/extra.json: + mode: '000644' + owner: root + group: root + content: !Ref 'ExtraJson' + commands: + mkdir: + command: mkdir -p /etc/chef/ohai/hints + touch: + command: touch /etc/chef/ohai/hints/ec2.json + jq: + command: /usr/local/bin/jq --argfile f1 /tmp/dna.json --argfile f2 /tmp/extra.json + -n '$f1 + $f2 | .cfncluster = $f1.cfncluster + $f2.cfncluster' > /etc/chef/dna.json + || ( echo "jq not installed"; cp /tmp/dna.json /etc/chef/dna.json + ) + getCookbooks: + commands: + berk: + command: '. /tmp/proxy.sh; for d in `ls /tmp/cookbooks`; do cd /tmp/cookbooks/$d;LANG=en_US.UTF-8 + /opt/chef/embedded/bin/berks vendor /etc/chef/cookbooks; done ' + cwd: /tmp/cookbooks + env: + HOME: /tmp + chefPrepEnv: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json --override-runlist cfncluster::_prep_env + cwd: /etc/chef + shellRunPreInstall: + commands: + runpreinstall: + command: /opt/cfncluster/scripts/fetch_and_run -preinstall + chefConfig: + commands: + chef: + command: chef-client --local-mode --config /etc/chef/client.rb --log_level + auto --force-formatter --no-color --chef-zero-port 8889 --json-attributes + /etc/chef/dna.json + cwd: /etc/chef + shellRunPostInstall: + commands: + runpostinstall: + command: /opt/cfncluster/scripts/fetch_and_run -postinstall + shellForkClusterReadyInstall: + commands: + clusterreadyinstall: + command: /opt/cfncluster/scripts/fetch_and_run -clusterreadyinstall + signalComputeReady: + commands: + compute_ready: + command: /opt/cfncluster/scripts/compute_ready + AWS::CloudFormation::Designer: + id: d3204db6-c821-400d-bf31-6afd5739923c + ScaleUpPolicy2: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + Cooldown: !Ref 'ScalingCooldown' + ScalingAdjustment: !Ref 'ScalingAdjustment2' + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: !Ref 'ComputeFleet' + Metadata: + AWS::CloudFormation::Designer: + id: 89b41ceb-a2fb-4e17-a808-9bedc37b2b2c + AddCapacityAlarm2: + Type: AWS::CloudWatch::Alarm + Properties: + Threshold: !Ref 'ScalingThreshold2' + Period: !Ref 'ScalingPeriod' + EvaluationPeriods: !Ref 'ScalingEvaluationPeriods' + Statistic: Sum + AlarmActions: + - !Ref 'ScaleUpPolicy2' + Namespace: cfncluster + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: !Ref 'AWS::StackName' + MetricName: pending + Metadata: + AWS::CloudFormation::Designer: + id: 127f78c3-d47a-4770-8aad-3fd733c3e1b0 + ScaleUpPolicy: + Type: AWS::AutoScaling::ScalingPolicy + Properties: + Cooldown: !Ref 'ScalingCooldown' + ScalingAdjustment: !Ref 'ScalingAdjustment' + AdjustmentType: ChangeInCapacity + AutoScalingGroupName: !Ref 'ComputeFleet' + Metadata: + AWS::CloudFormation::Designer: + id: 8e63b592-a50f-4b89-a40b-e991adaef77b + AddCapacityAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + Threshold: !Ref 'ScalingThreshold' + Period: !Ref 'ScalingPeriod' + EvaluationPeriods: !Ref 'ScalingEvaluationPeriods' + Statistic: Average + AlarmActions: + - !Ref 'ScaleUpPolicy' + Namespace: cfncluster + ComparisonOperator: GreaterThanOrEqualToThreshold + Dimensions: + - Name: Stack + Value: !Ref 'AWS::StackName' + MetricName: pending + Metadata: + AWS::CloudFormation::Designer: + id: 23ed10e0-7726-4099-8370-4116c0bc78a2 + ComputeSubnet: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref 'VPCId' + CidrBlock: !Ref 'ComputeSubnetCidr' + Tags: + - Key: Network + Value: ComputeSubnet + AvailabilityZone: !Ref 'AvailabilityZone' + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: bb5a532f-bb43-472a-9965-c4054cd61cc9 + ComputeRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref 'VPCId' + Tags: + - Key: Application + Value: !Ref 'AWS::StackName' + - Key: Network + Value: ComputeSubnet + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 0824f592-a72b-4bb5-8c67-4ce6df509cdd + ComputeRoute: + Type: AWS::EC2::Route + Properties: + RouteTableId: !Ref 'ComputeRouteTable' + DestinationCidrBlock: 0.0.0.0/0 + NetworkInterfaceId: !Ref 'MasterENI' + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 596ddc76-44b9-4747-921c-c000c5b636c0 + ComputeSubnetRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: !Ref 'ComputeSubnet' + RouteTableId: !Ref 'ComputeRouteTable' + Condition: CreateComputeSubnetForCompute + Metadata: + AWS::CloudFormation::Designer: + id: 9dd9772e-03a3-4983-b28e-a52ac42f0ffe + MasterSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Enable access to the Master host + VpcId: !Ref 'VPCId' + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: '22' + ToPort: '22' + CidrIp: !Ref 'AccessFrom' + - IpProtocol: tcp + FromPort: '80' + ToPort: '80' + CidrIp: !Ref 'AccessFrom' + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 2904615e-c0ad-4325-b0c5-b9ea7c482b4e + MasterSecurityGroupIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + SourceSecurityGroupId: !Ref 'ComputeSecurityGroup' + GroupId: !Ref 'MasterSecurityGroup' + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 466554d3-0e10-4ae4-bfee-51b0aadbf923 + ComputeSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Allow access to resources in subnets behind front + VpcId: !Ref 'VPCId' + SecurityGroupIngress: + - SourceSecurityGroupId: !Ref 'MasterSecurityGroup' + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + Condition: CreateSecurityGroups + Metadata: + AWS::CloudFormation::Designer: + id: 619070e4-82e5-445a-93a0-4795ff69d61c + ComputeSecurityGroupIngress: + Type: AWS::EC2::SecurityGroupIngress + Properties: + IpProtocol: '-1' + FromPort: '0' + ToPort: '65535' + SourceSecurityGroupId: !Ref 'ComputeSecurityGroup' + GroupId: !Ref 'ComputeSecurityGroup' + Condition: CreateSecurityGroups + MasterENI: + Type: AWS::EC2::NetworkInterface + Properties: + Description: cfncluster Master Server + SubnetId: !Ref 'MasterSubnetId' + SourceDestCheck: 'false' + GroupSet: + - !If [CreateSecurityGroups, !Ref 'MasterSecurityGroup', !Ref 'AWS::NoValue'] + - !If [AddAdditionalSG, !Ref 'AdditionalSG', !Ref 'AWS::NoValue'] + - !If [UseExistingSecurityGroup, !Ref 'VPCSecurityGroupId', !Ref 'AWS::NoValue'] + Metadata: + AWS::CloudFormation::Designer: + id: 45360a4f-64aa-49f8-b016-78fe5ed86819 + AdditionalCfnStack: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: !Ref 'AdditionalCfnTemplate' + Condition: CreateSubStack + SharedVolume: + Type: AWS::EC2::Volume + Properties: + AvailabilityZone: !Ref 'AvailabilityZone' + VolumeType: !Ref 'VolumeType' + Size: !If [UseEBSSnapshot, !Ref 'AWS::NoValue', !Ref 'VolumeSize'] + SnapshotId: !If [UseEBSSnapshot, !Ref 'EBSSnapshotId', !Ref 'AWS::NoValue'] + Iops: !If [UseEBSPIOPS, !Ref 'VolumeIOPS', !Ref 'AWS::NoValue'] + Encrypted: !If [UseEBSEncryption, !Ref 'EBSEncryption', !Ref 'AWS::NoValue'] + KmsKeyId: !If [UseEBSKMSKey, !Ref 'EBSKMSKeyId', !Ref 'AWS::NoValue'] + Condition: CreateEBSVolume + Metadata: + AWS::CloudFormation::Designer: + id: c8d3da13-fd27-4377-b606-0f41728c17e1 + AssociateEIP: + Type: AWS::EC2::EIPAssociation + Properties: + AllocationId: !GetAtt [MasterEIP, AllocationId] + NetworkInterfaceId: !Ref 'MasterENI' + Condition: MasterPublicIp + Metadata: + AWS::CloudFormation::Designer: + id: e9ac6c41-8110-49b9-a92c-de57be8a292a + DynamicPlacementGroup: + Type: AWS::EC2::PlacementGroup + Properties: + Strategy: cluster + Condition: CreatePlacementGroup + Metadata: + AWS::CloudFormation::Designer: + id: ee5f3006-419e-4786-b527-e9503a662e5e +Outputs: + MasterPrivateIP: + Description: Private IP Address of the Master host + Value: !GetAtt [MasterServer, PrivateIp] + MasterPublicIP: + Description: Public IP Address of the Master host + Value: !GetAtt [MasterServer, PublicIp] + Condition: MasterPublicIp + GangliaPrivateURL: + Description: Private URL to access Ganglia + Value: !Join ['', ['http://', !GetAtt [MasterServer, PrivateIp], /ganglia/]] + GangliaPublicURL: + Description: Public URL to access Ganglia + Value: !Join ['', ['http://', !GetAtt [MasterServer, PublicIp], /ganglia/]] + Condition: MasterPublicIp diff --git a/docs/source/ami_development.rst b/docs/source/ami_development.rst index 19e291156b..aebbbb5a80 100644 --- a/docs/source/ami_development.rst +++ b/docs/source/ami_development.rst @@ -16,7 +16,7 @@ Steps This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubuntu machine you can easily get an `EC2 instance `_ running Ubuntu. -#. :code:`sudo apt-get install build-essential` +#. :code:`sudo apt-get -y install build-essential git` #. Go to https://downloads.chef.io/chef-dk, grab the latest version for your OS and install. For example: @@ -36,6 +36,7 @@ This guide is written assuming your OS is Ubuntu 14.04. If you don't have an Ubu sudo tar xf ~/go1.7.linux-amd64.tar.gz echo 'export GOPATH=~/work' >> ~/.bashrc echo 'export PATH=$GOPATH/bin:/usr/local/go/bin:$PATH' >> ~/.bashrc + . ~/.bashrc #. Install packer from source diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index eb9e4ad09f..9af4945bae 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -273,17 +273,17 @@ master_root_volume_size """"""""""""""""""""""" MasterServer root volume size in GB. (AMI must support growroot) -Defaults to 10 in default template. :: +Defaults to 15 in default template. :: - master_root_volume_size = 10 + master_root_volume_size = 15 compute_root_volume_size """""""""""""""""""""""" ComputeFleet root volume size in GB. (AMI must support growroot) -Defaults to 10 in default template. :: +Defaults to 15 in default template. :: - compute_root_volume_size = 10 + compute_root_volume_size = 15 base_os """"""" @@ -378,7 +378,7 @@ Tags are JSON formatted and should not have quotes outside the curly braces. See `AWS CloudFormation Resource Tags Type `_. :: - tags = {"key": "value", "key2", "value2"} + tags = {"key" : "value", "key2" : "value2"} .. _vpc_section: diff --git a/docs/source/iam.rst b/docs/source/iam.rst index 21e05cc2b2..c990181799 100644 --- a/docs/source/iam.rst +++ b/docs/source/iam.rst @@ -144,7 +144,8 @@ CfnClusterUserPolicy "ec2:DescribeVpcAttribute", "ec2:DescribeAddresses", "ec2:CreateTags", - "ec2:DescribeNetworkInterfaces" + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeAvailabilityZones" ], "Effect": "Allow", "Resource": "*" @@ -162,7 +163,10 @@ CfnClusterUserPolicy "ec2:CreateNetworkInterface", "ec2:CreateSecurityGroup", "ec2:ModifyVolumeAttribute", - "ec2:ModifyNetworkInterfaceAttribute" + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:DeleteNetworkInterface", + "ec2:DeleteVolume", + "ec2:TerminateInstances" ], "Effect": "Allow", "Resource": "*" @@ -200,7 +204,8 @@ CfnClusterUserPolicy { "Sid": "DynamoDBModify", "Action": [ - "dynamodb:CreateTable" + "dynamodb:CreateTable", + "dynamodb:DeleteTable" ], "Effect": "Allow", "Resource": "*" @@ -225,7 +230,8 @@ CfnClusterUserPolicy "Sid": "SQSModify", "Action": [ "sqs:CreateQueue", - "sqs:SetQueueAttributes" + "sqs:SetQueueAttributes", + "sqs:DeleteQueue" ], "Effect": "Allow", "Resource": "*" @@ -233,7 +239,8 @@ CfnClusterUserPolicy { "Sid": "SNSDescribe", "Action": [ - "sns:ListTopics" + "sns:ListTopics", + “sns:GetTopicAttributes" ], "Effect": "Allow", "Resource": "*" @@ -242,7 +249,8 @@ CfnClusterUserPolicy "Sid": "SNSModify", "Action": [ "sns:CreateTopic", - "sns:Subscribe" + "sns:Subscribe", + “sns:DeleteTopic” ], "Effect": "Allow", "Resource": "*" diff --git a/tests/test.sh b/tests/test.sh index d9bc48948d..ed783e0541 100644 --- a/tests/test.sh +++ b/tests/test.sh @@ -3,4 +3,6 @@ set -x echo $PATH -which cfncluster \ No newline at end of file +which cfncluster +cfncluster version +cfncluster --help