From 94bbe0683782b0455726f8f6683f3305843452c7 Mon Sep 17 00:00:00 2001 From: Hanwen Date: Tue, 1 Dec 2020 13:16:20 -0500 Subject: [PATCH] Set default instance type to free tier instance type When running pcluster in a region with free tier, default instance type is set to the free tier instance type. When running pcluster in the China (BJS) region or AWS GovCloud (US) regions, default instance type is t3.micro. Free tier is not available in the China (BJS) region and AWS GovCloud (US) regions. For more information about free tier, please see https://aws.amazon.com/free/free-tier-faqs/ Signed-off-by: Hanwen --- CHANGELOG.md | 3 + cli/src/pcluster/config/cfn_param_types.py | 29 +++++++++ cli/src/pcluster/config/mappings.py | 24 ++++---- cli/src/pcluster/config/param_types.py | 3 +- cli/src/pcluster/configure/easyconfig.py | 13 +++- cli/src/pcluster/createami.py | 9 --- cli/src/pcluster/examples/config | 4 +- cli/src/pcluster/utils.py | 20 +++++++ cli/tests/conftest.py | 14 +++++ cli/tests/pcluster/config/defaults.py | 6 +- .../pcluster/config/test_section_cluster.py | 12 +++- cli/tests/pcluster/config/test_utils.py | 59 +++++++++++++++++++ .../configure/test_pcluster_configure.py | 1 + .../test_bad_config_file/pcluster.config.ini | 3 +- .../pcluster.config.ini | 1 + .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 2 + .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 3 +- .../pcluster.config.ini | 1 + .../test_slurm_sit_full/expected_output.ini | 2 +- .../test_slurm_sit_simple/expected_output.ini | 2 +- .../expected_output.ini | 2 +- cloudformation/aws-parallelcluster.cfn.json | 2 - .../configs/common/common.yaml | 5 ++ .../configure/test_pcluster_configure.py | 45 ++++++++++++-- 30 files changed, 224 insertions(+), 59 deletions(-) create mode 100644 cli/tests/pcluster/config/test_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dbbe6324c5..d32f5a8971 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ CHANGELOG - Pull Amazon Linux Docker images from ECR when building docker image for `awsbatch` scheduler. This only applies to images built for `x86` architecture. +- Change the default of instance types from the hardcoded `t2.micro` to the free tier instance type + (`t2.micro` or `t3.micro` dependent on region). In regions without free tier, the default is `t3.micro`. + **BUG FIXES** diff --git a/cli/src/pcluster/config/cfn_param_types.py b/cli/src/pcluster/config/cfn_param_types.py index 7127323012..2559f8e203 100644 --- a/cli/src/pcluster/config/cfn_param_types.py +++ b/cli/src/pcluster/config/cfn_param_types.py @@ -24,6 +24,7 @@ error, get_availability_zone_of_subnet, get_cfn_param, + get_default_instance_type, get_ebs_snapshot_info, get_efs_mount_target_id, get_file_section_name, @@ -854,6 +855,34 @@ def to_cfn(self): return cfn_params +class ComputeInstanceTypeCfnParam(CfnParam): + """ + Class to manage the compute instance type parameter. + + We need this class in order to set the default instance type from a boto3 call. + """ + + def refresh(self): + """Get default value from a boto3 call for free tier instance type.""" + if not self.value: + scheduler = self.pcluster_config.get_section("cluster").get_param_value("scheduler") + if scheduler: + self.value = "optimal" if scheduler == "awsbatch" else get_default_instance_type() + + +class MasterInstanceTypeCfnParam(CfnParam): + """ + Class to manage the head node instance type parameter. + + We need this class in order to set the default instance type from a boto3 call. + """ + + def refresh(self): + """Get default value from a boto3 call for free tier instance type.""" + if not self.value: + self.value = get_default_instance_type() + + class TagsParam(JsonCfnParam): """ Class to manage the tags json configuration parameter. diff --git a/cli/src/pcluster/config/mappings.py b/cli/src/pcluster/config/mappings.py index 480ed8ee88..58717c96f1 100644 --- a/cli/src/pcluster/config/mappings.py +++ b/cli/src/pcluster/config/mappings.py @@ -21,6 +21,7 @@ ClusterCfnSection, ClusterConfigMetadataCfnParam, ComputeAvailabilityZoneCfnParam, + ComputeInstanceTypeCfnParam, DisableHyperThreadingCfnParam, EBSSettingsCfnParam, EFSCfnSection, @@ -29,6 +30,7 @@ IntCfnParam, MaintainInitialSizeCfnParam, MasterAvailabilityZoneCfnParam, + MasterInstanceTypeCfnParam, NetworkInterfacesCountCfnParam, QueueSizeCfnParam, SettingsCfnParam, @@ -753,14 +755,6 @@ "validators": [ec2_key_pair_validator], "update_policy": UpdatePolicy.UNSUPPORTED }), - ("base_os", { - "type": BaseOSCfnParam, - "cfn_param_mapping": "BaseOS", - "allowed_values": ["alinux", "alinux2", "ubuntu1604", "ubuntu1804", "centos7", "centos8"], - "validators": [base_os_validator, architecture_os_validator], - "required": True, - "update_policy": UpdatePolicy.UNSUPPORTED - }), ("scheduler", { "cfn_param_mapping": "Scheduler", "allowed_values": ["awsbatch", "sge", "slurm", "torque"], @@ -770,7 +764,7 @@ }), # Master ("master_instance_type", { - "default": "t2.micro", + "type": MasterInstanceTypeCfnParam, "cfn_param_mapping": "MasterInstanceType", "validators": [head_node_instance_type_validator, ec2_instance_type_validator], "update_policy": UpdatePolicy.UNSUPPORTED, @@ -786,6 +780,14 @@ action_needed=UpdatePolicy.ACTIONS_NEEDED["ebs_volume_update"] ) }), + ("base_os", { + "type": BaseOSCfnParam, + "cfn_param_mapping": "BaseOS", + "allowed_values": ["alinux", "alinux2", "ubuntu1604", "ubuntu1804", "centos7", "centos8"], + "validators": [base_os_validator, architecture_os_validator], + "required": True, + "update_policy": UpdatePolicy.UNSUPPORTED + }), # Compute fleet ("compute_root_volume_size", { "type": IntCfnParam, @@ -1024,9 +1026,7 @@ }), # Compute fleet ("compute_instance_type", { - "default": - lambda section: - "optimal" if section and section.get_param_value("scheduler") == "awsbatch" else "t2.micro", + "type": ComputeInstanceTypeCfnParam, "cfn_param_mapping": "ComputeInstanceType", "validators": [compute_instance_type_validator, instances_architecture_compatibility_validator], "update_policy": UpdatePolicy.COMPUTE_FLEET_STOP diff --git a/cli/src/pcluster/config/param_types.py b/cli/src/pcluster/config/param_types.py index ce1e425f34..f6db94ad2b 100644 --- a/cli/src/pcluster/config/param_types.py +++ b/cli/src/pcluster/config/param_types.py @@ -13,6 +13,7 @@ import re import sys from abc import abstractmethod +from collections import OrderedDict from enum import Enum from configparser import NoSectionError @@ -441,7 +442,7 @@ def __init__(self, section_definition, pcluster_config, section_label=None, pare self.parent_section = parent_section # initialize section parameters with default values - self.params = {} + self.params = OrderedDict({}) self._from_definition() @property diff --git a/cli/src/pcluster/configure/easyconfig.py b/cli/src/pcluster/configure/easyconfig.py index 362d548036..cb6ad45f3d 100644 --- a/cli/src/pcluster/configure/easyconfig.py +++ b/cli/src/pcluster/configure/easyconfig.py @@ -34,6 +34,7 @@ from pcluster.configure.utils import get_regions, get_resource_tag, handle_client_exception, prompt, prompt_iterable from pcluster.utils import ( error, + get_default_instance_type, get_region, get_supported_az_for_multi_instance_types, get_supported_az_for_one_instance_type, @@ -122,7 +123,7 @@ def configure(args): if pcluster_config.cluster_model == ClusterModel.HIT: error( "Configuration in file {0} cannot be overwritten. Please specify a different file path".format( - args.config_file + pcluster_config.config_file ) ) @@ -367,16 +368,22 @@ def prompt_os(self): def prompt_instance_types(self): """Ask for master_instance_type and compute_instance_type (if necessary).""" + default_master_instance_type = self.cluster_section.get_param_value("master_instance_type") + if not default_master_instance_type: + default_master_instance_type = get_default_instance_type() self.master_instance_type = prompt( "Master instance type", lambda x: _is_instance_type_supported_for_head_node(x) and x in get_supported_instance_types(), - default_value=self.cluster_section.get_param_value("master_instance_type"), + default_value=default_master_instance_type, ) if not self.is_aws_batch: + default_compute_instance_type = self.cluster_section.get_param_value("compute_instance_type") + if not default_compute_instance_type: + default_compute_instance_type = get_default_instance_type() self.compute_instance_type = prompt( "Compute instance type", lambda x: x in get_supported_compute_instance_types(self.scheduler), - default_value=self.cluster_section.get_param_value("compute_instance_type"), + default_value=default_compute_instance_type, ) # Cache availability zones offering the selected instance type(s) for later use self.cache_qualified_az() diff --git a/cli/src/pcluster/createami.py b/cli/src/pcluster/createami.py index 3d034fd7c6..b0e3e8cf18 100644 --- a/cli/src/pcluster/createami.py +++ b/cli/src/pcluster/createami.py @@ -358,12 +358,3 @@ def create_ami(args): _print_create_ami_results(results) if "tmp_dir" in locals() and tmp_dir: rmtree(tmp_dir) - - -def _get_default_template_url(region): - return ( - "https://{REGION}-aws-parallelcluster.s3.{REGION}.amazonaws.com{SUFFIX}/templates/" - "aws-parallelcluster-{VERSION}.cfn.json".format( - REGION=region, SUFFIX=".cn" if region.startswith("cn") else "", VERSION=utils.get_installed_version() - ) - ) diff --git a/cli/src/pcluster/examples/config b/cli/src/pcluster/examples/config index 201172f06d..11fd580250 100644 --- a/cli/src/pcluster/examples/config +++ b/cli/src/pcluster/examples/config @@ -24,10 +24,10 @@ key_name = mykey # (defaults to https://s3.amazonaws.com/-aws-parallelcluster/templates/aws-parallelcluster-.cfn.json) #template_url = https://s3.amazonaws.com/-aws-parallelcluster/templates/aws-parallelcluster-.cfn.json # EC2 instance type for master node -# (defaults to t2.micro) +# (defaults to the free tier instance type of the region. If the region does not have free tier, default to t3.micro) #master_instance_type = t2.micro # EC2 instance type for compute nodes -# (defaults to t2.micro , 'optimal' when scheduler is awsbatch) +# (defaults to the free tier instance type of the region, 'optimal' when scheduler is awsbatch) #compute_instance_type = t2.micro # Initial number of EC2 instances to launch as compute nodes in the cluster for schedulers other than awsbatch. # (defaults to 2) diff --git a/cli/src/pcluster/utils.py b/cli/src/pcluster/utils.py index c479ad500d..f63f599803 100644 --- a/cli/src/pcluster/utils.py +++ b/cli/src/pcluster/utils.py @@ -1170,6 +1170,26 @@ def get_ebs_snapshot_info(ebs_snapshot_id, raise_exceptions=False): ) +def get_default_instance_type(): + """If current region support free tier, return the free tier instance type. Otherwise, return t3.micro .""" + if not hasattr(get_default_instance_type, "cache"): + get_default_instance_type.cache = {} + cache = get_default_instance_type.cache + region = os.environ.get("AWS_DEFAULT_REGION") + if region not in cache: + free_tier_instance_type = [] + for page in paginate_boto3( + boto3.client("ec2").describe_instance_types, + Filters=[ + {"Name": "free-tier-eligible", "Values": ["true"]}, + {"Name": "current-generation", "Values": ["true"]}, + ], + ): + free_tier_instance_type.append(page) + cache[region] = free_tier_instance_type[0]["InstanceType"] if free_tier_instance_type else "t3.micro" + return cache[region] + + class Cache: """Simple utility class providing a cache mechanism for expensive functions.""" diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index e1c4fbc8f3..0cad54ed55 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -19,6 +19,20 @@ def clear_env(): del os.environ["AWS_DEFAULT_REGION"] +@pytest.fixture(autouse=True) +def mock_default_instance(mocker, request): + """ + Mock get_default_instance_type for all tests. + + To disable the mock for certain tests, add annotation `@pytest.mark.nomockdefaultinstance` to the tests. + To disable the mock for an entire file, declare global var `pytestmark = pytest.mark.noassertnopendingresponses` + """ + if "nomockdefaultinstance" in request.keywords: + # skip mocking + return + mocker.patch("pcluster.config.cfn_param_types.get_default_instance_type", return_value="t2.micro") + + @pytest.fixture def failed_with_message(capsys): """Assert that the command exited with a specific error message.""" diff --git a/cli/tests/pcluster/config/defaults.py b/cli/tests/pcluster/config/defaults.py index 33e8bab722..73d0e4cae4 100644 --- a/cli/tests/pcluster/config/defaults.py +++ b/cli/tests/pcluster/config/defaults.py @@ -99,9 +99,9 @@ "shared_dir": "/shared", "placement_group": None, "placement": "compute", - "master_instance_type": "t2.micro", + "master_instance_type": None, "master_root_volume_size": 25, - "compute_instance_type": "t2.micro", + "compute_instance_type": None, "compute_root_volume_size": 25, "initial_queue_size": 0, "max_queue_size": 10, @@ -155,7 +155,7 @@ "base_os": None, # base_os does not have a default, but this is here to make testing easier "scheduler": None, # The cluster does not have a default, but this is here to make testing easier "shared_dir": "/shared", - "master_instance_type": "t2.micro", + "master_instance_type": None, "master_root_volume_size": 25, "compute_root_volume_size": 25, "proxy_server": None, diff --git a/cli/tests/pcluster/config/test_section_cluster.py b/cli/tests/pcluster/config/test_section_cluster.py index 2ccde19064..6736d6c106 100644 --- a/cli/tests/pcluster/config/test_section_cluster.py +++ b/cli/tests/pcluster/config/test_section_cluster.py @@ -38,6 +38,8 @@ "base_os": "alinux2", "scheduler": "slurm", "cluster_config_metadata": {"sections": {"cluster": ["custom_cluster_label"]}}, + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "custom_cluster_label", @@ -50,6 +52,8 @@ "additional_iam_policies": ["arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"], "base_os": "alinux2", "scheduler": "slurm", + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "default", @@ -86,6 +90,8 @@ "arn:aws:iam::aws:policy/AWSBatchFullAccess", "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy", ], + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "default", @@ -267,7 +273,6 @@ def test_hit_cluster_section_from_file(mocker, config_parser_dict, expected_dict ("placement", "cluster", "cluster", None), # Master # TODO add regex for master_instance_type - ("master_instance_type", None, "t2.micro", None), ("master_instance_type", "", "", None), ("master_instance_type", "test", "test", None), ("master_instance_type", "NONE", "NONE", None), @@ -281,7 +286,6 @@ def test_hit_cluster_section_from_file(mocker, config_parser_dict, expected_dict ("master_root_volume_size", "31", 31, None), # Compute fleet # TODO add regex for compute_instance_type - ("compute_instance_type", None, "t2.micro", None), ("compute_instance_type", "", "", None), ("compute_instance_type", "test", "test", None), ("compute_instance_type", "NONE", "NONE", None), @@ -543,7 +547,6 @@ def test_sit_cluster_param_from_file( ("shared_dir", "NONE", "NONE", None), # NONE is evaluated as a valid path # Master # TODO add regex for master_instance_type - ("master_instance_type", None, "t2.micro", None), ("master_instance_type", "", "", None), ("master_instance_type", "test", "test", None), ("master_instance_type", "NONE", "NONE", None), @@ -801,6 +804,9 @@ def test_sit_cluster_section_to_file(mocker, section_dict, expected_config_parse def test_cluster_section_to_cfn( mocker, cluster_section_definition, section_dict, expected_cfn_params, default_threads_per_core ): + section_dict["master_instance_type"] = "t2.micro" + if cluster_section_definition == CLUSTER_SIT: + section_dict["compute_instance_type"] = "t2.micro" utils.set_default_values_for_required_cluster_section_params(section_dict) utils.mock_pcluster_config(mocker) mocker.patch("pcluster.config.cfn_param_types.get_efs_mount_target_id", return_value="valid_mount_target_id") diff --git a/cli/tests/pcluster/config/test_utils.py b/cli/tests/pcluster/config/test_utils.py new file mode 100644 index 0000000000..88ae99f255 --- /dev/null +++ b/cli/tests/pcluster/config/test_utils.py @@ -0,0 +1,59 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +from assertpy import assert_that + +from pcluster.utils import get_default_instance_type +from tests.common import MockedBoto3Request + + +@pytest.fixture() +def boto3_stubber_path(): + return "pcluster.utils.boto3" + + +@pytest.mark.parametrize( + "region, free_tier_instance_type, default_instance_type, stub_boto3", + [ + ("us-east-1", "t2.micro", "t2.micro", True), + ("eu-north-1", "t3.micro", "t3.micro", True), + ("us-gov-east-1", None, "t3.micro", True), + # Retrieving free tier instance type again should use cache to reduce boto3 call + ("us-east-1", "t2.micro", "t2.micro", False), + ("eu-north-1", "t3.micro", "t3.micro", False), + ("us-gov-east-1", None, "t3.micro", False), + ], +) +@pytest.mark.nomockdefaultinstance +def test_get_default_instance(boto3_stubber, region, free_tier_instance_type, default_instance_type, stub_boto3): + os.environ["AWS_DEFAULT_REGION"] = region + if free_tier_instance_type: + response = {"InstanceTypes": [{"InstanceType": free_tier_instance_type}]} + else: + response = {"InstanceTypes": []} + if stub_boto3: + mocked_requests = [ + MockedBoto3Request( + method="describe_instance_types", + response=response, + expected_params={ + "Filters": [ + {"Name": "free-tier-eligible", "Values": ["true"]}, + {"Name": "current-generation", "Values": ["true"]}, + ] + }, + ) + ] + + boto3_stubber("ec2", mocked_requests) + assert_that(get_default_instance_type()).is_equal_to(default_instance_type) diff --git a/cli/tests/pcluster/configure/test_pcluster_configure.py b/cli/tests/pcluster/configure/test_pcluster_configure.py index 367f4fd2ce..85e492753b 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure.py +++ b/cli/tests/pcluster/configure/test_pcluster_configure.py @@ -382,6 +382,7 @@ def __init__(self, mocker, empty_region=False, partition="commercial", mock_avai _mock_list_vpcs_and_subnets(self.mocker, empty_region, partition) _mock_parallel_cluster_config(self.mocker) _mock_cache_availability_zones(self.mocker) + mocker.patch("pcluster.configure.easyconfig.get_default_instance_type", return_value="t2.micro") if mock_availability_zone: _mock_availability_zone(self.mocker) diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini index 584ef221cd..4bbcff5504 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini @@ -14,8 +14,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano #Invalid value type max_queue_size = 14 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini index e2fe3749f1..1f4800a4ca 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -6,6 +6,7 @@ key_name = key1 vpc_settings = default scheduler = torque master_instance_type = t2.nano +compute_instance_type = t2.micro max_queue_size = 14 initial_queue_size = 13 maintain_initial_size = true diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini index 95356e94e9..8c696b839e 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ key_name = key1 base_os = alinux2 vpc_settings = default scheduler = awsbatch -# Implied value -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini index 639f3e57a5..48e4bf5524 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini @@ -6,6 +6,8 @@ key_name = key3 vpc_settings = default scheduler = torque base_os = alinux2 +master_instance_type = t2.micro +compute_instance_type = t2.micro [vpc default] vpc_id = vpc-12345678 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini index d00170d6f6..f663bc15fc 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini index fc7f4ed2b3..bd2b285434 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini @@ -6,8 +6,7 @@ key_name = key1 vpc_settings = default scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini index fff2041ee2..a401a06e00 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini index 59256eeabd..170494afc4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini @@ -5,9 +5,8 @@ aws_region_name = eu-west-1 key_name = key1 vpc_settings = default scheduler = awsbatch -# Implied value base_os = alinux2 -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini index d00170d6f6..f663bc15fc 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini index 59256eeabd..170494afc4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini @@ -5,9 +5,8 @@ aws_region_name = eu-west-1 key_name = key1 vpc_settings = default scheduler = awsbatch -# Implied value base_os = alinux2 -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini index d80f4a8a9f..9b38fa7ec4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini @@ -14,6 +14,7 @@ key_name = key1 base_os = alinux scheduler = torque master_instance_type = t2.nano +compute_instance_type = t2.micro vpc_settings = default initial_queue_size = 13 max_queue_size = 14 diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini index 78222099bb..4aa0eafb7c 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini @@ -49,10 +49,10 @@ vpc_security_group_id = sg-0fa8d8e11dc6e9491 [cluster slurm-sit-full] key_name = test-key -base_os = centos7 scheduler = slurm master_instance_type = t2.large master_root_volume_size = 30 +base_os = centos7 compute_root_volume_size = 30 proxy_server = proxy ec2_iam_role = role diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini index e27b63153c..909995b851 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini @@ -15,9 +15,9 @@ sanity_check = true [cluster slurm-sit-simple] key_name = test -base_os = centos7 scheduler = slurm master_instance_type = c5.2xlarge +base_os = centos7 shared_dir = /test vpc_settings = public additional_iam_policies = arn:aws:iam::aws:policy/CloudWatchFullAccess diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini index 43d20fa990..1baaa68787 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini @@ -49,10 +49,10 @@ vpc_security_group_id = sg-0fa8d8e11dc6e9491 [cluster slurm-sit-full] key_name = test-key -base_os = centos7 scheduler = slurm master_instance_type = t2.large master_root_volume_size = 30 +base_os = centos7 compute_root_volume_size = 30 proxy_server = proxy ec2_iam_role = role diff --git a/cloudformation/aws-parallelcluster.cfn.json b/cloudformation/aws-parallelcluster.cfn.json index b287b0315a..3b27045bdd 100644 --- a/cloudformation/aws-parallelcluster.cfn.json +++ b/cloudformation/aws-parallelcluster.cfn.json @@ -9,13 +9,11 @@ "MasterInstanceType": { "Description": "MasterServer EC2 instance type", "Type": "String", - "Default": "t2.micro", "ConstraintDescription": "Must be a valid EC2 instance type, with support for HVM." }, "ComputeInstanceType": { "Description": "ComputeFleet EC2 instance type", "Type": "String", - "Default": "t2.micro", "ConstraintDescription": "Must be a valid EC2 instance type, with support for HVM." }, "MinSize": { diff --git a/tests/integration-tests/configs/common/common.yaml b/tests/integration-tests/configs/common/common.yaml index 2c3196c14b..fb6b5f59a6 100644 --- a/tests/integration-tests/configs/common/common.yaml +++ b/tests/integration-tests/configs/common/common.yaml @@ -69,6 +69,11 @@ configure: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["alinux2"] schedulers: ["slurm"] + test_pcluster_configure.py::test_region_without_t2micro: + dimensions: + - regions: ["eu-north-1"] # must be regions that do not have t2.micro + oss: ["centos7"] + schedulers: ["slurm"] create: test_create.py::test_create_wrong_os: dimensions: diff --git a/tests/integration-tests/tests/configure/test_pcluster_configure.py b/tests/integration-tests/tests/configure/test_pcluster_configure.py index 6eef0a28eb..1f7745e359 100644 --- a/tests/integration-tests/tests/configure/test_pcluster_configure.py +++ b/tests/integration-tests/tests/configure/test_pcluster_configure.py @@ -41,7 +41,6 @@ def test_pcluster_configure( vpc_stack.cfn_outputs["VpcId"], vpc_stack.cfn_outputs["PublicSubnetId"], vpc_stack.cfn_outputs["PrivateSubnetId"], - vpc_stack, ) assert_configure_workflow(region, stages, config_path) assert_config_contains_expected_values( @@ -92,7 +91,6 @@ def test_pcluster_configure_avoid_bad_subnets( # and use the first subnet in the remaining list of subnets "", "", - vpc_stack, omitted_subnets_num=1, ) assert_configure_workflow(region, stages, config_path) @@ -109,6 +107,44 @@ def test_pcluster_configure_avoid_bad_subnets( ) +def test_region_without_t2micro( + vpc_stack, + pcluster_config_reader, + key_name, + region, + os, + scheduler, + test_datadir, +): + """ + Verify the default instance type (free tier) is retrieved dynamically according to region. + In other words, t3.micro is retrieved when the region does not contain t2.micro + """ + config_path = test_datadir / "config.ini" + stages = orchestrate_pcluster_configure_stages( + region, + key_name, + scheduler, + os, + "", + vpc_stack.cfn_outputs["VpcId"], + vpc_stack.cfn_outputs["PublicSubnetId"], + vpc_stack.cfn_outputs["PrivateSubnetId"], + ) + assert_configure_workflow(region, stages, config_path) + assert_config_contains_expected_values( + region, + key_name, + scheduler, + os, + "", + vpc_stack.cfn_outputs["VpcId"], + vpc_stack.cfn_outputs["PublicSubnetId"], + vpc_stack.cfn_outputs["PrivateSubnetId"], + config_path, + ) + + def skip_if_unsupported_test_options_were_used(request): unsupported_options = get_unsupported_test_runner_options(request) if unsupported_options: @@ -205,7 +241,6 @@ def orchestrate_pcluster_configure_stages( vpc_id, headnode_subnet_id, compute_subnet_id, - vpc_stack, omitted_subnets_num=0, ): compute_units = "vcpus" if scheduler == "awsbatch" else "instances" @@ -220,8 +255,8 @@ def orchestrate_pcluster_configure_stages( {"prompt": r"Operating System \[alinux2\]: ", "response": os, "skip_for_batch": True}, {"prompt": fr"Minimum cluster size \({compute_units}\) \[0\]: ", "response": "1"}, {"prompt": fr"Maximum cluster size \({compute_units}\) \[10\]: ", "response": ""}, - {"prompt": r"Master instance type \[t2\.micro\]: ", "response": instance}, - {"prompt": r"Compute instance type \[t2\.micro\]: ", "response": instance, "skip_for_batch": True}, + {"prompt": r"Master instance type \[t.\.micro\]: ", "response": instance}, + {"prompt": r"Compute instance type \[t.\.micro\]: ", "response": instance, "skip_for_batch": True}, {"prompt": r"Automate VPC creation\? \(y/n\) \[n\]: ", "response": "n"}, {"prompt": r"VPC ID \[vpc-.+\]: ", "response": vpc_id}, {"prompt": r"Automate Subnet creation\? \(y/n\) \[y\]: ", "response": "n"},