diff --git a/CHANGELOG.md b/CHANGELOG.md index dd17a5f52a..1c1e25cb86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ CHANGELOG - Pull Amazon Linux Docker images from ECR when building docker image for `awsbatch` scheduler. This only applies to images built for `x86` architecture. - Use inclusive language in user facing messages and internal naming convention. +- Change the default of instance types from the hardcoded `t2.micro` to the free tier instance type + (`t2.micro` or `t3.micro` dependent on region). In regions without free tier, the default is `t3.micro`. + **BUG FIXES** diff --git a/cli/src/pcluster/config/cfn_param_types.py b/cli/src/pcluster/config/cfn_param_types.py index 3c7bf976c9..dfdb7f3816 100644 --- a/cli/src/pcluster/config/cfn_param_types.py +++ b/cli/src/pcluster/config/cfn_param_types.py @@ -24,6 +24,7 @@ error, get_availability_zone_of_subnet, get_cfn_param, + get_default_instance_type, get_ebs_snapshot_info, get_efs_mount_target_id, get_file_section_name, @@ -854,6 +855,34 @@ def to_cfn(self): return cfn_params +class ComputeInstanceTypeCfnParam(CfnParam): + """ + Class to manage the compute instance type parameter. + + We need this class in order to set the default instance type from a boto3 call. + """ + + def refresh(self): + """Get default value from a boto3 call for free tier instance type.""" + if not self.value: + scheduler = self.pcluster_config.get_section("cluster").get_param_value("scheduler") + if scheduler: + self.value = "optimal" if scheduler == "awsbatch" else get_default_instance_type() + + +class HeadNodeInstanceTypeCfnParam(CfnParam): + """ + Class to manage the head node instance type parameter. + + We need this class in order to set the default instance type from a boto3 call. + """ + + def refresh(self): + """Get default value from a boto3 call for free tier instance type.""" + if not self.value: + self.value = get_default_instance_type() + + class TagsParam(JsonCfnParam): """ Class to manage the tags json configuration parameter. diff --git a/cli/src/pcluster/config/mappings.py b/cli/src/pcluster/config/mappings.py index 2cf7221d79..768ec3255b 100644 --- a/cli/src/pcluster/config/mappings.py +++ b/cli/src/pcluster/config/mappings.py @@ -21,12 +21,14 @@ ClusterCfnSection, ClusterConfigMetadataCfnParam, ComputeAvailabilityZoneCfnParam, + ComputeInstanceTypeCfnParam, DisableHyperThreadingCfnParam, EBSSettingsCfnParam, EFSCfnSection, ExtraJsonCfnParam, FloatCfnParam, HeadNodeAvailabilityZoneCfnParam, + HeadNodeInstanceTypeCfnParam, IntCfnParam, MaintainInitialSizeCfnParam, NetworkInterfacesCountCfnParam, @@ -753,14 +755,6 @@ "validators": [ec2_key_pair_validator], "update_policy": UpdatePolicy.UNSUPPORTED }), - ("base_os", { - "type": BaseOSCfnParam, - "cfn_param_mapping": "BaseOS", - "allowed_values": ["alinux", "alinux2", "ubuntu1604", "ubuntu1804", "centos7", "centos8"], - "validators": [base_os_validator, architecture_os_validator], - "required": True, - "update_policy": UpdatePolicy.UNSUPPORTED - }), ("scheduler", { "cfn_param_mapping": "Scheduler", "allowed_values": ["awsbatch", "sge", "slurm", "torque"], @@ -770,7 +764,7 @@ }), # Head node ("master_instance_type", { - "default": "t2.micro", + "type": HeadNodeInstanceTypeCfnParam, "cfn_param_mapping": "MasterInstanceType", "validators": [head_node_instance_type_validator, ec2_instance_type_validator], "update_policy": UpdatePolicy.UNSUPPORTED, @@ -786,6 +780,14 @@ action_needed=UpdatePolicy.ACTIONS_NEEDED["ebs_volume_update"] ) }), + ("base_os", { + "type": BaseOSCfnParam, + "cfn_param_mapping": "BaseOS", + "allowed_values": ["alinux", "alinux2", "ubuntu1604", "ubuntu1804", "centos7", "centos8"], + "validators": [base_os_validator, architecture_os_validator], + "required": True, + "update_policy": UpdatePolicy.UNSUPPORTED + }), # Compute fleet ("compute_root_volume_size", { "type": IntCfnParam, @@ -1024,9 +1026,7 @@ }), # Compute fleet ("compute_instance_type", { - "default": - lambda section: - "optimal" if section and section.get_param_value("scheduler") == "awsbatch" else "t2.micro", + "type": ComputeInstanceTypeCfnParam, "cfn_param_mapping": "ComputeInstanceType", "validators": [compute_instance_type_validator, instances_architecture_compatibility_validator], "update_policy": UpdatePolicy.COMPUTE_FLEET_STOP diff --git a/cli/src/pcluster/config/param_types.py b/cli/src/pcluster/config/param_types.py index ce1e425f34..f6db94ad2b 100644 --- a/cli/src/pcluster/config/param_types.py +++ b/cli/src/pcluster/config/param_types.py @@ -13,6 +13,7 @@ import re import sys from abc import abstractmethod +from collections import OrderedDict from enum import Enum from configparser import NoSectionError @@ -441,7 +442,7 @@ def __init__(self, section_definition, pcluster_config, section_label=None, pare self.parent_section = parent_section # initialize section parameters with default values - self.params = {} + self.params = OrderedDict({}) self._from_definition() @property diff --git a/cli/src/pcluster/configure/easyconfig.py b/cli/src/pcluster/configure/easyconfig.py index ac707c3120..18720ecd6f 100644 --- a/cli/src/pcluster/configure/easyconfig.py +++ b/cli/src/pcluster/configure/easyconfig.py @@ -34,6 +34,7 @@ from pcluster.configure.utils import get_regions, get_resource_tag, handle_client_exception, prompt, prompt_iterable from pcluster.utils import ( error, + get_default_instance_type, get_region, get_supported_az_for_multi_instance_types, get_supported_az_for_one_instance_type, @@ -122,7 +123,7 @@ def configure(args): if pcluster_config.cluster_model == ClusterModel.HIT: error( "Configuration in file {0} cannot be overwritten. Please specify a different file path".format( - args.config_file + pcluster_config.config_file ) ) @@ -367,16 +368,22 @@ def prompt_os(self): def prompt_instance_types(self): """Ask for head_node_instance_type and compute_instance_type (if necessary).""" + default_head_node_instance_type = self.cluster_section.get_param_value("master_instance_type") + if not default_head_node_instance_type: + default_head_node_instance_type = get_default_instance_type() self.head_node_instance_type = prompt( "Head node instance type", lambda x: _is_instance_type_supported_for_head_node(x) and x in get_supported_instance_types(), - default_value=self.cluster_section.get_param_value("master_instance_type"), + default_value=default_head_node_instance_type, ) if not self.is_aws_batch: + default_compute_instance_type = self.cluster_section.get_param_value("compute_instance_type") + if not default_compute_instance_type: + default_compute_instance_type = get_default_instance_type() self.compute_instance_type = prompt( "Compute instance type", lambda x: x in get_supported_compute_instance_types(self.scheduler), - default_value=self.cluster_section.get_param_value("compute_instance_type"), + default_value=default_compute_instance_type, ) # Cache availability zones offering the selected instance type(s) for later use self.cache_qualified_az() diff --git a/cli/src/pcluster/createami.py b/cli/src/pcluster/createami.py index 3d034fd7c6..b0e3e8cf18 100644 --- a/cli/src/pcluster/createami.py +++ b/cli/src/pcluster/createami.py @@ -358,12 +358,3 @@ def create_ami(args): _print_create_ami_results(results) if "tmp_dir" in locals() and tmp_dir: rmtree(tmp_dir) - - -def _get_default_template_url(region): - return ( - "https://{REGION}-aws-parallelcluster.s3.{REGION}.amazonaws.com{SUFFIX}/templates/" - "aws-parallelcluster-{VERSION}.cfn.json".format( - REGION=region, SUFFIX=".cn" if region.startswith("cn") else "", VERSION=utils.get_installed_version() - ) - ) diff --git a/cli/src/pcluster/examples/config b/cli/src/pcluster/examples/config index 4179a54f56..f3494474ba 100644 --- a/cli/src/pcluster/examples/config +++ b/cli/src/pcluster/examples/config @@ -24,10 +24,10 @@ key_name = mykey # (defaults to https://s3.amazonaws.com/-aws-parallelcluster/templates/aws-parallelcluster-.cfn.json) #template_url = https://s3.amazonaws.com/-aws-parallelcluster/templates/aws-parallelcluster-.cfn.json # EC2 instance type for head node -# (defaults to t2.micro) +# (defaults to the free tier instance type of the region. If the region does not have free tier, default to t3.micro) #master_instance_type = t2.micro # EC2 instance type for compute nodes -# (defaults to t2.micro , 'optimal' when scheduler is awsbatch) +# (defaults to the free tier instance type of the region, 'optimal' when scheduler is awsbatch) #compute_instance_type = t2.micro # Initial number of EC2 instances to launch as compute nodes in the cluster for schedulers other than awsbatch. # (defaults to 2) diff --git a/cli/src/pcluster/utils.py b/cli/src/pcluster/utils.py index 62d7ba49cc..00844e2fb2 100644 --- a/cli/src/pcluster/utils.py +++ b/cli/src/pcluster/utils.py @@ -1170,6 +1170,26 @@ def get_ebs_snapshot_info(ebs_snapshot_id, raise_exceptions=False): ) +def get_default_instance_type(): + """If current region support free tier, return the free tier instance type. Otherwise, return t3.micro .""" + if not hasattr(get_default_instance_type, "cache"): + get_default_instance_type.cache = {} + cache = get_default_instance_type.cache + region = os.environ.get("AWS_DEFAULT_REGION") + if region not in cache: + free_tier_instance_type = [] + for page in paginate_boto3( + boto3.client("ec2").describe_instance_types, + Filters=[ + {"Name": "free-tier-eligible", "Values": ["true"]}, + {"Name": "current-generation", "Values": ["true"]}, + ], + ): + free_tier_instance_type.append(page) + cache[region] = free_tier_instance_type[0]["InstanceType"] if free_tier_instance_type else "t3.micro" + return cache[region] + + class Cache: """Simple utility class providing a cache mechanism for expensive functions.""" diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index e1c4fbc8f3..0cad54ed55 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -19,6 +19,20 @@ def clear_env(): del os.environ["AWS_DEFAULT_REGION"] +@pytest.fixture(autouse=True) +def mock_default_instance(mocker, request): + """ + Mock get_default_instance_type for all tests. + + To disable the mock for certain tests, add annotation `@pytest.mark.nomockdefaultinstance` to the tests. + To disable the mock for an entire file, declare global var `pytestmark = pytest.mark.noassertnopendingresponses` + """ + if "nomockdefaultinstance" in request.keywords: + # skip mocking + return + mocker.patch("pcluster.config.cfn_param_types.get_default_instance_type", return_value="t2.micro") + + @pytest.fixture def failed_with_message(capsys): """Assert that the command exited with a specific error message.""" diff --git a/cli/tests/pcluster/config/defaults.py b/cli/tests/pcluster/config/defaults.py index 33e8bab722..73d0e4cae4 100644 --- a/cli/tests/pcluster/config/defaults.py +++ b/cli/tests/pcluster/config/defaults.py @@ -99,9 +99,9 @@ "shared_dir": "/shared", "placement_group": None, "placement": "compute", - "master_instance_type": "t2.micro", + "master_instance_type": None, "master_root_volume_size": 25, - "compute_instance_type": "t2.micro", + "compute_instance_type": None, "compute_root_volume_size": 25, "initial_queue_size": 0, "max_queue_size": 10, @@ -155,7 +155,7 @@ "base_os": None, # base_os does not have a default, but this is here to make testing easier "scheduler": None, # The cluster does not have a default, but this is here to make testing easier "shared_dir": "/shared", - "master_instance_type": "t2.micro", + "master_instance_type": None, "master_root_volume_size": 25, "compute_root_volume_size": 25, "proxy_server": None, diff --git a/cli/tests/pcluster/config/test_section_cluster.py b/cli/tests/pcluster/config/test_section_cluster.py index c94840063c..b6f293767c 100644 --- a/cli/tests/pcluster/config/test_section_cluster.py +++ b/cli/tests/pcluster/config/test_section_cluster.py @@ -38,6 +38,8 @@ "base_os": "alinux2", "scheduler": "slurm", "cluster_config_metadata": {"sections": {"cluster": ["custom_cluster_label"]}}, + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "custom_cluster_label", @@ -50,6 +52,8 @@ "additional_iam_policies": ["arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"], "base_os": "alinux2", "scheduler": "slurm", + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "default", @@ -86,6 +90,8 @@ "arn:aws:iam::aws:policy/AWSBatchFullAccess", "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy", ], + "master_instance_type": "t2.micro", + "compute_instance_type": "t2.micro", }, ), "default", @@ -267,7 +273,6 @@ def test_hit_cluster_section_from_file(mocker, config_parser_dict, expected_dict ("placement", "cluster", "cluster", None), # Head node # TODO add regex for master_instance_type - ("master_instance_type", None, "t2.micro", None), ("master_instance_type", "", "", None), ("master_instance_type", "test", "test", None), ("master_instance_type", "NONE", "NONE", None), @@ -281,7 +286,6 @@ def test_hit_cluster_section_from_file(mocker, config_parser_dict, expected_dict ("master_root_volume_size", "31", 31, None), # Compute fleet # TODO add regex for compute_instance_type - ("compute_instance_type", None, "t2.micro", None), ("compute_instance_type", "", "", None), ("compute_instance_type", "test", "test", None), ("compute_instance_type", "NONE", "NONE", None), @@ -543,7 +547,6 @@ def test_sit_cluster_param_from_file( ("shared_dir", "NONE", "NONE", None), # NONE is evaluated as a valid path # Head node # TODO add regex for master_instance_type - ("master_instance_type", None, "t2.micro", None), ("master_instance_type", "", "", None), ("master_instance_type", "test", "test", None), ("master_instance_type", "NONE", "NONE", None), @@ -801,6 +804,9 @@ def test_sit_cluster_section_to_file(mocker, section_dict, expected_config_parse def test_cluster_section_to_cfn( mocker, cluster_section_definition, section_dict, expected_cfn_params, default_threads_per_core ): + section_dict["master_instance_type"] = "t2.micro" + if cluster_section_definition == CLUSTER_SIT: + section_dict["compute_instance_type"] = "t2.micro" utils.set_default_values_for_required_cluster_section_params(section_dict) utils.mock_pcluster_config(mocker) mocker.patch("pcluster.config.cfn_param_types.get_efs_mount_target_id", return_value="valid_mount_target_id") diff --git a/cli/tests/pcluster/config/test_utils.py b/cli/tests/pcluster/config/test_utils.py new file mode 100644 index 0000000000..88ae99f255 --- /dev/null +++ b/cli/tests/pcluster/config/test_utils.py @@ -0,0 +1,59 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest +from assertpy import assert_that + +from pcluster.utils import get_default_instance_type +from tests.common import MockedBoto3Request + + +@pytest.fixture() +def boto3_stubber_path(): + return "pcluster.utils.boto3" + + +@pytest.mark.parametrize( + "region, free_tier_instance_type, default_instance_type, stub_boto3", + [ + ("us-east-1", "t2.micro", "t2.micro", True), + ("eu-north-1", "t3.micro", "t3.micro", True), + ("us-gov-east-1", None, "t3.micro", True), + # Retrieving free tier instance type again should use cache to reduce boto3 call + ("us-east-1", "t2.micro", "t2.micro", False), + ("eu-north-1", "t3.micro", "t3.micro", False), + ("us-gov-east-1", None, "t3.micro", False), + ], +) +@pytest.mark.nomockdefaultinstance +def test_get_default_instance(boto3_stubber, region, free_tier_instance_type, default_instance_type, stub_boto3): + os.environ["AWS_DEFAULT_REGION"] = region + if free_tier_instance_type: + response = {"InstanceTypes": [{"InstanceType": free_tier_instance_type}]} + else: + response = {"InstanceTypes": []} + if stub_boto3: + mocked_requests = [ + MockedBoto3Request( + method="describe_instance_types", + response=response, + expected_params={ + "Filters": [ + {"Name": "free-tier-eligible", "Values": ["true"]}, + {"Name": "current-generation", "Values": ["true"]}, + ] + }, + ) + ] + + boto3_stubber("ec2", mocked_requests) + assert_that(get_default_instance_type()).is_equal_to(default_instance_type) diff --git a/cli/tests/pcluster/configure/test_pcluster_configure.py b/cli/tests/pcluster/configure/test_pcluster_configure.py index 5da9455901..cb4b43b12c 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure.py +++ b/cli/tests/pcluster/configure/test_pcluster_configure.py @@ -382,6 +382,7 @@ def __init__(self, mocker, empty_region=False, partition="commercial", mock_avai _mock_list_vpcs_and_subnets(self.mocker, empty_region, partition) _mock_parallel_cluster_config(self.mocker) _mock_cache_availability_zones(self.mocker) + mocker.patch("pcluster.configure.easyconfig.get_default_instance_type", return_value="t2.micro") if mock_availability_zone: _mock_availability_zone(self.mocker) diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini index 584ef221cd..4bbcff5504 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_bad_config_file/pcluster.config.ini @@ -14,8 +14,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano #Invalid value type max_queue_size = 14 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini index e2fe3749f1..1f4800a4ca 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -6,6 +6,7 @@ key_name = key1 vpc_settings = default scheduler = torque master_instance_type = t2.nano +compute_instance_type = t2.micro max_queue_size = 14 initial_queue_size = 13 maintain_initial_size = true diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini index 95356e94e9..8c696b839e 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_no_automation_yes_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ key_name = key1 base_os = alinux2 vpc_settings = default scheduler = awsbatch -# Implied value -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini index 639f3e57a5..48e4bf5524 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_region_env_overwrite_region_config/pcluster.config.ini @@ -6,6 +6,8 @@ key_name = key3 vpc_settings = default scheduler = torque base_os = alinux2 +master_instance_type = t2.micro +compute_instance_type = t2.micro [vpc default] vpc_id = vpc-12345678 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini index d00170d6f6..f663bc15fc 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini index fc7f4ed2b3..bd2b285434 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_empty_vpc/pcluster.config.ini @@ -6,8 +6,7 @@ key_name = key1 vpc_settings = default scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini index fff2041ee2..a401a06e00 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_no_awsbatch_no_errors_with_config_file/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini index 59256eeabd..170494afc4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_subnet_automation_yes_awsbatch_invalid_vpc/pcluster.config.ini @@ -5,9 +5,8 @@ aws_region_name = eu-west-1 key_name = key1 vpc_settings = default scheduler = awsbatch -# Implied value base_os = alinux2 -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini index d00170d6f6..f663bc15fc 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_no_awsbatch_no_errors/pcluster.config.ini @@ -7,8 +7,7 @@ vpc_settings = default # Implied value scheduler = sge base_os = centos7 -# Implied value -# compute_instance_type = t2.micro +compute_instance_type = t2.micro master_instance_type = t2.nano max_queue_size = 14 initial_queue_size = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini index 59256eeabd..170494afc4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_vpc_automation_yes_awsbatch_no_errors/pcluster.config.ini @@ -5,9 +5,8 @@ aws_region_name = eu-west-1 key_name = key1 vpc_settings = default scheduler = awsbatch -# Implied value base_os = alinux2 -# compute_instance_type = optimal +compute_instance_type = optimal master_instance_type = t2.nano max_vcpus = 14 min_vcpus = 13 diff --git a/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini b/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini index d80f4a8a9f..9b38fa7ec4 100644 --- a/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini +++ b/cli/tests/pcluster/configure/test_pcluster_configure/test_with_region_arg_with_config_file/pcluster.config.ini @@ -14,6 +14,7 @@ key_name = key1 base_os = alinux scheduler = torque master_instance_type = t2.nano +compute_instance_type = t2.micro vpc_settings = default initial_queue_size = 13 max_queue_size = 14 diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini index 78222099bb..4aa0eafb7c 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_full/expected_output.ini @@ -49,10 +49,10 @@ vpc_security_group_id = sg-0fa8d8e11dc6e9491 [cluster slurm-sit-full] key_name = test-key -base_os = centos7 scheduler = slurm master_instance_type = t2.large master_root_volume_size = 30 +base_os = centos7 compute_root_volume_size = 30 proxy_server = proxy ec2_iam_role = role diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini index e27b63153c..909995b851 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_sit_simple/expected_output.ini @@ -15,9 +15,9 @@ sanity_check = true [cluster slurm-sit-simple] key_name = test -base_os = centos7 scheduler = slurm master_instance_type = c5.2xlarge +base_os = centos7 shared_dir = /test vpc_settings = public additional_iam_policies = arn:aws:iam::aws:policy/CloudWatchFullAccess diff --git a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini index 43d20fa990..1baaa68787 100644 --- a/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini +++ b/cli/tests/pcluster_config/test_pcluster_config_convert/test_slurm_unrelated_sections/expected_output.ini @@ -49,10 +49,10 @@ vpc_security_group_id = sg-0fa8d8e11dc6e9491 [cluster slurm-sit-full] key_name = test-key -base_os = centos7 scheduler = slurm master_instance_type = t2.large master_root_volume_size = 30 +base_os = centos7 compute_root_volume_size = 30 proxy_server = proxy ec2_iam_role = role diff --git a/cloudformation/aws-parallelcluster.cfn.json b/cloudformation/aws-parallelcluster.cfn.json index 61f9f5dd34..e71999af29 100644 --- a/cloudformation/aws-parallelcluster.cfn.json +++ b/cloudformation/aws-parallelcluster.cfn.json @@ -9,13 +9,11 @@ "MasterInstanceType": { "Description": "Head node EC2 instance type", "Type": "String", - "Default": "t2.micro", "ConstraintDescription": "Must be a valid EC2 instance type, with support for HVM." }, "ComputeInstanceType": { "Description": "ComputeFleet EC2 instance type", "Type": "String", - "Default": "t2.micro", "ConstraintDescription": "Must be a valid EC2 instance type, with support for HVM." }, "MinSize": { diff --git a/tests/integration-tests/configs/common/common.yaml b/tests/integration-tests/configs/common/common.yaml index f51be906e7..d30339c18f 100644 --- a/tests/integration-tests/configs/common/common.yaml +++ b/tests/integration-tests/configs/common/common.yaml @@ -69,6 +69,11 @@ configure: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["alinux2"] schedulers: ["slurm"] + test_pcluster_configure.py::test_region_without_t2micro: + dimensions: + - regions: ["eu-north-1"] # must be regions that do not have t2.micro + oss: ["centos7"] + schedulers: ["slurm"] create: test_create.py::test_create_wrong_os: dimensions: diff --git a/tests/integration-tests/tests/configure/test_pcluster_configure.py b/tests/integration-tests/tests/configure/test_pcluster_configure.py index 0fa7bd180c..bc00100247 100644 --- a/tests/integration-tests/tests/configure/test_pcluster_configure.py +++ b/tests/integration-tests/tests/configure/test_pcluster_configure.py @@ -41,7 +41,6 @@ def test_pcluster_configure( vpc_stack.cfn_outputs["VpcId"], vpc_stack.cfn_outputs["PublicSubnetId"], vpc_stack.cfn_outputs["PrivateSubnetId"], - vpc_stack, ) assert_configure_workflow(region, stages, config_path) assert_config_contains_expected_values( @@ -92,7 +91,6 @@ def test_pcluster_configure_avoid_bad_subnets( # and use the first subnet in the remaining list of subnets "", "", - vpc_stack, omitted_subnets_num=1, ) assert_configure_workflow(region, stages, config_path) @@ -109,6 +107,44 @@ def test_pcluster_configure_avoid_bad_subnets( ) +def test_region_without_t2micro( + vpc_stack, + pcluster_config_reader, + key_name, + region, + os, + scheduler, + test_datadir, +): + """ + Verify the default instance type (free tier) is retrieved dynamically according to region. + In other words, t3.micro is retrieved when the region does not contain t2.micro + """ + config_path = test_datadir / "config.ini" + stages = orchestrate_pcluster_configure_stages( + region, + key_name, + scheduler, + os, + "", + vpc_stack.cfn_outputs["VpcId"], + vpc_stack.cfn_outputs["PublicSubnetId"], + vpc_stack.cfn_outputs["PrivateSubnetId"], + ) + assert_configure_workflow(region, stages, config_path) + assert_config_contains_expected_values( + region, + key_name, + scheduler, + os, + "", + vpc_stack.cfn_outputs["VpcId"], + vpc_stack.cfn_outputs["PublicSubnetId"], + vpc_stack.cfn_outputs["PrivateSubnetId"], + config_path, + ) + + def skip_if_unsupported_test_options_were_used(request): unsupported_options = get_unsupported_test_runner_options(request) if unsupported_options: @@ -205,7 +241,6 @@ def orchestrate_pcluster_configure_stages( vpc_id, headnode_subnet_id, compute_subnet_id, - vpc_stack, omitted_subnets_num=0, ): compute_units = "vcpus" if scheduler == "awsbatch" else "instances" @@ -220,8 +255,8 @@ def orchestrate_pcluster_configure_stages( {"prompt": r"Operating System \[alinux2\]: ", "response": os, "skip_for_batch": True}, {"prompt": fr"Minimum cluster size \({compute_units}\) \[0\]: ", "response": "1"}, {"prompt": fr"Maximum cluster size \({compute_units}\) \[10\]: ", "response": ""}, - {"prompt": r"Head node instance type \[t2\.micro\]: ", "response": instance}, - {"prompt": r"Compute instance type \[t2\.micro\]: ", "response": instance, "skip_for_batch": True}, + {"prompt": r"Head node instance type \[t.\.micro\]: ", "response": instance}, + {"prompt": r"Compute instance type \[t.\.micro\]: ", "response": instance, "skip_for_batch": True}, {"prompt": r"Automate VPC creation\? \(y/n\) \[n\]: ", "response": "n"}, {"prompt": r"VPC ID \[vpc-.+\]: ", "response": vpc_id}, {"prompt": r"Automate Subnet creation\? \(y/n\) \[y\]: ", "response": "n"},